From a7f93cec7999de04614d7adaf91c4348a3313e2e Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 29 Oct 2017 14:28:12 -0200 Subject: [benchmarks] add some orchestration to scalability tests --- .../scalability/test_controller/client/makefile | 101 ++++++++++++++++++--- .../scalability/test_controller/requirements.pip | 2 - .../scalability/test_controller/server/server.tac | 30 +++++- .../scalability/test_controller/server/user_dbs.py | 23 ++--- .../scalability/test_controller/server/utils.py | 22 +++++ 5 files changed, 146 insertions(+), 32 deletions(-) delete mode 100644 scripts/scalability/test_controller/requirements.pip create mode 100644 scripts/scalability/test_controller/server/utils.py (limited to 'scripts/scalability/test_controller') diff --git a/scripts/scalability/test_controller/client/makefile b/scripts/scalability/test_controller/client/makefile index 33e67e25..e0a719a0 100644 --- a/scripts/scalability/test_controller/client/makefile +++ b/scripts/scalability/test_controller/client/makefile @@ -1,19 +1,90 @@ -bench: upload download +# test_controller/client/makefile +# =============================== +# +# This file is part of the client-side infrastructure for Server Scalability +# Tests. The client-side is responsible for orchestrating the benchmarking by +# setting up server resources needed for the tests. +# +# What this file does +# ------------------- +# +# This makefile knows how to: +# +# - Start and stop server-side system resource monitoring. +# +# - Setup server-side resources for tests (user dbs, access tokens and blobs). +# +# - Orchestrate test runs using the client-side machinery. +# +# - Generate reports and save test results. +# +# Running tests +# ------------- +# +# To run tests, make sure a Test Controller Server is reachable at $(URI) and +# run `make`. -upload: - curl -X POST "http://127.0.0.1:7001/blobs?action=delete" - fl-run-bench -f test_Blobs.py Blobs.test_upload - fl-build-report --html results/blobs-bench.xml -download: - curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10&amount=5000" - fl-run-bench -f test_Blobs.py Blobs.test_download - fl-build-report --html results/blobs-bench.xml +URI = https://giraffe.cdev.bitmask.net:7001 +CREATE = 5000 -test_upload: - curl -X POST "http://127.0.0.1:7001/blobs?action=delete" - fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload -test_download: - curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10" - fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download +#------------------# +# main test target # +#------------------# + +test: bench + +bench: bench-cpu bench-mem + +bench-cpu: start-cpu bench-upload bench-download stop-cpu + +bench-mem: start-mem bench-upload bench-download stop-mem + +bench-upload: create-users delete-blobs + fl-run-bench -f test_Blobs.py Blobs.test_upload + fl-build-report --html results/blobs-bench.xml + +bench-download: create-users create-blobs + fl-run-bench -f test_Blobs.py Blobs.test_download + fl-build-report --html results/blobs-bench.xml + +# for quick tests only +bench-upload-quick: create-users delete-blobs + curl -X POST "$(URI)/blobs?action=delete" + fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload + +# for quick tests only +bench-download-quick: create-users create-blobs + curl -X POST "$(URI)/blobs?action=create&size=10" + fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download + +#---------------------# +# resource monitoring # +#---------------------# + +start-cpu: + curl -X POST $(URI)/cpu + +stop-cpu: + curl -X GET $(URI)/cpu + +start-mem: + curl -X POST $(URI)/mem + +stop-mem: + curl -X GET $(URI)/mem + + +#--------------# +# server setup # +#--------------# + +create-users: + curl -X POST $(URI)/users?create=$(CREATE) + +delete-blobs: + curl -X POST "$(URI)/blobs?action=delete" + +create-blobs: + curl -X POST "$(URI)/blobs?action=create&size=10&amount=5000" diff --git a/scripts/scalability/test_controller/requirements.pip b/scripts/scalability/test_controller/requirements.pip deleted file mode 100644 index 224b581e..00000000 --- a/scripts/scalability/test_controller/requirements.pip +++ /dev/null @@ -1,2 +0,0 @@ -psutil -twisted diff --git a/scripts/scalability/test_controller/server/server.tac b/scripts/scalability/test_controller/server/server.tac index d5176319..72b1df10 100644 --- a/scripts/scalability/test_controller/server/server.tac +++ b/scripts/scalability/test_controller/server/server.tac @@ -49,6 +49,7 @@ from twisted.logger import Logger from test_controller.server.user_dbs import ensure_dbs from test_controller.server.blobs import create_blobs from test_controller.server.blobs import delete_blobs +from test_controller.server.utils import get_soledad_server_pid DEFAULT_HTTP_PORT = 7001 @@ -62,6 +63,9 @@ logger = Logger(__name__) # class ResourceWatcher(object): + """ + A generic system resource watcher. Real watchers inherit from this class. + """ def __init__(self, pid): logger.info('%s started for process with PID %d' @@ -95,6 +99,9 @@ class ResourceWatcher(object): class CpuWatcher(ResourceWatcher): + """ + A watcher that trackes cpu usage of a process. + """ interval = 1 @@ -118,6 +125,9 @@ def _std(l): class MemoryWatcher(ResourceWatcher): + """ + A watcher that tracks the memory usage of a process. + """ interval = 0.1 @@ -161,7 +171,9 @@ class InvalidPidError(Exception): class MonitorResource(resource.Resource): """ - A generic resource-monitor web resource. + A generic web resource to monitor system resources. Which system resource + it allows monitoring depends on the watcher class passed to the + constructor. """ isLeaf = 1 @@ -173,7 +185,7 @@ class MonitorResource(resource.Resource): def _get_pid(self, request): if 'pid' not in request.args: - raise MissingPidError() + return get_soledad_server_pid() pid = request.args['pid'].pop() if not pid.isdigit(): raise InvalidPidError(pid) @@ -206,7 +218,11 @@ class MonitorResource(resource.Resource): self.watcher.stop() -class SetupResource(resource.Resource): +class UsersResource(resource.Resource): + """ + Allow controlling the creation of server-side user databases and access + tokens. + """ def render_POST(self, request): create = request.args.get('create') or [1000] @@ -228,6 +244,9 @@ class SetupResource(resource.Resource): class BlobsResource(resource.Resource): + """ + Allow controlling the creation and deletion of server-side blobs. + """ def render_POST(self, request): action = (request.args.get('action') or ['create']).pop() @@ -255,12 +274,15 @@ class BlobsResource(resource.Resource): class Root(resource.Resource): + """ + The root resource that gives access to all others. + """ def __init__(self): resource.Resource.__init__(self) self.putChild('mem', MonitorResource(MemoryWatcher)) self.putChild('cpu', MonitorResource(CpuWatcher)) - self.putChild('setup', SetupResource()) + self.putChild('users', UsersResource()) self.putChild('blobs', BlobsResource()) diff --git a/scripts/scalability/test_controller/server/user_dbs.py b/scripts/scalability/test_controller/server/user_dbs.py index 70fbd96d..c1012cc0 100755 --- a/scripts/scalability/test_controller/server/user_dbs.py +++ b/scripts/scalability/test_controller/server/user_dbs.py @@ -15,6 +15,7 @@ from urlparse import urljoin from twisted.internet import reactor, defer from twisted.logger import Logger + COUCH_URL = "http://127.0.0.1:5984" CREATE = 1000 @@ -56,10 +57,10 @@ def _req(method, *args, **kwargs): @defer.inlineCallbacks -def delete_dbs(dbs): +def delete_dbs(couch_url, dbs): deferreds = [] for db in dbs: - d = semaphore.run(_req, 'delete', urljoin(COUCH_URL, db)) + d = semaphore.run(_req, 'delete', urljoin(couch_url, db)) logfun = partial(_log, 'table', db, 'deleted') d.addCallback(logfun) deferreds.append(d) @@ -69,10 +70,10 @@ def delete_dbs(dbs): @defer.inlineCallbacks -def create_dbs(dbs): +def create_dbs(couch_url, dbs): deferreds = [] for db in dbs: - d = semaphore.run(_req, 'put', urljoin(COUCH_URL, db)) + d = semaphore.run(_req, 'put', urljoin(couch_url, db)) logfun = partial(_log, 'table', db, 'created') d.addCallback(logfun) deferreds.append(d) @@ -98,13 +99,13 @@ def _create_token(res, url, user_id): defer.returnValue(res) -def create_tokens(create): +def create_tokens(couch_url, create): deferreds = [] tokens_db = _get_tokens_db_name() for i in xrange(create): user_id = str(i) token = sha512('%s-token' % user_id).hexdigest() - url = '/'.join([COUCH_URL, tokens_db, token]) + url = '/'.join([couch_url, tokens_db, token]) d = semaphore.run(_req, 'get', url) d.addCallback(_create_token, url, user_id) logfun = partial(_log, 'token', user_id, 'created') @@ -115,15 +116,15 @@ def create_tokens(create): @defer.inlineCallbacks def ensure_dbs(couch_url=COUCH_URL, create=CREATE): - dbs = get_db_names(create) - yield delete_dbs(dbs) - yield create_dbs(dbs) - yield create_tokens(create) + # dbs = get_db_names(create) + # yield delete_dbs(couch_url, dbs) + # yield create_dbs(couch_url, dbs) + yield create_tokens(couch_url, create) @defer.inlineCallbacks def main(couch_url, create): - yield ensure_dbs(couch_url, create) + yield ensure_dbs(couch_url=couch_url, create=create) reactor.stop() diff --git a/scripts/scalability/test_controller/server/utils.py b/scripts/scalability/test_controller/server/utils.py new file mode 100644 index 00000000..a8274490 --- /dev/null +++ b/scripts/scalability/test_controller/server/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +from subprocess import check_output +from psutil import Process + +args = set([ + '/usr/bin/twistd', + '--python=/usr/lib/python2.7/dist-packages/leap/soledad/server/server.tac', +]) + + +def get_soledad_server_pid(): + output = check_output(['pidof', 'python']) + for pid in output.split(): + proc = Process(int(pid)) + cmdline = proc.cmdline() + if args.issubset(set(cmdline)): + return int(pid) + + +if __name__ == '__main__': + print get_soledad_server_pid() -- cgit v1.2.3