diff options
| -rw-r--r-- | scripts/scalability/README.rst | 36 | ||||
| -rw-r--r-- | scripts/scalability/makefile | 63 | ||||
| -rw-r--r-- | scripts/scalability/setup.py | 19 | ||||
| -rw-r--r-- | scripts/scalability/test_controller/client/makefile | 101 | ||||
| -rw-r--r-- | scripts/scalability/test_controller/requirements.pip | 2 | ||||
| -rw-r--r-- | scripts/scalability/test_controller/server/server.tac | 30 | ||||
| -rwxr-xr-x | scripts/scalability/test_controller/server/user_dbs.py | 23 | ||||
| -rw-r--r-- | scripts/scalability/test_controller/server/utils.py | 22 |
8 files changed, 252 insertions, 44 deletions
diff --git a/scripts/scalability/README.rst b/scripts/scalability/README.rst new file mode 100644 index 00000000..a2d891a7 --- /dev/null +++ b/scripts/scalability/README.rst @@ -0,0 +1,36 @@ +Blobs Server Scalability Tests +============================== + +This folder contains code for running benchmark tests for determining Soledad +Server Blobs service scalability. This consists basically in launching a number +of parallel blobs upload or download requests, to assess the number of requests +per second that the server is able to answer. + +When analyzing results, it's important to have in mind that the size of the +blobs that are transferred as well as the bandwith of both the server and the +client used for the test are resources that have high impact in the results. + +Test Controller server +---------------------- + +In the server, run the following to have an instance of the Test Controller +server running: + + make install-server + make start-server + +And, if you want to see the logs, use: + + make log + +Alternativelly, use `make start-server-nodaemon` to avoid detaching from the +terminal. + +Test Controller client +---------------------- + +Make sure an instance of the Test Controller Server is reachable at $(URI), +and run: + + make install-client + make run-test diff --git a/scripts/scalability/makefile b/scripts/scalability/makefile index 0a248d37..0067c650 100644 --- a/scripts/scalability/makefile +++ b/scripts/scalability/makefile @@ -1,14 +1,53 @@ +# Test Controller for Server Scalability Tests +# ============================================ +# +# This makefile knows how to install server and client components of the Test +# Controller, as well as to trigger a run of the benchmarks. +# +# Test Controller server +# ---------------------- +# +# In the server, run the following to have an instance of the Test Controller +# server running: +# +# make install-server +# make start-server +# +# And, if you want to see the logs, use: +# +# make log +# +# Alternativelly, use `make start-server-nodaemon` to avoid detaching from the +# terminal. +# +# Test Controller client +# ---------------------- +# +# Make sure an instance of the Test Controller Server is reachable at $(URI), +# and run: +# +# make install-client +# make run-test + + +URI = https://giraffe.cdev.bitmask.net:7001 + PIDFILE = /tmp/test_controller.pid LOGFILE = /tmp/test_controller.log TACFILE = ./test_controller/server/server.tac -HTTP_PORT = 7001 -RESOURCE = cpu -CREATE = 1000 -start: + +#----------------# +# Server targets # +#----------------# + +install-server: + pip install ".[server]" + +start-server: twistd --pidfile=$(PIDFILE) --logfile=$(LOGFILE) --python=$(TACFILE) -nodaemon: +start-server-nodaemon: twistd --nodaemon --python=$(TACFILE) kill: @@ -19,11 +58,13 @@ log: restart: kill start -post: - curl -X POST http://127.0.0.1:$(HTTP_PORT)/$(RESOURCE)?pid=$(PID) -get: - curl -X GET http://127.0.0.1:$(HTTP_PORT)/$(RESOURCE) +#----------------# +# Client targets # +#----------------# + +install-client: + pip install ".[client]" -setup: - curl -X POST http://127.0.0.1:$(HTTP_PORT)/setup?create=$(CREATE) +test: + (cd test_controller/client && make test URI=$(URI)) diff --git a/scripts/scalability/setup.py b/scripts/scalability/setup.py index e76437db..66f0789a 100644 --- a/scripts/scalability/setup.py +++ b/scripts/scalability/setup.py @@ -1,9 +1,25 @@ """ -A Test Controller application. +A Scalability Test Controller application. """ from setuptools import setup, find_packages + +client = [ + 'funkload', +] + +server = [ + 'psutil', + 'twisted', + 'leap.soledad', +] + +extras = { + 'client': client, + 'server': server, +} + setup( name="test-controller", version="0.1", @@ -11,4 +27,5 @@ setup( packages=find_packages(), include_package_data=True, zip_safe=False, + extras_require=extras, ) diff --git a/scripts/scalability/test_controller/client/makefile b/scripts/scalability/test_controller/client/makefile index 33e67e25..e0a719a0 100644 --- a/scripts/scalability/test_controller/client/makefile +++ b/scripts/scalability/test_controller/client/makefile @@ -1,19 +1,90 @@ -bench: upload download +# test_controller/client/makefile +# =============================== +# +# This file is part of the client-side infrastructure for Server Scalability +# Tests. The client-side is responsible for orchestrating the benchmarking by +# setting up server resources needed for the tests. +# +# What this file does +# ------------------- +# +# This makefile knows how to: +# +# - Start and stop server-side system resource monitoring. +# +# - Setup server-side resources for tests (user dbs, access tokens and blobs). +# +# - Orchestrate test runs using the client-side machinery. +# +# - Generate reports and save test results. +# +# Running tests +# ------------- +# +# To run tests, make sure a Test Controller Server is reachable at $(URI) and +# run `make`. -upload: - curl -X POST "http://127.0.0.1:7001/blobs?action=delete" - fl-run-bench -f test_Blobs.py Blobs.test_upload - fl-build-report --html results/blobs-bench.xml -download: - curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10&amount=5000" - fl-run-bench -f test_Blobs.py Blobs.test_download - fl-build-report --html results/blobs-bench.xml +URI = https://giraffe.cdev.bitmask.net:7001 +CREATE = 5000 -test_upload: - curl -X POST "http://127.0.0.1:7001/blobs?action=delete" - fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload -test_download: - curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10" - fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download +#------------------# +# main test target # +#------------------# + +test: bench + +bench: bench-cpu bench-mem + +bench-cpu: start-cpu bench-upload bench-download stop-cpu + +bench-mem: start-mem bench-upload bench-download stop-mem + +bench-upload: create-users delete-blobs + fl-run-bench -f test_Blobs.py Blobs.test_upload + fl-build-report --html results/blobs-bench.xml + +bench-download: create-users create-blobs + fl-run-bench -f test_Blobs.py Blobs.test_download + fl-build-report --html results/blobs-bench.xml + +# for quick tests only +bench-upload-quick: create-users delete-blobs + curl -X POST "$(URI)/blobs?action=delete" + fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload + +# for quick tests only +bench-download-quick: create-users create-blobs + curl -X POST "$(URI)/blobs?action=create&size=10" + fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download + +#---------------------# +# resource monitoring # +#---------------------# + +start-cpu: + curl -X POST $(URI)/cpu + +stop-cpu: + curl -X GET $(URI)/cpu + +start-mem: + curl -X POST $(URI)/mem + +stop-mem: + curl -X GET $(URI)/mem + + +#--------------# +# server setup # +#--------------# + +create-users: + curl -X POST $(URI)/users?create=$(CREATE) + +delete-blobs: + curl -X POST "$(URI)/blobs?action=delete" + +create-blobs: + curl -X POST "$(URI)/blobs?action=create&size=10&amount=5000" diff --git a/scripts/scalability/test_controller/requirements.pip b/scripts/scalability/test_controller/requirements.pip deleted file mode 100644 index 224b581e..00000000 --- a/scripts/scalability/test_controller/requirements.pip +++ /dev/null @@ -1,2 +0,0 @@ -psutil -twisted diff --git a/scripts/scalability/test_controller/server/server.tac b/scripts/scalability/test_controller/server/server.tac index d5176319..72b1df10 100644 --- a/scripts/scalability/test_controller/server/server.tac +++ b/scripts/scalability/test_controller/server/server.tac @@ -49,6 +49,7 @@ from twisted.logger import Logger from test_controller.server.user_dbs import ensure_dbs from test_controller.server.blobs import create_blobs from test_controller.server.blobs import delete_blobs +from test_controller.server.utils import get_soledad_server_pid DEFAULT_HTTP_PORT = 7001 @@ -62,6 +63,9 @@ logger = Logger(__name__) # class ResourceWatcher(object): + """ + A generic system resource watcher. Real watchers inherit from this class. + """ def __init__(self, pid): logger.info('%s started for process with PID %d' @@ -95,6 +99,9 @@ class ResourceWatcher(object): class CpuWatcher(ResourceWatcher): + """ + A watcher that trackes cpu usage of a process. + """ interval = 1 @@ -118,6 +125,9 @@ def _std(l): class MemoryWatcher(ResourceWatcher): + """ + A watcher that tracks the memory usage of a process. + """ interval = 0.1 @@ -161,7 +171,9 @@ class InvalidPidError(Exception): class MonitorResource(resource.Resource): """ - A generic resource-monitor web resource. + A generic web resource to monitor system resources. Which system resource + it allows monitoring depends on the watcher class passed to the + constructor. """ isLeaf = 1 @@ -173,7 +185,7 @@ class MonitorResource(resource.Resource): def _get_pid(self, request): if 'pid' not in request.args: - raise MissingPidError() + return get_soledad_server_pid() pid = request.args['pid'].pop() if not pid.isdigit(): raise InvalidPidError(pid) @@ -206,7 +218,11 @@ class MonitorResource(resource.Resource): self.watcher.stop() -class SetupResource(resource.Resource): +class UsersResource(resource.Resource): + """ + Allow controlling the creation of server-side user databases and access + tokens. + """ def render_POST(self, request): create = request.args.get('create') or [1000] @@ -228,6 +244,9 @@ class SetupResource(resource.Resource): class BlobsResource(resource.Resource): + """ + Allow controlling the creation and deletion of server-side blobs. + """ def render_POST(self, request): action = (request.args.get('action') or ['create']).pop() @@ -255,12 +274,15 @@ class BlobsResource(resource.Resource): class Root(resource.Resource): + """ + The root resource that gives access to all others. + """ def __init__(self): resource.Resource.__init__(self) self.putChild('mem', MonitorResource(MemoryWatcher)) self.putChild('cpu', MonitorResource(CpuWatcher)) - self.putChild('setup', SetupResource()) + self.putChild('users', UsersResource()) self.putChild('blobs', BlobsResource()) diff --git a/scripts/scalability/test_controller/server/user_dbs.py b/scripts/scalability/test_controller/server/user_dbs.py index 70fbd96d..c1012cc0 100755 --- a/scripts/scalability/test_controller/server/user_dbs.py +++ b/scripts/scalability/test_controller/server/user_dbs.py @@ -15,6 +15,7 @@ from urlparse import urljoin from twisted.internet import reactor, defer from twisted.logger import Logger + COUCH_URL = "http://127.0.0.1:5984" CREATE = 1000 @@ -56,10 +57,10 @@ def _req(method, *args, **kwargs): @defer.inlineCallbacks -def delete_dbs(dbs): +def delete_dbs(couch_url, dbs): deferreds = [] for db in dbs: - d = semaphore.run(_req, 'delete', urljoin(COUCH_URL, db)) + d = semaphore.run(_req, 'delete', urljoin(couch_url, db)) logfun = partial(_log, 'table', db, 'deleted') d.addCallback(logfun) deferreds.append(d) @@ -69,10 +70,10 @@ def delete_dbs(dbs): @defer.inlineCallbacks -def create_dbs(dbs): +def create_dbs(couch_url, dbs): deferreds = [] for db in dbs: - d = semaphore.run(_req, 'put', urljoin(COUCH_URL, db)) + d = semaphore.run(_req, 'put', urljoin(couch_url, db)) logfun = partial(_log, 'table', db, 'created') d.addCallback(logfun) deferreds.append(d) @@ -98,13 +99,13 @@ def _create_token(res, url, user_id): defer.returnValue(res) -def create_tokens(create): +def create_tokens(couch_url, create): deferreds = [] tokens_db = _get_tokens_db_name() for i in xrange(create): user_id = str(i) token = sha512('%s-token' % user_id).hexdigest() - url = '/'.join([COUCH_URL, tokens_db, token]) + url = '/'.join([couch_url, tokens_db, token]) d = semaphore.run(_req, 'get', url) d.addCallback(_create_token, url, user_id) logfun = partial(_log, 'token', user_id, 'created') @@ -115,15 +116,15 @@ def create_tokens(create): @defer.inlineCallbacks def ensure_dbs(couch_url=COUCH_URL, create=CREATE): - dbs = get_db_names(create) - yield delete_dbs(dbs) - yield create_dbs(dbs) - yield create_tokens(create) + # dbs = get_db_names(create) + # yield delete_dbs(couch_url, dbs) + # yield create_dbs(couch_url, dbs) + yield create_tokens(couch_url, create) @defer.inlineCallbacks def main(couch_url, create): - yield ensure_dbs(couch_url, create) + yield ensure_dbs(couch_url=couch_url, create=create) reactor.stop() diff --git a/scripts/scalability/test_controller/server/utils.py b/scripts/scalability/test_controller/server/utils.py new file mode 100644 index 00000000..a8274490 --- /dev/null +++ b/scripts/scalability/test_controller/server/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +from subprocess import check_output +from psutil import Process + +args = set([ + '/usr/bin/twistd', + '--python=/usr/lib/python2.7/dist-packages/leap/soledad/server/server.tac', +]) + + +def get_soledad_server_pid(): + output = check_output(['pidof', 'python']) + for pid in output.split(): + proc = Process(int(pid)) + cmdline = proc.cmdline() + if args.issubset(set(cmdline)): + return int(pid) + + +if __name__ == '__main__': + print get_soledad_server_pid() |