summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--scripts/scalability/README.rst36
-rw-r--r--scripts/scalability/makefile63
-rw-r--r--scripts/scalability/setup.py19
-rw-r--r--scripts/scalability/test_controller/client/makefile101
-rw-r--r--scripts/scalability/test_controller/requirements.pip2
-rw-r--r--scripts/scalability/test_controller/server/server.tac30
-rwxr-xr-xscripts/scalability/test_controller/server/user_dbs.py23
-rw-r--r--scripts/scalability/test_controller/server/utils.py22
8 files changed, 252 insertions, 44 deletions
diff --git a/scripts/scalability/README.rst b/scripts/scalability/README.rst
new file mode 100644
index 00000000..a2d891a7
--- /dev/null
+++ b/scripts/scalability/README.rst
@@ -0,0 +1,36 @@
+Blobs Server Scalability Tests
+==============================
+
+This folder contains code for running benchmark tests for determining Soledad
+Server Blobs service scalability. This consists basically in launching a number
+of parallel blobs upload or download requests, to assess the number of requests
+per second that the server is able to answer.
+
+When analyzing results, it's important to have in mind that the size of the
+blobs that are transferred as well as the bandwith of both the server and the
+client used for the test are resources that have high impact in the results.
+
+Test Controller server
+----------------------
+
+In the server, run the following to have an instance of the Test Controller
+server running:
+
+ make install-server
+ make start-server
+
+And, if you want to see the logs, use:
+
+ make log
+
+Alternativelly, use `make start-server-nodaemon` to avoid detaching from the
+terminal.
+
+Test Controller client
+----------------------
+
+Make sure an instance of the Test Controller Server is reachable at $(URI),
+and run:
+
+ make install-client
+ make run-test
diff --git a/scripts/scalability/makefile b/scripts/scalability/makefile
index 0a248d37..0067c650 100644
--- a/scripts/scalability/makefile
+++ b/scripts/scalability/makefile
@@ -1,14 +1,53 @@
+# Test Controller for Server Scalability Tests
+# ============================================
+#
+# This makefile knows how to install server and client components of the Test
+# Controller, as well as to trigger a run of the benchmarks.
+#
+# Test Controller server
+# ----------------------
+#
+# In the server, run the following to have an instance of the Test Controller
+# server running:
+#
+# make install-server
+# make start-server
+#
+# And, if you want to see the logs, use:
+#
+# make log
+#
+# Alternativelly, use `make start-server-nodaemon` to avoid detaching from the
+# terminal.
+#
+# Test Controller client
+# ----------------------
+#
+# Make sure an instance of the Test Controller Server is reachable at $(URI),
+# and run:
+#
+# make install-client
+# make run-test
+
+
+URI = https://giraffe.cdev.bitmask.net:7001
+
PIDFILE = /tmp/test_controller.pid
LOGFILE = /tmp/test_controller.log
TACFILE = ./test_controller/server/server.tac
-HTTP_PORT = 7001
-RESOURCE = cpu
-CREATE = 1000
-start:
+
+#----------------#
+# Server targets #
+#----------------#
+
+install-server:
+ pip install ".[server]"
+
+start-server:
twistd --pidfile=$(PIDFILE) --logfile=$(LOGFILE) --python=$(TACFILE)
-nodaemon:
+start-server-nodaemon:
twistd --nodaemon --python=$(TACFILE)
kill:
@@ -19,11 +58,13 @@ log:
restart: kill start
-post:
- curl -X POST http://127.0.0.1:$(HTTP_PORT)/$(RESOURCE)?pid=$(PID)
-get:
- curl -X GET http://127.0.0.1:$(HTTP_PORT)/$(RESOURCE)
+#----------------#
+# Client targets #
+#----------------#
+
+install-client:
+ pip install ".[client]"
-setup:
- curl -X POST http://127.0.0.1:$(HTTP_PORT)/setup?create=$(CREATE)
+test:
+ (cd test_controller/client && make test URI=$(URI))
diff --git a/scripts/scalability/setup.py b/scripts/scalability/setup.py
index e76437db..66f0789a 100644
--- a/scripts/scalability/setup.py
+++ b/scripts/scalability/setup.py
@@ -1,9 +1,25 @@
"""
-A Test Controller application.
+A Scalability Test Controller application.
"""
from setuptools import setup, find_packages
+
+client = [
+ 'funkload',
+]
+
+server = [
+ 'psutil',
+ 'twisted',
+ 'leap.soledad',
+]
+
+extras = {
+ 'client': client,
+ 'server': server,
+}
+
setup(
name="test-controller",
version="0.1",
@@ -11,4 +27,5 @@ setup(
packages=find_packages(),
include_package_data=True,
zip_safe=False,
+ extras_require=extras,
)
diff --git a/scripts/scalability/test_controller/client/makefile b/scripts/scalability/test_controller/client/makefile
index 33e67e25..e0a719a0 100644
--- a/scripts/scalability/test_controller/client/makefile
+++ b/scripts/scalability/test_controller/client/makefile
@@ -1,19 +1,90 @@
-bench: upload download
+# test_controller/client/makefile
+# ===============================
+#
+# This file is part of the client-side infrastructure for Server Scalability
+# Tests. The client-side is responsible for orchestrating the benchmarking by
+# setting up server resources needed for the tests.
+#
+# What this file does
+# -------------------
+#
+# This makefile knows how to:
+#
+# - Start and stop server-side system resource monitoring.
+#
+# - Setup server-side resources for tests (user dbs, access tokens and blobs).
+#
+# - Orchestrate test runs using the client-side machinery.
+#
+# - Generate reports and save test results.
+#
+# Running tests
+# -------------
+#
+# To run tests, make sure a Test Controller Server is reachable at $(URI) and
+# run `make`.
-upload:
- curl -X POST "http://127.0.0.1:7001/blobs?action=delete"
- fl-run-bench -f test_Blobs.py Blobs.test_upload
- fl-build-report --html results/blobs-bench.xml
-download:
- curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10&amount=5000"
- fl-run-bench -f test_Blobs.py Blobs.test_download
- fl-build-report --html results/blobs-bench.xml
+URI = https://giraffe.cdev.bitmask.net:7001
+CREATE = 5000
-test_upload:
- curl -X POST "http://127.0.0.1:7001/blobs?action=delete"
- fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload
-test_download:
- curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10"
- fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download
+#------------------#
+# main test target #
+#------------------#
+
+test: bench
+
+bench: bench-cpu bench-mem
+
+bench-cpu: start-cpu bench-upload bench-download stop-cpu
+
+bench-mem: start-mem bench-upload bench-download stop-mem
+
+bench-upload: create-users delete-blobs
+ fl-run-bench -f test_Blobs.py Blobs.test_upload
+ fl-build-report --html results/blobs-bench.xml
+
+bench-download: create-users create-blobs
+ fl-run-bench -f test_Blobs.py Blobs.test_download
+ fl-build-report --html results/blobs-bench.xml
+
+# for quick tests only
+bench-upload-quick: create-users delete-blobs
+ curl -X POST "$(URI)/blobs?action=delete"
+ fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload
+
+# for quick tests only
+bench-download-quick: create-users create-blobs
+ curl -X POST "$(URI)/blobs?action=create&size=10"
+ fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download
+
+#---------------------#
+# resource monitoring #
+#---------------------#
+
+start-cpu:
+ curl -X POST $(URI)/cpu
+
+stop-cpu:
+ curl -X GET $(URI)/cpu
+
+start-mem:
+ curl -X POST $(URI)/mem
+
+stop-mem:
+ curl -X GET $(URI)/mem
+
+
+#--------------#
+# server setup #
+#--------------#
+
+create-users:
+ curl -X POST $(URI)/users?create=$(CREATE)
+
+delete-blobs:
+ curl -X POST "$(URI)/blobs?action=delete"
+
+create-blobs:
+ curl -X POST "$(URI)/blobs?action=create&size=10&amount=5000"
diff --git a/scripts/scalability/test_controller/requirements.pip b/scripts/scalability/test_controller/requirements.pip
deleted file mode 100644
index 224b581e..00000000
--- a/scripts/scalability/test_controller/requirements.pip
+++ /dev/null
@@ -1,2 +0,0 @@
-psutil
-twisted
diff --git a/scripts/scalability/test_controller/server/server.tac b/scripts/scalability/test_controller/server/server.tac
index d5176319..72b1df10 100644
--- a/scripts/scalability/test_controller/server/server.tac
+++ b/scripts/scalability/test_controller/server/server.tac
@@ -49,6 +49,7 @@ from twisted.logger import Logger
from test_controller.server.user_dbs import ensure_dbs
from test_controller.server.blobs import create_blobs
from test_controller.server.blobs import delete_blobs
+from test_controller.server.utils import get_soledad_server_pid
DEFAULT_HTTP_PORT = 7001
@@ -62,6 +63,9 @@ logger = Logger(__name__)
#
class ResourceWatcher(object):
+ """
+ A generic system resource watcher. Real watchers inherit from this class.
+ """
def __init__(self, pid):
logger.info('%s started for process with PID %d'
@@ -95,6 +99,9 @@ class ResourceWatcher(object):
class CpuWatcher(ResourceWatcher):
+ """
+ A watcher that trackes cpu usage of a process.
+ """
interval = 1
@@ -118,6 +125,9 @@ def _std(l):
class MemoryWatcher(ResourceWatcher):
+ """
+ A watcher that tracks the memory usage of a process.
+ """
interval = 0.1
@@ -161,7 +171,9 @@ class InvalidPidError(Exception):
class MonitorResource(resource.Resource):
"""
- A generic resource-monitor web resource.
+ A generic web resource to monitor system resources. Which system resource
+ it allows monitoring depends on the watcher class passed to the
+ constructor.
"""
isLeaf = 1
@@ -173,7 +185,7 @@ class MonitorResource(resource.Resource):
def _get_pid(self, request):
if 'pid' not in request.args:
- raise MissingPidError()
+ return get_soledad_server_pid()
pid = request.args['pid'].pop()
if not pid.isdigit():
raise InvalidPidError(pid)
@@ -206,7 +218,11 @@ class MonitorResource(resource.Resource):
self.watcher.stop()
-class SetupResource(resource.Resource):
+class UsersResource(resource.Resource):
+ """
+ Allow controlling the creation of server-side user databases and access
+ tokens.
+ """
def render_POST(self, request):
create = request.args.get('create') or [1000]
@@ -228,6 +244,9 @@ class SetupResource(resource.Resource):
class BlobsResource(resource.Resource):
+ """
+ Allow controlling the creation and deletion of server-side blobs.
+ """
def render_POST(self, request):
action = (request.args.get('action') or ['create']).pop()
@@ -255,12 +274,15 @@ class BlobsResource(resource.Resource):
class Root(resource.Resource):
+ """
+ The root resource that gives access to all others.
+ """
def __init__(self):
resource.Resource.__init__(self)
self.putChild('mem', MonitorResource(MemoryWatcher))
self.putChild('cpu', MonitorResource(CpuWatcher))
- self.putChild('setup', SetupResource())
+ self.putChild('users', UsersResource())
self.putChild('blobs', BlobsResource())
diff --git a/scripts/scalability/test_controller/server/user_dbs.py b/scripts/scalability/test_controller/server/user_dbs.py
index 70fbd96d..c1012cc0 100755
--- a/scripts/scalability/test_controller/server/user_dbs.py
+++ b/scripts/scalability/test_controller/server/user_dbs.py
@@ -15,6 +15,7 @@ from urlparse import urljoin
from twisted.internet import reactor, defer
from twisted.logger import Logger
+
COUCH_URL = "http://127.0.0.1:5984"
CREATE = 1000
@@ -56,10 +57,10 @@ def _req(method, *args, **kwargs):
@defer.inlineCallbacks
-def delete_dbs(dbs):
+def delete_dbs(couch_url, dbs):
deferreds = []
for db in dbs:
- d = semaphore.run(_req, 'delete', urljoin(COUCH_URL, db))
+ d = semaphore.run(_req, 'delete', urljoin(couch_url, db))
logfun = partial(_log, 'table', db, 'deleted')
d.addCallback(logfun)
deferreds.append(d)
@@ -69,10 +70,10 @@ def delete_dbs(dbs):
@defer.inlineCallbacks
-def create_dbs(dbs):
+def create_dbs(couch_url, dbs):
deferreds = []
for db in dbs:
- d = semaphore.run(_req, 'put', urljoin(COUCH_URL, db))
+ d = semaphore.run(_req, 'put', urljoin(couch_url, db))
logfun = partial(_log, 'table', db, 'created')
d.addCallback(logfun)
deferreds.append(d)
@@ -98,13 +99,13 @@ def _create_token(res, url, user_id):
defer.returnValue(res)
-def create_tokens(create):
+def create_tokens(couch_url, create):
deferreds = []
tokens_db = _get_tokens_db_name()
for i in xrange(create):
user_id = str(i)
token = sha512('%s-token' % user_id).hexdigest()
- url = '/'.join([COUCH_URL, tokens_db, token])
+ url = '/'.join([couch_url, tokens_db, token])
d = semaphore.run(_req, 'get', url)
d.addCallback(_create_token, url, user_id)
logfun = partial(_log, 'token', user_id, 'created')
@@ -115,15 +116,15 @@ def create_tokens(create):
@defer.inlineCallbacks
def ensure_dbs(couch_url=COUCH_URL, create=CREATE):
- dbs = get_db_names(create)
- yield delete_dbs(dbs)
- yield create_dbs(dbs)
- yield create_tokens(create)
+ # dbs = get_db_names(create)
+ # yield delete_dbs(couch_url, dbs)
+ # yield create_dbs(couch_url, dbs)
+ yield create_tokens(couch_url, create)
@defer.inlineCallbacks
def main(couch_url, create):
- yield ensure_dbs(couch_url, create)
+ yield ensure_dbs(couch_url=couch_url, create=create)
reactor.stop()
diff --git a/scripts/scalability/test_controller/server/utils.py b/scripts/scalability/test_controller/server/utils.py
new file mode 100644
index 00000000..a8274490
--- /dev/null
+++ b/scripts/scalability/test_controller/server/utils.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+from subprocess import check_output
+from psutil import Process
+
+args = set([
+ '/usr/bin/twistd',
+ '--python=/usr/lib/python2.7/dist-packages/leap/soledad/server/server.tac',
+])
+
+
+def get_soledad_server_pid():
+ output = check_output(['pidof', 'python'])
+ for pid in output.split():
+ proc = Process(int(pid))
+ cmdline = proc.cmdline()
+ if args.issubset(set(cmdline)):
+ return int(pid)
+
+
+if __name__ == '__main__':
+ print get_soledad_server_pid()