[benchmarks] add some orchestration to scalability tests

8 files changed, 252 insertions, 44 deletions

diff --git a/scripts/scalability/README.rst b/scripts/scalability/README.rst
new file mode 100644
index 00000000..a2d891a7
--- /dev/null
+++ b/scripts/scalability/README.rst
@@ -0,0 +1,36 @@
+Blobs Server Scalability Tests
+==============================
+
+This folder contains code for running benchmark tests for determining Soledad
+Server Blobs service scalability. This consists basically in launching a number
+of parallel blobs upload or download requests, to assess the number of requests
+per second that the server is able to answer. 
+
+When analyzing results, it's important to have in mind that the size of the
+blobs that are transferred as well as the bandwith of both the server and the
+client used for the test are resources that have high impact in the results.
+
+Test Controller server
+----------------------
+
+In the server, run the following to have an instance of the Test Controller
+server running:
+
+  make install-server
+  make start-server
+
+And, if you want to see the logs, use:
+
+  make log 
+
+Alternativelly, use `make start-server-nodaemon` to avoid detaching from the
+terminal.
+
+Test Controller client
+----------------------
+
+Make sure an instance of the Test Controller Server is reachable at $(URI),
+and run:
+
+  make install-client
+  make run-test
diff --git a/scripts/scalability/makefile b/scripts/scalability/makefile
index 0a248d37..0067c650 100644
--- a/scripts/scalability/makefile
+++ b/scripts/scalability/makefile
@@ -1,14 +1,53 @@
+# Test Controller for Server Scalability Tests
+# ============================================
+#
+# This makefile knows how to install server and client components of the Test
+# Controller, as well as to trigger a run of the benchmarks.
+#
+# Test Controller server
+# ----------------------
+#
+# In the server, run the following to have an instance of the Test Controller
+# server running:
+#
+#   make install-server
+#   make start-server
+#
+# And, if you want to see the logs, use:
+#
+#   make log
+#
+# Alternativelly, use `make start-server-nodaemon` to avoid detaching from the
+# terminal.
+#
+# Test Controller client
+# ----------------------
+#
+# Make sure an instance of the Test Controller Server is reachable at $(URI),
+# and run:
+#
+#   make install-client
+#   make run-test
+
+
+URI = https://giraffe.cdev.bitmask.net:7001
+
 PIDFILE   = /tmp/test_controller.pid
 LOGFILE   = /tmp/test_controller.log
 TACFILE   = ./test_controller/server/server.tac
-HTTP_PORT = 7001
-RESOURCE  = cpu
-CREATE    = 1000
 
-start:
+
+#----------------#
+# Server targets #
+#----------------#
+
+install-server:
+	pip install ".[server]"
+
+start-server:
 	twistd --pidfile=$(PIDFILE) --logfile=$(LOGFILE) --python=$(TACFILE)
 
-nodaemon:
+start-server-nodaemon:
 	twistd --nodaemon --python=$(TACFILE)
 
 kill:
@@ -19,11 +58,13 @@ log:
 
 restart: kill start
 
-post:
-	curl -X POST http://127.0.0.1:$(HTTP_PORT)/$(RESOURCE)?pid=$(PID)
 
-get:
-	curl -X GET http://127.0.0.1:$(HTTP_PORT)/$(RESOURCE)
+#----------------#
+# Client targets #
+#----------------#
+
+install-client:
+	pip install ".[client]"
 
-setup:
-	curl -X POST http://127.0.0.1:$(HTTP_PORT)/setup?create=$(CREATE)
+test:
+	(cd test_controller/client && make test URI=$(URI))
diff --git a/scripts/scalability/setup.py b/scripts/scalability/setup.py
index e76437db..66f0789a 100644
--- a/scripts/scalability/setup.py
+++ b/scripts/scalability/setup.py
@@ -1,9 +1,25 @@
 """
-A Test Controller application.
+A Scalability Test Controller application.
 """
 
 from setuptools import setup, find_packages
 
+
+client = [
+    'funkload',
+]
+
+server = [
+    'psutil',
+    'twisted',
+    'leap.soledad',
+]
+
+extras = {
+    'client': client,
+    'server': server,
+}
+
 setup(
     name="test-controller",
     version="0.1",
@@ -11,4 +27,5 @@ setup(
     packages=find_packages(),
     include_package_data=True,
     zip_safe=False,
+    extras_require=extras,
 )
diff --git a/scripts/scalability/test_controller/client/makefile b/scripts/scalability/test_controller/client/makefile
index 33e67e25..e0a719a0 100644
--- a/scripts/scalability/test_controller/client/makefile
+++ b/scripts/scalability/test_controller/client/makefile
@@ -1,19 +1,90 @@
-bench: upload download
+# test_controller/client/makefile
+# ===============================
+#
+# This file is part of the client-side infrastructure for Server Scalability
+# Tests. The client-side is responsible for orchestrating the benchmarking by
+# setting up server resources needed for the tests.
+#
+# What this file does
+# -------------------
+#
+# This makefile knows how to:
+#
+#   - Start and stop server-side system resource monitoring.
+#
+#   - Setup server-side resources for tests (user dbs, access tokens and blobs).
+#
+#   - Orchestrate test runs using the client-side machinery.
+#
+#   - Generate reports and save test results.
+#
+# Running tests
+# -------------
+#
+# To run tests, make sure a Test Controller Server is reachable at $(URI) and
+# run `make`.
 
-upload:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=delete"
-	fl-run-bench -f test_Blobs.py Blobs.test_upload
-	fl-build-report --html results/blobs-bench.xml
 
-download:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10&amount=5000"
-	fl-run-bench -f test_Blobs.py Blobs.test_download
-	fl-build-report --html results/blobs-bench.xml
+URI    = https://giraffe.cdev.bitmask.net:7001
+CREATE = 5000
 
-test_upload:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=delete"
-	fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload
 
-test_download:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10"
-	fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download
+#------------------#
+# main test target #
+#------------------#
+
+test: bench
+
+bench: bench-cpu bench-mem
+
+bench-cpu: start-cpu bench-upload bench-download stop-cpu
+
+bench-mem: start-mem bench-upload bench-download stop-mem
+
+bench-upload: create-users delete-blobs
+	 fl-run-bench -f test_Blobs.py Blobs.test_upload
+	 fl-build-report --html results/blobs-bench.xml
+
+bench-download: create-users create-blobs
+	 fl-run-bench -f test_Blobs.py Blobs.test_download
+	 fl-build-report --html results/blobs-bench.xml
+
+# for quick tests only
+bench-upload-quick: create-users delete-blobs
+	 curl -X POST "$(URI)/blobs?action=delete"
+	 fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload
+
+# for quick tests only
+bench-download-quick: create-users create-blobs
+	 curl -X POST "$(URI)/blobs?action=create&size=10"
+	 fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download
+
+#---------------------#
+# resource monitoring #
+#---------------------#
+
+start-cpu:
+	 curl -X POST $(URI)/cpu
+
+stop-cpu:
+	 curl -X GET $(URI)/cpu
+
+start-mem:
+	 curl -X POST $(URI)/mem
+
+stop-mem:
+	 curl -X GET $(URI)/mem
+
+
+#--------------#
+# server setup #
+#--------------#
+
+create-users:
+	 curl -X POST $(URI)/users?create=$(CREATE)
+
+delete-blobs:
+	 curl -X POST "$(URI)/blobs?action=delete"
+
+create-blobs:
+	 curl -X POST "$(URI)/blobs?action=create&size=10&amount=5000"
diff --git a/scripts/scalability/test_controller/requirements.pip b/scripts/scalability/test_controller/requirements.pip
deleted file mode 100644
index 224b581e..00000000
--- a/scripts/scalability/test_controller/requirements.pip
+++ /dev/null
@@ -1,2 +0,0 @@
-psutil
-twisted
diff --git a/scripts/scalability/test_controller/server/server.tac b/scripts/scalability/test_controller/server/server.tac
index d5176319..72b1df10 100644
--- a/scripts/scalability/test_controller/server/server.tac
+++ b/scripts/scalability/test_controller/server/server.tac
@@ -49,6 +49,7 @@ from twisted.logger import Logger
 from test_controller.server.user_dbs import ensure_dbs
 from test_controller.server.blobs import create_blobs
 from test_controller.server.blobs import delete_blobs
+from test_controller.server.utils import get_soledad_server_pid
 
 
 DEFAULT_HTTP_PORT = 7001
@@ -62,6 +63,9 @@ logger = Logger(__name__)
 #
 
 class ResourceWatcher(object):
+    """
+    A generic system resource watcher. Real watchers inherit from this class.
+    """
 
     def __init__(self, pid):
         logger.info('%s started for process with PID %d'
@@ -95,6 +99,9 @@ class ResourceWatcher(object):
 
 
 class CpuWatcher(ResourceWatcher):
+    """
+    A watcher that trackes cpu usage of a process.
+    """
 
     interval = 1
 
@@ -118,6 +125,9 @@ def _std(l):
 
 
 class MemoryWatcher(ResourceWatcher):
+    """
+    A watcher that tracks the memory usage of a process.
+    """
 
     interval = 0.1
 
@@ -161,7 +171,9 @@ class InvalidPidError(Exception):
 
 class MonitorResource(resource.Resource):
     """
-    A generic resource-monitor web resource.
+    A generic web resource to monitor system resources. Which system resource
+    it allows monitoring depends on the watcher class passed to the
+    constructor.
     """
 
     isLeaf = 1
@@ -173,7 +185,7 @@ class MonitorResource(resource.Resource):
 
     def _get_pid(self, request):
         if 'pid' not in request.args:
-            raise MissingPidError()
+            return get_soledad_server_pid()
         pid = request.args['pid'].pop()
         if not pid.isdigit():
             raise InvalidPidError(pid)
@@ -206,7 +218,11 @@ class MonitorResource(resource.Resource):
             self.watcher.stop()
 
 
-class SetupResource(resource.Resource):
+class UsersResource(resource.Resource):
+    """
+    Allow controlling the creation of server-side user databases and access
+    tokens.
+    """
 
     def render_POST(self, request):
         create = request.args.get('create') or [1000]
@@ -228,6 +244,9 @@ class SetupResource(resource.Resource):
 
 
 class BlobsResource(resource.Resource):
+    """
+    Allow controlling the creation and deletion of server-side blobs.
+    """
 
     def render_POST(self, request):
         action = (request.args.get('action') or ['create']).pop()
@@ -255,12 +274,15 @@ class BlobsResource(resource.Resource):
 
 
 class Root(resource.Resource):
+    """
+    The root resource that gives access to all others.
+    """
 
     def __init__(self):
         resource.Resource.__init__(self)
         self.putChild('mem', MonitorResource(MemoryWatcher))
         self.putChild('cpu', MonitorResource(CpuWatcher))
-        self.putChild('setup', SetupResource())
+        self.putChild('users', UsersResource())
         self.putChild('blobs', BlobsResource())
 
 
diff --git a/scripts/scalability/test_controller/server/user_dbs.py b/scripts/scalability/test_controller/server/user_dbs.py
index 70fbd96d..c1012cc0 100755
--- a/scripts/scalability/test_controller/server/user_dbs.py
+++ b/scripts/scalability/test_controller/server/user_dbs.py
@@ -15,6 +15,7 @@ from urlparse import urljoin
 from twisted.internet import reactor, defer
 from twisted.logger import Logger
 
+
 COUCH_URL = "http://127.0.0.1:5984"
 CREATE = 1000
 
@@ -56,10 +57,10 @@ def _req(method, *args, **kwargs):
 
 
 @defer.inlineCallbacks
-def delete_dbs(dbs):
+def delete_dbs(couch_url, dbs):
     deferreds = []
     for db in dbs:
-        d = semaphore.run(_req, 'delete', urljoin(COUCH_URL, db))
+        d = semaphore.run(_req, 'delete', urljoin(couch_url, db))
         logfun = partial(_log, 'table', db, 'deleted')
         d.addCallback(logfun)
         deferreds.append(d)
@@ -69,10 +70,10 @@ def delete_dbs(dbs):
 
 
 @defer.inlineCallbacks
-def create_dbs(dbs):
+def create_dbs(couch_url, dbs):
     deferreds = []
     for db in dbs:
-        d = semaphore.run(_req, 'put', urljoin(COUCH_URL, db))
+        d = semaphore.run(_req, 'put', urljoin(couch_url, db))
         logfun = partial(_log, 'table', db, 'created')
         d.addCallback(logfun)
         deferreds.append(d)
@@ -98,13 +99,13 @@ def _create_token(res, url, user_id):
     defer.returnValue(res)
 
 
-def create_tokens(create):
+def create_tokens(couch_url, create):
     deferreds = []
     tokens_db = _get_tokens_db_name()
     for i in xrange(create):
         user_id = str(i)
         token = sha512('%s-token' % user_id).hexdigest()
-        url = '/'.join([COUCH_URL, tokens_db, token])
+        url = '/'.join([couch_url, tokens_db, token])
         d = semaphore.run(_req, 'get', url)
         d.addCallback(_create_token, url, user_id)
         logfun = partial(_log, 'token', user_id, 'created')
@@ -115,15 +116,15 @@ def create_tokens(create):
 
 @defer.inlineCallbacks
 def ensure_dbs(couch_url=COUCH_URL, create=CREATE):
-    dbs = get_db_names(create)
-    yield delete_dbs(dbs)
-    yield create_dbs(dbs)
-    yield create_tokens(create)
+    # dbs = get_db_names(create)
+    # yield delete_dbs(couch_url, dbs)
+    # yield create_dbs(couch_url, dbs)
+    yield create_tokens(couch_url, create)
 
 
 @defer.inlineCallbacks
 def main(couch_url, create):
-    yield ensure_dbs(couch_url, create)
+    yield ensure_dbs(couch_url=couch_url, create=create)
     reactor.stop()
 
 
diff --git a/scripts/scalability/test_controller/server/utils.py b/scripts/scalability/test_controller/server/utils.py
new file mode 100644
index 00000000..a8274490
--- /dev/null
+++ b/scripts/scalability/test_controller/server/utils.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+from subprocess import check_output
+from psutil import Process
+
+args = set([
+    '/usr/bin/twistd',
+    '--python=/usr/lib/python2.7/dist-packages/leap/soledad/server/server.tac',
+])
+
+
+def get_soledad_server_pid():
+    output = check_output(['pidof', 'python'])
+    for pid in output.split():
+        proc = Process(int(pid))
+        cmdline = proc.cmdline()
+        if args.issubset(set(cmdline)):
+            return int(pid)
+
+
+if __name__ == '__main__':
+    print get_soledad_server_pid()