[benchmarks] add some orchestration to scalability tests

5 files changed, 146 insertions, 32 deletions

diff --git a/scripts/scalability/test_controller/client/makefile b/scripts/scalability/test_controller/client/makefile
index 33e67e25..e0a719a0 100644
--- a/scripts/scalability/test_controller/client/makefile
+++ b/scripts/scalability/test_controller/client/makefile
@@ -1,19 +1,90 @@
-bench: upload download
+# test_controller/client/makefile
+# ===============================
+#
+# This file is part of the client-side infrastructure for Server Scalability
+# Tests. The client-side is responsible for orchestrating the benchmarking by
+# setting up server resources needed for the tests.
+#
+# What this file does
+# -------------------
+#
+# This makefile knows how to:
+#
+#   - Start and stop server-side system resource monitoring.
+#
+#   - Setup server-side resources for tests (user dbs, access tokens and blobs).
+#
+#   - Orchestrate test runs using the client-side machinery.
+#
+#   - Generate reports and save test results.
+#
+# Running tests
+# -------------
+#
+# To run tests, make sure a Test Controller Server is reachable at $(URI) and
+# run `make`.
 
-upload:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=delete"
-	fl-run-bench -f test_Blobs.py Blobs.test_upload
-	fl-build-report --html results/blobs-bench.xml
 
-download:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10&amount=5000"
-	fl-run-bench -f test_Blobs.py Blobs.test_download
-	fl-build-report --html results/blobs-bench.xml
+URI    = https://giraffe.cdev.bitmask.net:7001
+CREATE = 5000
 
-test_upload:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=delete"
-	fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload
 
-test_download:
-	curl -X POST "http://127.0.0.1:7001/blobs?action=create&size=10"
-	fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download
+#------------------#
+# main test target #
+#------------------#
+
+test: bench
+
+bench: bench-cpu bench-mem
+
+bench-cpu: start-cpu bench-upload bench-download stop-cpu
+
+bench-mem: start-mem bench-upload bench-download stop-mem
+
+bench-upload: create-users delete-blobs
+	 fl-run-bench -f test_Blobs.py Blobs.test_upload
+	 fl-build-report --html results/blobs-bench.xml
+
+bench-download: create-users create-blobs
+	 fl-run-bench -f test_Blobs.py Blobs.test_download
+	 fl-build-report --html results/blobs-bench.xml
+
+# for quick tests only
+bench-upload-quick: create-users delete-blobs
+	 curl -X POST "$(URI)/blobs?action=delete"
+	 fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_upload
+
+# for quick tests only
+bench-download-quick: create-users create-blobs
+	 curl -X POST "$(URI)/blobs?action=create&size=10"
+	 fl-run-bench -c 1 --duration 10 -f test_Blobs.py Blobs.test_download
+
+#---------------------#
+# resource monitoring #
+#---------------------#
+
+start-cpu:
+	 curl -X POST $(URI)/cpu
+
+stop-cpu:
+	 curl -X GET $(URI)/cpu
+
+start-mem:
+	 curl -X POST $(URI)/mem
+
+stop-mem:
+	 curl -X GET $(URI)/mem
+
+
+#--------------#
+# server setup #
+#--------------#
+
+create-users:
+	 curl -X POST $(URI)/users?create=$(CREATE)
+
+delete-blobs:
+	 curl -X POST "$(URI)/blobs?action=delete"
+
+create-blobs:
+	 curl -X POST "$(URI)/blobs?action=create&size=10&amount=5000"
diff --git a/scripts/scalability/test_controller/requirements.pip b/scripts/scalability/test_controller/requirements.pip
deleted file mode 100644
index 224b581e..00000000
--- a/scripts/scalability/test_controller/requirements.pip
+++ /dev/null
@@ -1,2 +0,0 @@
-psutil
-twisted
diff --git a/scripts/scalability/test_controller/server/server.tac b/scripts/scalability/test_controller/server/server.tac
index d5176319..72b1df10 100644
--- a/scripts/scalability/test_controller/server/server.tac
+++ b/scripts/scalability/test_controller/server/server.tac
@@ -49,6 +49,7 @@ from twisted.logger import Logger
 from test_controller.server.user_dbs import ensure_dbs
 from test_controller.server.blobs import create_blobs
 from test_controller.server.blobs import delete_blobs
+from test_controller.server.utils import get_soledad_server_pid
 
 
 DEFAULT_HTTP_PORT = 7001
@@ -62,6 +63,9 @@ logger = Logger(__name__)
 #
 
 class ResourceWatcher(object):
+    """
+    A generic system resource watcher. Real watchers inherit from this class.
+    """
 
     def __init__(self, pid):
         logger.info('%s started for process with PID %d'
@@ -95,6 +99,9 @@ class ResourceWatcher(object):
 
 
 class CpuWatcher(ResourceWatcher):
+    """
+    A watcher that trackes cpu usage of a process.
+    """
 
     interval = 1
 
@@ -118,6 +125,9 @@ def _std(l):
 
 
 class MemoryWatcher(ResourceWatcher):
+    """
+    A watcher that tracks the memory usage of a process.
+    """
 
     interval = 0.1
 
@@ -161,7 +171,9 @@ class InvalidPidError(Exception):
 
 class MonitorResource(resource.Resource):
     """
-    A generic resource-monitor web resource.
+    A generic web resource to monitor system resources. Which system resource
+    it allows monitoring depends on the watcher class passed to the
+    constructor.
     """
 
     isLeaf = 1
@@ -173,7 +185,7 @@ class MonitorResource(resource.Resource):
 
     def _get_pid(self, request):
         if 'pid' not in request.args:
-            raise MissingPidError()
+            return get_soledad_server_pid()
         pid = request.args['pid'].pop()
         if not pid.isdigit():
             raise InvalidPidError(pid)
@@ -206,7 +218,11 @@ class MonitorResource(resource.Resource):
             self.watcher.stop()
 
 
-class SetupResource(resource.Resource):
+class UsersResource(resource.Resource):
+    """
+    Allow controlling the creation of server-side user databases and access
+    tokens.
+    """
 
     def render_POST(self, request):
         create = request.args.get('create') or [1000]
@@ -228,6 +244,9 @@ class SetupResource(resource.Resource):
 
 
 class BlobsResource(resource.Resource):
+    """
+    Allow controlling the creation and deletion of server-side blobs.
+    """
 
     def render_POST(self, request):
         action = (request.args.get('action') or ['create']).pop()
@@ -255,12 +274,15 @@ class BlobsResource(resource.Resource):
 
 
 class Root(resource.Resource):
+    """
+    The root resource that gives access to all others.
+    """
 
     def __init__(self):
         resource.Resource.__init__(self)
         self.putChild('mem', MonitorResource(MemoryWatcher))
         self.putChild('cpu', MonitorResource(CpuWatcher))
-        self.putChild('setup', SetupResource())
+        self.putChild('users', UsersResource())
         self.putChild('blobs', BlobsResource())
 
 
diff --git a/scripts/scalability/test_controller/server/user_dbs.py b/scripts/scalability/test_controller/server/user_dbs.py
index 70fbd96d..c1012cc0 100755
--- a/scripts/scalability/test_controller/server/user_dbs.py
+++ b/scripts/scalability/test_controller/server/user_dbs.py
@@ -15,6 +15,7 @@ from urlparse import urljoin
 from twisted.internet import reactor, defer
 from twisted.logger import Logger
 
+
 COUCH_URL = "http://127.0.0.1:5984"
 CREATE = 1000
 
@@ -56,10 +57,10 @@ def _req(method, *args, **kwargs):
 
 
 @defer.inlineCallbacks
-def delete_dbs(dbs):
+def delete_dbs(couch_url, dbs):
     deferreds = []
     for db in dbs:
-        d = semaphore.run(_req, 'delete', urljoin(COUCH_URL, db))
+        d = semaphore.run(_req, 'delete', urljoin(couch_url, db))
         logfun = partial(_log, 'table', db, 'deleted')
         d.addCallback(logfun)
         deferreds.append(d)
@@ -69,10 +70,10 @@ def delete_dbs(dbs):
 
 
 @defer.inlineCallbacks
-def create_dbs(dbs):
+def create_dbs(couch_url, dbs):
     deferreds = []
     for db in dbs:
-        d = semaphore.run(_req, 'put', urljoin(COUCH_URL, db))
+        d = semaphore.run(_req, 'put', urljoin(couch_url, db))
         logfun = partial(_log, 'table', db, 'created')
         d.addCallback(logfun)
         deferreds.append(d)
@@ -98,13 +99,13 @@ def _create_token(res, url, user_id):
     defer.returnValue(res)
 
 
-def create_tokens(create):
+def create_tokens(couch_url, create):
     deferreds = []
     tokens_db = _get_tokens_db_name()
     for i in xrange(create):
         user_id = str(i)
         token = sha512('%s-token' % user_id).hexdigest()
-        url = '/'.join([COUCH_URL, tokens_db, token])
+        url = '/'.join([couch_url, tokens_db, token])
         d = semaphore.run(_req, 'get', url)
         d.addCallback(_create_token, url, user_id)
         logfun = partial(_log, 'token', user_id, 'created')
@@ -115,15 +116,15 @@ def create_tokens(create):
 
 @defer.inlineCallbacks
 def ensure_dbs(couch_url=COUCH_URL, create=CREATE):
-    dbs = get_db_names(create)
-    yield delete_dbs(dbs)
-    yield create_dbs(dbs)
-    yield create_tokens(create)
+    # dbs = get_db_names(create)
+    # yield delete_dbs(couch_url, dbs)
+    # yield create_dbs(couch_url, dbs)
+    yield create_tokens(couch_url, create)
 
 
 @defer.inlineCallbacks
 def main(couch_url, create):
-    yield ensure_dbs(couch_url, create)
+    yield ensure_dbs(couch_url=couch_url, create=create)
     reactor.stop()
 
 
diff --git a/scripts/scalability/test_controller/server/utils.py b/scripts/scalability/test_controller/server/utils.py
new file mode 100644
index 00000000..a8274490
--- /dev/null
+++ b/scripts/scalability/test_controller/server/utils.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+from subprocess import check_output
+from psutil import Process
+
+args = set([
+    '/usr/bin/twistd',
+    '--python=/usr/lib/python2.7/dist-packages/leap/soledad/server/server.tac',
+])
+
+
+def get_soledad_server_pid():
+    output = check_output(['pidof', 'python'])
+    for pid in output.split():
+        proc = Process(int(pid))
+        cmdline = proc.cmdline()
+        if args.issubset(set(cmdline)):
+            return int(pid)
+
+
+if __name__ == '__main__':
+    print get_soledad_server_pid()