[benchmarks] add responsiveness test with watchdog

6 files changed, 150 insertions, 14 deletions

diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py
index 25fe5134..377fc606 100644
--- a/testing/tests/benchmarks/conftest.py
+++ b/testing/tests/benchmarks/conftest.py
@@ -1,10 +1,8 @@
-import base64
 import functools
 import numpy
 import os
 import psutil
 import pytest
-import random
 import threading
 import time
 
@@ -37,16 +35,6 @@ def pytest_collection_modifyitems(items):
 #
 
 @pytest.fixture()
-def payload():
-    def generate(size):
-        random.seed(1337)  # same seed to avoid different bench results
-        payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
-        # encode as base64 to avoid ascii encode/decode errors
-        return base64.b64encode(payload_bytes)[:size]  # remove b64 overhead
-    return generate
-
-
-@pytest.fixture()
 def txbenchmark(monitored_benchmark):
     def blockOnThread(*args, **kwargs):
         return threads.deferToThread(
diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py
index e9641eba..6ce97625 100644
--- a/testing/tests/conftest.py
+++ b/testing/tests/conftest.py
@@ -1,8 +1,10 @@
 import glob
+import base64
 import json
 import os
 import pytest
 import re
+import random
 import requests
 import signal
 import socket
@@ -331,3 +333,17 @@ if 'pytest_benchmark' in sys.modules:
         """
         hostname = os.environ.get('HOST_HOSTNAME', socket.gethostname())
         machine_info['host'] = hostname
+
+
+#
+# benchmark/responsiveness fixtures
+#
+
+@pytest.fixture()
+def payload():
+    def generate(size):
+        random.seed(1337)  # same seed to avoid different bench results
+        payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
+        # encode as base64 to avoid ascii encode/decode errors
+        return base64.b64encode(payload_bytes)[:size]  # remove b64 overhead
+    return generate
diff --git a/testing/tests/responsiveness/conftest.py b/testing/tests/responsiveness/conftest.py
new file mode 100644
index 00000000..6b2687b1
--- /dev/null
+++ b/testing/tests/responsiveness/conftest.py
@@ -0,0 +1,22 @@
+import pytest
+
+from watchdog import Watchdog
+
+
+def _post_results(dog):
+    print("\n")
+    print("+" * 50)
+    print(dog.seconds_blocked)
+    print("+" * 50)
+
+
+@pytest.fixture
+def watchdog(request):
+    dog = Watchdog()
+    dog_d = dog.start()
+    request.addfinalizer(lambda: _post_results(dog))
+
+    def _run(deferred_fun):
+        deferred_fun().addCallback(lambda _: dog.stop())
+        return dog_d
+    return _run
diff --git a/testing/tests/responsiveness/test_responsiveness.py b/testing/tests/responsiveness/test_responsiveness.py
new file mode 100644
index 00000000..c8844d94
--- /dev/null
+++ b/testing/tests/responsiveness/test_responsiveness.py
@@ -0,0 +1,50 @@
+import pytest
+
+from twisted.internet import defer
+
+
+@pytest.inlineCallbacks
+def load_up(client, amount, payload):
+    # create a bunch of local documents
+    deferreds = []
+    for i in xrange(amount):
+        deferreds.append(client.create_doc({'content': payload}))
+    yield defer.gatherResults(deferreds)
+
+
+def create_upload(amount, size):
+
+    @pytest.mark.responsiveness
+    @pytest.inlineCallbacks
+    def _test(soledad_client, payload, watchdog):
+
+        client = soledad_client()
+        yield load_up(client, amount, payload(size))
+        yield watchdog(lambda: client.sync())
+
+    return _test
+
+
+test_responsiveness_upload_10_1000k = create_upload(10, 1000 * 1000)
+test_responsiveness_upload_100_100k = create_upload(100, 100 * 1000)
+test_responsiveness_upload_1000_10k = create_upload(1000, 10 * 1000)
+
+
+def create_download(downloads, size):
+
+    @pytest.mark.responsiveness
+    @pytest.inlineCallbacks
+    def _test(soledad_client, payload, watchdog):
+        client = soledad_client()
+        yield load_up(client, downloads, payload(size))
+        yield client.sync()
+
+        clean_client = soledad_client(force_fresh_db=True)
+        yield watchdog(clean_client.sync)
+
+    return _test
+
+
+test_responsiveness_download_10_1000k = create_download(10, 1000 * 1000)
+test_responsiveness_download_100_100k = create_download(100, 100 * 1000)
+test_responsiveness_download_1000_10k = create_download(1000, 10 * 1000)
diff --git a/testing/tests/responsiveness/watchdog.py b/testing/tests/responsiveness/watchdog.py
new file mode 100644
index 00000000..88f4fa67
--- /dev/null
+++ b/testing/tests/responsiveness/watchdog.py
@@ -0,0 +1,53 @@
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+from twisted.internet.threads import deferToThread
+
+
+class Watchdog(object):
+
+    DEBUG = False
+
+    def __init__(self, delay=0.01):
+        self.delay = delay
+        self.loop_call = LoopingCall.withCount(self.watch)
+        self.blocked = 0
+        self.checks = []
+        self.d = None
+
+    def start(self):
+        self.debug("\n[watchdog] starting")
+        self.loop_call.start(self.delay)
+        self.d = defer.Deferred()
+        return self.d
+
+    def watch(self, count):
+        self.debug("[watchdog] watching (%d)" % count)
+        if (self.loop_call.running):
+            self.checks.append(deferToThread(self._check, count))
+
+    def _check(self, count):
+        # self.debug("[watchdog] _checking (%d)" % count)
+        if count > 1:
+            self.blocked += count
+
+    def stop(self):
+        # delay the actual stop so we make sure at least one check watch will
+        # run in the reactor.
+        reactor.callLater(2 * self.delay, self._stop)
+
+    @defer.inlineCallbacks
+    def _stop(self):
+        if not self.loop_call.running:
+            return
+
+        self.loop_call.stop()
+        yield defer.gatherResults(self.checks)
+        self.d.callback(None)
+
+    @property
+    def seconds_blocked(self):
+        return self.blocked * self.delay
+
+    def debug(self, s):
+        if self.DEBUG:
+            print(s)
diff --git a/testing/tox.ini b/testing/tox.ini
index cbfa7dde..6bc82b8e 100644
--- a/testing/tox.ini
+++ b/testing/tox.ini
@@ -6,7 +6,7 @@ skipsdist=True
 basepython = python2.7
 commands = 
     ./ensure-pysqlcipher-has-usleep.sh
-    py.test -x --ignore=tests/benchmarks \
+    py.test -x --ignore=tests/benchmarks --ignore=tests/responsiveness \
                    --cov-report=html \
                    --cov-report=term \
                    --cov=leap.soledad \
@@ -37,7 +37,7 @@ install_command = pip install {opts} {packages}
 
 [testenv:py34]
 basepython = python3.4
-commands = py.test --ignore=tests/benchmarks \
+commands = py.test --ignore=tests/benchmarks --ignore=tests/responsiveness \
                    --cov-report=html \
                    --cov-report=term \
                    --cov=leap.soledad \
@@ -80,6 +80,13 @@ commands =
     py.test --benchmark-only --watch-memory {posargs}
 passenv = HOST_HOSTNAME
 
+[testenv:responsiveness]
+deps =
+    {[testenv:benchmark]deps}
+commands =
+    ./ensure-pysqlcipher-has-usleep.sh
+    pytest -m responsiveness {posargs}
+
 [testenv:code-check]
 changedir = ..
 deps =