summaryrefslogtreecommitdiff
path: root/testing
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2017-07-27 17:12:04 -0300
committerVictor Shyba <victor1984@riseup.net>2017-09-11 16:24:30 -0300
commit99cffc7388d53f9aaf5b8890401ba8ddc5b29178 (patch)
treef98282d0818ebe048b49ffc0112a1843305497a2 /testing
parent70aa789c94b0dcced535cee86243e2dd4113b28a (diff)
[benchmarks] add responsiveness test with watchdog
Diffstat (limited to 'testing')
-rw-r--r--testing/tests/benchmarks/conftest.py12
-rw-r--r--testing/tests/conftest.py16
-rw-r--r--testing/tests/responsiveness/conftest.py22
-rw-r--r--testing/tests/responsiveness/test_responsiveness.py50
-rw-r--r--testing/tests/responsiveness/watchdog.py53
-rw-r--r--testing/tox.ini11
6 files changed, 150 insertions, 14 deletions
diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py
index 25fe5134..377fc606 100644
--- a/testing/tests/benchmarks/conftest.py
+++ b/testing/tests/benchmarks/conftest.py
@@ -1,10 +1,8 @@
-import base64
import functools
import numpy
import os
import psutil
import pytest
-import random
import threading
import time
@@ -37,16 +35,6 @@ def pytest_collection_modifyitems(items):
#
@pytest.fixture()
-def payload():
- def generate(size):
- random.seed(1337) # same seed to avoid different bench results
- payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
- # encode as base64 to avoid ascii encode/decode errors
- return base64.b64encode(payload_bytes)[:size] # remove b64 overhead
- return generate
-
-
-@pytest.fixture()
def txbenchmark(monitored_benchmark):
def blockOnThread(*args, **kwargs):
return threads.deferToThread(
diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py
index e9641eba..6ce97625 100644
--- a/testing/tests/conftest.py
+++ b/testing/tests/conftest.py
@@ -1,8 +1,10 @@
import glob
+import base64
import json
import os
import pytest
import re
+import random
import requests
import signal
import socket
@@ -331,3 +333,17 @@ if 'pytest_benchmark' in sys.modules:
"""
hostname = os.environ.get('HOST_HOSTNAME', socket.gethostname())
machine_info['host'] = hostname
+
+
+#
+# benchmark/responsiveness fixtures
+#
+
+@pytest.fixture()
+def payload():
+ def generate(size):
+ random.seed(1337) # same seed to avoid different bench results
+ payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
+ # encode as base64 to avoid ascii encode/decode errors
+ return base64.b64encode(payload_bytes)[:size] # remove b64 overhead
+ return generate
diff --git a/testing/tests/responsiveness/conftest.py b/testing/tests/responsiveness/conftest.py
new file mode 100644
index 00000000..6b2687b1
--- /dev/null
+++ b/testing/tests/responsiveness/conftest.py
@@ -0,0 +1,22 @@
+import pytest
+
+from watchdog import Watchdog
+
+
+def _post_results(dog):
+ print("\n")
+ print("+" * 50)
+ print(dog.seconds_blocked)
+ print("+" * 50)
+
+
+@pytest.fixture
+def watchdog(request):
+ dog = Watchdog()
+ dog_d = dog.start()
+ request.addfinalizer(lambda: _post_results(dog))
+
+ def _run(deferred_fun):
+ deferred_fun().addCallback(lambda _: dog.stop())
+ return dog_d
+ return _run
diff --git a/testing/tests/responsiveness/test_responsiveness.py b/testing/tests/responsiveness/test_responsiveness.py
new file mode 100644
index 00000000..c8844d94
--- /dev/null
+++ b/testing/tests/responsiveness/test_responsiveness.py
@@ -0,0 +1,50 @@
+import pytest
+
+from twisted.internet import defer
+
+
+@pytest.inlineCallbacks
+def load_up(client, amount, payload):
+ # create a bunch of local documents
+ deferreds = []
+ for i in xrange(amount):
+ deferreds.append(client.create_doc({'content': payload}))
+ yield defer.gatherResults(deferreds)
+
+
+def create_upload(amount, size):
+
+ @pytest.mark.responsiveness
+ @pytest.inlineCallbacks
+ def _test(soledad_client, payload, watchdog):
+
+ client = soledad_client()
+ yield load_up(client, amount, payload(size))
+ yield watchdog(lambda: client.sync())
+
+ return _test
+
+
+test_responsiveness_upload_10_1000k = create_upload(10, 1000 * 1000)
+test_responsiveness_upload_100_100k = create_upload(100, 100 * 1000)
+test_responsiveness_upload_1000_10k = create_upload(1000, 10 * 1000)
+
+
+def create_download(downloads, size):
+
+ @pytest.mark.responsiveness
+ @pytest.inlineCallbacks
+ def _test(soledad_client, payload, watchdog):
+ client = soledad_client()
+ yield load_up(client, downloads, payload(size))
+ yield client.sync()
+
+ clean_client = soledad_client(force_fresh_db=True)
+ yield watchdog(clean_client.sync)
+
+ return _test
+
+
+test_responsiveness_download_10_1000k = create_download(10, 1000 * 1000)
+test_responsiveness_download_100_100k = create_download(100, 100 * 1000)
+test_responsiveness_download_1000_10k = create_download(1000, 10 * 1000)
diff --git a/testing/tests/responsiveness/watchdog.py b/testing/tests/responsiveness/watchdog.py
new file mode 100644
index 00000000..88f4fa67
--- /dev/null
+++ b/testing/tests/responsiveness/watchdog.py
@@ -0,0 +1,53 @@
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+from twisted.internet.threads import deferToThread
+
+
+class Watchdog(object):
+
+ DEBUG = False
+
+ def __init__(self, delay=0.01):
+ self.delay = delay
+ self.loop_call = LoopingCall.withCount(self.watch)
+ self.blocked = 0
+ self.checks = []
+ self.d = None
+
+ def start(self):
+ self.debug("\n[watchdog] starting")
+ self.loop_call.start(self.delay)
+ self.d = defer.Deferred()
+ return self.d
+
+ def watch(self, count):
+ self.debug("[watchdog] watching (%d)" % count)
+ if (self.loop_call.running):
+ self.checks.append(deferToThread(self._check, count))
+
+ def _check(self, count):
+ # self.debug("[watchdog] _checking (%d)" % count)
+ if count > 1:
+ self.blocked += count
+
+ def stop(self):
+ # delay the actual stop so we make sure at least one check watch will
+ # run in the reactor.
+ reactor.callLater(2 * self.delay, self._stop)
+
+ @defer.inlineCallbacks
+ def _stop(self):
+ if not self.loop_call.running:
+ return
+
+ self.loop_call.stop()
+ yield defer.gatherResults(self.checks)
+ self.d.callback(None)
+
+ @property
+ def seconds_blocked(self):
+ return self.blocked * self.delay
+
+ def debug(self, s):
+ if self.DEBUG:
+ print(s)
diff --git a/testing/tox.ini b/testing/tox.ini
index cbfa7dde..6bc82b8e 100644
--- a/testing/tox.ini
+++ b/testing/tox.ini
@@ -6,7 +6,7 @@ skipsdist=True
basepython = python2.7
commands =
./ensure-pysqlcipher-has-usleep.sh
- py.test -x --ignore=tests/benchmarks \
+ py.test -x --ignore=tests/benchmarks --ignore=tests/responsiveness \
--cov-report=html \
--cov-report=term \
--cov=leap.soledad \
@@ -37,7 +37,7 @@ install_command = pip install {opts} {packages}
[testenv:py34]
basepython = python3.4
-commands = py.test --ignore=tests/benchmarks \
+commands = py.test --ignore=tests/benchmarks --ignore=tests/responsiveness \
--cov-report=html \
--cov-report=term \
--cov=leap.soledad \
@@ -80,6 +80,13 @@ commands =
py.test --benchmark-only --watch-memory {posargs}
passenv = HOST_HOSTNAME
+[testenv:responsiveness]
+deps =
+ {[testenv:benchmark]deps}
+commands =
+ ./ensure-pysqlcipher-has-usleep.sh
+ pytest -m responsiveness {posargs}
+
[testenv:code-check]
changedir = ..
deps =