diff options
| -rw-r--r-- | testing/tests/benchmarks/conftest.py | 12 | ||||
| -rw-r--r-- | testing/tests/conftest.py | 16 | ||||
| -rw-r--r-- | testing/tests/responsiveness/conftest.py | 22 | ||||
| -rw-r--r-- | testing/tests/responsiveness/test_responsiveness.py | 50 | ||||
| -rw-r--r-- | testing/tests/responsiveness/watchdog.py | 53 | ||||
| -rw-r--r-- | testing/tox.ini | 11 |
6 files changed, 150 insertions, 14 deletions
diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py index 25fe5134..377fc606 100644 --- a/testing/tests/benchmarks/conftest.py +++ b/testing/tests/benchmarks/conftest.py @@ -1,10 +1,8 @@ -import base64 import functools import numpy import os import psutil import pytest -import random import threading import time @@ -37,16 +35,6 @@ def pytest_collection_modifyitems(items): # @pytest.fixture() -def payload(): - def generate(size): - random.seed(1337) # same seed to avoid different bench results - payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) - # encode as base64 to avoid ascii encode/decode errors - return base64.b64encode(payload_bytes)[:size] # remove b64 overhead - return generate - - -@pytest.fixture() def txbenchmark(monitored_benchmark): def blockOnThread(*args, **kwargs): return threads.deferToThread( diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index e9641eba..6ce97625 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -1,8 +1,10 @@ import glob +import base64 import json import os import pytest import re +import random import requests import signal import socket @@ -331,3 +333,17 @@ if 'pytest_benchmark' in sys.modules: """ hostname = os.environ.get('HOST_HOSTNAME', socket.gethostname()) machine_info['host'] = hostname + + +# +# benchmark/responsiveness fixtures +# + +@pytest.fixture() +def payload(): + def generate(size): + random.seed(1337) # same seed to avoid different bench results + payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) + # encode as base64 to avoid ascii encode/decode errors + return base64.b64encode(payload_bytes)[:size] # remove b64 overhead + return generate diff --git a/testing/tests/responsiveness/conftest.py b/testing/tests/responsiveness/conftest.py new file mode 100644 index 00000000..6b2687b1 --- /dev/null +++ b/testing/tests/responsiveness/conftest.py @@ -0,0 +1,22 @@ +import pytest + +from watchdog import Watchdog + + +def _post_results(dog): + print("\n") + print("+" * 50) + print(dog.seconds_blocked) + print("+" * 50) + + +@pytest.fixture +def watchdog(request): + dog = Watchdog() + dog_d = dog.start() + request.addfinalizer(lambda: _post_results(dog)) + + def _run(deferred_fun): + deferred_fun().addCallback(lambda _: dog.stop()) + return dog_d + return _run diff --git a/testing/tests/responsiveness/test_responsiveness.py b/testing/tests/responsiveness/test_responsiveness.py new file mode 100644 index 00000000..c8844d94 --- /dev/null +++ b/testing/tests/responsiveness/test_responsiveness.py @@ -0,0 +1,50 @@ +import pytest + +from twisted.internet import defer + + +@pytest.inlineCallbacks +def load_up(client, amount, payload): + # create a bunch of local documents + deferreds = [] + for i in xrange(amount): + deferreds.append(client.create_doc({'content': payload})) + yield defer.gatherResults(deferreds) + + +def create_upload(amount, size): + + @pytest.mark.responsiveness + @pytest.inlineCallbacks + def _test(soledad_client, payload, watchdog): + + client = soledad_client() + yield load_up(client, amount, payload(size)) + yield watchdog(lambda: client.sync()) + + return _test + + +test_responsiveness_upload_10_1000k = create_upload(10, 1000 * 1000) +test_responsiveness_upload_100_100k = create_upload(100, 100 * 1000) +test_responsiveness_upload_1000_10k = create_upload(1000, 10 * 1000) + + +def create_download(downloads, size): + + @pytest.mark.responsiveness + @pytest.inlineCallbacks + def _test(soledad_client, payload, watchdog): + client = soledad_client() + yield load_up(client, downloads, payload(size)) + yield client.sync() + + clean_client = soledad_client(force_fresh_db=True) + yield watchdog(clean_client.sync) + + return _test + + +test_responsiveness_download_10_1000k = create_download(10, 1000 * 1000) +test_responsiveness_download_100_100k = create_download(100, 100 * 1000) +test_responsiveness_download_1000_10k = create_download(1000, 10 * 1000) diff --git a/testing/tests/responsiveness/watchdog.py b/testing/tests/responsiveness/watchdog.py new file mode 100644 index 00000000..88f4fa67 --- /dev/null +++ b/testing/tests/responsiveness/watchdog.py @@ -0,0 +1,53 @@ +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall +from twisted.internet.threads import deferToThread + + +class Watchdog(object): + + DEBUG = False + + def __init__(self, delay=0.01): + self.delay = delay + self.loop_call = LoopingCall.withCount(self.watch) + self.blocked = 0 + self.checks = [] + self.d = None + + def start(self): + self.debug("\n[watchdog] starting") + self.loop_call.start(self.delay) + self.d = defer.Deferred() + return self.d + + def watch(self, count): + self.debug("[watchdog] watching (%d)" % count) + if (self.loop_call.running): + self.checks.append(deferToThread(self._check, count)) + + def _check(self, count): + # self.debug("[watchdog] _checking (%d)" % count) + if count > 1: + self.blocked += count + + def stop(self): + # delay the actual stop so we make sure at least one check watch will + # run in the reactor. + reactor.callLater(2 * self.delay, self._stop) + + @defer.inlineCallbacks + def _stop(self): + if not self.loop_call.running: + return + + self.loop_call.stop() + yield defer.gatherResults(self.checks) + self.d.callback(None) + + @property + def seconds_blocked(self): + return self.blocked * self.delay + + def debug(self, s): + if self.DEBUG: + print(s) diff --git a/testing/tox.ini b/testing/tox.ini index cbfa7dde..6bc82b8e 100644 --- a/testing/tox.ini +++ b/testing/tox.ini @@ -6,7 +6,7 @@ skipsdist=True basepython = python2.7 commands = ./ensure-pysqlcipher-has-usleep.sh - py.test -x --ignore=tests/benchmarks \ + py.test -x --ignore=tests/benchmarks --ignore=tests/responsiveness \ --cov-report=html \ --cov-report=term \ --cov=leap.soledad \ @@ -37,7 +37,7 @@ install_command = pip install {opts} {packages} [testenv:py34] basepython = python3.4 -commands = py.test --ignore=tests/benchmarks \ +commands = py.test --ignore=tests/benchmarks --ignore=tests/responsiveness \ --cov-report=html \ --cov-report=term \ --cov=leap.soledad \ @@ -80,6 +80,13 @@ commands = py.test --benchmark-only --watch-memory {posargs} passenv = HOST_HOSTNAME +[testenv:responsiveness] +deps = + {[testenv:benchmark]deps} +commands = + ./ensure-pysqlcipher-has-usleep.sh + pytest -m responsiveness {posargs} + [testenv:code-check] changedir = .. deps = |