summaryrefslogtreecommitdiff
path: root/tests/benchmarks
diff options
context:
space:
mode:
Diffstat (limited to 'tests/benchmarks')
-rw-r--r--tests/benchmarks/README.md51
-rw-r--r--tests/benchmarks/assets/cert_default.conf15
-rw-r--r--tests/benchmarks/conftest.py154
-rw-r--r--tests/benchmarks/pytest.ini2
-rw-r--r--tests/benchmarks/test_crypto.py109
-rw-r--r--tests/benchmarks/test_legacy_vs_blobs.py305
-rw-r--r--tests/benchmarks/test_misc.py9
-rw-r--r--tests/benchmarks/test_resources.py50
-rw-r--r--tests/benchmarks/test_sqlcipher.py47
-rw-r--r--tests/benchmarks/test_sqlite_blobs_backend.py82
-rw-r--r--tests/benchmarks/test_sync.py92
11 files changed, 916 insertions, 0 deletions
diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md
new file mode 100644
index 00000000..b2465a78
--- /dev/null
+++ b/tests/benchmarks/README.md
@@ -0,0 +1,51 @@
+Benchmark tests
+===============
+
+This folder contains benchmark tests for Soledad. It aims to provide a fair
+account on the time and resources taken to perform some actions.
+
+These benchmarks are built on top of `pytest-benchmark`, a `pytest` fixture that
+provides means for running test functions multiple times and generating
+reports. The results are printed to screen and also posted to elasticsearch.
+
+`pytest-benchmark` runs tests multiple times so it can provide meaningful
+statistics for the time taken for a tipical run of a test function. The number
+of times that the test is run can be manually or automatically configured. When
+automatically configured, the number of runs is decided by taking into account
+multiple `pytest-benchmark` configuration parameters. See the following page
+for more details on how `pytest-benchmark` works:
+
+ https://pytest-benchmark.readthedocs.io/en/stable/calibration.html
+
+Some graphs and analysis resulting from these tests can be seen on:
+
+ https://benchmarks.leap.se/
+
+
+Resource consumption
+--------------------
+
+For each test, CPU and memory usage statistics are also collected, by querying
+`cpu_percent()` and `memory_percent()` from `psutil.Process` for the current
+test process. Some notes about the current resource consumption estimation process:
+
+* Currently, resources are measured for the whole set of rounds that a test
+ function is run. That means that the CPU and memory percentage include the
+ `pytest` and `pytest-benchmark` machinery overhead. Anyway, for now this might
+ provide a fair approximation of per-run test function resource usage.
+
+* CPU is measured before and after the run of the benchmark function and
+ returns the percentage that the currnet process occupied of the CPU time
+ between the two calls.
+
+* Memory is sampled during the benchmark run by a separate thread. Sampling
+ interval might have to be configured on a per-test basis, as different tests
+ take different times to execute (from milliseconds to tens of seconds). For
+ now, an interval of 0.1s seems to cover all tests.
+
+
+Benchmarks website
+------------------
+
+To update the benchmarks website, see the documentation in
+``../../../docs/misc/benchmarks-website.rst``.
diff --git a/tests/benchmarks/assets/cert_default.conf b/tests/benchmarks/assets/cert_default.conf
new file mode 100644
index 00000000..8043cea3
--- /dev/null
+++ b/tests/benchmarks/assets/cert_default.conf
@@ -0,0 +1,15 @@
+[ req ]
+default_bits = 1024
+default_keyfile = keyfile.pem
+distinguished_name = req_distinguished_name
+prompt = no
+output_password = mypass
+
+[ req_distinguished_name ]
+C = GB
+ST = Test State or Province
+L = Test Locality
+O = Organization Name
+OU = Organizational Unit Name
+CN = localhost
+emailAddress = test@email.address
diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py
new file mode 100644
index 00000000..80eccb08
--- /dev/null
+++ b/tests/benchmarks/conftest.py
@@ -0,0 +1,154 @@
+import functools
+import numpy
+import os
+import psutil
+import pytest
+import threading
+import time
+
+from twisted.internet import threads, reactor
+
+
+#
+# pytest customizations
+#
+
+# mark benchmark tests using their group names (thanks ionelmc! :)
+def pytest_collection_modifyitems(items, config):
+ for item in items:
+ bench = item.get_marker("benchmark")
+ if bench and bench.kwargs.get('group'):
+ group = bench.kwargs['group']
+ marker = getattr(pytest.mark, 'benchmark_' + group)
+ item.add_marker(marker)
+
+ subdir = config.getoption('subdir')
+ if subdir == 'benchmarks':
+ # we have to manually setup the events server in order to be able to
+ # signal events. This is usually done by the enclosing application
+ # using soledad client (i.e. bitmask client).
+ from leap.common.events import server
+ server.ensure_server()
+
+
+#
+# benchmark fixtures
+#
+
+@pytest.fixture()
+def txbenchmark(monitored_benchmark):
+ def blockOnThread(*args, **kwargs):
+ return threads.deferToThread(
+ monitored_benchmark, threads.blockingCallFromThread,
+ reactor, *args, **kwargs)
+ return blockOnThread
+
+
+@pytest.fixture()
+def txbenchmark_with_setup(monitored_benchmark_with_setup):
+ def blockOnThreadWithSetup(setup, f, *args, **kwargs):
+ def blocking_runner(*args, **kwargs):
+ return threads.blockingCallFromThread(reactor, f, *args, **kwargs)
+
+ def blocking_setup():
+ args = threads.blockingCallFromThread(reactor, setup)
+ try:
+ return tuple(arg for arg in args), {}
+ except TypeError:
+ return ((args,), {}) if args else None
+
+ def bench():
+ return monitored_benchmark_with_setup(
+ blocking_runner, setup=blocking_setup,
+ rounds=4, warmup_rounds=1, iterations=1,
+ args=args, kwargs=kwargs)
+ return threads.deferToThread(bench)
+ return blockOnThreadWithSetup
+
+
+#
+# resource monitoring
+#
+
+class ResourceWatcher(threading.Thread):
+
+ sampling_interval = 0.1
+
+ def __init__(self, watch_memory):
+ threading.Thread.__init__(self)
+ self.process = psutil.Process(os.getpid())
+ self.running = False
+ # monitored resources
+ self.cpu_percent = None
+ self.watch_memory = watch_memory
+ self.memory_samples = []
+ self.memory_percent = None
+
+ def run(self):
+ self.running = True
+ self.process.cpu_percent()
+ # decide how long to sleep based on need to sample memory
+ sleep = self.sampling_interval if not self.watch_memory else 1
+ while self.running:
+ if self.watch_memory:
+ sample = self.process.memory_percent(memtype='rss')
+ self.memory_samples.append(sample)
+ time.sleep(sleep)
+
+ def stop(self):
+ self.running = False
+ self.join()
+ # save cpu usage info
+ self.cpu_percent = self.process.cpu_percent()
+ # save memory usage info
+ if self.watch_memory:
+ memory_percent = {
+ 'sampling_interval': self.sampling_interval,
+ 'samples': self.memory_samples,
+ 'stats': {},
+ }
+ for stat in 'max', 'min', 'mean', 'std':
+ fun = getattr(numpy, stat)
+ memory_percent['stats'][stat] = fun(self.memory_samples)
+ self.memory_percent = memory_percent
+
+
+def _monitored_benchmark(benchmark_fixture, benchmark_function, request,
+ *args, **kwargs):
+ # setup resource monitoring
+ watch_memory = _watch_memory(request)
+ watcher = ResourceWatcher(watch_memory)
+ watcher.start()
+ # run benchmarking function
+ benchmark_function(*args, **kwargs)
+ # store results
+ watcher.stop()
+ benchmark_fixture.extra_info.update({
+ 'cpu_percent': watcher.cpu_percent
+ })
+ if watch_memory:
+ benchmark_fixture.extra_info.update({
+ 'memory_percent': watcher.memory_percent,
+ })
+ # add docstring info
+ if request.scope == 'function':
+ fun = request.function
+ doc = fun.__doc__ or ''
+ benchmark_fixture.extra_info.update({'doc': doc.strip()})
+
+
+def _watch_memory(request):
+ return request.config.getoption('--watch-memory')
+
+
+@pytest.fixture
+def monitored_benchmark(benchmark, request):
+ return functools.partial(
+ _monitored_benchmark, benchmark, benchmark, request)
+
+
+@pytest.fixture
+def monitored_benchmark_with_setup(benchmark, request, *args, **kwargs):
+ return functools.partial(
+ _monitored_benchmark, benchmark, benchmark.pedantic, request,
+ *args, **kwargs)
diff --git a/tests/benchmarks/pytest.ini b/tests/benchmarks/pytest.ini
new file mode 100644
index 00000000..7a0508ce
--- /dev/null
+++ b/tests/benchmarks/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+twisted = yes
diff --git a/tests/benchmarks/test_crypto.py b/tests/benchmarks/test_crypto.py
new file mode 100644
index 00000000..3be447a5
--- /dev/null
+++ b/tests/benchmarks/test_crypto.py
@@ -0,0 +1,109 @@
+"""
+Benchmarks for crypto operations.
+If you don't want to stress your local machine too much, you can pass the
+SIZE_LIMT environment variable.
+
+For instance, to keep the maximum payload at 1MB:
+
+SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py
+"""
+import pytest
+import os
+import json
+from uuid import uuid4
+
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.client import _crypto
+
+LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000)))
+
+
+def create_doc_encryption(size):
+ @pytest.mark.benchmark(group="test_crypto_encrypt_doc")
+ @pytest.inlineCallbacks
+ def test_doc_encryption(soledad_client, txbenchmark, payload):
+ """
+ Encrypt a document of a given size.
+ """
+ crypto = soledad_client()._crypto
+
+ DOC_CONTENT = {'payload': payload(size)}
+ doc = SoledadDocument(
+ doc_id=uuid4().hex, rev='rev',
+ json=json.dumps(DOC_CONTENT))
+
+ yield txbenchmark(crypto.encrypt_doc, doc)
+ return test_doc_encryption
+
+
+# TODO this test is really bullshit, because it's still including
+# the json serialization.
+
+def create_doc_decryption(size):
+ @pytest.inlineCallbacks
+ @pytest.mark.benchmark(group="test_crypto_decrypt_doc")
+ def test_doc_decryption(soledad_client, txbenchmark, payload):
+ """
+ Decrypt a document of a given size.
+ """
+ crypto = soledad_client()._crypto
+
+ DOC_CONTENT = {'payload': payload(size)}
+ doc = SoledadDocument(
+ doc_id=uuid4().hex, rev='rev',
+ json=json.dumps(DOC_CONTENT))
+
+ encrypted_doc = yield crypto.encrypt_doc(doc)
+ doc.set_json(encrypted_doc)
+
+ yield txbenchmark(crypto.decrypt_doc, doc)
+ return test_doc_decryption
+
+
+def create_raw_encryption(size):
+ @pytest.mark.benchmark(group="test_crypto_raw_encrypt")
+ def test_raw_encrypt(monitored_benchmark, payload):
+ """
+ Encrypt raw payload using default mode from crypto module.
+ """
+ key = payload(32)
+ monitored_benchmark(_crypto.encrypt_sym, payload(size), key)
+ return test_raw_encrypt
+
+
+def create_raw_decryption(size):
+ @pytest.mark.benchmark(group="test_crypto_raw_decrypt")
+ def test_raw_decrypt(monitored_benchmark, payload):
+ """
+ Decrypt raw payload using default mode from crypto module.
+ """
+ key = payload(32)
+ iv, ciphertext = _crypto.encrypt_sym(payload(size), key)
+ monitored_benchmark(_crypto.decrypt_sym, ciphertext, key, iv)
+ return test_raw_decrypt
+
+
+# Create the TESTS in the global namespace, they'll be picked by the benchmark
+# plugin.
+
+encryption_tests = [
+ ('10k', 1E4),
+ ('100k', 1E5),
+ ('500k', 5E5),
+ ('1M', 1E6),
+ ('10M', 1E7),
+ ('50M', 5E7),
+]
+
+for name, size in encryption_tests:
+ if size < LIMIT:
+ sz = int(size)
+ globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz)
+ globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz)
+
+
+for name, size in encryption_tests:
+ if size < LIMIT:
+ sz = int(size)
+ globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz)
+ globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz)
diff --git a/tests/benchmarks/test_legacy_vs_blobs.py b/tests/benchmarks/test_legacy_vs_blobs.py
new file mode 100644
index 00000000..47d6482c
--- /dev/null
+++ b/tests/benchmarks/test_legacy_vs_blobs.py
@@ -0,0 +1,305 @@
+# "Legacy" versus "Incoming blobs" pipeline comparison
+# ====================================================
+#
+# This benchmarking aims to compare the legacy and new mail incoming pipeline,
+# to asses performance improvements brought by the introduction of blobs.
+#
+# We use the following sizes in these tests:
+#
+# - headers: 4 KB
+# - metadata: 0.1 KB
+# - flags: 0.5 KB
+# - content: variable
+#
+# "Legacy" incoming mail pipeline:
+#
+# - email arrives at MX.
+# - MX encrypts to public key and puts into couch.
+# - pubkey encrypted doc is synced to soledad client as "incoming".
+# - bitmask mail processes "incoming" and generates 3 metadocs + 1 payload
+# doc per message.
+# - soledad client syncs 4 documents back to server.
+#
+# "Incoming blobs" mail pipeline:
+#
+# - email arrives at MX.
+# - MX encyrpts to public key and puts into soledad server.
+# - soledad server writes a blob to filesystem.
+# - soledad client gets the incoming blob from server and generates 3
+# metadocs + 1 blob.
+# - soledad client syncs 3 meta documents and 1 blob back to server.
+#
+# Some notes about the tests in this file:
+#
+# - This is a simulation of the legacy and new incoming mail pipelines.
+# There is no actual mail processing operation done (i.e. no pubkey crypto,
+# no mail parsing), only usual soledad document manipulation and sync (with
+# local 1network and crypto).
+#
+# - Each test simulates a whole incoming mail pipeline, including get new
+# incoming messages from server, create new documents that represent the
+# parsed message, and synchronize those back to the server.
+#
+# - These tests are disabled by default because it doesn't make much sense to
+# have them run automatically for all commits in the repository. Instead,
+# we will run them manually for specific releases and store results and
+# analisys in a subfolder.
+
+import base64
+import pytest
+import random
+import sys
+import treq
+import uuid
+
+from io import BytesIO
+
+from twisted.internet.defer import gatherResults
+from twisted.internet.defer import returnValue
+from twisted.internet.defer import DeferredSemaphore
+
+from leap.soledad.common.blobs import Flags
+from leap.soledad.client._db.blobs import BlobDoc
+
+
+def payload(size):
+ random.seed(1337) # same seed to avoid different bench results
+ payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
+ # encode as base64 to avoid ascii encode/decode errors
+ return base64.b64encode(payload_bytes)[:size] # remove b64 overhead
+
+
+PARTS = {
+ 'headers': payload(4000),
+ 'metadata': payload(100),
+ 'flags': payload(500),
+}
+
+
+#
+# "Legacy" incoming mail pipeline.
+#
+
+@pytest.inlineCallbacks
+def load_up_legacy(client, amount, content):
+ # make sure there are no document from previous runs
+ yield client.sync()
+ _, docs = yield client.get_all_docs()
+ deferreds = []
+ for doc in docs:
+ d = client.delete_doc(doc)
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+ yield client.sync()
+
+ # create a bunch of local documents representing email messages
+ deferreds = []
+ for i in xrange(amount):
+ deferreds.append(client.create_doc(content))
+ yield gatherResults(deferreds)
+ yield client.sync()
+
+
+@pytest.inlineCallbacks
+def process_incoming_docs(client, docs):
+ deferreds = []
+ for doc in docs:
+
+ # create fake documents that represent message
+ for name in PARTS.keys():
+ d = client.create_doc({name: doc.content[name]})
+ deferreds.append(d)
+
+ # create one document with content
+ key = 'content'
+ d = client.create_doc({key: doc.content[key]})
+ deferreds.append(d)
+
+ # delete the old incoming document
+ d = client.delete_doc(doc)
+ deferreds.append(d)
+
+ # wait for all operatios to succeed
+ yield gatherResults(deferreds)
+
+
+def create_legacy_test(amount, size):
+ group = 'test_legacy_vs_blobs_%d_%dk' % (amount, (size / 1000))
+
+ @pytest.inlineCallbacks
+ @pytest.mark.skip(reason="avoid running for all commits")
+ @pytest.mark.benchmark(group=group)
+ def test(soledad_client, txbenchmark_with_setup):
+ client = soledad_client()
+
+ # setup the content of initial documents representing incoming emails
+ content = {'content': payload(size), 'incoming': True}
+ for name, data in PARTS.items():
+ content[name] = data
+
+ @pytest.inlineCallbacks
+ def setup():
+ yield load_up_legacy(client, amount, content)
+ clean_client = soledad_client(force_fresh_db=True)
+ yield clean_client.create_index('incoming', 'bool(incoming)')
+ returnValue(clean_client)
+
+ @pytest.inlineCallbacks
+ def legacy_pipeline(client):
+ yield client.sync()
+ docs = yield client.get_from_index('incoming', '1')
+ yield process_incoming_docs(client, docs)
+ yield client.sync()
+
+ yield txbenchmark_with_setup(setup, legacy_pipeline)
+ return test
+
+
+# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change
+# the number of docs or the doc sizes for the tests below.
+test_legacy_10_1000k = create_legacy_test(10, 1000 * 1000)
+test_legacy_100_100k = create_legacy_test(100, 100 * 1000)
+test_legacy_1000_10k = create_legacy_test(1000, 10 * 1000)
+
+
+#
+# "Incoming blobs" mail pipeline:
+#
+
+# used to limit the amount of concurrent accesses to the blob manager
+semaphore = DeferredSemaphore(2)
+
+
+# deliver data to a user by using the incoming api at given url.
+def deliver_using_incoming_api(url, user_uuid, token, data):
+ auth = 'Token %s' % base64.b64encode('%s:%s' % (user_uuid, token))
+ uri = "%s/incoming/%s/%s?namespace=MX" % (url, user_uuid, uuid.uuid4().hex)
+ return treq.put(uri, headers={'Authorization': auth}, data=BytesIO(data))
+
+
+# deliver data to a user by faking incoming using blobs
+@pytest.inlineCallbacks
+def deliver_using_blobs(client, fd):
+ # put
+ blob_id = uuid.uuid4().hex
+ doc = BlobDoc(fd, blob_id=blob_id)
+ size = sys.getsizeof(fd)
+ yield client.blobmanager.put(doc, size, namespace='MX')
+ # and flag
+ flags = [Flags.PENDING]
+ yield client.blobmanager.set_flags(blob_id, flags, namespace='MX')
+
+
+def reclaim_free_space(client):
+ return client.blobmanager.local.dbpool.runQuery("VACUUM")
+
+
+@pytest.inlineCallbacks
+def load_up_blobs(client, amount, data):
+ # make sure there are no document from previous runs
+ yield client.sync()
+ _, docs = yield client.get_all_docs()
+ deferreds = []
+ for doc in docs:
+ d = client.delete_doc(doc)
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+ yield client.sync()
+
+ # delete all payload from blobs db and server
+ for namespace in ['MX', 'payload']:
+ ids = yield client.blobmanager.remote_list(namespace=namespace)
+ deferreds = []
+ for blob_id in ids:
+ d = semaphore.run(
+ client.blobmanager.delete, blob_id, namespace=namespace)
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+
+ # create a bunch of incoming blobs
+ deferreds = []
+ for i in xrange(amount):
+ # choose method of delivery based in test being local or remote
+ if '127.0.0.1' in client.server_url:
+ fun = deliver_using_incoming_api
+ args = (client.server_url, client.uuid, client.token, data)
+ else:
+ fun = deliver_using_blobs
+ args = (client, BytesIO(data))
+ d = semaphore.run(fun, *args)
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+
+ # empty local blobs db
+ yield client.blobmanager.local.dbpool.runQuery(
+ "DELETE FROM blobs WHERE 1;")
+ yield reclaim_free_space(client)
+
+
+@pytest.inlineCallbacks
+def process_incoming_blobs(client, pending):
+ # process items
+ deferreds = []
+ for item in pending:
+ d = process_one_incoming_blob(client, item)
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+
+
+@pytest.inlineCallbacks
+def process_one_incoming_blob(client, item):
+ fd = yield semaphore.run(
+ client.blobmanager.get, item, namespace='MX')
+
+ # create metadata docs
+ deferreds = []
+ for name, data in PARTS.items():
+ d = client.create_doc({name: data})
+ deferreds.append(d)
+
+ # put the incoming blob as it would be done after mail processing
+ doc = BlobDoc(fd, blob_id=uuid.uuid4().hex)
+ size = sys.getsizeof(fd)
+ d = semaphore.run(
+ client.blobmanager.put, doc, size, namespace='payload')
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+
+ # delete incoming blob
+ yield semaphore.run(
+ client.blobmanager.delete, item, namespace='MX')
+
+
+def create_blobs_test(amount, size):
+ group = 'test_legacy_vs_blobs_%d_%dk' % (amount, (size / 1000))
+
+ @pytest.inlineCallbacks
+ @pytest.mark.skip(reason="avoid running for all commits")
+ @pytest.mark.benchmark(group=group)
+ def test(soledad_client, txbenchmark_with_setup):
+ client = soledad_client()
+ blob_payload = payload(size)
+
+ @pytest.inlineCallbacks
+ def setup():
+ yield load_up_blobs(client, amount, blob_payload)
+ returnValue(soledad_client(force_fresh_db=True))
+
+ @pytest.inlineCallbacks
+ def blobs_pipeline(client):
+ pending = yield client.blobmanager.remote_list(
+ namespace='MX', filter_flags=Flags.PENDING)
+ yield process_incoming_blobs(client, pending)
+ # reclaim_free_space(client)
+ yield client.sync()
+ yield client.blobmanager.send_missing(namespace='payload')
+
+ yield txbenchmark_with_setup(setup, blobs_pipeline)
+ return test
+
+
+# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change
+# the number of docs or the doc sizes for the tests below.
+test_blobs_10_1000k = create_blobs_test(10, 1000 * 1000)
+test_blobs_100_100k = create_blobs_test(100, 100 * 1000)
+test_blobs_1000_10k = create_blobs_test(1000, 10 * 1000)
diff --git a/tests/benchmarks/test_misc.py b/tests/benchmarks/test_misc.py
new file mode 100644
index 00000000..8b2178b9
--- /dev/null
+++ b/tests/benchmarks/test_misc.py
@@ -0,0 +1,9 @@
+import pytest
+
+
+@pytest.mark.benchmark(group="test_instance")
+def test_initialization(soledad_client, monitored_benchmark):
+ """
+ Soledad client object initialization.
+ """
+ monitored_benchmark(soledad_client)
diff --git a/tests/benchmarks/test_resources.py b/tests/benchmarks/test_resources.py
new file mode 100644
index 00000000..173edbd1
--- /dev/null
+++ b/tests/benchmarks/test_resources.py
@@ -0,0 +1,50 @@
+import pytest
+import random
+import time
+
+from decimal import Decimal
+
+
+def bellardBig(n):
+ # http://en.wikipedia.org/wiki/Bellard%27s_formula
+ pi = Decimal(0)
+ k = 0
+ while k < n:
+ pi += (Decimal(-1) ** k / (1024 ** k)) * (
+ Decimal(256) / (10 * k + 1) +
+ Decimal(1) / (10 * k + 9) -
+ Decimal(64) / (10 * k + 3) -
+ Decimal(32) / (4 * k + 1) -
+ Decimal(4) / (10 * k + 5) -
+ Decimal(4) / (10 * k + 7) -
+ Decimal(1) / (4 * k + 3))
+ k += 1
+ pi = pi * 1 / (2 ** 6)
+ return pi
+
+
+@pytest.mark.skip(reason='not a real use case, used only for instrumentation')
+def test_cpu_intensive(monitored_benchmark):
+
+ def _cpu_intensive():
+ sleep = [random.uniform(0.5, 1.5) for _ in xrange(3)]
+ while sleep:
+ t = sleep.pop()
+ time.sleep(t)
+ bellardBig(int((10 ** 3) * t))
+
+ monitored_benchmark(_cpu_intensive)
+
+
+@pytest.mark.skip(reason='not a real use case, used only for instrumentation')
+def test_memory_intensive(monitored_benchmark):
+
+ def _memory_intensive():
+ sleep = [random.uniform(0.5, 1.5) for _ in xrange(3)]
+ bigdata = ""
+ while sleep:
+ t = sleep.pop()
+ bigdata += "b" * 10 * int(10E6)
+ time.sleep(t)
+
+ monitored_benchmark(_memory_intensive)
diff --git a/tests/benchmarks/test_sqlcipher.py b/tests/benchmarks/test_sqlcipher.py
new file mode 100644
index 00000000..9108084c
--- /dev/null
+++ b/tests/benchmarks/test_sqlcipher.py
@@ -0,0 +1,47 @@
+'''
+Tests SoledadClient/SQLCipher interaction
+'''
+import pytest
+
+from twisted.internet.defer import gatherResults
+
+
+def load_up(client, amount, payload, defer=True):
+ results = [client.create_doc({'content': payload}) for _ in xrange(amount)]
+ if defer:
+ return gatherResults(results)
+
+
+def build_test_sqlcipher_async_create(amount, size):
+ @pytest.inlineCallbacks
+ @pytest.mark.benchmark(group="test_sqlcipher_async_create")
+ def test(soledad_client, txbenchmark_with_setup, payload):
+ """
+ Create many documents of a given size concurrently.
+ """
+ client = soledad_client()
+ yield txbenchmark_with_setup(
+ lambda: None, load_up, client, amount, payload(size))
+ return test
+
+
+def build_test_sqlcipher_create(amount, size):
+ @pytest.mark.skip(reason="this test is lengthy and not a real use case")
+ @pytest.mark.benchmark(group="test_sqlcipher_create")
+ def test(soledad_client, monitored_benchmark, payload):
+ """
+ Create many documents of a given size serially.
+ """
+ client = soledad_client()._dbsyncer
+ monitored_benchmark(
+ load_up, client, amount, payload(size), defer=False)
+ return test
+
+
+test_async_create_10_1000k = build_test_sqlcipher_async_create(10, 1000 * 1000)
+test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000)
+test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000)
+# synchronous
+test_create_10_1000k = build_test_sqlcipher_create(10, 1000 * 1000)
+test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000)
+test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000)
diff --git a/tests/benchmarks/test_sqlite_blobs_backend.py b/tests/benchmarks/test_sqlite_blobs_backend.py
new file mode 100644
index 00000000..e02cacad
--- /dev/null
+++ b/tests/benchmarks/test_sqlite_blobs_backend.py
@@ -0,0 +1,82 @@
+import pytest
+import os
+
+from uuid import uuid4
+from io import BytesIO
+
+from twisted.internet.defer import gatherResults
+from twisted.internet.defer import DeferredSemaphore
+
+from leap.soledad.client._db.blobs import SQLiteBlobBackend
+
+
+semaphore = DeferredSemaphore(2)
+
+
+#
+# put
+#
+
+def put(backend, amount, data):
+ deferreds = []
+ for _ in xrange(amount):
+ blob_id = uuid4().hex
+ fd = BytesIO(data)
+ size = len(data)
+ d = semaphore.run(backend.put, blob_id, fd, size)
+ deferreds.append(d)
+ return gatherResults(deferreds)
+
+
+def create_put_test(amount, size):
+
+ @pytest.inlineCallbacks
+ @pytest.mark.sqlite_blobs_backend_put
+ def test(txbenchmark, payload, tmpdir):
+ dbpath = os.path.join(tmpdir.strpath, 'blobs.db')
+ backend = SQLiteBlobBackend(dbpath, key='123')
+ data = payload(size)
+ yield txbenchmark(put, backend, amount, data)
+
+ return test
+
+
+test_sqlite_blobs_backend_put_1_10000k = create_put_test(1, 10000 * 1000)
+test_sqlite_blobs_backend_put_10_1000k = create_put_test(10, 1000 * 1000)
+test_sqlite_blobs_backend_put_100_100k = create_put_test(100, 100 * 1000)
+test_sqlite_blobs_backend_put_1000_10k = create_put_test(1000, 10 * 1000)
+
+
+#
+# put
+#
+
+@pytest.inlineCallbacks
+def get(backend):
+ local = yield backend.list()
+ deferreds = []
+ for blob_id in local:
+ d = backend.get(blob_id)
+ deferreds.append(d)
+ yield gatherResults(deferreds)
+
+
+def create_get_test(amount, size):
+
+ @pytest.inlineCallbacks
+ @pytest.mark.sqlite_blobs_backend_get
+ def test(txbenchmark, payload, tmpdir):
+ dbpath = os.path.join(tmpdir.strpath, 'blobs.db')
+ backend = SQLiteBlobBackend(dbpath, key='123')
+ data = payload(size)
+
+ yield put(backend, amount, data)
+ yield txbenchmark(get, backend)
+
+ return test
+
+
+test_sqlite_blobs_backend_get_1_10000k = create_get_test(1, 10000 * 1000)
+test_sqlite_blobs_backend_get_10_1000k = create_get_test(10, 1000 * 1000)
+test_sqlite_blobs_backend_get_100_100k = create_get_test(100, 100 * 1000)
+test_sqlite_blobs_backend_get_1000_10k = create_get_test(1000, 10 * 1000)
diff --git a/tests/benchmarks/test_sync.py b/tests/benchmarks/test_sync.py
new file mode 100644
index 00000000..45506d77
--- /dev/null
+++ b/tests/benchmarks/test_sync.py
@@ -0,0 +1,92 @@
+import pytest
+from twisted.internet.defer import gatherResults
+
+
+@pytest.inlineCallbacks
+def load_up(client, amount, payload):
+ # create a bunch of local documents
+ deferreds = []
+ for i in xrange(amount):
+ deferreds.append(client.create_doc({'content': payload}))
+ yield gatherResults(deferreds)
+
+
+# Each test created with this function will:
+#
+# - get a fresh client.
+# - iterate:
+# - setup: create N docs of a certain size
+# - benchmark: sync() -- uploads N docs.
+def create_upload(uploads, size):
+ @pytest.inlineCallbacks
+ @pytest.mark.benchmark(group="test_upload")
+ def test(soledad_client, txbenchmark_with_setup, payload):
+ """
+ Upload many documents of a given size.
+ """
+ client = soledad_client()
+
+ def setup():
+ return load_up(client, uploads, payload(size))
+
+ yield txbenchmark_with_setup(setup, client.sync)
+ return test
+
+
+# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change
+# the number of docs or the doc sizes for the tests below.
+test_upload_10_1000k = create_upload(10, 1000 * 1000)
+test_upload_100_100k = create_upload(100, 100 * 1000)
+test_upload_1000_10k = create_upload(1000, 10 * 1000)
+
+
+# Each test created with this function will:
+#
+# - get a fresh client.
+# - create N docs of a certain size
+# - sync (uploads those docs)
+# - iterate:
+# - setup: get a fresh client with empty local db
+# - benchmark: sync() -- downloads N docs.
+def create_download(downloads, size):
+ @pytest.inlineCallbacks
+ @pytest.mark.benchmark(group="test_download")
+ def test(soledad_client, txbenchmark_with_setup, payload):
+ """
+ Download many documents of the same size.
+ """
+ client = soledad_client()
+
+ yield load_up(client, downloads, payload(size))
+ yield client.sync()
+ # We could create them directly on couch, but sending them
+ # ensures we are dealing with properly encrypted docs
+
+ def setup():
+ return soledad_client(force_fresh_db=True)
+
+ def sync(clean_client):
+ return clean_client.sync()
+ yield txbenchmark_with_setup(setup, sync)
+ return test
+
+
+# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change
+# the number of docs or the doc sizes for the tests below.
+test_download_10_1000k = create_download(10, 1000 * 1000)
+test_download_100_100k = create_download(100, 100 * 1000)
+test_download_1000_10k = create_download(1000, 10 * 1000)
+
+
+@pytest.inlineCallbacks
+@pytest.mark.benchmark(group="test_nothing_to_sync")
+def test_nothing_to_sync(soledad_client, txbenchmark_with_setup):
+ """
+ Sync two replicas that are already in sync.
+ """
+ def setup():
+ return soledad_client()
+
+ def sync(clean_client):
+ return clean_client.sync()
+ yield txbenchmark_with_setup(setup, sync)