From 378a07113a713a7c25f0fb8510d18ecdae2198bd Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 17 Nov 2016 22:35:21 -0200 Subject: [test] rename benchmark tests directory and tag --- testing/tests/benchmarks/assets/cert_default.conf | 15 ++++ testing/tests/benchmarks/conftest.py | 57 +++++++++++++ testing/tests/benchmarks/pytest.ini | 2 + testing/tests/benchmarks/test_crypto.py | 99 +++++++++++++++++++++++ testing/tests/benchmarks/test_misc.py | 8 ++ testing/tests/benchmarks/test_sqlcipher.py | 40 +++++++++ testing/tests/benchmarks/test_sync.py | 64 +++++++++++++++ 7 files changed, 285 insertions(+) create mode 100644 testing/tests/benchmarks/assets/cert_default.conf create mode 100644 testing/tests/benchmarks/conftest.py create mode 100644 testing/tests/benchmarks/pytest.ini create mode 100644 testing/tests/benchmarks/test_crypto.py create mode 100644 testing/tests/benchmarks/test_misc.py create mode 100644 testing/tests/benchmarks/test_sqlcipher.py create mode 100644 testing/tests/benchmarks/test_sync.py (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/assets/cert_default.conf b/testing/tests/benchmarks/assets/cert_default.conf new file mode 100644 index 00000000..8043cea3 --- /dev/null +++ b/testing/tests/benchmarks/assets/cert_default.conf @@ -0,0 +1,15 @@ +[ req ] +default_bits = 1024 +default_keyfile = keyfile.pem +distinguished_name = req_distinguished_name +prompt = no +output_password = mypass + +[ req_distinguished_name ] +C = GB +ST = Test State or Province +L = Test Locality +O = Organization Name +OU = Organizational Unit Name +CN = localhost +emailAddress = test@email.address diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py new file mode 100644 index 00000000..a9cc3464 --- /dev/null +++ b/testing/tests/benchmarks/conftest.py @@ -0,0 +1,57 @@ +import pytest +import random +import base64 + +from twisted.internet import threads, reactor + + +# we have to manually setup the events server in order to be able to signal +# events. This is usually done by the enclosing application using soledad +# client (i.e. bitmask client). +from leap.common.events import server +server.ensure_server() + + +def pytest_addoption(parser): + parser.addoption( + "--num-docs", type="int", default=100, + help="the number of documents to use in performance tests") + + +@pytest.fixture() +def payload(): + def generate(size): + random.seed(1337) # same seed to avoid different bench results + payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) + # encode as base64 to avoid ascii encode/decode errors + return base64.b64encode(payload_bytes)[:size] # remove b64 overhead + return generate + + +@pytest.fixture() +def txbenchmark(benchmark): + def blockOnThread(*args, **kwargs): + return threads.deferToThread( + benchmark, threads.blockingCallFromThread, + reactor, *args, **kwargs) + return blockOnThread + + +@pytest.fixture() +def txbenchmark_with_setup(benchmark): + def blockOnThreadWithSetup(setup, f): + def blocking_runner(*args, **kwargs): + return threads.blockingCallFromThread(reactor, f, *args, **kwargs) + + def blocking_setup(): + args = threads.blockingCallFromThread(reactor, setup) + try: + return tuple(arg for arg in args), {} + except TypeError: + return ((args,), {}) if args else None + + def bench(): + return benchmark.pedantic(blocking_runner, setup=blocking_setup, + rounds=4, warmup_rounds=1) + return threads.deferToThread(bench) + return blockOnThreadWithSetup diff --git a/testing/tests/benchmarks/pytest.ini b/testing/tests/benchmarks/pytest.ini new file mode 100644 index 00000000..7a0508ce --- /dev/null +++ b/testing/tests/benchmarks/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +twisted = yes diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py new file mode 100644 index 00000000..ab586bea --- /dev/null +++ b/testing/tests/benchmarks/test_crypto.py @@ -0,0 +1,99 @@ +""" +Benchmarks for crypto operations. +If you don't want to stress your local machine too much, you can pass the +SIZE_LIMT environment variable. + +For instance, to keep the maximum payload at 1MB: + +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +""" +import pytest +import os +import json +from uuid import uuid4 + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import _crypto + +LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) + + +pytestmark = pytest.mark.benchmark + + +def create_doc_encryption(size): + @pytest.mark.benchmark(group="test_crypto_encrypt_doc") + def test_doc_encryption(soledad_client, benchmark, payload): + crypto = soledad_client()._crypto + + DOC_CONTENT = {'payload': payload(size)} + doc = SoledadDocument( + doc_id=uuid4().hex, rev='rev', + json=json.dumps(DOC_CONTENT)) + + benchmark(crypto.encrypt_doc, doc) + return test_doc_encryption + + +# TODO this test is really bullshit, because it's still including +# the json serialization. + +def create_doc_decryption(size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_crypto_decrypt_doc") + def test_doc_decryption(soledad_client, benchmark, payload): + crypto = soledad_client()._crypto + + DOC_CONTENT = {'payload': payload(size)} + doc = SoledadDocument( + doc_id=uuid4().hex, rev='rev', + json=json.dumps(DOC_CONTENT)) + + encrypted_doc = yield crypto.encrypt_doc(doc) + doc.set_json(encrypted_doc) + + benchmark(crypto.decrypt_doc, doc) + return test_doc_decryption + + +def create_raw_encryption(size): + @pytest.mark.benchmark(group="test_crypto_raw_encrypt") + def test_raw_encrypt(benchmark, payload): + key = payload(32) + benchmark(_crypto.encrypt_sym, payload(size), key) + return test_raw_encrypt + + +def create_raw_decryption(size): + @pytest.mark.benchmark(group="test_crypto_raw_decrypt") + def test_raw_decrypt(benchmark, payload): + key = payload(32) + iv, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv) + return test_raw_decrypt + + +# Create the TESTS in the global namespace, they'll be picked by the benchmark +# plugin. + +encryption_tests = [ + ('10k', 1E4), + ('100k', 1E5), + ('500k', 5E5), + ('1M', 1E6), + ('10M', 1E7), + ('50M', 5E7), +] + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) + globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) + + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz) + globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz) diff --git a/testing/tests/benchmarks/test_misc.py b/testing/tests/benchmarks/test_misc.py new file mode 100644 index 00000000..2f32ad7c --- /dev/null +++ b/testing/tests/benchmarks/test_misc.py @@ -0,0 +1,8 @@ +import pytest + +pytestmark = pytest.mark.benchmark + + +@pytest.mark.benchmark(group="test_instance") +def test_initialization(soledad_client, benchmark): + benchmark(soledad_client) diff --git a/testing/tests/benchmarks/test_sqlcipher.py b/testing/tests/benchmarks/test_sqlcipher.py new file mode 100644 index 00000000..7f8842bd --- /dev/null +++ b/testing/tests/benchmarks/test_sqlcipher.py @@ -0,0 +1,40 @@ +''' +Tests SoledadClient/SQLCipher interaction +''' +import pytest + +from twisted.internet.defer import gatherResults + +pytestmark = pytest.mark.benchmark + + +def load_up(client, amount, payload, defer=True): + results = [client.create_doc({'content': payload}) for _ in xrange(amount)] + if defer: + return gatherResults(results) + + +def build_test_sqlcipher_async_create(amount, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_sqlcipher_async_create") + def test(soledad_client, txbenchmark, payload): + client = soledad_client() + yield txbenchmark(load_up, client, amount, payload(size)) + return test + + +def build_test_sqlcipher_create(amount, size): + @pytest.mark.benchmark(group="test_sqlcipher_create") + def test(soledad_client, benchmark, payload): + client = soledad_client()._dbsyncer + benchmark(load_up, client, amount, payload(size), defer=False) + return test + + +test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500 * 1000) +test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) +test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000) +# synchronous +test_create_20_500k = build_test_sqlcipher_create(20, 500 * 1000) +test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) +test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py new file mode 100644 index 00000000..88afe9f8 --- /dev/null +++ b/testing/tests/benchmarks/test_sync.py @@ -0,0 +1,64 @@ +import pytest + +pytestmark = pytest.mark.benchmark + + +@pytest.inlineCallbacks +def load_up(client, amount, payload): + # create a bunch of local documents + for i in xrange(amount): + yield client.create_doc({'content': payload}) + + +def create_upload(uploads, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_upload") + def test(soledad_client, txbenchmark_with_setup, payload): + client = soledad_client() + + def setup(): + return load_up(client, uploads, payload(size)) + + yield txbenchmark_with_setup(setup, client.sync) + return test + + +test_upload_20_500k = create_upload(20, 500 * 1000) +test_upload_100_100k = create_upload(100, 100 * 1000) +test_upload_1000_10k = create_upload(1000, 10 * 1000) + + +def create_download(downloads, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_download") + def test(soledad_client, txbenchmark_with_setup, payload): + client = soledad_client() + + yield load_up(client, downloads, payload(size)) + yield client.sync() + # We could create them directly on couch, but sending them + # ensures we are dealing with properly encrypted docs + + def setup(): + return soledad_client() + + def sync(clean_client): + return clean_client.sync() + yield txbenchmark_with_setup(setup, sync) + return test + + +test_download_20_500k = create_download(20, 500 * 1000) +test_download_100_100k = create_download(100, 100 * 1000) +test_download_1000_10k = create_download(1000, 10 * 1000) + + +@pytest.inlineCallbacks +@pytest.mark.benchmark(group="test_nothing_to_sync") +def test_nothing_to_sync(soledad_client, txbenchmark_with_setup): + def setup(): + return soledad_client() + + def sync(clean_client): + return clean_client.sync() + yield txbenchmark_with_setup(setup, sync) -- cgit v1.2.3 From 63c33b1b20e013571fb870205302bbc9e4a06e23 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 19 Nov 2016 21:52:30 -0300 Subject: [tests] use options instead of marks When we use marks the new pytests from benchmarks folder are collected and ignored, but this causes trial to fail sometimes. Using --ignore avoids it from being loaded while --benchmark-only will properly select the benchmarks for tox, as intended. --- testing/tests/benchmarks/test_crypto.py | 3 --- testing/tests/benchmarks/test_misc.py | 2 -- testing/tests/benchmarks/test_sqlcipher.py | 2 -- testing/tests/benchmarks/test_sync.py | 2 -- 4 files changed, 9 deletions(-) (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index ab586bea..367c3b5b 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -18,9 +18,6 @@ from leap.soledad.client import _crypto LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) -pytestmark = pytest.mark.benchmark - - def create_doc_encryption(size): @pytest.mark.benchmark(group="test_crypto_encrypt_doc") def test_doc_encryption(soledad_client, benchmark, payload): diff --git a/testing/tests/benchmarks/test_misc.py b/testing/tests/benchmarks/test_misc.py index 2f32ad7c..ead48adf 100644 --- a/testing/tests/benchmarks/test_misc.py +++ b/testing/tests/benchmarks/test_misc.py @@ -1,7 +1,5 @@ import pytest -pytestmark = pytest.mark.benchmark - @pytest.mark.benchmark(group="test_instance") def test_initialization(soledad_client, benchmark): diff --git a/testing/tests/benchmarks/test_sqlcipher.py b/testing/tests/benchmarks/test_sqlcipher.py index 7f8842bd..39c9e3ad 100644 --- a/testing/tests/benchmarks/test_sqlcipher.py +++ b/testing/tests/benchmarks/test_sqlcipher.py @@ -5,8 +5,6 @@ import pytest from twisted.internet.defer import gatherResults -pytestmark = pytest.mark.benchmark - def load_up(client, amount, payload, defer=True): results = [client.create_doc({'content': payload}) for _ in xrange(amount)] diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index 88afe9f8..1bf6cc21 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -1,7 +1,5 @@ import pytest -pytestmark = pytest.mark.benchmark - @pytest.inlineCallbacks def load_up(client, amount, payload): -- cgit v1.2.3 From bfe330a7eaad1c51640dbbb91be233a65d2a4bd7 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 19 Nov 2016 21:54:13 -0300 Subject: [tests] fixes test_crypto bench encrypt returns a deferred and needs the adapted benchmark runner. --- testing/tests/benchmarks/test_crypto.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 367c3b5b..75ad9a30 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -20,7 +20,8 @@ LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) def create_doc_encryption(size): @pytest.mark.benchmark(group="test_crypto_encrypt_doc") - def test_doc_encryption(soledad_client, benchmark, payload): + @pytest.inlineCallbacks + def test_doc_encryption(soledad_client, txbenchmark, payload): crypto = soledad_client()._crypto DOC_CONTENT = {'payload': payload(size)} @@ -28,7 +29,7 @@ def create_doc_encryption(size): doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) - benchmark(crypto.encrypt_doc, doc) + yield txbenchmark(crypto.encrypt_doc, doc) return test_doc_encryption -- cgit v1.2.3 From 42082cfa648ec10612823086e72dc2a70a0e773c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 18:09:26 -0300 Subject: [feature] make _crypto stream on decryption We are already doing this on encryption, now we can stream also from decryption. This unblocks the reactor and will be valuable for blobs-io. --- testing/tests/benchmarks/test_crypto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 75ad9a30..8ee9b899 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -39,7 +39,7 @@ def create_doc_encryption(size): def create_doc_decryption(size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_crypto_decrypt_doc") - def test_doc_decryption(soledad_client, benchmark, payload): + def test_doc_decryption(soledad_client, txbenchmark, payload): crypto = soledad_client()._crypto DOC_CONTENT = {'payload': payload(size)} @@ -50,7 +50,7 @@ def create_doc_decryption(size): encrypted_doc = yield crypto.encrypt_doc(doc) doc.set_json(encrypted_doc) - benchmark(crypto.decrypt_doc, doc) + yield txbenchmark(crypto.decrypt_doc, doc) return test_doc_decryption -- cgit v1.2.3 From 67917656589d08a84f98ff675f6aeade809e1faf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 29 Nov 2016 02:04:57 -0300 Subject: [feature] speed up sync benchmark setup code We aren't testing huge payloads on CI, so it doesn't make sense to insert docs one by one. 'gatherResults' can speed up bench setup. --- testing/tests/benchmarks/test_sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index 1bf6cc21..1501d74b 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -1,11 +1,14 @@ import pytest +from twisted.internet.defer import gatherResults @pytest.inlineCallbacks def load_up(client, amount, payload): # create a bunch of local documents + deferreds = [] for i in xrange(amount): - yield client.create_doc({'content': payload}) + deferreds.append(client.create_doc({'content': payload})) + yield gatherResults(deferreds) def create_upload(uploads, size): -- cgit v1.2.3 From 349a49d2be011a428023a4ece14001fda57e65c4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 6 Dec 2016 23:16:28 -0300 Subject: [feature] use GCM instead of CTR+HMAC Resolves: #8668 - client: substitute usage of CTR mode + HMAC by GCM cipher mode Signed-off-by: Victor Shyba --- testing/tests/benchmarks/test_crypto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 8ee9b899..631ac041 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -66,8 +66,8 @@ def create_raw_decryption(size): @pytest.mark.benchmark(group="test_crypto_raw_decrypt") def test_raw_decrypt(benchmark, payload): key = payload(32) - iv, ciphertext = _crypto.encrypt_sym(payload(size), key) - benchmark(_crypto.decrypt_sym, ciphertext, key, iv) + iv, tag, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv, tag) return test_raw_decrypt -- cgit v1.2.3 From b3fcc5c5bddc73475596c4fe74e3402f0d5c021a Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 7 Dec 2016 01:24:53 -0300 Subject: [feature] Add retro compat on secrets.py ciphers Integrated the secrets's JSON key that specifies ciphers into _crypto and added optional GCM. Also added a test to check if both cipher types can be imported. Resolves: #8680 Signed-off-by: Victor Shyba --- testing/tests/benchmarks/test_crypto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'testing/tests/benchmarks') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 631ac041..8ee9b899 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -66,8 +66,8 @@ def create_raw_decryption(size): @pytest.mark.benchmark(group="test_crypto_raw_decrypt") def test_raw_decrypt(benchmark, payload): key = payload(32) - iv, tag, ciphertext = _crypto.encrypt_sym(payload(size), key) - benchmark(_crypto.decrypt_sym, ciphertext, key, iv, tag) + iv, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv) return test_raw_decrypt -- cgit v1.2.3