From d86831e4cd3e77f340618168528e62cf4dafb5d7 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 29 Aug 2016 00:05:45 -0300 Subject: [test] randomize payload We were using 'x'*size as payload, but on real usage the payload will be random. This commit randomizes the payload using a predefined seed, so the random payload will be the same across benchmarks. Using random payloads also improves accuracy of compression or encoding impacts and we will be evaluating those changes for resouce usage issues. Also note that base64 is used on payload. That was needed for utf8 safety, but overhead was removed to leave payloads as defined by benchmarks. Base64 was chosen also due its popular usage on MIME encoding, which is used on mail attachments (our current scenario). --- testing/tests/perf/conftest.py | 12 ++++++++++++ testing/tests/perf/test_crypto.py | 22 +++++++++++----------- testing/tests/perf/test_encdecpool.py | 8 ++++---- testing/tests/perf/test_sqlcipher.py | 13 ++++++------- testing/tests/perf/test_sync.py | 13 ++++++------- 5 files changed, 39 insertions(+), 29 deletions(-) (limited to 'testing') diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py index 9abd0c54..3681025f 100644 --- a/testing/tests/perf/conftest.py +++ b/testing/tests/perf/conftest.py @@ -2,6 +2,8 @@ import json import os import pytest import requests +import random +import base64 import signal import time @@ -43,6 +45,16 @@ DEFAULT_CERTKEY = 'soledad_certkey.pem' DEFAULT_TOKEN = 'an-auth-token' +@pytest.fixture() +def payload(): + def generate(size): + random.seed(1337) # same seed to avoid different bench results + payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) + # encode as base64 to avoid ascii encode/decode errors + return base64.b64encode(payload_bytes)[:size] # remove b64 overhead + return generate + + # # soledad_dbs fixture: provides all databases needed by soledad server in a per # module scope (same databases for all tests in this module). diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py index a32ef593..be00560b 100644 --- a/testing/tests/perf/test_crypto.py +++ b/testing/tests/perf/test_crypto.py @@ -8,10 +8,10 @@ from leap.soledad.client.crypto import decrypt_sym def create_doc_encryption(size): @pytest.mark.benchmark(group="test_crypto_encrypt_doc") - def test_doc_encryption(soledad_client, benchmark): + def test_doc_encryption(soledad_client, benchmark, payload): crypto = soledad_client()._crypto - DOC_CONTENT = {'payload': 'x'*size} + DOC_CONTENT = {'payload': payload(size)} doc = SoledadDocument( doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) @@ -22,10 +22,10 @@ def create_doc_encryption(size): def create_doc_decryption(size): @pytest.mark.benchmark(group="test_crypto_decrypt_doc") - def test_doc_decryption(soledad_client, benchmark): + def test_doc_decryption(soledad_client, benchmark, payload): crypto = soledad_client()._crypto - DOC_CONTENT = {'payload': 'x'*size} + DOC_CONTENT = {'payload': payload(size)} doc = SoledadDocument( doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) @@ -49,21 +49,21 @@ test_decrypt_doc_1M = create_doc_decryption(1000*1000) test_decrypt_doc_10M = create_doc_decryption(10*1000*1000) test_decrypt_doc_50M = create_doc_decryption(50*1000*1000) -KEY = 'x'*32 - def create_raw_encryption(size): @pytest.mark.benchmark(group="test_crypto_raw_encrypt") - def test_raw_encrypt(benchmark): - benchmark(encrypt_sym, 'x'*size, KEY) + def test_raw_encrypt(benchmark, payload): + key = payload(32) + benchmark(encrypt_sym, payload(size), key) return test_raw_encrypt def create_raw_decryption(size): @pytest.mark.benchmark(group="test_crypto_raw_decrypt") - def test_raw_decrypt(benchmark): - iv, ciphertext = encrypt_sym('x'*size, KEY) - benchmark(decrypt_sym, ciphertext, KEY, iv) + def test_raw_decrypt(benchmark, payload): + key = payload(32) + iv, ciphertext = encrypt_sym(payload(size), key) + benchmark(decrypt_sym, ciphertext, key, iv) return test_raw_decrypt diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py index 3c501084..77091a41 100644 --- a/testing/tests/perf/test_encdecpool.py +++ b/testing/tests/perf/test_encdecpool.py @@ -11,8 +11,8 @@ from leap.soledad.common.document import SoledadDocument def create_encrypt(amount, size): @pytest.mark.benchmark(group="test_pool_encrypt") @pytest.inlineCallbacks - def test(soledad_client, txbenchmark_with_setup, request): - DOC_CONTENT = {'payload': 'x'*size} + def test(soledad_client, txbenchmark_with_setup, request, payload): + DOC_CONTENT = {'payload': payload(size)} def setup(): client = soledad_client() @@ -41,8 +41,8 @@ test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000) def create_decrypt(amount, size): @pytest.mark.benchmark(group="test_pool_decrypt") @pytest.inlineCallbacks - def test(soledad_client, txbenchmark_with_setup, request): - DOC_CONTENT = {'payload': 'x'*size} + def test(soledad_client, txbenchmark_with_setup, request, payload): + DOC_CONTENT = {'payload': payload(size)} client = soledad_client() def setup(): diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py index 1fce1c3e..e7a54228 100644 --- a/testing/tests/perf/test_sqlcipher.py +++ b/testing/tests/perf/test_sqlcipher.py @@ -6,9 +6,8 @@ import pytest from twisted.internet.defer import gatherResults -def load_up(client, amount, size, defer=True): - content = 'x'*size - results = [client.create_doc({'content': content}) for _ in xrange(amount)] +def load_up(client, amount, payload, defer=True): + results = [client.create_doc({'content': payload}) for _ in xrange(amount)] if defer: return gatherResults(results) @@ -16,17 +15,17 @@ def load_up(client, amount, size, defer=True): def build_test_sqlcipher_async_create(amount, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_sqlcipher_async_create") - def test(soledad_client, txbenchmark): + def test(soledad_client, txbenchmark, payload): client = soledad_client() - yield txbenchmark(load_up, client, amount, size) + yield txbenchmark(load_up, client, amount, payload(size)) return test def build_test_sqlcipher_create(amount, size): @pytest.mark.benchmark(group="test_sqlcipher_create") - def test(soledad_client, benchmark): + def test(soledad_client, benchmark, payload): client = soledad_client()._dbsyncer - benchmark(load_up, client, amount, size, defer=False) + benchmark(load_up, client, amount, payload(size), defer=False) return test diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 0be9d12f..0b48a0b9 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -3,12 +3,11 @@ import pytest from twisted.internet.defer import gatherResults -def load_up(client, amount, size): - content = 'x'*size +def load_up(client, amount, payload): deferreds = [] # create a bunch of local documents for i in xrange(amount): - d = client.create_doc({'content': content}) + d = client.create_doc({'content': payload}) deferreds.append(d) d = gatherResults(deferreds) d.addCallback(lambda _: None) @@ -18,11 +17,11 @@ def load_up(client, amount, size): def create_upload(uploads, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_upload") - def test(soledad_client, txbenchmark_with_setup): + def test(soledad_client, txbenchmark_with_setup, payload): client = soledad_client() def setup(): - return load_up(client, uploads, size) + return load_up(client, uploads, payload(size)) yield txbenchmark_with_setup(setup, client.sync) return test @@ -36,10 +35,10 @@ test_upload_1000_10k = create_upload(1000, 10*1000) def create_download(downloads, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_download") - def test(soledad_client, txbenchmark_with_setup): + def test(soledad_client, txbenchmark_with_setup, payload): client = soledad_client() - yield load_up(client, downloads, size) + yield load_up(client, downloads, payload(size)) yield client.sync() # We could create them directly on couch, but sending them # ensures we are dealing with properly encrypted docs -- cgit v1.2.3