[test] randomize payload

We were using 'x'*size as payload, but on real usage the payload will be
random. This commit randomizes the payload using a predefined seed, so
the random payload will be the same across benchmarks.
Using random payloads also improves accuracy of compression or encoding
impacts and we will be evaluating those changes for resouce usage
issues.
Also note that base64 is used on payload. That was needed for utf8
safety, but overhead was removed to leave payloads as defined by
benchmarks.
Base64 was chosen also due its popular usage on MIME encoding, which is
used on mail attachments (our current scenario).

5 files changed, 39 insertions, 29 deletions

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 9abd0c54..3681025f 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -2,6 +2,8 @@ import json
 import os
 import pytest
 import requests
+import random
+import base64
 import signal
 import time
 
@@ -43,6 +45,16 @@ DEFAULT_CERTKEY = 'soledad_certkey.pem'
 DEFAULT_TOKEN = 'an-auth-token'
 
 
+@pytest.fixture()
+def payload():
+    def generate(size):
+        random.seed(1337)  # same seed to avoid different bench results
+        payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
+        # encode as base64 to avoid ascii encode/decode errors
+        return base64.b64encode(payload_bytes)[:size]  # remove b64 overhead
+    return generate
+
+
 #
 # soledad_dbs fixture: provides all databases needed by soledad server in a per
 # module scope (same databases for all tests in this module).
diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py
index a32ef593..be00560b 100644
--- a/testing/tests/perf/test_crypto.py
+++ b/testing/tests/perf/test_crypto.py
@@ -8,10 +8,10 @@ from leap.soledad.client.crypto import decrypt_sym
 
 def create_doc_encryption(size):
     @pytest.mark.benchmark(group="test_crypto_encrypt_doc")
-    def test_doc_encryption(soledad_client, benchmark):
+    def test_doc_encryption(soledad_client, benchmark, payload):
         crypto = soledad_client()._crypto
 
-        DOC_CONTENT = {'payload': 'x'*size}
+        DOC_CONTENT = {'payload': payload(size)}
         doc = SoledadDocument(
             doc_id=uuid4().hex, rev='rev',
             json=json.dumps(DOC_CONTENT))
@@ -22,10 +22,10 @@ def create_doc_encryption(size):
 
 def create_doc_decryption(size):
     @pytest.mark.benchmark(group="test_crypto_decrypt_doc")
-    def test_doc_decryption(soledad_client, benchmark):
+    def test_doc_decryption(soledad_client, benchmark, payload):
         crypto = soledad_client()._crypto
 
-        DOC_CONTENT = {'payload': 'x'*size}
+        DOC_CONTENT = {'payload': payload(size)}
         doc = SoledadDocument(
             doc_id=uuid4().hex, rev='rev',
             json=json.dumps(DOC_CONTENT))
@@ -49,21 +49,21 @@ test_decrypt_doc_1M = create_doc_decryption(1000*1000)
 test_decrypt_doc_10M = create_doc_decryption(10*1000*1000)
 test_decrypt_doc_50M = create_doc_decryption(50*1000*1000)
 
-KEY = 'x'*32
-
 
 def create_raw_encryption(size):
     @pytest.mark.benchmark(group="test_crypto_raw_encrypt")
-    def test_raw_encrypt(benchmark):
-        benchmark(encrypt_sym, 'x'*size, KEY)
+    def test_raw_encrypt(benchmark, payload):
+        key = payload(32)
+        benchmark(encrypt_sym, payload(size), key)
     return test_raw_encrypt
 
 
 def create_raw_decryption(size):
     @pytest.mark.benchmark(group="test_crypto_raw_decrypt")
-    def test_raw_decrypt(benchmark):
-        iv, ciphertext = encrypt_sym('x'*size, KEY)
-        benchmark(decrypt_sym, ciphertext, KEY, iv)
+    def test_raw_decrypt(benchmark, payload):
+        key = payload(32)
+        iv, ciphertext = encrypt_sym(payload(size), key)
+        benchmark(decrypt_sym, ciphertext, key, iv)
     return test_raw_decrypt
 
 
diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index 3c501084..77091a41 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -11,8 +11,8 @@ from leap.soledad.common.document import SoledadDocument
 def create_encrypt(amount, size):
     @pytest.mark.benchmark(group="test_pool_encrypt")
     @pytest.inlineCallbacks
-    def test(soledad_client, txbenchmark_with_setup, request):
-        DOC_CONTENT = {'payload': 'x'*size}
+    def test(soledad_client, txbenchmark_with_setup, request, payload):
+        DOC_CONTENT = {'payload': payload(size)}
 
         def setup():
             client = soledad_client()
@@ -41,8 +41,8 @@ test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000)
 def create_decrypt(amount, size):
     @pytest.mark.benchmark(group="test_pool_decrypt")
     @pytest.inlineCallbacks
-    def test(soledad_client, txbenchmark_with_setup, request):
-        DOC_CONTENT = {'payload': 'x'*size}
+    def test(soledad_client, txbenchmark_with_setup, request, payload):
+        DOC_CONTENT = {'payload': payload(size)}
         client = soledad_client()
 
         def setup():
diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py
index 1fce1c3e..e7a54228 100644
--- a/testing/tests/perf/test_sqlcipher.py
+++ b/testing/tests/perf/test_sqlcipher.py
@@ -6,9 +6,8 @@ import pytest
 from twisted.internet.defer import gatherResults
 
 
-def load_up(client, amount, size, defer=True):
-    content = 'x'*size
-    results = [client.create_doc({'content': content}) for _ in xrange(amount)]
+def load_up(client, amount, payload, defer=True):
+    results = [client.create_doc({'content': payload}) for _ in xrange(amount)]
     if defer:
         return gatherResults(results)
 
@@ -16,17 +15,17 @@ def load_up(client, amount, size, defer=True):
 def build_test_sqlcipher_async_create(amount, size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_sqlcipher_async_create")
-    def test(soledad_client, txbenchmark):
+    def test(soledad_client, txbenchmark, payload):
         client = soledad_client()
-        yield txbenchmark(load_up, client, amount, size)
+        yield txbenchmark(load_up, client, amount, payload(size))
     return test
 
 
 def build_test_sqlcipher_create(amount, size):
     @pytest.mark.benchmark(group="test_sqlcipher_create")
-    def test(soledad_client, benchmark):
+    def test(soledad_client, benchmark, payload):
         client = soledad_client()._dbsyncer
-        benchmark(load_up, client, amount, size, defer=False)
+        benchmark(load_up, client, amount, payload(size), defer=False)
     return test
 
 
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 0be9d12f..0b48a0b9 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -3,12 +3,11 @@ import pytest
 from twisted.internet.defer import gatherResults
 
 
-def load_up(client, amount, size):
-    content = 'x'*size
+def load_up(client, amount, payload):
     deferreds = []
     # create a bunch of local documents
     for i in xrange(amount):
-        d = client.create_doc({'content': content})
+        d = client.create_doc({'content': payload})
         deferreds.append(d)
     d = gatherResults(deferreds)
     d.addCallback(lambda _: None)
@@ -18,11 +17,11 @@ def load_up(client, amount, size):
 def create_upload(uploads, size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_upload")
-    def test(soledad_client, txbenchmark_with_setup):
+    def test(soledad_client, txbenchmark_with_setup, payload):
         client = soledad_client()
 
         def setup():
-            return load_up(client, uploads, size)
+            return load_up(client, uploads, payload(size))
 
         yield txbenchmark_with_setup(setup, client.sync)
     return test
@@ -36,10 +35,10 @@ test_upload_1000_10k = create_upload(1000, 10*1000)
 def create_download(downloads, size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_download")
-    def test(soledad_client, txbenchmark_with_setup):
+    def test(soledad_client, txbenchmark_with_setup, payload):
         client = soledad_client()
 
-        yield load_up(client, downloads, size)
+        yield load_up(client, downloads, payload(size))
         yield client.sync()
         # We could create them directly on couch, but sending them
         # ensures we are dealing with properly encrypted docs