From 793180533e4f19b364145c61939d6cad07dd851a Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Sun, 24 Jul 2016 07:56:52 -0300
Subject: [test] add pytest initial setup for performance tests

---
 testing/tests/perf/assets/cert_default.conf |  15 +++
 testing/tests/perf/conftest.py              | 143 ++++++++++++++++++++++++++++
 testing/tests/perf/pytest.ini               |   2 +
 testing/tests/perf/test_sync.py             |  43 +++++++++
 4 files changed, 203 insertions(+)
 create mode 100644 testing/tests/perf/assets/cert_default.conf
 create mode 100644 testing/tests/perf/conftest.py
 create mode 100644 testing/tests/perf/pytest.ini
 create mode 100644 testing/tests/perf/test_sync.py

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/assets/cert_default.conf b/testing/tests/perf/assets/cert_default.conf
new file mode 100644
index 00000000..8043cea3
--- /dev/null
+++ b/testing/tests/perf/assets/cert_default.conf
@@ -0,0 +1,15 @@
+[ req ]
+default_bits           = 1024
+default_keyfile        = keyfile.pem
+distinguished_name     = req_distinguished_name
+prompt                 = no
+output_password        = mypass
+
+[ req_distinguished_name ]
+C                      = GB
+ST                     = Test State or Province
+L                      = Test Locality
+O                      = Organization Name
+OU                     = Organizational Unit Name
+CN                     = localhost
+emailAddress           = test@email.address
diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
new file mode 100644
index 00000000..c66f2863
--- /dev/null
+++ b/testing/tests/perf/conftest.py
@@ -0,0 +1,143 @@
+import json
+import os
+import pytest
+import requests
+import signal
+import time
+
+from hashlib import sha512
+from subprocess import call
+
+from leap.soledad.client import Soledad
+from leap.soledad.common.couch import CouchDatabase
+
+from leap.common.events import server
+server.ensure_server()
+
+
+DEFAULT_UUID = '0'
+DEFAULT_PASSPHRASE = '123'
+
+DEFAULT_URL = 'http://127.0.0.1:2424'
+DEFAULT_PRIVKEY = 'soledad_privkey.pem'
+DEFAULT_CERTKEY = 'soledad_certkey.pem'
+DEFAULT_TOKEN = 'an-auth-token'
+
+
+@pytest.fixture
+def certificate(tmpdir):
+    privkey = os.path.join(tmpdir.strpath, 'privkey.pem')
+    certkey = os.path.join(tmpdir.strpath, 'certkey.pem')
+    call([
+        'openssl',
+        'req',
+        '-x509',
+        '-sha256',
+        '-nodes',
+        '-days', '365',
+        '-newkey', 'rsa:2048',
+        '-config', './assets/cert_default.conf',  # TODO: fix basedir
+        '-keyout', privkey,
+        '-out', certkey])
+    return privkey, certkey
+
+
+def get_pid(pidfile):
+    if not os.path.isfile(pidfile):
+        return 0
+    try:
+        with open(pidfile) as f:
+            return int(f.read())
+    except IOError:
+        return 0
+
+
+class CouchUserDatabase(object):
+
+    def __init__(self):
+        url = 'http://127.0.0.1:5984/'
+        self._user_db_url = url + 'user-%s' % DEFAULT_UUID
+        self._token_db_url = url + _token_dbname()
+        self._shared_db_url = url + 'shared'
+
+    def setup(self):
+        CouchDatabase.open_database(
+            url=self._user_db_url, create=True, replica_uid=None)
+        requests.put(self._token_db_url)
+        requests.put(self._shared_db_url)
+        self._add_token()
+
+    def _add_token(self):
+        token = sha512(DEFAULT_TOKEN).hexdigest()
+        content = {'type': 'Token', 'user_id': DEFAULT_UUID}
+        requests.put(
+            self._token_db_url + '/' + token, data=json.dumps(content))
+
+    def teardown(self):
+        requests.delete(self._user_db_url)
+        requests.delete(self._token_db_url)
+        requests.delete(self._shared_db_url)
+
+
+@pytest.fixture(scope='function')
+def couchdb_user_db(request):
+    db = CouchUserDatabase()
+    db.setup()
+    request.addfinalizer(db.teardown)
+    return db
+
+
+def _token_dbname():
+    dbname = 'tokens_' + \
+        str(int(time.time() / (30 * 24 * 3600)))
+    return dbname
+
+
+class SoledadServer(object):
+
+    def __init__(self, tmpdir):
+        self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid')
+        self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log')
+
+    def start(self):
+        call([
+            'twistd',
+            '--logfile=%s' % self._logfile,
+            '--pidfile=%s' % self._pidfile,
+            'web',
+            '--wsgi=leap.soledad.server.application',
+            '--port=2424'
+        ])
+
+    def stop(self):
+        pid = get_pid(self._pidfile)
+        os.kill(pid, signal.SIGKILL)
+
+
+@pytest.fixture
+def soledad_server(tmpdir, couchdb_user_db, request):
+    server = SoledadServer(tmpdir)
+    server.start()
+    request.addfinalizer(server.stop)
+    return server
+
+
+@pytest.fixture()
+def soledad_client(tmpdir, soledad_server):
+    uuid = DEFAULT_UUID
+    passphrase = DEFAULT_PASSPHRASE
+    secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid)
+    local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid)
+    server_url = DEFAULT_URL
+    token = DEFAULT_TOKEN
+
+    # get a soledad instance
+    return Soledad(
+        uuid,
+        unicode(passphrase),
+        secrets_path=secrets_path,
+        local_db_path=local_db_path,
+        server_url=server_url,
+        cert_file=None,
+        auth_token=token,
+        defer_encryption=True)
diff --git a/testing/tests/perf/pytest.ini b/testing/tests/perf/pytest.ini
new file mode 100644
index 00000000..7a0508ce
--- /dev/null
+++ b/testing/tests/perf/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+twisted = yes
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
new file mode 100644
index 00000000..1e29a86a
--- /dev/null
+++ b/testing/tests/perf/test_sync.py
@@ -0,0 +1,43 @@
+import pytest
+
+from twisted.internet.defer import gatherResults
+
+from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.document import ServerDocument
+
+
+@pytest.inlineCallbacks
+def test_upload(soledad_client):
+    # create a bunch of local documents
+    uploads = 100
+    deferreds = []
+    for i in xrange(uploads):
+        d = soledad_client.create_doc({'upload': True})
+        deferreds.append(d)
+    yield gatherResults(deferreds)
+
+    # synchronize
+    yield soledad_client.sync()
+
+    # check that documents reached the remote database
+    remote = CouchDatabase('http://127.0.0.1:5984', 'user-0')
+    remote_count, _ = remote.get_all_docs()
+    assert remote_count == uploads
+
+
+@pytest.inlineCallbacks
+def test_download(soledad_client):
+    # create a bunch of remote documents
+    downloads = 100
+    remote = CouchDatabase('http://127.0.0.1:5984', 'user-0')
+    for i in xrange(downloads):
+        doc = ServerDocument('doc-%d' % i, 'replica:1')
+        doc.content = {'download': True}
+        remote.save_document(None, doc, i)
+
+    # synchronize
+    yield soledad_client.sync()
+
+    # check that documents reached the local database
+    local_count, docs = yield soledad_client.get_all_docs()
+    assert local_count == downloads
-- 
cgit v1.2.3


From 4073d6a8542121504ef83b9cc02ecff94e041e32 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 25 Jul 2016 05:44:20 -0300
Subject: [test] use pytest fixture scopes to provide per module soledad server
 for perf tests

---
 testing/tests/perf/conftest.py | 110 +++++++++++++++++++++++++----------------
 1 file changed, 68 insertions(+), 42 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index c66f2863..05f91a45 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -11,6 +11,9 @@ from subprocess import call
 from leap.soledad.client import Soledad
 from leap.soledad.common.couch import CouchDatabase
 
+# we have to manually setup the events server in order to be able to signal
+# events. This is usually done by the enclosing application using soledad
+# client (i.e. bitmask client).
 from leap.common.events import server
 server.ensure_server()
 
@@ -24,48 +27,31 @@ DEFAULT_CERTKEY = 'soledad_certkey.pem'
 DEFAULT_TOKEN = 'an-auth-token'
 
 
-@pytest.fixture
-def certificate(tmpdir):
-    privkey = os.path.join(tmpdir.strpath, 'privkey.pem')
-    certkey = os.path.join(tmpdir.strpath, 'certkey.pem')
-    call([
-        'openssl',
-        'req',
-        '-x509',
-        '-sha256',
-        '-nodes',
-        '-days', '365',
-        '-newkey', 'rsa:2048',
-        '-config', './assets/cert_default.conf',  # TODO: fix basedir
-        '-keyout', privkey,
-        '-out', certkey])
-    return privkey, certkey
+#
+# soledad_dbs fixture: provides all databases needed by soledad server in a per
+# module scope (same databases for all tests in this module).
+#
 
-
-def get_pid(pidfile):
-    if not os.path.isfile(pidfile):
-        return 0
-    try:
-        with open(pidfile) as f:
-            return int(f.read())
-    except IOError:
-        return 0
+def _token_dbname():
+    dbname = 'tokens_' + \
+        str(int(time.time() / (30 * 24 * 3600)))
+    return dbname
 
 
-class CouchUserDatabase(object):
+class SoledadDatabases(object):
 
     def __init__(self):
         url = 'http://127.0.0.1:5984/'
-        self._user_db_url = url + 'user-%s' % DEFAULT_UUID
         self._token_db_url = url + _token_dbname()
         self._shared_db_url = url + 'shared'
 
     def setup(self):
-        CouchDatabase.open_database(
-            url=self._user_db_url, create=True, replica_uid=None)
+        self._create_dbs()
+        self._add_token()
+
+    def _create_dbs(self):
         requests.put(self._token_db_url)
         requests.put(self._shared_db_url)
-        self._add_token()
 
     def _add_token(self):
         token = sha512(DEFAULT_TOKEN).hexdigest()
@@ -74,28 +60,64 @@ class CouchUserDatabase(object):
             self._token_db_url + '/' + token, data=json.dumps(content))
 
     def teardown(self):
-        requests.delete(self._user_db_url)
         requests.delete(self._token_db_url)
         requests.delete(self._shared_db_url)
 
 
+@pytest.fixture(scope='module')
+def soledad_dbs(request):
+    db = SoledadDatabases()
+    db.setup()
+    request.addfinalizer(db.teardown)
+    return db
+
+
+#
+# user_db fixture: provides an empty database for a given user in a per
+# function scope.
+#
+
+class UserDatabase(object):
+
+    def __init__(self):
+        url = 'http://127.0.0.1:5984/'
+        self._user_db_url = url + 'user-%s' % DEFAULT_UUID
+
+    def setup(self):
+        CouchDatabase.open_database(
+            url=self._user_db_url, create=True, replica_uid=None)
+
+    def teardown(self):
+        requests.delete(self._user_db_url)
+
+
 @pytest.fixture(scope='function')
-def couchdb_user_db(request):
-    db = CouchUserDatabase()
+def user_db(request):
+    db = UserDatabase()
     db.setup()
     request.addfinalizer(db.teardown)
     return db
 
 
-def _token_dbname():
-    dbname = 'tokens_' + \
-        str(int(time.time() / (30 * 24 * 3600)))
-    return dbname
+def get_pid(pidfile):
+    if not os.path.isfile(pidfile):
+        return 0
+    try:
+        with open(pidfile) as f:
+            return int(f.read())
+    except IOError:
+        return 0
+
 
+#
+# soledad_server fixture: provides a running soledad server in a per module
+# context (same soledad server for all tests in this module).
+#
 
 class SoledadServer(object):
 
-    def __init__(self, tmpdir):
+    def __init__(self, tmpdir_factory):
+        tmpdir = tmpdir_factory.mktemp('soledad-server')
         self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid')
         self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log')
 
@@ -114,16 +136,20 @@ class SoledadServer(object):
         os.kill(pid, signal.SIGKILL)
 
 
-@pytest.fixture
-def soledad_server(tmpdir, couchdb_user_db, request):
-    server = SoledadServer(tmpdir)
+@pytest.fixture(scope='module')
+def soledad_server(tmpdir_factory, request):
+    server = SoledadServer(tmpdir_factory)
     server.start()
     request.addfinalizer(server.stop)
     return server
 
 
+#
+# soledad_client fixture: provides a clean soledad client for a test function.
+#
+
 @pytest.fixture()
-def soledad_client(tmpdir, soledad_server):
+def soledad_client(tmpdir, soledad_server, user_db, soledad_dbs):
     uuid = DEFAULT_UUID
     passphrase = DEFAULT_PASSPHRASE
     secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid)
-- 
cgit v1.2.3


From c40a2bf488e03bef14d440ab1a847afab6f5fb76 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 25 Jul 2016 05:44:47 -0300
Subject: [test] add some payload to perf sync tests

---
 testing/tests/perf/test_sync.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 1e29a86a..fbe3b877 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -6,13 +6,16 @@ from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.document import ServerDocument
 
 
+content = ' ' * 10000
+
+
 @pytest.inlineCallbacks
 def test_upload(soledad_client):
     # create a bunch of local documents
     uploads = 100
     deferreds = []
     for i in xrange(uploads):
-        d = soledad_client.create_doc({'upload': True})
+        d = soledad_client.create_doc({'upload': True, 'content': content})
         deferreds.append(d)
     yield gatherResults(deferreds)
 
@@ -32,7 +35,7 @@ def test_download(soledad_client):
     remote = CouchDatabase('http://127.0.0.1:5984', 'user-0')
     for i in xrange(downloads):
         doc = ServerDocument('doc-%d' % i, 'replica:1')
-        doc.content = {'download': True}
+        doc.content = {'download': True, 'content': content}
         remote.save_document(None, doc, i)
 
     # synchronize
-- 
cgit v1.2.3


From 1dc48b603869db1bfcd9c0f86ae3973b715f9222 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 25 Jul 2016 06:58:26 -0300
Subject: [test] allow custom couch url for perf tests

---
 testing/tests/perf/conftest.py  | 47 +++++++++++++++++++++++++++++++----------
 testing/tests/perf/test_sync.py | 10 +++++----
 2 files changed, 42 insertions(+), 15 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 05f91a45..85a48059 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -7,10 +7,12 @@ import time
 
 from hashlib import sha512
 from subprocess import call
+from urlparse import urljoin
 
 from leap.soledad.client import Soledad
 from leap.soledad.common.couch import CouchDatabase
 
+
 # we have to manually setup the events server in order to be able to signal
 # events. This is usually done by the enclosing application using soledad
 # client (i.e. bitmask client).
@@ -18,6 +20,16 @@ from leap.common.events import server
 server.ensure_server()
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--couch-url", type="string", default="http://127.0.0.1:5984",
+        help="the url for the couch server to be used during tests")
+
+
+#
+# default options for all tests
+#
+
 DEFAULT_UUID = '0'
 DEFAULT_PASSPHRASE = '123'
 
@@ -40,10 +52,9 @@ def _token_dbname():
 
 class SoledadDatabases(object):
 
-    def __init__(self):
-        url = 'http://127.0.0.1:5984/'
-        self._token_db_url = url + _token_dbname()
-        self._shared_db_url = url + 'shared'
+    def __init__(self, url):
+        self._token_db_url = urljoin(url, _token_dbname())
+        self._shared_db_url = urljoin(url, 'shared')
 
     def setup(self):
         self._create_dbs()
@@ -66,7 +77,8 @@ class SoledadDatabases(object):
 
 @pytest.fixture(scope='module')
 def soledad_dbs(request):
-    db = SoledadDatabases()
+    couch_url = request.config.option.couch_url
+    db = SoledadDatabases(couch_url)
     db.setup()
     request.addfinalizer(db.teardown)
     return db
@@ -79,9 +91,8 @@ def soledad_dbs(request):
 
 class UserDatabase(object):
 
-    def __init__(self):
-        url = 'http://127.0.0.1:5984/'
-        self._user_db_url = url + 'user-%s' % DEFAULT_UUID
+    def __init__(self, url):
+        self._user_db_url = urljoin(url, 'user-%s' % DEFAULT_UUID)
 
     def setup(self):
         CouchDatabase.open_database(
@@ -93,7 +104,8 @@ class UserDatabase(object):
 
 @pytest.fixture(scope='function')
 def user_db(request):
-    db = UserDatabase()
+    couch_url = request.config.option.couch_url
+    db = UserDatabase(couch_url)
     db.setup()
     request.addfinalizer(db.teardown)
     return db
@@ -116,12 +128,15 @@ def get_pid(pidfile):
 
 class SoledadServer(object):
 
-    def __init__(self, tmpdir_factory):
+    def __init__(self, tmpdir_factory, couch_url):
         tmpdir = tmpdir_factory.mktemp('soledad-server')
         self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid')
         self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log')
+        self._couch_url = couch_url
 
     def start(self):
+        self._create_conf_file()
+        # start the server
         call([
             'twistd',
             '--logfile=%s' % self._logfile,
@@ -131,6 +146,15 @@ class SoledadServer(object):
             '--port=2424'
         ])
 
+    def _create_conf_file(self):
+        if not os.access('/etc', os.W_OK):
+            return
+        if not os.path.isdir('/etc/soledad'):
+            os.mkdir('/etc/soledad')
+        with open('/etc/soledad/soledad-server.conf', 'w') as f:
+            content = '[soledad-server]\ncouch_url = %s' % self._couch_url
+            f.write(content)
+
     def stop(self):
         pid = get_pid(self._pidfile)
         os.kill(pid, signal.SIGKILL)
@@ -138,7 +162,8 @@ class SoledadServer(object):
 
 @pytest.fixture(scope='module')
 def soledad_server(tmpdir_factory, request):
-    server = SoledadServer(tmpdir_factory)
+    couch_url = request.config.option.couch_url
+    server = SoledadServer(tmpdir_factory, couch_url)
     server.start()
     request.addfinalizer(server.stop)
     return server
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index fbe3b877..9de733fb 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -10,7 +10,7 @@ content = ' ' * 10000
 
 
 @pytest.inlineCallbacks
-def test_upload(soledad_client):
+def test_upload(soledad_client, request):
     # create a bunch of local documents
     uploads = 100
     deferreds = []
@@ -23,16 +23,18 @@ def test_upload(soledad_client):
     yield soledad_client.sync()
 
     # check that documents reached the remote database
-    remote = CouchDatabase('http://127.0.0.1:5984', 'user-0')
+    url = request.config.getoption('--couch-url')
+    remote = CouchDatabase(url, 'user-0')
     remote_count, _ = remote.get_all_docs()
     assert remote_count == uploads
 
 
 @pytest.inlineCallbacks
-def test_download(soledad_client):
+def test_download(soledad_client, request):
     # create a bunch of remote documents
     downloads = 100
-    remote = CouchDatabase('http://127.0.0.1:5984', 'user-0')
+    url = request.config.getoption('--couch-url')
+    remote = CouchDatabase(url, 'user-0')
     for i in xrange(downloads):
         doc = ServerDocument('doc-%d' % i, 'replica:1')
         doc.content = {'download': True, 'content': content}
-- 
cgit v1.2.3


From de5cd462cc3f04275e22d9267ecb8e6c2b23dfda Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 25 Jul 2016 21:34:23 -0300
Subject: [test] allow passing number of docs on command line on perf tests

---
 testing/tests/perf/conftest.py  | 6 ++++++
 testing/tests/perf/test_sync.py | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 85a48059..463c791a 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -26,6 +26,12 @@ def pytest_addoption(parser):
         help="the url for the couch server to be used during tests")
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--num-docs", type="int", default=100,
+        help="the number of documents to use in performance tests")
+
+
 #
 # default options for all tests
 #
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 9de733fb..45af9a91 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -12,7 +12,7 @@ content = ' ' * 10000
 @pytest.inlineCallbacks
 def test_upload(soledad_client, request):
     # create a bunch of local documents
-    uploads = 100
+    uploads = request.config.option.num_docs
     deferreds = []
     for i in xrange(uploads):
         d = soledad_client.create_doc({'upload': True, 'content': content})
@@ -32,7 +32,7 @@ def test_upload(soledad_client, request):
 @pytest.inlineCallbacks
 def test_download(soledad_client, request):
     # create a bunch of remote documents
-    downloads = 100
+    downloads = request.config.option.num_docs
     url = request.config.getoption('--couch-url')
     remote = CouchDatabase(url, 'user-0')
     for i in xrange(downloads):
-- 
cgit v1.2.3


From c889ba67158850763394fc6087b0837716866cd1 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor1984@riseup.net>
Date: Wed, 27 Jul 2016 18:13:01 -0300
Subject: [test] remove duplicated function declaration

`pytest_addoption` was declared twice making the second declaration
replace the first, thus removing couch url parameter.
---
 testing/tests/perf/conftest.py | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 463c791a..5ec047e4 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -24,9 +24,6 @@ def pytest_addoption(parser):
     parser.addoption(
         "--couch-url", type="string", default="http://127.0.0.1:5984",
         help="the url for the couch server to be used during tests")
-
-
-def pytest_addoption(parser):
     parser.addoption(
         "--num-docs", type="int", default=100,
         help="the number of documents to use in performance tests")
-- 
cgit v1.2.3


From f0f3e0358a01708eb048d8eaf463361e682be466 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 9 Aug 2016 21:08:13 -0300
Subject: [test] Adds pytest-benchmark adapted to Twisted

Adapted pytest-benchmark to Twisted as it's synchronous and added
fixtures for benchmarking.
---
 testing/tests/perf/conftest.py  |  99 +++++++++++++++++++++-----------
 testing/tests/perf/test_sync.py | 121 +++++++++++++++++++++++++++++-----------
 2 files changed, 154 insertions(+), 66 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 5ec047e4..8a75d0ae 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -6,8 +6,10 @@ import signal
 import time
 
 from hashlib import sha512
+from uuid import uuid4
 from subprocess import call
 from urlparse import urljoin
+from twisted.internet import threads, reactor
 
 from leap.soledad.client import Soledad
 from leap.soledad.common.couch import CouchDatabase
@@ -59,17 +61,17 @@ class SoledadDatabases(object):
         self._token_db_url = urljoin(url, _token_dbname())
         self._shared_db_url = urljoin(url, 'shared')
 
-    def setup(self):
+    def setup(self, uuid):
         self._create_dbs()
-        self._add_token()
+        self._add_token(uuid)
 
     def _create_dbs(self):
         requests.put(self._token_db_url)
         requests.put(self._shared_db_url)
 
-    def _add_token(self):
+    def _add_token(self, uuid):
         token = sha512(DEFAULT_TOKEN).hexdigest()
-        content = {'type': 'Token', 'user_id': DEFAULT_UUID}
+        content = {'type': 'Token', 'user_id': uuid}
         requests.put(
             self._token_db_url + '/' + token, data=json.dumps(content))
 
@@ -81,37 +83,41 @@ class SoledadDatabases(object):
 @pytest.fixture(scope='module')
 def soledad_dbs(request):
     couch_url = request.config.option.couch_url
-    db = SoledadDatabases(couch_url)
-    db.setup()
-    request.addfinalizer(db.teardown)
-    return db
+
+    def create(uuid=DEFAULT_UUID):
+        db = SoledadDatabases(couch_url)
+        request.addfinalizer(db.teardown)
+        return db.setup(uuid)
+    return create
 
 
 #
-# user_db fixture: provides an empty database for a given user in a per
+# remote_db fixture: provides an empty database for a given user in a per
 # function scope.
 #
 
 class UserDatabase(object):
 
-    def __init__(self, url):
-        self._user_db_url = urljoin(url, 'user-%s' % DEFAULT_UUID)
+    def __init__(self, url, uuid):
+        self._remote_db_url = urljoin(url, 'user-%s' % uuid)
 
     def setup(self):
-        CouchDatabase.open_database(
-            url=self._user_db_url, create=True, replica_uid=None)
+        return CouchDatabase.open_database(
+            url=self._remote_db_url, create=True, replica_uid=None)
 
     def teardown(self):
-        requests.delete(self._user_db_url)
+        requests.delete(self._remote_db_url)
 
 
 @pytest.fixture(scope='function')
-def user_db(request):
+def remote_db(request):
     couch_url = request.config.option.couch_url
-    db = UserDatabase(couch_url)
-    db.setup()
-    request.addfinalizer(db.teardown)
-    return db
+
+    def create(uuid=DEFAULT_UUID):
+        db = UserDatabase(couch_url, uuid)
+        request.addfinalizer(db.teardown)
+        return db.setup()
+    return create
 
 
 def get_pid(pidfile):
@@ -172,26 +178,55 @@ def soledad_server(tmpdir_factory, request):
     return server
 
 
+@pytest.fixture(scope='function')
+def txbenchmark(benchmark):
+    def blockOnThread(*args, **kwargs):
+        return threads.deferToThread(
+                benchmark, threads.blockingCallFromThread,
+                reactor, *args, **kwargs)
+    return blockOnThread
+
+
+@pytest.fixture(scope='function')
+def txbenchmark_with_setup(benchmark):
+    def blockOnThreadWithSetup(setup, f):
+        def blocking_runner(*args, **kwargs):
+            return threads.blockingCallFromThread(reactor, f, *args, **kwargs)
+
+        def blocking_setup():
+            return threads.blockingCallFromThread(reactor, setup)
+
+        def bench():
+            return benchmark.pedantic(blocking_runner, setup=blocking_setup,
+                                      rounds=4, warmup_rounds=1)
+        return threads.deferToThread(bench)
+    return blockOnThreadWithSetup
+
+
 #
 # soledad_client fixture: provides a clean soledad client for a test function.
 #
 
 @pytest.fixture()
-def soledad_client(tmpdir, soledad_server, user_db, soledad_dbs):
-    uuid = DEFAULT_UUID
+def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs):
     passphrase = DEFAULT_PASSPHRASE
-    secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid)
-    local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid)
     server_url = DEFAULT_URL
     token = DEFAULT_TOKEN
 
     # get a soledad instance
-    return Soledad(
-        uuid,
-        unicode(passphrase),
-        secrets_path=secrets_path,
-        local_db_path=local_db_path,
-        server_url=server_url,
-        cert_file=None,
-        auth_token=token,
-        defer_encryption=True)
+    def create(new=False):
+        uuid = uuid4().hex if new else DEFAULT_UUID
+        secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid4().hex)
+        local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid4().hex)
+        remote_db(uuid)
+        soledad_dbs(uuid)
+        return Soledad(
+            uuid,
+            unicode(passphrase),
+            secrets_path=secrets_path,
+            local_db_path=local_db_path,
+            server_url=server_url,
+            cert_file=None,
+            auth_token=token,
+            defer_encryption=True)
+    return create
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 45af9a91..ea109d05 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -2,47 +2,100 @@ import pytest
 
 from twisted.internet.defer import gatherResults
 
-from leap.soledad.common.couch import CouchDatabase
-from leap.soledad.common.document import ServerDocument
+
+def load_up(client, amount, size):
+    content = 'x'*size
+    deferreds = []
+    # create a bunch of local documents
+    for i in xrange(amount):
+        d = client.create_doc({'content': content})
+        deferreds.append(d)
+    d = gatherResults(deferreds)
+    d.addCallback(lambda _: None)
+    return d
 
 
-content = ' ' * 10000
+@pytest.inlineCallbacks
+@pytest.mark.benchmark(group="test_upload")
+def test_upload_20_500k(soledad_client, txbenchmark_with_setup):
+    uploads, size, client = 20, 500*1000, soledad_client()
+
+    def setup():
+        return load_up(client, uploads, size)
+
+    yield txbenchmark_with_setup(setup, client.sync)
 
 
 @pytest.inlineCallbacks
-def test_upload(soledad_client, request):
-    # create a bunch of local documents
-    uploads = request.config.option.num_docs
-    deferreds = []
-    for i in xrange(uploads):
-        d = soledad_client.create_doc({'upload': True, 'content': content})
-        deferreds.append(d)
-    yield gatherResults(deferreds)
+@pytest.mark.benchmark(group="test_upload")
+def test_upload_100_100k(soledad_client, txbenchmark_with_setup):
+    uploads, size, client = 100, 100*1000, soledad_client()
+
+    def setup():
+        return load_up(client, uploads, size)
+
+    yield txbenchmark_with_setup(setup, client.sync)
+
+
+@pytest.inlineCallbacks
+@pytest.mark.benchmark(group="test_upload")
+def test_upload_1000_10k(soledad_client, txbenchmark_with_setup):
+    uploads, size, client = 1000, 10*1000, soledad_client()
 
-    # synchronize
-    yield soledad_client.sync()
+    def setup():
+        return load_up(client, uploads, size)
 
-    # check that documents reached the remote database
-    url = request.config.getoption('--couch-url')
-    remote = CouchDatabase(url, 'user-0')
-    remote_count, _ = remote.get_all_docs()
-    assert remote_count == uploads
+    yield txbenchmark_with_setup(setup, client.sync)
 
 
 @pytest.inlineCallbacks
-def test_download(soledad_client, request):
-    # create a bunch of remote documents
-    downloads = request.config.option.num_docs
-    url = request.config.getoption('--couch-url')
-    remote = CouchDatabase(url, 'user-0')
-    for i in xrange(downloads):
-        doc = ServerDocument('doc-%d' % i, 'replica:1')
-        doc.content = {'download': True, 'content': content}
-        remote.save_document(None, doc, i)
-
-    # synchronize
-    yield soledad_client.sync()
-
-    # check that documents reached the local database
-    local_count, docs = yield soledad_client.get_all_docs()
-    assert local_count == downloads
+@pytest.mark.benchmark(group="test_download")
+def test_download_20_500k(soledad_client, txbenchmark_with_setup):
+    downloads, size, client = 20, 500*1000, soledad_client()
+
+    yield load_up(client, downloads, size)
+    yield client.sync()
+
+    def setup():
+        clean_client = soledad_client()
+        return (clean_client,), {}
+
+    def sync(clean_client):
+        return clean_client.sync()
+    yield txbenchmark_with_setup(setup, sync)
+
+
+@pytest.inlineCallbacks
+@pytest.mark.benchmark(group="test_download")
+def test_download_100_100k(soledad_client, txbenchmark_with_setup):
+    downloads, size, client = 100, 100*1000, soledad_client()
+
+    yield load_up(client, downloads, size)
+    yield client.sync()
+    # We could create them directly on remote db, but sending them
+    # ensures we are dealing with properly encrypted docs
+
+    def setup():
+        clean_client = soledad_client()
+        return (clean_client,), {}
+
+    def sync(clean_client):
+        return clean_client.sync()
+    yield txbenchmark_with_setup(setup, sync)
+
+
+@pytest.inlineCallbacks
+@pytest.mark.benchmark(group="test_download")
+def test_download_1000_10k(soledad_client, txbenchmark_with_setup):
+    downloads, size, client = 1000, 10*1000, soledad_client()
+
+    yield load_up(client, downloads, size)
+    yield client.sync()
+
+    def setup():
+        clean_client = soledad_client()
+        return (clean_client,), {}
+
+    def sync(clean_client):
+        return clean_client.sync()
+    yield txbenchmark_with_setup(setup, sync)
-- 
cgit v1.2.3


From 6639cf0d00fa5bdfc0f43d4dea5c5055776130b8 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 9 Aug 2016 23:54:33 -0300
Subject: [test] use a nested func to simplify scenarios

If we have many scenarios (like 20/500k, 100/100k, 1000,10k) then making
a nested function to generate tests based on scenario parameters
simplifies the code a lot.
---
 testing/tests/perf/conftest.py  |   6 ++-
 testing/tests/perf/test_sync.py | 104 ++++++++++++----------------------------
 2 files changed, 35 insertions(+), 75 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 8a75d0ae..4b2186db 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -208,7 +208,7 @@ def txbenchmark_with_setup(benchmark):
 #
 
 @pytest.fixture()
-def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs):
+def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request):
     passphrase = DEFAULT_PASSPHRASE
     server_url = DEFAULT_URL
     token = DEFAULT_TOKEN
@@ -220,7 +220,7 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs):
         local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid4().hex)
         remote_db(uuid)
         soledad_dbs(uuid)
-        return Soledad(
+        soledad_client = Soledad(
             uuid,
             unicode(passphrase),
             secrets_path=secrets_path,
@@ -229,4 +229,6 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs):
             cert_file=None,
             auth_token=token,
             defer_encryption=True)
+        request.addfinalizer(soledad_client.close)
+        return soledad_client
     return create
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index ea109d05..146f1394 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -15,87 +15,45 @@ def load_up(client, amount, size):
     return d
 
 
-@pytest.inlineCallbacks
-@pytest.mark.benchmark(group="test_upload")
-def test_upload_20_500k(soledad_client, txbenchmark_with_setup):
-    uploads, size, client = 20, 500*1000, soledad_client()
+def create_upload(uploads, size):
+    @pytest.inlineCallbacks
+    @pytest.mark.benchmark(group="test_upload")
+    def test(soledad_client, txbenchmark_with_setup):
+        client = soledad_client()
 
-    def setup():
-        return load_up(client, uploads, size)
+        def setup():
+            return load_up(client, uploads, size)
 
-    yield txbenchmark_with_setup(setup, client.sync)
+        yield txbenchmark_with_setup(setup, client.sync)
+    return test
 
 
-@pytest.inlineCallbacks
-@pytest.mark.benchmark(group="test_upload")
-def test_upload_100_100k(soledad_client, txbenchmark_with_setup):
-    uploads, size, client = 100, 100*1000, soledad_client()
+test_upload_20_500k = create_upload(20, 500*1000)
+test_upload_100_100k = create_upload(100, 100*1000)
+test_upload_1000_10k = create_upload(1000, 10*1000)
 
-    def setup():
-        return load_up(client, uploads, size)
 
-    yield txbenchmark_with_setup(setup, client.sync)
+def create_download(downloads, size):
+    @pytest.inlineCallbacks
+    @pytest.mark.benchmark(group="test_download")
+    def test(soledad_client, txbenchmark_with_setup):
+        client = soledad_client()
 
+        yield load_up(client, downloads, size)
+        yield client.sync()
+        # We could create them directly on couch, but sending them
+        # ensures we are dealing with properly encrypted docs
 
-@pytest.inlineCallbacks
-@pytest.mark.benchmark(group="test_upload")
-def test_upload_1000_10k(soledad_client, txbenchmark_with_setup):
-    uploads, size, client = 1000, 10*1000, soledad_client()
+        def setup():
+            clean_client = soledad_client()
+            return (clean_client,), {}
 
-    def setup():
-        return load_up(client, uploads, size)
+        def sync(clean_client):
+            return clean_client.sync()
+        yield txbenchmark_with_setup(setup, sync)
+    return test
 
-    yield txbenchmark_with_setup(setup, client.sync)
 
-
-@pytest.inlineCallbacks
-@pytest.mark.benchmark(group="test_download")
-def test_download_20_500k(soledad_client, txbenchmark_with_setup):
-    downloads, size, client = 20, 500*1000, soledad_client()
-
-    yield load_up(client, downloads, size)
-    yield client.sync()
-
-    def setup():
-        clean_client = soledad_client()
-        return (clean_client,), {}
-
-    def sync(clean_client):
-        return clean_client.sync()
-    yield txbenchmark_with_setup(setup, sync)
-
-
-@pytest.inlineCallbacks
-@pytest.mark.benchmark(group="test_download")
-def test_download_100_100k(soledad_client, txbenchmark_with_setup):
-    downloads, size, client = 100, 100*1000, soledad_client()
-
-    yield load_up(client, downloads, size)
-    yield client.sync()
-    # We could create them directly on remote db, but sending them
-    # ensures we are dealing with properly encrypted docs
-
-    def setup():
-        clean_client = soledad_client()
-        return (clean_client,), {}
-
-    def sync(clean_client):
-        return clean_client.sync()
-    yield txbenchmark_with_setup(setup, sync)
-
-
-@pytest.inlineCallbacks
-@pytest.mark.benchmark(group="test_download")
-def test_download_1000_10k(soledad_client, txbenchmark_with_setup):
-    downloads, size, client = 1000, 10*1000, soledad_client()
-
-    yield load_up(client, downloads, size)
-    yield client.sync()
-
-    def setup():
-        clean_client = soledad_client()
-        return (clean_client,), {}
-
-    def sync(clean_client):
-        return clean_client.sync()
-    yield txbenchmark_with_setup(setup, sync)
+test_download_20_500k = create_download(20, 500*1000)
+test_download_100_100k = create_download(100, 100*1000)
+test_download_1000_10k = create_download(1000, 10*1000)
-- 
cgit v1.2.3


From 7c811131771af33370aa04b33dc70f6ed2cc637a Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 9 Aug 2016 23:57:18 -0300
Subject: [test] adds sqlcipher create tests

Creating 20/500k, 100/100k and 1000/10k.
---
 testing/tests/perf/test_sqlcipher.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 testing/tests/perf/test_sqlcipher.py

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py
new file mode 100644
index 00000000..8e82d172
--- /dev/null
+++ b/testing/tests/perf/test_sqlcipher.py
@@ -0,0 +1,32 @@
+'''
+Tests SoledadClient/SQLCipher interaction
+'''
+import pytest
+
+from twisted.internet.defer import gatherResults
+
+
+def load_up(client, amount, size):
+    content = 'x'*size
+    deferreds = []
+    # create a bunch of local documents
+    for i in xrange(amount):
+        d = client.create_doc({'content': content})
+        deferreds.append(d)
+    d = gatherResults(deferreds)
+    d.addCallback(lambda _: None)
+    return d
+
+
+def build_test_sqlcipher_create(amount, size):
+    @pytest.inlineCallbacks
+    @pytest.mark.benchmark(group="test_sqlcipher_create")
+    def test(soledad_client, txbenchmark):
+        client = soledad_client()
+        yield txbenchmark(load_up, client, amount, size)
+    return test
+
+
+test_create_20_500k = build_test_sqlcipher_create(20, 500*1000)
+test_create_100_100k = build_test_sqlcipher_create(100, 100*1000)
+test_create_1000_10k = build_test_sqlcipher_create(1000, 10*1000)
-- 
cgit v1.2.3


From 481fa255bf3cb53fd932bd984cd40d097ca7bb61 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 15 Aug 2016 15:42:23 -0300
Subject: [test] adds sqlcipher synchronous tests

---
 testing/tests/perf/test_sqlcipher.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py
index 8e82d172..511080a1 100644
--- a/testing/tests/perf/test_sqlcipher.py
+++ b/testing/tests/perf/test_sqlcipher.py
@@ -6,27 +6,40 @@ import pytest
 from twisted.internet.defer import gatherResults
 
 
-def load_up(client, amount, size):
+def load_up(client, amount, size, defer=True):
     content = 'x'*size
     deferreds = []
     # create a bunch of local documents
     for i in xrange(amount):
         d = client.create_doc({'content': content})
         deferreds.append(d)
-    d = gatherResults(deferreds)
-    d.addCallback(lambda _: None)
-    return d
+    if defer:
+        d = gatherResults(deferreds)
+        d.addCallback(lambda _: None)
+        return d
 
 
-def build_test_sqlcipher_create(amount, size):
+def build_test_sqlcipher_async_create(amount, size):
     @pytest.inlineCallbacks
-    @pytest.mark.benchmark(group="test_sqlcipher_create")
+    @pytest.mark.benchmark(group="test_sqlcipher_async_create")
     def test(soledad_client, txbenchmark):
         client = soledad_client()
         yield txbenchmark(load_up, client, amount, size)
     return test
 
 
+def build_test_sqlcipher_create(amount, size):
+    @pytest.mark.benchmark(group="test_sqlcipher_create")
+    def test(soledad_client, benchmark):
+        client = soledad_client()._dbsyncer
+        benchmark(load_up, client, amount, size, defer=False)
+    return test
+
+
+test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500*1000)
+test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100*1000)
+test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10*1000)
+# synchronous
 test_create_20_500k = build_test_sqlcipher_create(20, 500*1000)
 test_create_100_100k = build_test_sqlcipher_create(100, 100*1000)
 test_create_1000_10k = build_test_sqlcipher_create(1000, 10*1000)
-- 
cgit v1.2.3


From 6f5bc4d7301147c35a9651cf423804c6de252647 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 19 Aug 2016 17:04:27 -0300
Subject: [test] make all nested fixture function-scoped

function is the default scope, so there is no need to pass this
parameter. Previously, one of the scopes was 'module', but it is a
nested function that fires on demand, so it should clean up itself from
test to test in order to avoid conflict while putting.
---
 testing/tests/perf/conftest.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 4b2186db..0b66263a 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -80,7 +80,7 @@ class SoledadDatabases(object):
         requests.delete(self._shared_db_url)
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture()
 def soledad_dbs(request):
     couch_url = request.config.option.couch_url
 
@@ -109,7 +109,7 @@ class UserDatabase(object):
         requests.delete(self._remote_db_url)
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture()
 def remote_db(request):
     couch_url = request.config.option.couch_url
 
@@ -178,7 +178,7 @@ def soledad_server(tmpdir_factory, request):
     return server
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture()
 def txbenchmark(benchmark):
     def blockOnThread(*args, **kwargs):
         return threads.deferToThread(
@@ -187,7 +187,7 @@ def txbenchmark(benchmark):
     return blockOnThread
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture()
 def txbenchmark_with_setup(benchmark):
     def blockOnThreadWithSetup(setup, f):
         def blocking_runner(*args, **kwargs):
-- 
cgit v1.2.3


From 8e3d85950bb54b7818f0356387bba81769787f13 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 19 Aug 2016 17:06:25 -0300
Subject: [test] remove fixed default uuid

Use a new one to avoid reusing the same database.
---
 testing/tests/perf/conftest.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 0b66263a..fca9c58d 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -35,7 +35,6 @@ def pytest_addoption(parser):
 # default options for all tests
 #
 
-DEFAULT_UUID = '0'
 DEFAULT_PASSPHRASE = '123'
 
 DEFAULT_URL = 'http://127.0.0.1:2424'
@@ -84,7 +83,7 @@ class SoledadDatabases(object):
 def soledad_dbs(request):
     couch_url = request.config.option.couch_url
 
-    def create(uuid=DEFAULT_UUID):
+    def create(uuid):
         db = SoledadDatabases(couch_url)
         request.addfinalizer(db.teardown)
         return db.setup(uuid)
@@ -113,7 +112,7 @@ class UserDatabase(object):
 def remote_db(request):
     couch_url = request.config.option.couch_url
 
-    def create(uuid=DEFAULT_UUID):
+    def create(uuid):
         db = UserDatabase(couch_url, uuid)
         request.addfinalizer(db.teardown)
         return db.setup()
@@ -212,14 +211,20 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request):
     passphrase = DEFAULT_PASSPHRASE
     server_url = DEFAULT_URL
     token = DEFAULT_TOKEN
+    default_uuid = uuid4().hex
+    remote_db(default_uuid)
+    soledad_dbs(default_uuid)
 
     # get a soledad instance
     def create(new=False):
-        uuid = uuid4().hex if new else DEFAULT_UUID
         secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid4().hex)
         local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid4().hex)
-        remote_db(uuid)
-        soledad_dbs(uuid)
+        if new:
+            uuid = uuid4().hex
+            remote_db(uuid)
+            soledad_dbs(uuid)
+        else:
+            uuid = default_uuid
         soledad_client = Soledad(
             uuid,
             unicode(passphrase),
-- 
cgit v1.2.3


From c8c82b6663d122b0933a8459c4710c914edf7f84 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 19 Aug 2016 17:07:04 -0300
Subject: [test] sync without changes

Syncing without any changes was reported as slow. This benchmark will
help measure it.
---
 testing/tests/perf/test_sync.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 146f1394..668ceae7 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -57,3 +57,15 @@ def create_download(downloads, size):
 test_download_20_500k = create_download(20, 500*1000)
 test_download_100_100k = create_download(100, 100*1000)
 test_download_1000_10k = create_download(1000, 10*1000)
+
+
+@pytest.inlineCallbacks
+@pytest.mark.benchmark(group="test_nothing_to_sync")
+def test_nothing_to_sync(soledad_client, txbenchmark_with_setup):
+    def setup():
+        clean_client = soledad_client()
+        return (clean_client,), {}
+
+    def sync(clean_client):
+        return clean_client.sync()
+    yield txbenchmark_with_setup(setup, sync)
-- 
cgit v1.2.3


From 1dc2c18f02e62644da00cb0e3326f357953f5c84 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 19 Aug 2016 17:08:20 -0300
Subject: [test] test soledad instatiation time

It has a heavy scrypt hashing processing with room for improvement.
---
 testing/tests/perf/test_misc.py | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 testing/tests/perf/test_misc.py

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_misc.py b/testing/tests/perf/test_misc.py
new file mode 100644
index 00000000..c2f8e3b1
--- /dev/null
+++ b/testing/tests/perf/test_misc.py
@@ -0,0 +1,6 @@
+import pytest
+
+
+@pytest.mark.benchmark(group="test_instance")
+def test_instance(soledad_client, benchmark):
+    benchmark(soledad_client)
-- 
cgit v1.2.3


From b75165567539dcd59873395049ce2210776aa166 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 20 Aug 2016 00:42:34 -0300
Subject: [test] adds encdecpool tests

Most of them are commented as memory usage is going out of control for
now.
---
 testing/tests/perf/test_encdecpool.py | 84 +++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 testing/tests/perf/test_encdecpool.py

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
new file mode 100644
index 00000000..dbbbea89
--- /dev/null
+++ b/testing/tests/perf/test_encdecpool.py
@@ -0,0 +1,84 @@
+import pytest
+import json
+from uuid import uuid4
+from twisted.internet.defer import gatherResults
+from leap.soledad.client.encdecpool import SyncEncrypterPool
+from leap.soledad.client.encdecpool import SyncDecrypterPool
+from leap.soledad.common.document import SoledadDocument
+
+
+def create_encrypt(amount, size):
+    @pytest.mark.benchmark(group="test_pool_encrypt")
+    @pytest.inlineCallbacks
+    def test(soledad_client, txbenchmark_with_setup, request):
+        DOC_CONTENT = {'payload': 'x'*size}
+
+        def setup():
+            client = soledad_client()
+            pool = SyncEncrypterPool(client._crypto, client._sync_db)
+            pool.start()
+            request.addfinalizer(pool.stop)
+            return (pool,), {}
+
+        @pytest.inlineCallbacks
+        def put_and_wait(pool):
+            doc_ids = []
+            deferreds = []
+            for _ in xrange(amount):
+                doc = SoledadDocument(
+                    doc_id=uuid4().hex, rev='rev',
+                    json=json.dumps(DOC_CONTENT))
+                deferreds.append(pool.encrypt_doc(doc))
+                doc_ids.append(doc.doc_id)
+            yield gatherResults(deferreds)
+
+        yield txbenchmark_with_setup(setup, put_and_wait)
+    return test
+
+test_encrypt_1000_10k = create_encrypt(1000, 10*1000)
+# test_encrypt_1000_500k = create_encrypt(1000, 500*1000)
+# test_encrypt_1000_1M = create_encrypt(1000, 1000*1000)
+# test_encrypt_1000_10M = create_encrypt(1000, 10*1000*1000)
+
+
+def create_decrypt(amount, size):
+    @pytest.mark.benchmark(group="test_pool_decrypt")
+    @pytest.inlineCallbacks
+    def test(soledad_client, txbenchmark_with_setup, request):
+        DOC_CONTENT = {'payload': 'x'*size}
+        client = soledad_client()
+
+        def setup():
+            pool = SyncDecrypterPool(
+                client._crypto,
+                client._sync_db,
+                source_replica_uid=client._dbpool.replica_uid,
+                insert_doc_cb=lambda x, y, z: False)  # ignored
+            pool.start(amount)
+            request.addfinalizer(pool.stop)
+            crypto = client._crypto
+            docs = []
+            for _ in xrange(amount):
+                doc = SoledadDocument(
+                    doc_id=uuid4().hex, rev='rev',
+                    json=json.dumps(DOC_CONTENT))
+                encrypted_content = json.loads(crypto.encrypt_doc(doc))
+                docs.append((doc.doc_id, encrypted_content))
+            return (pool, docs), {}
+
+        def put_and_wait(pool, docs):
+            deferreds = []  # fires on completion
+            for idx, (doc_id, content) in enumerate(docs, 1):
+                deferreds.append(pool.insert_encrypted_received_doc(
+                    doc_id, 'rev', content, idx, "trans_id", idx))
+            return gatherResults(deferreds)
+
+        yield txbenchmark_with_setup(setup, put_and_wait)
+    return test
+
+test_decrypt_1000_10k = create_decrypt(1000, 10*1000)
+test_decrypt_1000_100k = create_decrypt(1000, 10*1000)
+# memory issues ahead
+# test_decrypt_1000_500k = create_decrypt(1000, 500*1000)
+# test_decrypt_1000_1M = create_decrypt(1000, 1000*1000)
+# test_decrypt_1000_10M = create_decrypt(1000, 10*1000*1000)
-- 
cgit v1.2.3


From 585d6b2461869594210639548549aa6be336e752 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 20 Aug 2016 01:27:47 -0300
Subject: [test] adds test for SoledadCrypto

10k, 100k, 500k, 1m, 10m and 50m for encryption and decryption of a
whole document.
---
 testing/tests/perf/test_crypto.py     | 48 +++++++++++++++++++++++++++++++++++
 testing/tests/perf/test_encdecpool.py | 18 ++++++-------
 2 files changed, 57 insertions(+), 9 deletions(-)
 create mode 100644 testing/tests/perf/test_crypto.py

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py
new file mode 100644
index 00000000..84c89a0d
--- /dev/null
+++ b/testing/tests/perf/test_crypto.py
@@ -0,0 +1,48 @@
+import pytest
+import json
+from uuid import uuid4
+from leap.soledad.common.document import SoledadDocument
+
+
+def create_doc_encryption(size):
+    @pytest.mark.benchmark(group="test_crypto_encrypt_doc")
+    def test_doc_encryption(soledad_client, benchmark):
+        crypto = soledad_client()._crypto
+
+        DOC_CONTENT = {'payload': 'x'*size}
+        doc = SoledadDocument(
+            doc_id=uuid4().hex, rev='rev',
+            json=json.dumps(DOC_CONTENT))
+
+        benchmark(crypto.encrypt_doc, doc)
+    return test_doc_encryption
+
+
+def create_doc_decryption(size):
+    @pytest.mark.benchmark(group="test_crypto_decrypt_doc")
+    def test_doc_decryption(soledad_client, benchmark):
+        crypto = soledad_client()._crypto
+
+        DOC_CONTENT = {'payload': 'x'*size}
+        doc = SoledadDocument(
+            doc_id=uuid4().hex, rev='rev',
+            json=json.dumps(DOC_CONTENT))
+        encrypted_doc = crypto.encrypt_doc(doc)
+        doc.set_json(encrypted_doc)
+
+        benchmark(crypto.decrypt_doc, doc)
+    return test_doc_decryption
+
+
+test_encrypt_doc_10k = create_doc_encryption(10*1000)
+test_encrypt_doc_100k = create_doc_encryption(100*1000)
+test_encrypt_doc_500k = create_doc_encryption(500*1000)
+test_encrypt_doc_1M = create_doc_encryption(1000*1000)
+test_encrypt_doc_10M = create_doc_encryption(10*1000*1000)
+test_encrypt_doc_50M = create_doc_encryption(50*1000*1000)
+test_decrypt_doc_10k = create_doc_decryption(10*1000)
+test_decrypt_doc_100k = create_doc_decryption(100*1000)
+test_decrypt_doc_500k = create_doc_decryption(500*1000)
+test_decrypt_doc_1M = create_doc_decryption(1000*1000)
+test_decrypt_doc_10M = create_doc_decryption(10*1000*1000)
+test_decrypt_doc_50M = create_doc_decryption(50*1000*1000)
diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index dbbbea89..abc58e35 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -35,10 +35,10 @@ def create_encrypt(amount, size):
         yield txbenchmark_with_setup(setup, put_and_wait)
     return test
 
-test_encrypt_1000_10k = create_encrypt(1000, 10*1000)
-# test_encrypt_1000_500k = create_encrypt(1000, 500*1000)
-# test_encrypt_1000_1M = create_encrypt(1000, 1000*1000)
-# test_encrypt_1000_10M = create_encrypt(1000, 10*1000*1000)
+test_encdecpool_encrypt_1000_10k = create_encrypt(1000, 10*1000)
+# test_encdecpool_encrypt_1000_500k = create_encrypt(1000, 500*1000)
+# test_encdecpool_encrypt_1000_1M = create_encrypt(1000, 1000*1000)
+# test_encdecpool_encrypt_1000_10M = create_encrypt(1000, 10*1000*1000)
 
 
 def create_decrypt(amount, size):
@@ -76,9 +76,9 @@ def create_decrypt(amount, size):
         yield txbenchmark_with_setup(setup, put_and_wait)
     return test
 
-test_decrypt_1000_10k = create_decrypt(1000, 10*1000)
-test_decrypt_1000_100k = create_decrypt(1000, 10*1000)
+test_encdecpool_decrypt_1000_10k = create_decrypt(1000, 10*1000)
+test_encdecpool_decrypt_1000_100k = create_decrypt(1000, 10*1000)
 # memory issues ahead
-# test_decrypt_1000_500k = create_decrypt(1000, 500*1000)
-# test_decrypt_1000_1M = create_decrypt(1000, 1000*1000)
-# test_decrypt_1000_10M = create_decrypt(1000, 10*1000*1000)
+# test_encdecpool_decrypt_1000_500k = create_decrypt(1000, 500*1000)
+# test_encdecpool_decrypt_1000_1M = create_decrypt(1000, 1000*1000)
+# test_encdecpool_decrypt_1000_10M = create_decrypt(1000, 10*1000*1000)
-- 
cgit v1.2.3


From fa4bf209bef0ca4fd6145c8d518c3d99f770cb65 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 20 Aug 2016 01:42:13 -0300
Subject: [test] adds tests for raw encryption

Hypothesis: raw vs doc
Added the same sizes set (10k, 100k, 500k, 1M, 10M, 50M) as the document
crypto test, so we can compare how close to raw the higher level
operation is.
---
 testing/tests/perf/test_crypto.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py
index 84c89a0d..a32ef593 100644
--- a/testing/tests/perf/test_crypto.py
+++ b/testing/tests/perf/test_crypto.py
@@ -2,6 +2,8 @@ import pytest
 import json
 from uuid import uuid4
 from leap.soledad.common.document import SoledadDocument
+from leap.soledad.client.crypto import encrypt_sym
+from leap.soledad.client.crypto import decrypt_sym
 
 
 def create_doc_encryption(size):
@@ -46,3 +48,34 @@ test_decrypt_doc_500k = create_doc_decryption(500*1000)
 test_decrypt_doc_1M = create_doc_decryption(1000*1000)
 test_decrypt_doc_10M = create_doc_decryption(10*1000*1000)
 test_decrypt_doc_50M = create_doc_decryption(50*1000*1000)
+
+KEY = 'x'*32
+
+
+def create_raw_encryption(size):
+    @pytest.mark.benchmark(group="test_crypto_raw_encrypt")
+    def test_raw_encrypt(benchmark):
+        benchmark(encrypt_sym, 'x'*size, KEY)
+    return test_raw_encrypt
+
+
+def create_raw_decryption(size):
+    @pytest.mark.benchmark(group="test_crypto_raw_decrypt")
+    def test_raw_decrypt(benchmark):
+        iv, ciphertext = encrypt_sym('x'*size, KEY)
+        benchmark(decrypt_sym, ciphertext, KEY, iv)
+    return test_raw_decrypt
+
+
+test_encrypt_raw_10k = create_raw_encryption(10*1000)
+test_encrypt_raw_100k = create_raw_encryption(100*1000)
+test_encrypt_raw_500k = create_raw_encryption(500*1000)
+test_encrypt_raw_1M = create_raw_encryption(1000*1000)
+test_encrypt_raw_10M = create_raw_encryption(10*1000*1000)
+test_encrypt_raw_50M = create_raw_encryption(50*1000*1000)
+test_decrypt_raw_10k = create_raw_decryption(10*1000)
+test_decrypt_raw_100k = create_raw_decryption(100*1000)
+test_decrypt_raw_500k = create_raw_decryption(500*1000)
+test_decrypt_raw_1M = create_raw_decryption(1000*1000)
+test_decrypt_raw_10M = create_raw_decryption(10*1000*1000)
+test_decrypt_raw_50M = create_raw_decryption(50*1000*1000)
-- 
cgit v1.2.3


From cf91133809bab11ee43f20178944f91b1466bfd5 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 20 Aug 2016 02:09:50 -0300
Subject: [test] calibrate encdecpool bench for memory

1000 docs at 100k~500k are exploding memory (4Gb+4Gb swap).
Changed for 100 docs in order to be able to get measures on higher
loads. Now its 10k, 100k and 500k
---
 testing/tests/perf/test_encdecpool.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index abc58e35..681b909a 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -35,10 +35,9 @@ def create_encrypt(amount, size):
         yield txbenchmark_with_setup(setup, put_and_wait)
     return test
 
-test_encdecpool_encrypt_1000_10k = create_encrypt(1000, 10*1000)
-# test_encdecpool_encrypt_1000_500k = create_encrypt(1000, 500*1000)
-# test_encdecpool_encrypt_1000_1M = create_encrypt(1000, 1000*1000)
-# test_encdecpool_encrypt_1000_10M = create_encrypt(1000, 10*1000*1000)
+test_encdecpool_encrypt_100_10k = create_encrypt(100, 10*1000)
+test_encdecpool_encrypt_100_100k = create_encrypt(100, 100*1000)
+test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000)
 
 
 def create_decrypt(amount, size):
@@ -76,9 +75,6 @@ def create_decrypt(amount, size):
         yield txbenchmark_with_setup(setup, put_and_wait)
     return test
 
-test_encdecpool_decrypt_1000_10k = create_decrypt(1000, 10*1000)
-test_encdecpool_decrypt_1000_100k = create_decrypt(1000, 10*1000)
-# memory issues ahead
-# test_encdecpool_decrypt_1000_500k = create_decrypt(1000, 500*1000)
-# test_encdecpool_decrypt_1000_1M = create_decrypt(1000, 1000*1000)
-# test_encdecpool_decrypt_1000_10M = create_decrypt(1000, 10*1000*1000)
+test_encdecpool_decrypt_100_10k = create_decrypt(100, 10*1000)
+test_encdecpool_decrypt_100_100k = create_decrypt(100, 100*1000)
+test_encdecpool_decrypt_100_500k = create_decrypt(100, 500*1000)
-- 
cgit v1.2.3


From 4f5ecb4c719a3a842d852fbaab549d2881d6528f Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 22 Aug 2016 18:00:52 -0300
Subject: [test] make txbench ignore kwargs for readability

They arent used so far and using empty dicts to make them work is ugly.
Removing it leaves the return function on setup code clean and readable.
---
 testing/tests/perf/conftest.py        | 6 +++++-
 testing/tests/perf/test_encdecpool.py | 4 ++--
 testing/tests/perf/test_sync.py       | 6 ++----
 3 files changed, 9 insertions(+), 7 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index fca9c58d..68e0fb38 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -193,7 +193,11 @@ def txbenchmark_with_setup(benchmark):
             return threads.blockingCallFromThread(reactor, f, *args, **kwargs)
 
         def blocking_setup():
-            return threads.blockingCallFromThread(reactor, setup)
+            args = threads.blockingCallFromThread(reactor, setup)
+            try:
+                return tuple(arg for arg in args), {}
+            except TypeError:
+                    return ((args,), {}) if args else None
 
         def bench():
             return benchmark.pedantic(blocking_runner, setup=blocking_setup,
diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index 681b909a..4eb990a8 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -18,7 +18,7 @@ def create_encrypt(amount, size):
             pool = SyncEncrypterPool(client._crypto, client._sync_db)
             pool.start()
             request.addfinalizer(pool.stop)
-            return (pool,), {}
+            return pool
 
         @pytest.inlineCallbacks
         def put_and_wait(pool):
@@ -63,7 +63,7 @@ def create_decrypt(amount, size):
                     json=json.dumps(DOC_CONTENT))
                 encrypted_content = json.loads(crypto.encrypt_doc(doc))
                 docs.append((doc.doc_id, encrypted_content))
-            return (pool, docs), {}
+            return pool, docs
 
         def put_and_wait(pool, docs):
             deferreds = []  # fires on completion
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 668ceae7..0be9d12f 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -45,8 +45,7 @@ def create_download(downloads, size):
         # ensures we are dealing with properly encrypted docs
 
         def setup():
-            clean_client = soledad_client()
-            return (clean_client,), {}
+            return soledad_client()
 
         def sync(clean_client):
             return clean_client.sync()
@@ -63,8 +62,7 @@ test_download_1000_10k = create_download(1000, 10*1000)
 @pytest.mark.benchmark(group="test_nothing_to_sync")
 def test_nothing_to_sync(soledad_client, txbenchmark_with_setup):
     def setup():
-        clean_client = soledad_client()
-        return (clean_client,), {}
+        return soledad_client()
 
     def sync(clean_client):
         return clean_client.sync()
-- 
cgit v1.2.3


From 71b13eda4ec5a577df1ccf4b07e26cbd87eaaa3e Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 22 Aug 2016 18:52:51 -0300
Subject: [tests] remove unused new=False logic

---
 testing/tests/perf/conftest.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 68e0fb38..9abd0c54 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -220,17 +220,11 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request):
     soledad_dbs(default_uuid)
 
     # get a soledad instance
-    def create(new=False):
+    def create():
         secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid4().hex)
         local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid4().hex)
-        if new:
-            uuid = uuid4().hex
-            remote_db(uuid)
-            soledad_dbs(uuid)
-        else:
-            uuid = default_uuid
         soledad_client = Soledad(
-            uuid,
+            default_uuid,
             unicode(passphrase),
             secrets_path=secrets_path,
             local_db_path=local_db_path,
-- 
cgit v1.2.3


From bad25ba9a5e7e5296b79544f50cafc47599a76b9 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 22 Aug 2016 18:53:49 -0300
Subject: [tests] move doc creation to setup

Otherwise it will add unrelated overhead to results.
---
 testing/tests/perf/test_encdecpool.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index 4eb990a8..919fdaa7 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -18,19 +18,16 @@ def create_encrypt(amount, size):
             pool = SyncEncrypterPool(client._crypto, client._sync_db)
             pool.start()
             request.addfinalizer(pool.stop)
-            return pool
+            docs = [
+                SoledadDocument(doc_id=uuid4().hex, rev='rev',
+                                json=json.dumps(DOC_CONTENT))
+                for _ in xrange(amount)
+            ]
+            return pool, docs
 
         @pytest.inlineCallbacks
-        def put_and_wait(pool):
-            doc_ids = []
-            deferreds = []
-            for _ in xrange(amount):
-                doc = SoledadDocument(
-                    doc_id=uuid4().hex, rev='rev',
-                    json=json.dumps(DOC_CONTENT))
-                deferreds.append(pool.encrypt_doc(doc))
-                doc_ids.append(doc.doc_id)
-            yield gatherResults(deferreds)
+        def put_and_wait(pool, docs):
+            yield gatherResults([pool.encrypt_doc(doc) for doc in docs])
 
         yield txbenchmark_with_setup(setup, put_and_wait)
     return test
-- 
cgit v1.2.3


From a64095ad5e6c6a36221444a34d8d56c0ae1c6150 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 22 Aug 2016 19:22:15 -0300
Subject: [test] refactor load_up for readability

defer parameter wasnt clear
---
 testing/tests/perf/test_sqlcipher.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py
index 511080a1..1fce1c3e 100644
--- a/testing/tests/perf/test_sqlcipher.py
+++ b/testing/tests/perf/test_sqlcipher.py
@@ -8,15 +8,9 @@ from twisted.internet.defer import gatherResults
 
 def load_up(client, amount, size, defer=True):
     content = 'x'*size
-    deferreds = []
-    # create a bunch of local documents
-    for i in xrange(amount):
-        d = client.create_doc({'content': content})
-        deferreds.append(d)
+    results = [client.create_doc({'content': content}) for _ in xrange(amount)]
     if defer:
-        d = gatherResults(deferreds)
-        d.addCallback(lambda _: None)
-        return d
+        return gatherResults(results)
 
 
 def build_test_sqlcipher_async_create(amount, size):
-- 
cgit v1.2.3


From b847ffe282fe0e08783447359cbbf745a2c8f376 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 23 Aug 2016 15:40:08 -0300
Subject: [test] point issue #7370 as reason for low values

We are using lower values on test_encdecpool due high memory usage,
described in #7370. Added a comment to explain it.
---
 testing/tests/perf/test_encdecpool.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index 919fdaa7..3c501084 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -5,6 +5,7 @@ from twisted.internet.defer import gatherResults
 from leap.soledad.client.encdecpool import SyncEncrypterPool
 from leap.soledad.client.encdecpool import SyncDecrypterPool
 from leap.soledad.common.document import SoledadDocument
+# FIXME: test load is low due issue #7370, higher values will get out of memory
 
 
 def create_encrypt(amount, size):
-- 
cgit v1.2.3


From bebbaad4988c4f4ec26d37791f7738ea27719fca Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 23 Aug 2016 15:53:04 -0300
Subject: [test] test_instance -> test_initialization

This isnt a test, but a benchmark. Initialization sounds more like an
operation while instance is just something.
---
 testing/tests/perf/test_misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/test_misc.py b/testing/tests/perf/test_misc.py
index c2f8e3b1..ead48adf 100644
--- a/testing/tests/perf/test_misc.py
+++ b/testing/tests/perf/test_misc.py
@@ -2,5 +2,5 @@ import pytest
 
 
 @pytest.mark.benchmark(group="test_instance")
-def test_instance(soledad_client, benchmark):
+def test_initialization(soledad_client, benchmark):
     benchmark(soledad_client)
-- 
cgit v1.2.3


From d86831e4cd3e77f340618168528e62cf4dafb5d7 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 29 Aug 2016 00:05:45 -0300
Subject: [test] randomize payload

We were using 'x'*size as payload, but on real usage the payload will be
random. This commit randomizes the payload using a predefined seed, so
the random payload will be the same across benchmarks.
Using random payloads also improves accuracy of compression or encoding
impacts and we will be evaluating those changes for resouce usage
issues.
Also note that base64 is used on payload. That was needed for utf8
safety, but overhead was removed to leave payloads as defined by
benchmarks.
Base64 was chosen also due its popular usage on MIME encoding, which is
used on mail attachments (our current scenario).
---
 testing/tests/perf/conftest.py        | 12 ++++++++++++
 testing/tests/perf/test_crypto.py     | 22 +++++++++++-----------
 testing/tests/perf/test_encdecpool.py |  8 ++++----
 testing/tests/perf/test_sqlcipher.py  | 13 ++++++-------
 testing/tests/perf/test_sync.py       | 13 ++++++-------
 5 files changed, 39 insertions(+), 29 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 9abd0c54..3681025f 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -2,6 +2,8 @@ import json
 import os
 import pytest
 import requests
+import random
+import base64
 import signal
 import time
 
@@ -43,6 +45,16 @@ DEFAULT_CERTKEY = 'soledad_certkey.pem'
 DEFAULT_TOKEN = 'an-auth-token'
 
 
+@pytest.fixture()
+def payload():
+    def generate(size):
+        random.seed(1337)  # same seed to avoid different bench results
+        payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size))
+        # encode as base64 to avoid ascii encode/decode errors
+        return base64.b64encode(payload_bytes)[:size]  # remove b64 overhead
+    return generate
+
+
 #
 # soledad_dbs fixture: provides all databases needed by soledad server in a per
 # module scope (same databases for all tests in this module).
diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py
index a32ef593..be00560b 100644
--- a/testing/tests/perf/test_crypto.py
+++ b/testing/tests/perf/test_crypto.py
@@ -8,10 +8,10 @@ from leap.soledad.client.crypto import decrypt_sym
 
 def create_doc_encryption(size):
     @pytest.mark.benchmark(group="test_crypto_encrypt_doc")
-    def test_doc_encryption(soledad_client, benchmark):
+    def test_doc_encryption(soledad_client, benchmark, payload):
         crypto = soledad_client()._crypto
 
-        DOC_CONTENT = {'payload': 'x'*size}
+        DOC_CONTENT = {'payload': payload(size)}
         doc = SoledadDocument(
             doc_id=uuid4().hex, rev='rev',
             json=json.dumps(DOC_CONTENT))
@@ -22,10 +22,10 @@ def create_doc_encryption(size):
 
 def create_doc_decryption(size):
     @pytest.mark.benchmark(group="test_crypto_decrypt_doc")
-    def test_doc_decryption(soledad_client, benchmark):
+    def test_doc_decryption(soledad_client, benchmark, payload):
         crypto = soledad_client()._crypto
 
-        DOC_CONTENT = {'payload': 'x'*size}
+        DOC_CONTENT = {'payload': payload(size)}
         doc = SoledadDocument(
             doc_id=uuid4().hex, rev='rev',
             json=json.dumps(DOC_CONTENT))
@@ -49,21 +49,21 @@ test_decrypt_doc_1M = create_doc_decryption(1000*1000)
 test_decrypt_doc_10M = create_doc_decryption(10*1000*1000)
 test_decrypt_doc_50M = create_doc_decryption(50*1000*1000)
 
-KEY = 'x'*32
-
 
 def create_raw_encryption(size):
     @pytest.mark.benchmark(group="test_crypto_raw_encrypt")
-    def test_raw_encrypt(benchmark):
-        benchmark(encrypt_sym, 'x'*size, KEY)
+    def test_raw_encrypt(benchmark, payload):
+        key = payload(32)
+        benchmark(encrypt_sym, payload(size), key)
     return test_raw_encrypt
 
 
 def create_raw_decryption(size):
     @pytest.mark.benchmark(group="test_crypto_raw_decrypt")
-    def test_raw_decrypt(benchmark):
-        iv, ciphertext = encrypt_sym('x'*size, KEY)
-        benchmark(decrypt_sym, ciphertext, KEY, iv)
+    def test_raw_decrypt(benchmark, payload):
+        key = payload(32)
+        iv, ciphertext = encrypt_sym(payload(size), key)
+        benchmark(decrypt_sym, ciphertext, key, iv)
     return test_raw_decrypt
 
 
diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py
index 3c501084..77091a41 100644
--- a/testing/tests/perf/test_encdecpool.py
+++ b/testing/tests/perf/test_encdecpool.py
@@ -11,8 +11,8 @@ from leap.soledad.common.document import SoledadDocument
 def create_encrypt(amount, size):
     @pytest.mark.benchmark(group="test_pool_encrypt")
     @pytest.inlineCallbacks
-    def test(soledad_client, txbenchmark_with_setup, request):
-        DOC_CONTENT = {'payload': 'x'*size}
+    def test(soledad_client, txbenchmark_with_setup, request, payload):
+        DOC_CONTENT = {'payload': payload(size)}
 
         def setup():
             client = soledad_client()
@@ -41,8 +41,8 @@ test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000)
 def create_decrypt(amount, size):
     @pytest.mark.benchmark(group="test_pool_decrypt")
     @pytest.inlineCallbacks
-    def test(soledad_client, txbenchmark_with_setup, request):
-        DOC_CONTENT = {'payload': 'x'*size}
+    def test(soledad_client, txbenchmark_with_setup, request, payload):
+        DOC_CONTENT = {'payload': payload(size)}
         client = soledad_client()
 
         def setup():
diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py
index 1fce1c3e..e7a54228 100644
--- a/testing/tests/perf/test_sqlcipher.py
+++ b/testing/tests/perf/test_sqlcipher.py
@@ -6,9 +6,8 @@ import pytest
 from twisted.internet.defer import gatherResults
 
 
-def load_up(client, amount, size, defer=True):
-    content = 'x'*size
-    results = [client.create_doc({'content': content}) for _ in xrange(amount)]
+def load_up(client, amount, payload, defer=True):
+    results = [client.create_doc({'content': payload}) for _ in xrange(amount)]
     if defer:
         return gatherResults(results)
 
@@ -16,17 +15,17 @@ def load_up(client, amount, size, defer=True):
 def build_test_sqlcipher_async_create(amount, size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_sqlcipher_async_create")
-    def test(soledad_client, txbenchmark):
+    def test(soledad_client, txbenchmark, payload):
         client = soledad_client()
-        yield txbenchmark(load_up, client, amount, size)
+        yield txbenchmark(load_up, client, amount, payload(size))
     return test
 
 
 def build_test_sqlcipher_create(amount, size):
     @pytest.mark.benchmark(group="test_sqlcipher_create")
-    def test(soledad_client, benchmark):
+    def test(soledad_client, benchmark, payload):
         client = soledad_client()._dbsyncer
-        benchmark(load_up, client, amount, size, defer=False)
+        benchmark(load_up, client, amount, payload(size), defer=False)
     return test
 
 
diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py
index 0be9d12f..0b48a0b9 100644
--- a/testing/tests/perf/test_sync.py
+++ b/testing/tests/perf/test_sync.py
@@ -3,12 +3,11 @@ import pytest
 from twisted.internet.defer import gatherResults
 
 
-def load_up(client, amount, size):
-    content = 'x'*size
+def load_up(client, amount, payload):
     deferreds = []
     # create a bunch of local documents
     for i in xrange(amount):
-        d = client.create_doc({'content': content})
+        d = client.create_doc({'content': payload})
         deferreds.append(d)
     d = gatherResults(deferreds)
     d.addCallback(lambda _: None)
@@ -18,11 +17,11 @@ def load_up(client, amount, size):
 def create_upload(uploads, size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_upload")
-    def test(soledad_client, txbenchmark_with_setup):
+    def test(soledad_client, txbenchmark_with_setup, payload):
         client = soledad_client()
 
         def setup():
-            return load_up(client, uploads, size)
+            return load_up(client, uploads, payload(size))
 
         yield txbenchmark_with_setup(setup, client.sync)
     return test
@@ -36,10 +35,10 @@ test_upload_1000_10k = create_upload(1000, 10*1000)
 def create_download(downloads, size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_download")
-    def test(soledad_client, txbenchmark_with_setup):
+    def test(soledad_client, txbenchmark_with_setup, payload):
         client = soledad_client()
 
-        yield load_up(client, downloads, size)
+        yield load_up(client, downloads, payload(size))
         yield client.sync()
         # We could create them directly on couch, but sending them
         # ensures we are dealing with properly encrypted docs
-- 
cgit v1.2.3


From 4e06eb370b99f2d343e96f774a3ad9b8b77c9548 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 3 Oct 2016 19:27:42 -0300
Subject: [feature] check for user dbs couch schema versions

---
 testing/tests/perf/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'testing/tests/perf')

diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 3681025f..5ac1f3c0 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -193,8 +193,8 @@ def soledad_server(tmpdir_factory, request):
 def txbenchmark(benchmark):
     def blockOnThread(*args, **kwargs):
         return threads.deferToThread(
-                benchmark, threads.blockingCallFromThread,
-                reactor, *args, **kwargs)
+            benchmark, threads.blockingCallFromThread,
+            reactor, *args, **kwargs)
     return blockOnThread
 
 
-- 
cgit v1.2.3