[test] add speed tests for gpg/wrapper init/enc/dec

7 files changed, 428 insertions, 5 deletions

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..0a3db77
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,37 @@
+# This makefile is currently intended to make it easy to generate the
+# benchmarking graphs.
+
+RESULTS_FILE = tests/results.json
+GRAPH_PREFIX = benchmark
+
+GRAPH_FILE = $(GRAPH_PREFIX)-test_gpg_init.svg
+
+all: $(GRAPH_FILE)
+
+#
+# rules for generating one graph with the results of all speed tests
+#
+
+$(RESULTS_FILE):
+	tox -v test_gpg_speed.py -- -v --pdb -s \
+	    --benchmark-max-time=2.0 \
+	    --benchmark-json=$(subst tests/,,$@)
+
+$(GRAPH_FILE): $(RESULTS_FILE)
+	py.test-benchmark compare $< --histogram $(GRAPH_PREFIX)
+
+
+#
+# rule for generating one graph for each graph
+#
+
+test:
+	tox -v test_gpg_speed.py -- -v --pdb -s \
+	    --benchmark-histogram=gpg_speed \
+	    --benchmark-storage=./graphs/ \
+	    --benchmark-save=keymanager_gpg_speed \
+
+clean:
+	rm -f $(RESULTS_FILE) $(GRAPH_PREFIX)*.svg
+
+.PHONY: all test graph
diff --git a/tests/TODO b/tests/TODO
new file mode 100644
index 0000000..9f596a1
--- /dev/null
+++ b/tests/TODO
@@ -0,0 +1 @@
+- tune test params (max time, number of iterations, etc)
diff --git a/tests/common.py b/tests/common.py
index 8eb5d4e..9d16991 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -265,8 +265,51 @@ RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
 =JTFu
 -----END PGP PRIVATE KEY BLOCK-----
 """
+CIPHERTEXT = """
+-----BEGIN PGP MESSAGE-----
+Version: GnuPG v1
+
+hQIMAyj9aG/xtZOwAQ/+OtgYlCULNaCqSnzDIAIO5Swsg7fLFIErSWF/4ngkNxPk
+GqL3/2/HvlLY6blsmn5RU2AK4vo5Dc5s2lQU3PwqsaVKNoqkn1G9bMqsoIQhP4vW
+M6c+KdxtA8cQggENpx0kF8FXdvV+GclChfK38TDJUrJLnksfo+UP9rI2BsIpHpiS
+sfikVLuUc83+2hTozuTOVARNG7x58hhnR7wPtbgm/6AwVNvU0SQDEDJXi43MFRbP
+9VboUHIRZLbeIwlxFHj3umh4f8rca4jSwiWxDna2YRAFBZrFiyiGAnISZQc9Nh4d
+jLqa7rMeLSfTigkOXVYdVjEgx/wsbsDjgJ2f6TUrfP6RWen3/4223ctfeL2x4r2V
+6ej2cbbR61qQx2FR4HV4XRaSg3tV+Ytz0dklrvcL0PQVdSsJhDmRV4pNgLSt0/bA
+E6F2hk/S3sjnXZRMvXNb3SRQk2R0Svn4j/8ft/8VC+h5jkHO2G5K3DbAHkS2MPJm
+bLwx4LMPohZstS75OFKZv63nzdOYIz5gPSy9A4IC1VHqM4d3T99cqY8XOJOqRkr5
+TCYOxOHF4z6N2nRkzVXoDoTfC6+qx9bWuvGhoj5WjFG1I19e+L1IVVvpYKo6LAOa
+7Dn+IXe3XnCGGHn3AusJgyeqt/3Z5HgzsSPMKIh2WfUKERZZ+vrxTXIf4Ko+2tbS
+aQEpuiQUr9zE4z8eT+brGJNl5mx9c1qYCSPjGIGUCt8fTOKruMEkWBQHMln05qpF
+nFMYk7JXtYaArGrC1QPRVHOvgUTB6Vx7KVHikzsEXGLF6ywBnZYzeh8ah/FtxHG9
+o+vTTcScQkxVcw==
+=qYFS
+-----END PGP MESSAGE-----
+"""
+SIGNEDTEXT = """
+-----BEGIN PGP MESSAGE-----
+Version: GnuPG v1
+
+owHtwYszEwAcAOBtbMxrxOWRhp2aIq/Ligubt2Ynbh45c6Sb6mZbmeFY18qi7lY4
+VHMlXENey2M9PCKvW5FHpesKXSycavNcxOr/6H7fV2Ssg0Ah3YKTnBwnxLPIJqLd
+GSyXlXKe7crL5sU1I6j2AAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAf+AGylYXgUQh
+MGhUXDOCijDAmroFJzk5TohnsZm4v7GUm3nKlmm81iD29kxTiej+o721Di7tHisO
+RQNzteoFD7aa5r8tfnUnptG7YHjV10I7mKLYnuDFCJJxrPxmUo7v0haOsKS4GqYe
+snts8+eyUaU/rjVh/Z5ygNnEovajBQGWeYnTpLcc6fsF7a7TmNzQndW3FjKnUitb
+P0lGTGQ1kfV0RdakLsOYpNAU8qNWZTYSVcimX9kHhlliY8WQXUMEVn9/GMEkXPV5
+5GLQiuVAYXqQ9YPXmJ64UYOds8Nzxpe62vu2enMwSLQ3Odo8p1CJiBqjOAtSnegs
+2sM1T1odsnqFk3zUiO2bwRfqdS40qNjX5vkbZBceiXM9mRedWaXRHE6L9PMpDnhK
+iTY5Hag99XXq25v1Y0gmt2aPWnXrOyNUKuo9UrrzrlhhvxyJo3Xhmi70SbKJ83KZ
+F+rkogA/nM/8GKWT4VkQLw/16KKmM8q8pjpGB3UtDsV3L1df6UnlL5LN+lOFL0QO
+AZQDXE3aL8fy8B9aeri4CJmLrxvElwhH3Ad2Kp6XykVZnc8wT1q39OTFS8tfXsb8
+lKyZcmy47m2x5TKf+LpAqxKNraHf+CS+xiS7Et0gvbtrNSNN26ffprRdtWZFdByX
+H0SiTbvNExzL8zhEXo/QldiyWT+e6GwpOadPGL9xIpe8Uc0k/A6uonvy8f8A
+=wZ2N
+-----END PGP MESSAGE-----
+"""
 
 # key 7FEE575A: public key "anotheruser <anotheruser@leap.se>"
+KEY_FINGERPRINT_2 = "F6E2B572ADB84EA58BD2E9A57F9DFA687FEE575A"
 PUBLIC_KEY_2 = """
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1.4.10 (GNU/Linux)
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..0c428a0
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,51 @@
+import pytest
+
+from leap.keymanager.keys import build_key_from_dict
+from common import ADDRESS
+from common import KEY_FINGERPRINT
+from common import PUBLIC_KEY
+from common import PRIVATE_KEY
+from common import ADDRESS_2
+from common import KEY_FINGERPRINT_2
+from common import PUBLIC_KEY_2
+from common import PRIVATE_KEY_2
+from leap.keymanager.wrapper import TempGPGWrapper
+
+
+@pytest.fixture
+def wrapper(keys=None):
+    return TempGPGWrapper(keys=keys)
+
+
+def _get_key(address, key_fingerprint, key_data, private):
+    kdict = {
+        'uids': [address],
+        'fingerprint': key_fingerprint,
+        'key_data': key_data,
+        'private': private,
+        'length': 4096,
+        'expiry_date': 0,
+        'refreshed_at': 1311239602,
+    }
+    key = build_key_from_dict(kdict)
+    return key
+
+
+@pytest.fixture
+def public_key():
+    return _get_key(ADDRESS, KEY_FINGERPRINT, PUBLIC_KEY, False)
+
+
+@pytest.fixture
+def public_key_2():
+    return _get_key(ADDRESS_2, KEY_FINGERPRINT_2, PUBLIC_KEY_2, False)
+
+
+@pytest.fixture
+def openpgp_keys():
+    return [
+        _get_key(ADDRESS, KEY_FINGERPRINT, PUBLIC_KEY, False),
+        _get_key(ADDRESS_2, KEY_FINGERPRINT_2, PUBLIC_KEY_2, False),
+        _get_key(ADDRESS, KEY_FINGERPRINT, PRIVATE_KEY, True),
+        _get_key(ADDRESS_2, KEY_FINGERPRINT_2, PRIVATE_KEY_2, True),
+    ]
diff --git a/tests/test_gpg_speed.py b/tests/test_gpg_speed.py
new file mode 100644
index 0000000..e028cae
--- /dev/null
+++ b/tests/test_gpg_speed.py
@@ -0,0 +1,286 @@
+import commands
+import pytest
+
+from functools import partial
+from gnupg import GPG
+from leap.keymanager.wrapper import TempGPGWrapper
+
+from common import CIPHERTEXT
+from common import SIGNEDTEXT
+
+
+GROUP_INIT = 'initialization only'
+GROUP_CRYPTO = 'crypto only'
+GROUP_INIT_AND_CRYPTO = 'initialization and crypto'
+
+
+# the gnupg module gets the binary version each time the GPG object is
+# initialized. In some platforms this takes hundreds of milliseconds, and
+# sometimes more than a second. This is currently a known bug. For making it
+# evident, we provide a way to bypass the version check by monkeypatching the
+# actual function that does the job.
+
+def get_gpg_version():
+    output = commands.getoutput(
+        'gpg --list-config --with-colons | grep version')
+    version = output.split(':').pop()
+    return version
+
+
+GPG_VERSION = get_gpg_version()
+
+
+def mock_gpg_get_version(monkeypatch):
+    def _setver(self):
+        self.binary_version = GPG_VERSION
+    monkeypatch.setattr(
+        GPG, '_check_sane_and_get_gpg_version', _setver)
+
+
+#
+# generic speed test creator
+#
+
+def create_test(fun, num_keys=0, mock_get_version=True, init=None, group=None):
+
+    @pytest.mark.benchmark(group=group)
+    def test(tmpdir, benchmark, openpgp_keys, monkeypatch):
+
+        if mock_get_version:
+            mock_gpg_get_version(monkeypatch)
+
+        if init:
+            res = init(tmpdir, benchmark, openpgp_keys, monkeypatch, num_keys)
+            benchmark(fun, res)
+        else:
+            benchmark(
+                fun, tmpdir, benchmark, openpgp_keys, monkeypatch, num_keys)
+
+    return test
+
+
+#
+# gpg initializarion: 0, 1 and 2 keys
+#
+
+def gpg_init_only(tmpdir, benchmark, openpgp_keys, monkeypatch, num_keys):
+    keys = openpgp_keys[0:num_keys]
+    gpg = GPG(homedir=tmpdir.dirname)
+    for key in keys:
+        gpg.import_keys(key.key_data)
+
+
+test_gpg_init_nokey_slow = create_test(
+    gpg_init_only, num_keys=0,
+    mock_get_version=False,
+    group=GROUP_INIT)
+test_gpg_init_1key_slow = create_test(
+    gpg_init_only, num_keys=1,
+    mock_get_version=False,
+    group=GROUP_INIT)
+test_gpg_init_2keys_slow = create_test(
+    gpg_init_only, num_keys=2,
+    mock_get_version=False,
+    group=GROUP_INIT)
+
+test_gpg_init_nokey = create_test(
+    gpg_init_only, num_keys=0,
+    group=GROUP_INIT)
+test_gpg_init_1key = create_test(
+    gpg_init_only, num_keys=1,
+    group=GROUP_INIT)
+test_gpg_init_2keys = create_test(
+    gpg_init_only, num_keys=2,
+    group=GROUP_INIT)
+
+
+#
+# wrapper initialization: 0, 1 and 2 keys
+#
+
+def wrapper_init_only(tmpdir, benchmark, openpgp_keys, monkeypatch, num_keys):
+    keys = openpgp_keys[0:num_keys]
+    wrapper = TempGPGWrapper(keys=keys)
+    with wrapper as gpg:
+        assert GPG == type(gpg)
+
+
+test_wrapper_init_nokey_slow = create_test(
+    wrapper_init_only, num_keys=0,
+    mock_get_version=False,
+    group=GROUP_INIT)
+test_wrapper_init_1key_slow = create_test(
+    wrapper_init_only, num_keys=1,
+    mock_get_version=False,
+    group=GROUP_INIT)
+test_wrapper_init_2keys_slow = create_test(
+    wrapper_init_only, num_keys=2,
+    mock_get_version=False,
+    group=GROUP_INIT)
+
+test_wrapper_init_nokey = create_test(
+    wrapper_init_only, num_keys=0,
+    group=GROUP_INIT)
+test_wrapper_init_1key = create_test(
+    wrapper_init_only, num_keys=1,
+    group=GROUP_INIT)
+test_wrapper_init_2keys = create_test(
+    wrapper_init_only, num_keys=2,
+    group=GROUP_INIT)
+
+
+#
+# initialization + encryption
+#
+
+PLAINTEXT = ' ' * 10000  # 10 KB
+
+
+def gpg_init_exec(fun, tmpdir, benchmark, openpgp_keys, monkeypatch, _):
+    pubkey = openpgp_keys[0]
+    privkey = openpgp_keys[2]  # this is PRIVATE_KEY
+    gpg = GPG(homedir=tmpdir.dirname)
+    gpg.import_keys(pubkey.key_data)
+    gpg.import_keys(privkey.key_data)
+    fun((gpg, pubkey, privkey))
+
+
+def wrapper_init_exec(fun, tmpdir, benchmark, openpgp_keys, monkeypatch, _):
+    pubkey = openpgp_keys[0]
+    privkey = openpgp_keys[2]
+    wrapper = TempGPGWrapper(keys=[pubkey, privkey])
+    wrapper._build_keyring()
+    fun((wrapper._gpg, pubkey, privkey))
+
+
+def gpg_enc(res):
+    gpg, pubkey, _ = res
+    ciphertext = gpg.encrypt(PLAINTEXT, pubkey.fingerprint)
+    assert ciphertext.ok
+    assert len(ciphertext.data)
+
+
+test_gpg_init_enc = create_test(
+    partial(gpg_init_exec, gpg_enc),
+    group=GROUP_INIT_AND_CRYPTO)
+test_wrapper_init_enc = create_test(
+    partial(wrapper_init_exec, gpg_enc),
+    group=GROUP_INIT_AND_CRYPTO)
+
+
+#
+# initialization + decryption
+#
+
+def gpg_dec(res):
+    gpg, _, _ = res
+    plaintext = gpg.decrypt(CIPHERTEXT)
+    assert plaintext.ok
+    assert len(plaintext.data)
+
+
+test_gpg_init_dec = create_test(
+    partial(gpg_init_exec, gpg_dec),
+    group=GROUP_INIT_AND_CRYPTO)
+test_wrapper_init_dec = create_test(
+    partial(wrapper_init_exec, gpg_dec),
+    group=GROUP_INIT_AND_CRYPTO)
+
+
+#
+# initialization + sign
+#
+
+def gpg_sign(res):
+    gpg, _, privkey = res
+    gpg.import_keys(privkey.key_data)
+    signed = gpg.sign(PLAINTEXT, default_key=privkey.fingerprint)
+    assert signed.status == 'begin signing'
+    assert len(signed.data) > len(PLAINTEXT)
+    assert '-----BEGIN PGP SIGNATURE-----' in signed.data
+    assert '-----END PGP SIGNATURE-----' in signed.data
+
+
+test_gpg_init_sign = create_test(
+    partial(gpg_init_exec, gpg_sign),
+    group=GROUP_INIT_AND_CRYPTO)
+test_wrapper_init_sign = create_test(
+    partial(wrapper_init_exec, gpg_sign),
+    group=GROUP_INIT_AND_CRYPTO)
+
+
+#
+# initialization + verify
+#
+
+def gpg_verify(res):
+    gpg, _, privkey = res
+    signed = gpg.verify(SIGNEDTEXT)
+    assert signed.valid
+
+
+test_gpg_init_verify = create_test(
+    partial(gpg_init_exec, gpg_verify),
+    group=GROUP_INIT_AND_CRYPTO)
+test_wrapper_init_verify = create_test(
+    partial(wrapper_init_exec, gpg_verify),
+    group=GROUP_INIT_AND_CRYPTO)
+
+
+#
+# encryption only
+#
+
+def gpg_init(tmpdir, benchmark, openpgp_keys, monkeypatch, _):
+    pubkey = openpgp_keys[0]
+    privkey = openpgp_keys[2]  # this is PRIVATE_KEY
+    gpg = GPG(homedir=tmpdir.dirname)
+    gpg.import_keys(pubkey.key_data)
+    gpg.import_keys(privkey.key_data)
+    return gpg, pubkey, privkey
+
+
+def wrapper_init(tmpdir, benchmark, openpgp_keys, monkeypatch, _):
+    pubkey = openpgp_keys[0]
+    privkey = openpgp_keys[2]
+    wrapper = TempGPGWrapper(keys=[pubkey, privkey])
+    wrapper._build_keyring()
+    return wrapper._gpg, pubkey, privkey
+
+
+test_gpg_enc = create_test(
+    gpg_enc, init=gpg_init, group=GROUP_CRYPTO)
+test_wrapper_enc = create_test(
+    gpg_enc, init=wrapper_init, group=GROUP_CRYPTO)
+
+
+#
+# decryption only
+#
+
+test_gpg_dec = create_test(
+    gpg_dec,
+    init=gpg_init, group=GROUP_CRYPTO)
+test_wrapper_dec = create_test(
+    gpg_dec,
+    init=wrapper_init, group=GROUP_CRYPTO)
+
+
+#
+# sign only
+#
+
+test_gpg_sign = create_test(
+    gpg_sign, init=gpg_init, group=GROUP_CRYPTO)
+test_wrapper_sign = create_test(
+    gpg_sign, init=wrapper_init, group=GROUP_CRYPTO)
+
+
+#
+# verify only
+#
+
+test_gpg_verify = create_test(
+    gpg_verify, init=gpg_init, group=GROUP_CRYPTO)
+test_wrapper_verify = create_test(
+    gpg_verify, init=wrapper_init, group=GROUP_CRYPTO)
diff --git a/tests/test_openpgp.py b/tests/test_openpgp.py
index 1f78ad4..60d2319 100644
--- a/tests/test_openpgp.py
+++ b/tests/test_openpgp.py
@@ -23,16 +23,13 @@ Tests for the OpenPGP support on Key Manager.
 
 from datetime import datetime
 from mock import Mock
-from twisted.internet.defer import inlineCallbacks, gatherResults, succeed
+from twisted.internet.defer import inlineCallbacks, succeed
 
 from leap.keymanager import (
     KeyNotFound,
     openpgp,
 )
-from leap.keymanager.documents import (
-    TYPE_FINGERPRINT_PRIVATE_INDEX,
-    TYPE_ADDRESS_PRIVATE_INDEX,
-)
+from leap.keymanager.documents import TYPE_FINGERPRINT_PRIVATE_INDEX
 from leap.keymanager.keys import OpenPGPKey
 
 from common import (
diff --git a/tox.ini b/tox.ini
index 7667788..af67f56 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,8 +3,15 @@ envlist = py27
 
 [testenv]
 commands = py.test {posargs}
+changedir = tests
 deps =
     pytest
+    pytest-benchmark
+# need the next 2 for graphs, but new version changed api a bit and is
+# incompatible with pytest-benchmark, so we pin version <2.1
+# (https://github.com/ionelmc/pytest-benchmark/issues/50).
+    pygal<2.1
+    pygaljs
     pdbpp
     mock
     setuptools-trial
@@ -12,3 +19,4 @@ deps =
     leap.soledad.client
 setenv =
     HOME=/tmp
+    TERM=xterm