diff options
| author | drebs <drebs@leap.se> | 2016-12-22 17:36:15 -0200 | 
|---|---|---|
| committer | drebs <drebs@leap.se> | 2016-12-22 17:36:15 -0200 | 
| commit | 8a463796bbaba3979234b0699d140947581421e7 (patch) | |
| tree | d1e2ea96ed91ac66c7e52a30d16246a498ae9ed6 /testing/tests | |
| parent | f072f18f317ea31e66c7890d672b5d2fd9f3ef14 (diff) | |
| parent | e360a3a75999503cf45bfbbad69970a452cf3d32 (diff) | |
Merge tag '0.9.2'
Tag version 0.9.2
# gpg: Signature made Thu 22 Dec 2016 05:33:30 PM BRST
# gpg:                using RSA key 0x6071E70DCACC60B2
# gpg: Good signature from "drebs (work key) <db@leap.se>" [ultimate]
# gpg:                 aka "drebs (work key) <drebs@leap.se>" [ultimate]
# Impressão da chave primária: 9F73 295B 6306 E06F 3151  99AE 6071 E70D CACC 60B2
Diffstat (limited to 'testing/tests')
24 files changed, 793 insertions, 1196 deletions
| diff --git a/testing/tests/perf/assets/cert_default.conf b/testing/tests/benchmarks/assets/cert_default.conf index 8043cea3..8043cea3 100644 --- a/testing/tests/perf/assets/cert_default.conf +++ b/testing/tests/benchmarks/assets/cert_default.conf diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py new file mode 100644 index 00000000..a9cc3464 --- /dev/null +++ b/testing/tests/benchmarks/conftest.py @@ -0,0 +1,57 @@ +import pytest +import random +import base64 + +from twisted.internet import threads, reactor + + +# we have to manually setup the events server in order to be able to signal +# events. This is usually done by the enclosing application using soledad +# client (i.e. bitmask client). +from leap.common.events import server +server.ensure_server() + + +def pytest_addoption(parser): +    parser.addoption( +        "--num-docs", type="int", default=100, +        help="the number of documents to use in performance tests") + + +@pytest.fixture() +def payload(): +    def generate(size): +        random.seed(1337)  # same seed to avoid different bench results +        payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) +        # encode as base64 to avoid ascii encode/decode errors +        return base64.b64encode(payload_bytes)[:size]  # remove b64 overhead +    return generate + + +@pytest.fixture() +def txbenchmark(benchmark): +    def blockOnThread(*args, **kwargs): +        return threads.deferToThread( +            benchmark, threads.blockingCallFromThread, +            reactor, *args, **kwargs) +    return blockOnThread + + +@pytest.fixture() +def txbenchmark_with_setup(benchmark): +    def blockOnThreadWithSetup(setup, f): +        def blocking_runner(*args, **kwargs): +            return threads.blockingCallFromThread(reactor, f, *args, **kwargs) + +        def blocking_setup(): +            args = threads.blockingCallFromThread(reactor, setup) +            try: +                return tuple(arg for arg in args), {} +            except TypeError: +                    return ((args,), {}) if args else None + +        def bench(): +            return benchmark.pedantic(blocking_runner, setup=blocking_setup, +                                      rounds=4, warmup_rounds=1) +        return threads.deferToThread(bench) +    return blockOnThreadWithSetup diff --git a/testing/tests/perf/pytest.ini b/testing/tests/benchmarks/pytest.ini index 7a0508ce..7a0508ce 100644 --- a/testing/tests/perf/pytest.ini +++ b/testing/tests/benchmarks/pytest.ini diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py new file mode 100644 index 00000000..8ee9b899 --- /dev/null +++ b/testing/tests/benchmarks/test_crypto.py @@ -0,0 +1,97 @@ +""" +Benchmarks for crypto operations. +If you don't want to stress your local machine too much, you can pass the +SIZE_LIMT environment variable. + +For instance, to keep the maximum payload at 1MB: + +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +""" +import pytest +import os +import json +from uuid import uuid4 + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import _crypto + +LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) + + +def create_doc_encryption(size): +    @pytest.mark.benchmark(group="test_crypto_encrypt_doc") +    @pytest.inlineCallbacks +    def test_doc_encryption(soledad_client, txbenchmark, payload): +        crypto = soledad_client()._crypto + +        DOC_CONTENT = {'payload': payload(size)} +        doc = SoledadDocument( +            doc_id=uuid4().hex, rev='rev', +            json=json.dumps(DOC_CONTENT)) + +        yield txbenchmark(crypto.encrypt_doc, doc) +    return test_doc_encryption + + +# TODO this test is really bullshit, because it's still including +# the json serialization. + +def create_doc_decryption(size): +    @pytest.inlineCallbacks +    @pytest.mark.benchmark(group="test_crypto_decrypt_doc") +    def test_doc_decryption(soledad_client, txbenchmark, payload): +        crypto = soledad_client()._crypto + +        DOC_CONTENT = {'payload': payload(size)} +        doc = SoledadDocument( +            doc_id=uuid4().hex, rev='rev', +            json=json.dumps(DOC_CONTENT)) + +        encrypted_doc = yield crypto.encrypt_doc(doc) +        doc.set_json(encrypted_doc) + +        yield txbenchmark(crypto.decrypt_doc, doc) +    return test_doc_decryption + + +def create_raw_encryption(size): +    @pytest.mark.benchmark(group="test_crypto_raw_encrypt") +    def test_raw_encrypt(benchmark, payload): +        key = payload(32) +        benchmark(_crypto.encrypt_sym, payload(size), key) +    return test_raw_encrypt + + +def create_raw_decryption(size): +    @pytest.mark.benchmark(group="test_crypto_raw_decrypt") +    def test_raw_decrypt(benchmark, payload): +        key = payload(32) +        iv, ciphertext = _crypto.encrypt_sym(payload(size), key) +        benchmark(_crypto.decrypt_sym, ciphertext, key, iv) +    return test_raw_decrypt + + +# Create the TESTS in the global namespace, they'll be picked by the benchmark +# plugin. + +encryption_tests = [ +    ('10k', 1E4), +    ('100k', 1E5), +    ('500k', 5E5), +    ('1M', 1E6), +    ('10M', 1E7), +    ('50M', 5E7), +] + +for name, size in encryption_tests: +    if size < LIMIT: +        sz = int(size) +        globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) +        globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) + + +for name, size in encryption_tests: +    if size < LIMIT: +        sz = int(size) +        globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz) +        globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz) diff --git a/testing/tests/perf/test_misc.py b/testing/tests/benchmarks/test_misc.py index ead48adf..ead48adf 100644 --- a/testing/tests/perf/test_misc.py +++ b/testing/tests/benchmarks/test_misc.py diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/benchmarks/test_sqlcipher.py index e7a54228..39c9e3ad 100644 --- a/testing/tests/perf/test_sqlcipher.py +++ b/testing/tests/benchmarks/test_sqlcipher.py @@ -29,10 +29,10 @@ def build_test_sqlcipher_create(amount, size):      return test -test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500*1000) -test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100*1000) -test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10*1000) +test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500 * 1000) +test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) +test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000)  # synchronous -test_create_20_500k = build_test_sqlcipher_create(20, 500*1000) -test_create_100_100k = build_test_sqlcipher_create(100, 100*1000) -test_create_1000_10k = build_test_sqlcipher_create(1000, 10*1000) +test_create_20_500k = build_test_sqlcipher_create(20, 500 * 1000) +test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) +test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/testing/tests/perf/test_sync.py b/testing/tests/benchmarks/test_sync.py index 0b48a0b9..1501d74b 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -1,17 +1,14 @@  import pytest -  from twisted.internet.defer import gatherResults +@pytest.inlineCallbacks  def load_up(client, amount, payload): -    deferreds = []      # create a bunch of local documents +    deferreds = []      for i in xrange(amount): -        d = client.create_doc({'content': payload}) -        deferreds.append(d) -    d = gatherResults(deferreds) -    d.addCallback(lambda _: None) -    return d +        deferreds.append(client.create_doc({'content': payload})) +    yield gatherResults(deferreds)  def create_upload(uploads, size): @@ -27,9 +24,9 @@ def create_upload(uploads, size):      return test -test_upload_20_500k = create_upload(20, 500*1000) -test_upload_100_100k = create_upload(100, 100*1000) -test_upload_1000_10k = create_upload(1000, 10*1000) +test_upload_20_500k = create_upload(20, 500 * 1000) +test_upload_100_100k = create_upload(100, 100 * 1000) +test_upload_1000_10k = create_upload(1000, 10 * 1000)  def create_download(downloads, size): @@ -52,9 +49,9 @@ def create_download(downloads, size):      return test -test_download_20_500k = create_download(20, 500*1000) -test_download_100_100k = create_download(100, 100*1000) -test_download_1000_10k = create_download(1000, 10*1000) +test_download_20_500k = create_download(20, 500 * 1000) +test_download_100_100k = create_download(100, 100 * 1000) +test_download_1000_10k = create_download(1000, 10 * 1000)  @pytest.inlineCallbacks diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py index c25ff8ca..9b4a175f 100644 --- a/testing/tests/client/test_aux_methods.py +++ b/testing/tests/client/test_aux_methods.py @@ -21,10 +21,10 @@ import os  from twisted.internet import defer -from leap.soledad.common.errors import DatabaseAccessError  from leap.soledad.client import Soledad  from leap.soledad.client.adbapi import U1DBConnectionPool  from leap.soledad.client.secrets import PassphraseTooShort +from leap.soledad.client.secrets import SecretsException  from test_soledad.util import BaseSoledadTest @@ -108,7 +108,7 @@ class AuxMethodsTestCase(BaseSoledadTest):          sol.change_passphrase(u'654321')          sol.close() -        with self.assertRaises(DatabaseAccessError): +        with self.assertRaises(SecretsException):              self._soledad_instance(                  'leap@leap.se',                  passphrase=u'123', diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 77252b46..49a61438 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -17,47 +17,173 @@  """  Tests for cryptographic related stuff.  """ -import os -import hashlib  import binascii +import base64 +import hashlib +import json +import os + +from io import BytesIO + +import pytest + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend +from cryptography.exceptions import InvalidTag -from leap.soledad.client import crypto  from leap.soledad.common.document import SoledadDocument  from test_soledad.util import BaseSoledadTest -from leap.soledad.common.crypto import WrongMacError -from leap.soledad.common.crypto import UnknownMacMethodError -from leap.soledad.common.crypto import ENC_JSON_KEY -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.crypto import MAC_KEY -from leap.soledad.common.crypto import MAC_METHOD_KEY +from leap.soledad.client import _crypto + +from twisted.trial import unittest +from twisted.internet import defer + + +snowden1 = ( +    "You can't come up against " +    "the world's most powerful intelligence " +    "agencies and not accept the risk. " +    "If they want to get you, over time " +    "they will.") + + +class AESTest(unittest.TestCase): + +    def test_chunked_encryption(self): +        key = 'A' * 32 + +        fd = BytesIO() +        aes = _crypto.AESWriter(key, _buffer=fd) +        iv = aes.iv + +        data = snowden1 +        block = 16 + +        for i in range(len(data) / block): +            chunk = data[i * block:(i + 1) * block] +            aes.write(chunk) +        aes.end() + +        ciphertext_chunked = fd.getvalue() +        ciphertext, tag = _aes_encrypt(key, iv, data) + +        assert ciphertext_chunked == ciphertext + +    def test_decrypt(self): +        key = 'A' * 32 +        iv = 'A' * 16 + +        data = snowden1 +        block = 16 + +        ciphertext, tag = _aes_encrypt(key, iv, data) + +        fd = BytesIO() +        aes = _crypto.AESWriter(key, iv, fd, tag=tag) + +        for i in range(len(ciphertext) / block): +            chunk = ciphertext[i * block:(i + 1) * block] +            aes.write(chunk) +        aes.end() + +        cleartext_chunked = fd.getvalue() +        assert cleartext_chunked == data + + +class BlobTestCase(unittest.TestCase): + +    class doc_info: +        doc_id = 'D-deadbeef' +        rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + +    @defer.inlineCallbacks +    def test_blob_encryptor(self): + +        inf = BytesIO(snowden1) + +        blob = _crypto.BlobEncryptor( +            self.doc_info, inf, +            secret='A' * 96) +        encrypted = yield blob.encrypt() +        preamble, ciphertext = _crypto._split(encrypted.getvalue()) +        ciphertext = ciphertext[:-16] -class EncryptedSyncTestCase(BaseSoledadTest): +        assert len(preamble) == _crypto.PACMAN.size +        unpacked_data = _crypto.PACMAN.unpack(preamble) +        magic, sch, meth, ts, iv, doc_id, rev = unpacked_data +        assert magic == _crypto.BLOB_SIGNATURE_MAGIC +        assert sch == 1 +        assert meth == _crypto.ENC_METHOD.aes_256_gcm +        assert iv == blob.iv +        assert doc_id == 'D-deadbeef' +        assert rev == self.doc_info.rev -    """ -    Tests that guarantee that data will always be encrypted when syncing. -    """ +        aes_key = _crypto._get_sym_key_for_doc( +            self.doc_info.doc_id, 'A' * 96) +        assert ciphertext == _aes_encrypt(aes_key, blob.iv, snowden1)[0] -    def test_encrypt_decrypt_json(self): +        decrypted = _aes_decrypt(aes_key, blob.iv, blob.tag, ciphertext, +                                 preamble) +        assert str(decrypted) == snowden1 + +    @defer.inlineCallbacks +    def test_blob_decryptor(self): + +        inf = BytesIO(snowden1) + +        blob = _crypto.BlobEncryptor( +            self.doc_info, inf, +            secret='A' * 96) +        ciphertext = yield blob.encrypt() + +        decryptor = _crypto.BlobDecryptor( +            self.doc_info, ciphertext, +            secret='A' * 96) +        decrypted = yield decryptor.decrypt() +        assert decrypted == snowden1 + +    @defer.inlineCallbacks +    def test_encrypt_and_decrypt(self):          """ -        Test encrypting and decrypting documents. +        Check that encrypting and decrypting gives same doc.          """ -        simpledoc = {'key': 'val'} -        doc1 = SoledadDocument(doc_id='id') -        doc1.content = simpledoc - -        # encrypt doc -        doc1.set_json(self._soledad._crypto.encrypt_doc(doc1)) -        # assert content is different and includes keys -        self.assertNotEqual( -            simpledoc, doc1.content, -            'incorrect document encryption') -        self.assertTrue(ENC_JSON_KEY in doc1.content) -        self.assertTrue(ENC_SCHEME_KEY in doc1.content) -        # decrypt doc -        doc1.set_json(self._soledad._crypto.decrypt_doc(doc1)) -        self.assertEqual( -            simpledoc, doc1.content, 'incorrect document encryption') +        crypto = _crypto.SoledadCrypto('A' * 96) +        payload = {'key': 'someval'} +        doc1 = SoledadDocument('id1', '1', json.dumps(payload)) + +        encrypted = yield crypto.encrypt_doc(doc1) +        assert encrypted != payload +        assert 'raw' in encrypted +        doc2 = SoledadDocument('id1', '1') +        doc2.set_json(encrypted) +        assert _crypto.is_symmetrically_encrypted(encrypted) +        decrypted = yield crypto.decrypt_doc(doc2) +        assert len(decrypted) != 0 +        assert json.loads(decrypted) == payload + +    @defer.inlineCallbacks +    def test_decrypt_with_wrong_tag_raises(self): +        """ +        Trying to decrypt a document with wrong MAC should raise. +        """ +        crypto = _crypto.SoledadCrypto('A' * 96) +        payload = {'key': 'someval'} +        doc1 = SoledadDocument('id1', '1', json.dumps(payload)) + +        encrypted = yield crypto.encrypt_doc(doc1) +        encdict = json.loads(encrypted) +        preamble, raw = _crypto._split(str(encdict['raw'])) +        # mess with tag +        messed = raw[:-16] + '0' * 16 + +        preamble = base64.urlsafe_b64encode(preamble) +        newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed)) +        doc2 = SoledadDocument('id1', '1') +        doc2.set_json(json.dumps({"raw": str(newraw)})) + +        with pytest.raises(_crypto.InvalidBlob): +            yield crypto.decrypt_doc(doc2)  class RecoveryDocumentTestCase(BaseSoledadTest): @@ -74,13 +200,14 @@ class RecoveryDocumentTestCase(BaseSoledadTest):          encrypted_secret = rd[              self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]          self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) -        self.assertTrue( -            encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') +        self.assertEquals( +            _crypto.ENC_METHOD.aes_256_gcm, +            encrypted_secret[self._soledad.secrets.CIPHER_KEY])          self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret)          self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) -    def test_import_recovery_document(self): -        rd = self._soledad.secrets._export_recovery_document() +    def test_import_recovery_document(self, cipher='aes256'): +        rd = self._soledad.secrets._export_recovery_document(cipher)          s = self._soledad_instance()          s.secrets._import_recovery_document(rd)          s.secrets.set_secret_id(self._soledad.secrets._secret_id) @@ -89,6 +216,14 @@ class RecoveryDocumentTestCase(BaseSoledadTest):                           'Failed settinng secret for symmetric encryption.')          s.close() +    def test_import_GCM_recovery_document(self): +        cipher = self._soledad.secrets.CIPHER_AES256_GCM +        self.test_import_recovery_document(cipher) + +    def test_import_legacy_CTR_recovery_document(self): +        cipher = self._soledad.secrets.CIPHER_AES256 +        self.test_import_recovery_document(cipher) +  class SoledadSecretsTestCase(BaseSoledadTest): @@ -146,60 +281,21 @@ class SoledadSecretsTestCase(BaseSoledadTest):              "Should have a secret at this point") -class MacAuthTestCase(BaseSoledadTest): - -    def test_decrypt_with_wrong_mac_raises(self): -        """ -        Trying to decrypt a document with wrong MAC should raise. -        """ -        simpledoc = {'key': 'val'} -        doc = SoledadDocument(doc_id='id') -        doc.content = simpledoc -        # encrypt doc -        doc.set_json(self._soledad._crypto.encrypt_doc(doc)) -        self.assertTrue(MAC_KEY in doc.content) -        self.assertTrue(MAC_METHOD_KEY in doc.content) -        # mess with MAC -        doc.content[MAC_KEY] = '1234567890ABCDEF' -        # try to decrypt doc -        self.assertRaises( -            WrongMacError, -            self._soledad._crypto.decrypt_doc, doc) - -    def test_decrypt_with_unknown_mac_method_raises(self): -        """ -        Trying to decrypt a document with unknown MAC method should raise. -        """ -        simpledoc = {'key': 'val'} -        doc = SoledadDocument(doc_id='id') -        doc.content = simpledoc -        # encrypt doc -        doc.set_json(self._soledad._crypto.encrypt_doc(doc)) -        self.assertTrue(MAC_KEY in doc.content) -        self.assertTrue(MAC_METHOD_KEY in doc.content) -        # mess with MAC method -        doc.content[MAC_METHOD_KEY] = 'mymac' -        # try to decrypt doc -        self.assertRaises( -            UnknownMacMethodError, -            self._soledad._crypto.decrypt_doc, doc) - -  class SoledadCryptoAESTestCase(BaseSoledadTest):      def test_encrypt_decrypt_sym(self):          # generate 256-bit key          key = os.urandom(32) -        iv, cyphertext = crypto.encrypt_sym('data', key) +        iv, cyphertext = _crypto.encrypt_sym('data', key)          self.assertTrue(cyphertext is not None)          self.assertTrue(cyphertext != '')          self.assertTrue(cyphertext != 'data') -        plaintext = crypto.decrypt_sym(cyphertext, key, iv) +        plaintext = _crypto.decrypt_sym(cyphertext, key, iv)          self.assertEqual('data', plaintext) -    def test_decrypt_with_wrong_iv_fails(self): +    def test_decrypt_with_wrong_iv_raises(self):          key = os.urandom(32) -        iv, cyphertext = crypto.encrypt_sym('data', key) +        iv, cyphertext = _crypto.encrypt_sym('data', key)          self.assertTrue(cyphertext is not None)          self.assertTrue(cyphertext != '')          self.assertTrue(cyphertext != 'data') @@ -208,13 +304,13 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):          wrongiv = rawiv          while wrongiv == rawiv:              wrongiv = os.urandom(1) + rawiv[1:] -        plaintext = crypto.decrypt_sym( -            cyphertext, key, iv=binascii.b2a_base64(wrongiv)) -        self.assertNotEqual('data', plaintext) +        with pytest.raises(InvalidTag): +            _crypto.decrypt_sym( +                cyphertext, key, iv=binascii.b2a_base64(wrongiv)) -    def test_decrypt_with_wrong_key_fails(self): +    def test_decrypt_with_wrong_key_raises(self):          key = os.urandom(32) -        iv, cyphertext = crypto.encrypt_sym('data', key) +        iv, cyphertext = _crypto.encrypt_sym('data', key)          self.assertTrue(cyphertext is not None)          self.assertTrue(cyphertext != '')          self.assertTrue(cyphertext != 'data') @@ -222,5 +318,21 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):          # ensure keys are different in case we are extremely lucky          while wrongkey == key:              wrongkey = os.urandom(32) -        plaintext = crypto.decrypt_sym(cyphertext, wrongkey, iv) -        self.assertNotEqual('data', plaintext) +        with pytest.raises(InvalidTag): +            _crypto.decrypt_sym(cyphertext, wrongkey, iv) + + +def _aes_encrypt(key, iv, data): +    backend = default_backend() +    cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend) +    encryptor = cipher.encryptor() +    return encryptor.update(data) + encryptor.finalize(), encryptor.tag + + +def _aes_decrypt(key, iv, tag, data, aead=''): +    backend = default_backend() +    cipher = Cipher(algorithms.AES(key), modes.GCM(iv, tag), backend=backend) +    decryptor = cipher.decryptor() +    if aead: +        decryptor.authenticate_additional_data(aead) +    return decryptor.update(data) + decryptor.finalize() diff --git a/testing/tests/client/test_deprecated_crypto.py b/testing/tests/client/test_deprecated_crypto.py new file mode 100644 index 00000000..8ee3735c --- /dev/null +++ b/testing/tests/client/test_deprecated_crypto.py @@ -0,0 +1,91 @@ +import json +from twisted.internet import defer +from uuid import uuid4 +from urlparse import urljoin + +from leap.soledad.client import crypto as old_crypto +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common import crypto as common_crypto + +from test_soledad.u1db_tests import simple_doc +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import make_token_soledad_app +from test_soledad.u1db_tests import TestCaseWithServer + + +def deprecate_client_crypto(client): +    secret = client._crypto.secret +    _crypto = old_crypto.SoledadCrypto(secret) +    setattr(client._dbsyncer, '_crypto', _crypto) +    return client + + +def couch_database(couch_url, uuid): +    db = CouchDatabase(couch_url, "user-%s" % (uuid,)) +    return db + + +class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): + +    def setUp(self): +        SoledadWithCouchServerMixin.setUp(self) +        TestCaseWithServer.setUp(self) + +    def tearDown(self): +        SoledadWithCouchServerMixin.tearDown(self) +        TestCaseWithServer.tearDown(self) + +    @staticmethod +    def make_app_with_state(state): +        return make_token_soledad_app(state) + +    @defer.inlineCallbacks +    def test_touch_updates_remote_representation(self): +        self.startTwistedServer() +        user = 'user-' + uuid4().hex +        server_url = 'http://%s:%d' % (self.server_address) +        client = self._soledad_instance(user=user, server_url=server_url) +        deprecated_client = deprecate_client_crypto( +            self._soledad_instance(user=user, server_url=server_url)) + +        self.make_app() +        remote = self.request_state._create_database(replica_uid=client._uuid) +        remote = CouchDatabase.open_database( +            urljoin(self.couch_url, 'user-' + user), +            create=True) + +        # ensure remote db is empty +        gen, docs = remote.get_all_docs() +        assert gen == 0 +        assert len(docs) == 0 + +        # create a doc with deprecated client and sync +        yield deprecated_client.create_doc(json.loads(simple_doc)) +        yield deprecated_client.sync() + +        # check for doc in remote db +        gen, docs = remote.get_all_docs() +        assert gen == 1 +        assert len(docs) == 1 +        doc = docs.pop() +        content = doc.content +        assert common_crypto.ENC_JSON_KEY in content +        assert common_crypto.ENC_SCHEME_KEY in content +        assert common_crypto.ENC_METHOD_KEY in content +        assert common_crypto.ENC_IV_KEY in content +        assert common_crypto.MAC_KEY in content +        assert common_crypto.MAC_METHOD_KEY in content + +        # "touch" the document with a newer client and synx +        _, docs = yield client.get_all_docs() +        yield client.put_doc(doc) +        yield client.sync() + +        # check for newer representation of doc in remote db +        gen, docs = remote.get_all_docs() +        assert gen == 2 +        assert len(docs) == 1 +        doc = docs.pop() +        content = doc.content +        assert len(content) == 1 +        assert 'raw' in content diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 9e4319ac..1ff1cbb7 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -1,4 +1,29 @@ +import json +import os  import pytest +import requests +import signal +import time + +from hashlib import sha512 +from subprocess import call +from urlparse import urljoin +from uuid import uuid4 + +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.client import Soledad + + +# +# default options for all tests +# + +DEFAULT_PASSPHRASE = '123' + +DEFAULT_URL = 'http://127.0.0.1:2424' +DEFAULT_PRIVKEY = 'soledad_privkey.pem' +DEFAULT_CERTKEY = 'soledad_certkey.pem' +DEFAULT_TOKEN = 'an-auth-token'  def pytest_addoption(parser): @@ -16,3 +41,167 @@ def couch_url(request):  @pytest.fixture  def method_tmpdir(request, tmpdir):      request.instance.tempdir = tmpdir.strpath + + +# +# remote_db fixture: provides an empty database for a given user in a per +# function scope. +# + +class UserDatabase(object): + +    def __init__(self, url, uuid): +        self._remote_db_url = urljoin(url, 'user-%s' % uuid) + +    def setup(self): +        return CouchDatabase.open_database( +            url=self._remote_db_url, create=True, replica_uid=None) + +    def teardown(self): +        requests.delete(self._remote_db_url) + + +@pytest.fixture() +def remote_db(request): +    couch_url = request.config.option.couch_url + +    def create(uuid): +        db = UserDatabase(couch_url, uuid) +        request.addfinalizer(db.teardown) +        return db.setup() +    return create + + +def get_pid(pidfile): +    if not os.path.isfile(pidfile): +        return 0 +    try: +        with open(pidfile) as f: +            return int(f.read()) +    except IOError: +        return 0 + + +# +# soledad_server fixture: provides a running soledad server in a per module +# context (same soledad server for all tests in this module). +# + +class SoledadServer(object): + +    def __init__(self, tmpdir_factory, couch_url): +        tmpdir = tmpdir_factory.mktemp('soledad-server') +        self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid') +        self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log') +        self._couch_url = couch_url + +    def start(self): +        self._create_conf_file() +        # start the server +        call([ +            'twistd', +            '--logfile=%s' % self._logfile, +            '--pidfile=%s' % self._pidfile, +            'web', +            '--wsgi=leap.soledad.server.application.wsgi_application', +            '--port=2424' +        ]) + +    def _create_conf_file(self): +        if not os.access('/etc', os.W_OK): +            return +        if not os.path.isdir('/etc/soledad'): +            os.mkdir('/etc/soledad') +        with open('/etc/soledad/soledad-server.conf', 'w') as f: +            content = '[soledad-server]\ncouch_url = %s' % self._couch_url +            f.write(content) + +    def stop(self): +        pid = get_pid(self._pidfile) +        os.kill(pid, signal.SIGKILL) + + +@pytest.fixture(scope='module') +def soledad_server(tmpdir_factory, request): +    couch_url = request.config.option.couch_url +    server = SoledadServer(tmpdir_factory, couch_url) +    server.start() +    request.addfinalizer(server.stop) +    return server + + +# +# soledad_dbs fixture: provides all databases needed by soledad server in a per +# module scope (same databases for all tests in this module). +# + +def _token_dbname(): +    dbname = 'tokens_' + \ +        str(int(time.time() / (30 * 24 * 3600))) +    return dbname + + +class SoledadDatabases(object): + +    def __init__(self, url): +        self._token_db_url = urljoin(url, _token_dbname()) +        self._shared_db_url = urljoin(url, 'shared') + +    def setup(self, uuid): +        self._create_dbs() +        self._add_token(uuid) + +    def _create_dbs(self): +        requests.put(self._token_db_url) +        requests.put(self._shared_db_url) + +    def _add_token(self, uuid): +        token = sha512(DEFAULT_TOKEN).hexdigest() +        content = {'type': 'Token', 'user_id': uuid} +        requests.put( +            self._token_db_url + '/' + token, data=json.dumps(content)) + +    def teardown(self): +        requests.delete(self._token_db_url) +        requests.delete(self._shared_db_url) + + +@pytest.fixture() +def soledad_dbs(request): +    couch_url = request.config.option.couch_url + +    def create(uuid): +        db = SoledadDatabases(couch_url) +        request.addfinalizer(db.teardown) +        return db.setup(uuid) +    return create + + +# +# soledad_client fixture: provides a clean soledad client for a test function. +# + +@pytest.fixture() +def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): +    passphrase = DEFAULT_PASSPHRASE +    server_url = DEFAULT_URL +    token = DEFAULT_TOKEN +    default_uuid = uuid4().hex +    remote_db(default_uuid) +    soledad_dbs(default_uuid) + +    # get a soledad instance +    def create(): +        secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % default_uuid) +        local_db_path = os.path.join(tmpdir.strpath, '%s.db' % default_uuid) +        soledad_client = Soledad( +            default_uuid, +            unicode(passphrase), +            secrets_path=secrets_path, +            local_db_path=local_db_path, +            server_url=server_url, +            cert_file=None, +            auth_token=token) +        request.addfinalizer(soledad_client.close) +        return soledad_client +    return create diff --git a/testing/tests/couch/conftest.py b/testing/tests/couch/conftest.py deleted file mode 100644 index 1074f091..00000000 --- a/testing/tests/couch/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -import couchdb -import pytest -import random -import string - - -@pytest.fixture -def random_name(): -    return 'user-' + ''.join( -        random.choice( -            string.ascii_lowercase) for _ in range(10)) - - -class RandomDatabase(object): - -    def __init__(self, couch_url, name): -        self.couch_url = couch_url -        self.name = name -        self.server = couchdb.client.Server(couch_url) -        self.database = self.server.create(name) - -    def teardown(self): -        self.server.delete(self.name) - - -@pytest.fixture -def db(random_name, request): -    couch_url = request.config.getoption('--couch-url') -    db = RandomDatabase(couch_url, random_name) -    request.addfinalizer(db.teardown) -    return db diff --git a/testing/tests/couch/test_command.py b/testing/tests/couch/test_command.py index 68097fb1..9fb2c153 100644 --- a/testing/tests/couch/test_command.py +++ b/testing/tests/couch/test_command.py @@ -25,6 +25,7 @@ class CommandBasedDBCreationTest(unittest.TestCase):                            state.ensure_database, "user-1337")      def test_raises_unauthorized_by_default(self): -        state = couch_state.CouchServerState("url", check_schema_versions=False) +        state = couch_state.CouchServerState("url", +                                             check_schema_versions=False)          self.assertRaises(u1db_errors.Unauthorized,                            state.ensure_database, "user-1337") diff --git a/testing/tests/couch/test_state.py b/testing/tests/couch/test_state.py index e293b5b8..e5ac3704 100644 --- a/testing/tests/couch/test_state.py +++ b/testing/tests/couch/test_state.py @@ -1,25 +1,32 @@  import pytest -  from leap.soledad.common.couch import CONFIG_DOC_ID  from leap.soledad.common.couch import SCHEMA_VERSION  from leap.soledad.common.couch import SCHEMA_VERSION_KEY  from leap.soledad.common.couch.state import CouchServerState +from uuid import uuid4  from leap.soledad.common.errors import WrongCouchSchemaVersionError  from leap.soledad.common.errors import MissingCouchConfigDocumentError +from test_soledad.util import CouchDBTestCase + +class CouchDesignDocsTests(CouchDBTestCase): -def test_wrong_couch_version_raises(db): -    wrong_schema_version = SCHEMA_VERSION + 1 -    db.database.create( -        {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version}) -    with pytest.raises(WrongCouchSchemaVersionError): -        CouchServerState(db.couch_url, create_cmd='/bin/echo', -                         check_schema_versions=True) +    def setUp(self): +        CouchDBTestCase.setUp(self) +        self.db = self.couch_server.create('user-' + uuid4().hex) +        self.addCleanup(self.delete_db, self.db.name) +    def test_wrong_couch_version_raises(self): +        wrong_schema_version = SCHEMA_VERSION + 1 +        self.db.create( +            {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version}) +        with pytest.raises(WrongCouchSchemaVersionError): +            CouchServerState(self.couch_url, create_cmd='/bin/echo', +                             check_schema_versions=True) -def test_missing_config_doc_raises(db): -    db.database.create({}) -    with pytest.raises(MissingCouchConfigDocumentError): -        CouchServerState(db.couch_url, create_cmd='/bin/echo', -                         check_schema_versions=True) +    def test_missing_config_doc_raises(self): +        self.db.create({}) +        with pytest.raises(MissingCouchConfigDocumentError): +            CouchServerState(self.couch_url, create_cmd='/bin/echo', +                             check_schema_versions=True) diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py deleted file mode 100644 index 6fa6b2c0..00000000 --- a/testing/tests/perf/conftest.py +++ /dev/null @@ -1,249 +0,0 @@ -import json -import os -import pytest -import requests -import random -import base64 -import signal -import time - -from hashlib import sha512 -from uuid import uuid4 -from subprocess import call -from urlparse import urljoin -from twisted.internet import threads, reactor - -from leap.soledad.client import Soledad -from leap.soledad.common.couch import CouchDatabase - - -# we have to manually setup the events server in order to be able to signal -# events. This is usually done by the enclosing application using soledad -# client (i.e. bitmask client). -from leap.common.events import server -server.ensure_server() - - -def pytest_addoption(parser): -    parser.addoption( -        "--couch-url", type="string", default="http://127.0.0.1:5984", -        help="the url for the couch server to be used during tests") -    parser.addoption( -        "--num-docs", type="int", default=100, -        help="the number of documents to use in performance tests") - - -# -# default options for all tests -# - -DEFAULT_PASSPHRASE = '123' - -DEFAULT_URL = 'http://127.0.0.1:2424' -DEFAULT_PRIVKEY = 'soledad_privkey.pem' -DEFAULT_CERTKEY = 'soledad_certkey.pem' -DEFAULT_TOKEN = 'an-auth-token' - - -@pytest.fixture() -def payload(): -    def generate(size): -        random.seed(1337)  # same seed to avoid different bench results -        payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) -        # encode as base64 to avoid ascii encode/decode errors -        return base64.b64encode(payload_bytes)[:size]  # remove b64 overhead -    return generate - - -# -# soledad_dbs fixture: provides all databases needed by soledad server in a per -# module scope (same databases for all tests in this module). -# - -def _token_dbname(): -    dbname = 'tokens_' + \ -        str(int(time.time() / (30 * 24 * 3600))) -    return dbname - - -class SoledadDatabases(object): - -    def __init__(self, url): -        self._token_db_url = urljoin(url, _token_dbname()) -        self._shared_db_url = urljoin(url, 'shared') - -    def setup(self, uuid): -        self._create_dbs() -        self._add_token(uuid) - -    def _create_dbs(self): -        requests.put(self._token_db_url) -        requests.put(self._shared_db_url) - -    def _add_token(self, uuid): -        token = sha512(DEFAULT_TOKEN).hexdigest() -        content = {'type': 'Token', 'user_id': uuid} -        requests.put( -            self._token_db_url + '/' + token, data=json.dumps(content)) - -    def teardown(self): -        requests.delete(self._token_db_url) -        requests.delete(self._shared_db_url) - - -@pytest.fixture() -def soledad_dbs(request): -    couch_url = request.config.option.couch_url - -    def create(uuid): -        db = SoledadDatabases(couch_url) -        request.addfinalizer(db.teardown) -        return db.setup(uuid) -    return create - - -# -# remote_db fixture: provides an empty database for a given user in a per -# function scope. -# - -class UserDatabase(object): - -    def __init__(self, url, uuid): -        self._remote_db_url = urljoin(url, 'user-%s' % uuid) - -    def setup(self): -        return CouchDatabase.open_database( -            url=self._remote_db_url, create=True, replica_uid=None) - -    def teardown(self): -        requests.delete(self._remote_db_url) - - -@pytest.fixture() -def remote_db(request): -    couch_url = request.config.option.couch_url - -    def create(uuid): -        db = UserDatabase(couch_url, uuid) -        request.addfinalizer(db.teardown) -        return db.setup() -    return create - - -def get_pid(pidfile): -    if not os.path.isfile(pidfile): -        return 0 -    try: -        with open(pidfile) as f: -            return int(f.read()) -    except IOError: -        return 0 - - -# -# soledad_server fixture: provides a running soledad server in a per module -# context (same soledad server for all tests in this module). -# - -class SoledadServer(object): - -    def __init__(self, tmpdir_factory, couch_url): -        tmpdir = tmpdir_factory.mktemp('soledad-server') -        self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid') -        self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log') -        self._couch_url = couch_url - -    def start(self): -        self._create_conf_file() -        # start the server -        call([ -            'twistd', -            '--logfile=%s' % self._logfile, -            '--pidfile=%s' % self._pidfile, -            'web', -            '--wsgi=leap.soledad.server.application.wsgi_application', -            '--port=2424' -        ]) - -    def _create_conf_file(self): -        if not os.access('/etc', os.W_OK): -            return -        if not os.path.isdir('/etc/soledad'): -            os.mkdir('/etc/soledad') -        with open('/etc/soledad/soledad-server.conf', 'w') as f: -            content = '[soledad-server]\ncouch_url = %s' % self._couch_url -            f.write(content) - -    def stop(self): -        pid = get_pid(self._pidfile) -        os.kill(pid, signal.SIGKILL) - - -@pytest.fixture(scope='module') -def soledad_server(tmpdir_factory, request): -    couch_url = request.config.option.couch_url -    server = SoledadServer(tmpdir_factory, couch_url) -    server.start() -    request.addfinalizer(server.stop) -    return server - - -@pytest.fixture() -def txbenchmark(benchmark): -    def blockOnThread(*args, **kwargs): -        return threads.deferToThread( -            benchmark, threads.blockingCallFromThread, -            reactor, *args, **kwargs) -    return blockOnThread - - -@pytest.fixture() -def txbenchmark_with_setup(benchmark): -    def blockOnThreadWithSetup(setup, f): -        def blocking_runner(*args, **kwargs): -            return threads.blockingCallFromThread(reactor, f, *args, **kwargs) - -        def blocking_setup(): -            args = threads.blockingCallFromThread(reactor, setup) -            try: -                return tuple(arg for arg in args), {} -            except TypeError: -                    return ((args,), {}) if args else None - -        def bench(): -            return benchmark.pedantic(blocking_runner, setup=blocking_setup, -                                      rounds=4, warmup_rounds=1) -        return threads.deferToThread(bench) -    return blockOnThreadWithSetup - - -# -# soledad_client fixture: provides a clean soledad client for a test function. -# - -@pytest.fixture() -def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): -    passphrase = DEFAULT_PASSPHRASE -    server_url = DEFAULT_URL -    token = DEFAULT_TOKEN -    default_uuid = uuid4().hex -    remote_db(default_uuid) -    soledad_dbs(default_uuid) - -    # get a soledad instance -    def create(): -        secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid4().hex) -        local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid4().hex) -        soledad_client = Soledad( -            default_uuid, -            unicode(passphrase), -            secrets_path=secrets_path, -            local_db_path=local_db_path, -            server_url=server_url, -            cert_file=None, -            auth_token=token, -            defer_encryption=True) -        request.addfinalizer(soledad_client.close) -        return soledad_client -    return create diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py deleted file mode 100644 index be00560b..00000000 --- a/testing/tests/perf/test_crypto.py +++ /dev/null @@ -1,81 +0,0 @@ -import pytest -import json -from uuid import uuid4 -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.crypto import encrypt_sym -from leap.soledad.client.crypto import decrypt_sym - - -def create_doc_encryption(size): -    @pytest.mark.benchmark(group="test_crypto_encrypt_doc") -    def test_doc_encryption(soledad_client, benchmark, payload): -        crypto = soledad_client()._crypto - -        DOC_CONTENT = {'payload': payload(size)} -        doc = SoledadDocument( -            doc_id=uuid4().hex, rev='rev', -            json=json.dumps(DOC_CONTENT)) - -        benchmark(crypto.encrypt_doc, doc) -    return test_doc_encryption - - -def create_doc_decryption(size): -    @pytest.mark.benchmark(group="test_crypto_decrypt_doc") -    def test_doc_decryption(soledad_client, benchmark, payload): -        crypto = soledad_client()._crypto - -        DOC_CONTENT = {'payload': payload(size)} -        doc = SoledadDocument( -            doc_id=uuid4().hex, rev='rev', -            json=json.dumps(DOC_CONTENT)) -        encrypted_doc = crypto.encrypt_doc(doc) -        doc.set_json(encrypted_doc) - -        benchmark(crypto.decrypt_doc, doc) -    return test_doc_decryption - - -test_encrypt_doc_10k = create_doc_encryption(10*1000) -test_encrypt_doc_100k = create_doc_encryption(100*1000) -test_encrypt_doc_500k = create_doc_encryption(500*1000) -test_encrypt_doc_1M = create_doc_encryption(1000*1000) -test_encrypt_doc_10M = create_doc_encryption(10*1000*1000) -test_encrypt_doc_50M = create_doc_encryption(50*1000*1000) -test_decrypt_doc_10k = create_doc_decryption(10*1000) -test_decrypt_doc_100k = create_doc_decryption(100*1000) -test_decrypt_doc_500k = create_doc_decryption(500*1000) -test_decrypt_doc_1M = create_doc_decryption(1000*1000) -test_decrypt_doc_10M = create_doc_decryption(10*1000*1000) -test_decrypt_doc_50M = create_doc_decryption(50*1000*1000) - - -def create_raw_encryption(size): -    @pytest.mark.benchmark(group="test_crypto_raw_encrypt") -    def test_raw_encrypt(benchmark, payload): -        key = payload(32) -        benchmark(encrypt_sym, payload(size), key) -    return test_raw_encrypt - - -def create_raw_decryption(size): -    @pytest.mark.benchmark(group="test_crypto_raw_decrypt") -    def test_raw_decrypt(benchmark, payload): -        key = payload(32) -        iv, ciphertext = encrypt_sym(payload(size), key) -        benchmark(decrypt_sym, ciphertext, key, iv) -    return test_raw_decrypt - - -test_encrypt_raw_10k = create_raw_encryption(10*1000) -test_encrypt_raw_100k = create_raw_encryption(100*1000) -test_encrypt_raw_500k = create_raw_encryption(500*1000) -test_encrypt_raw_1M = create_raw_encryption(1000*1000) -test_encrypt_raw_10M = create_raw_encryption(10*1000*1000) -test_encrypt_raw_50M = create_raw_encryption(50*1000*1000) -test_decrypt_raw_10k = create_raw_decryption(10*1000) -test_decrypt_raw_100k = create_raw_decryption(100*1000) -test_decrypt_raw_500k = create_raw_decryption(500*1000) -test_decrypt_raw_1M = create_raw_decryption(1000*1000) -test_decrypt_raw_10M = create_raw_decryption(10*1000*1000) -test_decrypt_raw_50M = create_raw_decryption(50*1000*1000) diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py deleted file mode 100644 index 77091a41..00000000 --- a/testing/tests/perf/test_encdecpool.py +++ /dev/null @@ -1,78 +0,0 @@ -import pytest -import json -from uuid import uuid4 -from twisted.internet.defer import gatherResults -from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool -from leap.soledad.common.document import SoledadDocument -# FIXME: test load is low due issue #7370, higher values will get out of memory - - -def create_encrypt(amount, size): -    @pytest.mark.benchmark(group="test_pool_encrypt") -    @pytest.inlineCallbacks -    def test(soledad_client, txbenchmark_with_setup, request, payload): -        DOC_CONTENT = {'payload': payload(size)} - -        def setup(): -            client = soledad_client() -            pool = SyncEncrypterPool(client._crypto, client._sync_db) -            pool.start() -            request.addfinalizer(pool.stop) -            docs = [ -                SoledadDocument(doc_id=uuid4().hex, rev='rev', -                                json=json.dumps(DOC_CONTENT)) -                for _ in xrange(amount) -            ] -            return pool, docs - -        @pytest.inlineCallbacks -        def put_and_wait(pool, docs): -            yield gatherResults([pool.encrypt_doc(doc) for doc in docs]) - -        yield txbenchmark_with_setup(setup, put_and_wait) -    return test - -test_encdecpool_encrypt_100_10k = create_encrypt(100, 10*1000) -test_encdecpool_encrypt_100_100k = create_encrypt(100, 100*1000) -test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000) - - -def create_decrypt(amount, size): -    @pytest.mark.benchmark(group="test_pool_decrypt") -    @pytest.inlineCallbacks -    def test(soledad_client, txbenchmark_with_setup, request, payload): -        DOC_CONTENT = {'payload': payload(size)} -        client = soledad_client() - -        def setup(): -            pool = SyncDecrypterPool( -                client._crypto, -                client._sync_db, -                source_replica_uid=client._dbpool.replica_uid, -                insert_doc_cb=lambda x, y, z: False)  # ignored -            pool.start(amount) -            request.addfinalizer(pool.stop) -            crypto = client._crypto -            docs = [] -            for _ in xrange(amount): -                doc = SoledadDocument( -                    doc_id=uuid4().hex, rev='rev', -                    json=json.dumps(DOC_CONTENT)) -                encrypted_content = json.loads(crypto.encrypt_doc(doc)) -                docs.append((doc.doc_id, encrypted_content)) -            return pool, docs - -        def put_and_wait(pool, docs): -            deferreds = []  # fires on completion -            for idx, (doc_id, content) in enumerate(docs, 1): -                deferreds.append(pool.insert_encrypted_received_doc( -                    doc_id, 'rev', content, idx, "trans_id", idx)) -            return gatherResults(deferreds) - -        yield txbenchmark_with_setup(setup, put_and_wait) -    return test - -test_encdecpool_decrypt_100_10k = create_decrypt(100, 10*1000) -test_encdecpool_decrypt_100_100k = create_decrypt(100, 100*1000) -test_encdecpool_decrypt_100_500k = create_decrypt(100, 500*1000) diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 6bbcf002..6710caaf 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -41,7 +41,7 @@ from test_soledad.util import (      BaseSoledadTest,  ) -from leap.soledad.common import crypto +from leap.soledad.client import _crypto  from leap.soledad.client import Soledad  from leap.soledad.server.config import load_configuration  from leap.soledad.server.config import CONFIG_DEFAULTS @@ -412,13 +412,9 @@ class EncryptedSyncTestCase(                  self.assertEqual(soldoc.doc_id, couchdoc.doc_id)                  self.assertEqual(soldoc.rev, couchdoc.rev)                  couch_content = couchdoc.content.keys() -                self.assertEqual(6, len(couch_content)) -                self.assertTrue(crypto.ENC_JSON_KEY in couch_content) -                self.assertTrue(crypto.ENC_SCHEME_KEY in couch_content) -                self.assertTrue(crypto.ENC_METHOD_KEY in couch_content) -                self.assertTrue(crypto.ENC_IV_KEY in couch_content) -                self.assertTrue(crypto.MAC_KEY in couch_content) -                self.assertTrue(crypto.MAC_METHOD_KEY in couch_content) +                self.assertEqual(['raw'], couch_content) +                content = couchdoc.get_json() +                self.assertTrue(_crypto.is_symmetrically_encrypted(content))          d = sol1.get_all_docs()          d.addCallback(_db1AssertEmptyDocList) @@ -473,16 +469,6 @@ class EncryptedSyncTestCase(          """          return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëÃìïóòöõúùüñç') -    def test_sync_very_large_files(self): -        """ -        Test if Soledad can sync very large files. -        """ -        self.skipTest( -            "Work in progress. For reference, see: " -            "https://leap.se/code/issues/7370") -        length = 100 * (10 ** 6)  # 100 MB -        return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) -      def test_sync_many_small_files(self):          """          Test if Soledad can sync many smallfiles. diff --git a/testing/tests/sync/test_encdecpool.py b/testing/tests/sync/test_encdecpool.py deleted file mode 100644 index 4a32885e..00000000 --- a/testing/tests/sync/test_encdecpool.py +++ /dev/null @@ -1,306 +0,0 @@ -# -*- coding: utf-8 -*- -# test_encdecpool.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Tests for encryption and decryption pool. -""" -import json -from random import shuffle - -from mock import MagicMock -from twisted.internet.defer import inlineCallbacks - -from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool - -from leap.soledad.common.document import SoledadDocument -from test_soledad.util import BaseSoledadTest -from twisted.internet import defer - -DOC_ID = "mydoc" -DOC_REV = "rev" -DOC_CONTENT = {'simple': 'document'} - - -class TestSyncEncrypterPool(BaseSoledadTest): - -    def setUp(self): -        BaseSoledadTest.setUp(self) -        crypto = self._soledad._crypto -        sync_db = self._soledad._sync_db -        self._pool = SyncEncrypterPool(crypto, sync_db) -        self._pool.start() - -    def tearDown(self): -        self._pool.stop() -        BaseSoledadTest.tearDown(self) - -    @inlineCallbacks -    def test_get_encrypted_doc_returns_none(self): -        """ -        Test that trying to get an encrypted doc from the pool returns None if -        the document was never added for encryption. -        """ -        doc = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) -        self.assertIsNone(doc) - -    @inlineCallbacks -    def test_encrypt_doc_and_get_it_back(self): -        """ -        Test that the pool actually encrypts a document added to the queue. -        """ -        doc = SoledadDocument( -            doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - -        yield self._pool.encrypt_doc(doc) -        encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - -        self.assertIsNotNone(encrypted) - - -class TestSyncDecrypterPool(BaseSoledadTest): - -    def _insert_doc_cb(self, doc, gen, trans_id): -        """ -        Method used to mock the sync's return_doc_cb callback. -        """ -        self._inserted_docs.append((doc, gen, trans_id)) - -    def _setup_pool(self, sync_db=None): -        sync_db = sync_db or self._soledad._sync_db -        return SyncDecrypterPool( -            self._soledad._crypto, -            sync_db, -            source_replica_uid=self._soledad._dbpool.replica_uid, -            insert_doc_cb=self._insert_doc_cb) - -    def setUp(self): -        BaseSoledadTest.setUp(self) -        # setup the pool -        self._pool = self._setup_pool() -        # reset the inserted docs mock -        self._inserted_docs = [] - -    def tearDown(self): -        if self._pool.running: -            self._pool.stop() -        BaseSoledadTest.tearDown(self) - -    def test_insert_received_doc(self): -        """ -        Test that one document added to the pool is inserted using the -        callback. -        """ -        self._pool.start(1) -        self._pool.insert_received_doc( -            DOC_ID, DOC_REV, "{}", 1, "trans_id", 1) - -        def _assert_doc_was_inserted(_): -            self.assertEqual( -                self._inserted_docs, -                [(SoledadDocument(DOC_ID, DOC_REV, "{}"), 1, u"trans_id")]) - -        self._pool.deferred.addCallback(_assert_doc_was_inserted) -        return self._pool.deferred - -    def test_looping_control(self): -        """ -        Start and stop cleanly. -        """ -        self._pool.start(10) -        self.assertTrue(self._pool.running) -        self._pool.stop() -        self.assertFalse(self._pool.running) -        self.assertTrue(self._pool.deferred.called) - -    def test_sync_id_col_is_created_if_non_existing_in_docs_recvd_table(self): -        """ -        Test that docs_received table is migrated, and has the sync_id column -        """ -        mock_run_query = MagicMock(return_value=defer.succeed(None)) -        mock_sync_db = MagicMock() -        mock_sync_db.runQuery = mock_run_query -        pool = self._setup_pool(mock_sync_db) -        d = pool.start(10) -        pool.stop() - -        def assert_trial_to_create_sync_id_column(_): -            mock_run_query.assert_called_once_with( -                "ALTER TABLE docs_received ADD COLUMN sync_id") - -        d.addCallback(assert_trial_to_create_sync_id_column) -        return d - -    def test_insert_received_doc_many(self): -        """ -        Test that many documents added to the pool are inserted using the -        callback. -        """ -        many = 100 -        self._pool.start(many) - -        # insert many docs in the pool -        for i in xrange(many): -            gen = idx = i + 1 -            doc_id = "doc_id: %d" % idx -            rev = "rev: %d" % idx -            content = {'idx': idx} -            trans_id = "trans_id: %d" % idx -            self._pool.insert_received_doc( -                doc_id, rev, content, gen, trans_id, idx) - -        def _assert_doc_was_inserted(_): -            self.assertEqual(many, len(self._inserted_docs)) -            idx = 1 -            for doc, gen, trans_id in self._inserted_docs: -                expected_gen = idx -                expected_doc_id = "doc_id: %d" % idx -                expected_rev = "rev: %d" % idx -                expected_content = json.dumps({'idx': idx}) -                expected_trans_id = "trans_id: %d" % idx - -                self.assertEqual(expected_doc_id, doc.doc_id) -                self.assertEqual(expected_rev, doc.rev) -                self.assertEqual(expected_content, json.dumps(doc.content)) -                self.assertEqual(expected_gen, gen) -                self.assertEqual(expected_trans_id, trans_id) - -                idx += 1 - -        self._pool.deferred.addCallback(_assert_doc_was_inserted) -        return self._pool.deferred - -    def test_insert_encrypted_received_doc(self): -        """ -        Test that one encrypted document added to the pool is decrypted and -        inserted using the callback. -        """ -        crypto = self._soledad._crypto -        doc = SoledadDocument( -            doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) -        encrypted_content = json.loads(crypto.encrypt_doc(doc)) - -        # insert the encrypted document in the pool -        self._pool.start(1) -        self._pool.insert_encrypted_received_doc( -            DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1) - -        def _assert_doc_was_decrypted_and_inserted(_): -            self.assertEqual(1, len(self._inserted_docs)) -            self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")]) - -        self._pool.deferred.addCallback( -            _assert_doc_was_decrypted_and_inserted) -        return self._pool.deferred - -    @inlineCallbacks -    def test_processing_order(self): -        """ -        This test ensures that processing of documents only occur if there is -        a sequence in place. -        """ -        crypto = self._soledad._crypto - -        docs = [] -        for i in xrange(1, 10): -            i = str(i) -            doc = SoledadDocument( -                doc_id=DOC_ID + i, rev=DOC_REV + i, -                json=json.dumps(DOC_CONTENT)) -            encrypted_content = json.loads(crypto.encrypt_doc(doc)) -            docs.append((doc, encrypted_content)) - -        # insert the encrypted document in the pool -        yield self._pool.start(10)  # pool is expecting to process 10 docs -        self._pool._loop.stop()  # we are processing manually -        # first three arrives, forming a sequence -        for i, (doc, encrypted_content) in enumerate(docs[:3]): -            gen = idx = i + 1 -            yield self._pool.insert_encrypted_received_doc( -                doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx) - -        # last one arrives alone, so it can't be processed -        doc, encrypted_content = docs[-1] -        yield self._pool.insert_encrypted_received_doc( -            doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) - -        yield self._pool._decrypt_and_recurse() - -        self.assertEqual(3, self._pool._processed_docs) - -    def test_insert_encrypted_received_doc_many(self, many=100): -        """ -        Test that many encrypted documents added to the pool are decrypted and -        inserted using the callback. -        """ -        crypto = self._soledad._crypto -        self._pool.start(many) -        docs = [] - -        # insert many encrypted docs in the pool -        for i in xrange(many): -            gen = idx = i + 1 -            doc_id = "doc_id: %d" % idx -            rev = "rev: %d" % idx -            content = {'idx': idx} -            trans_id = "trans_id: %d" % idx - -            doc = SoledadDocument( -                doc_id=doc_id, rev=rev, json=json.dumps(content)) - -            encrypted_content = json.loads(crypto.encrypt_doc(doc)) -            docs.append((doc_id, rev, encrypted_content, gen, -                         trans_id, idx)) -        shuffle(docs) - -        for doc in docs: -            self._pool.insert_encrypted_received_doc(*doc) - -        def _assert_docs_were_decrypted_and_inserted(_): -            self.assertEqual(many, len(self._inserted_docs)) -            idx = 1 -            for doc, gen, trans_id in self._inserted_docs: -                expected_gen = idx -                expected_doc_id = "doc_id: %d" % idx -                expected_rev = "rev: %d" % idx -                expected_content = json.dumps({'idx': idx}) -                expected_trans_id = "trans_id: %d" % idx - -                self.assertEqual(expected_doc_id, doc.doc_id) -                self.assertEqual(expected_rev, doc.rev) -                self.assertEqual(expected_content, json.dumps(doc.content)) -                self.assertEqual(expected_gen, gen) -                self.assertEqual(expected_trans_id, trans_id) - -                idx += 1 - -        self._pool.deferred.addCallback( -            _assert_docs_were_decrypted_and_inserted) -        return self._pool.deferred - -    @inlineCallbacks -    def test_pool_reuse(self): -        """ -        The pool is reused between syncs, this test verifies that -        reusing is fine. -        """ -        for i in xrange(3): -            yield self.test_insert_encrypted_received_doc_many(5) -            self._inserted_docs = [] -            decrypted_docs = yield self._pool._get_docs(encrypted=False) -            # check that decrypted docs staging is clean -            self.assertEquals([], decrypted_docs) -            self._pool.stop() diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 3cbefc8b..26f63a40 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -27,8 +27,6 @@ from leap.soledad.common.l2db import sync  from leap.soledad.common.l2db import vectorclock  from leap.soledad.common.l2db import errors -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict  from leap.soledad.client.http_target import SoledadHTTPSyncTarget  from test_soledad import u1db_tests as tests @@ -545,13 +543,7 @@ class SQLCipherDatabaseSyncTests(          self.assertFalse(doc2.has_conflicts)          self.sync(self.db2, db3)          doc3 = db3.get_doc('the-doc') -        if ENC_SCHEME_KEY in doc3.content: -            _crypto = self._soledad._crypto -            key = _crypto.doc_passphrase(doc3.doc_id) -            secret = _crypto.secret -            doc3.set_json(decrypt_doc_dict( -                doc3.content, -                doc3.doc_id, doc3.rev, key, secret)) +          self.assertEqual(doc4.get_json(), doc3.get_json())          self.assertFalse(doc3.has_conflicts)          self.db1.close() @@ -713,15 +705,12 @@ def make_local_db_and_soledad_target(      test.startTwistedServer()      replica_uid = os.path.basename(path)      db = test.request_state._create_database(replica_uid) -    sync_db = test._soledad._sync_db -    sync_enc_pool = test._soledad._sync_enc_pool      st = soledad_sync_target(          test, db._dbname, -        source_replica_uid=source_replica_uid, -        sync_db=sync_db, -        sync_enc_pool=sync_enc_pool) +        source_replica_uid=source_replica_uid)      return db, st +  target_scenarios = [      ('leap', {          'create_db_and_target': make_local_db_and_soledad_target, diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index 5290003e..76757c5b 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -19,6 +19,7 @@ import threading  import time  from urlparse import urljoin +from mock import Mock  from twisted.internet import defer  from testscenarios import TestWithScenarios @@ -184,10 +185,9 @@ class TestSoledadDbSync(          target = soledad_sync_target(              self, self.db2._dbname,              source_replica_uid=self._soledad._dbpool.replica_uid) -        self.addCleanup(target.close)          return sync.SoledadSynchronizer(              self.db, -            target).sync(defer_decryption=False) +            target).sync()      @defer.inlineCallbacks      def test_db_sync(self): @@ -211,3 +211,21 @@ class TestSoledadDbSync(              self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False)      # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration + + +class TestSoledadSynchronizer(BaseSoledadTest): + +    def setUp(self): +        BaseSoledadTest.setUp(self) +        self.db = Mock() +        self.target = Mock() +        self.synchronizer = sync.SoledadSynchronizer( +            self.db, +            self.target) + +    def test_docs_by_gen_includes_deleted(self): +        changes = [('id', 'gen', 'trans')] +        docs_by_gen = self.synchronizer._docs_by_gen_from_changes(changes) +        f, args, kwargs = docs_by_gen[0][0] +        self.assertIn('include_deleted', kwargs) +        self.assertTrue(kwargs['include_deleted']) diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py deleted file mode 100644 index 4948aaf8..00000000 --- a/testing/tests/sync/test_sync_deferred.py +++ /dev/null @@ -1,196 +0,0 @@ -# test_sync_deferred.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: sync with deferred encryption/decryption. -""" -import time -import os -import random -import string -import shutil - -from urlparse import urljoin - -from twisted.internet import defer - -from leap.soledad.common import couch - -from leap.soledad.client import sync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from testscenarios import TestWithScenarios - -from test_soledad import u1db_tests as tests -from test_soledad.util import ADDRESS -from test_soledad.util import SoledadWithCouchServerMixin -from test_soledad.util import make_soledad_app -from test_soledad.util import soledad_sync_target - - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = True - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): - -    """ -    Another base class for testing the deferred encryption/decryption during -    the syncs, using the intermediate database. -    """ -    defer_sync_encryption = True - -    def setUp(self): -        SoledadWithCouchServerMixin.setUp(self) -        self.startTwistedServer() -        # config info -        self.db1_file = os.path.join(self.tempdir, "db1.u1db") -        os.unlink(self.db1_file) -        self.db_pass = DBPASS -        self.email = ADDRESS - -        # get a random prefix for each test, so we do not mess with -        # concurrency during initialization and shutting down of -        # each local db. -        self.rand_prefix = ''.join( -            map(lambda x: random.choice(string.ascii_letters), range(6))) - -        # open test dbs: db1 will be the local sqlcipher db (which -        # instantiates a syncdb). We use the self._soledad instance that was -        # already created on some setUp method. -        import binascii -        tohex = binascii.b2a_hex -        key = tohex(self._soledad.secrets.get_local_storage_key()) -        sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) -        dbpath = self._soledad._local_db_path - -        self.opts = SQLCipherOptions( -            dbpath, key, is_raw_key=True, create=False, -            defer_encryption=True, sync_db_key=sync_db_key) -        self.db1 = SQLCipherDatabase(self.opts) - -        self.db2 = self.request_state._create_database('test') - -    def tearDown(self): -        # XXX should not access "private" attrs -        shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) -        SoledadWithCouchServerMixin.tearDown(self) - - -class SyncTimeoutError(Exception): - -    """ -    Dummy exception to notify timeout during sync. -    """ -    pass - - -class TestSoledadDbSyncDeferredEncDecr( -        TestWithScenarios, -        BaseSoledadDeferredEncTest, -        tests.TestCaseWithServer): - -    """ -    Test db.sync remote sync shortcut. -    Case with deferred encryption and decryption: using the intermediate -    syncdb. -    """ - -    scenarios = [ -        ('http', { -            'make_app_with_state': make_soledad_app, -            'make_database_for_test': tests.make_memory_database_for_test, -        }), -    ] - -    oauth = False -    token = True - -    def setUp(self): -        """ -        Need to explicitely invoke inicialization on all bases. -        """ -        BaseSoledadDeferredEncTest.setUp(self) -        self.server = self.server_thread = None -        self.syncer = None - -    def tearDown(self): -        """ -        Need to explicitely invoke destruction on all bases. -        """ -        dbsyncer = getattr(self, 'dbsyncer', None) -        if dbsyncer: -            dbsyncer.close() -        BaseSoledadDeferredEncTest.tearDown(self) - -    def do_sync(self): -        """ -        Perform sync using SoledadSynchronizer, SoledadSyncTarget -        and Token auth. -        """ -        replica_uid = self._soledad._dbpool.replica_uid -        sync_db = self._soledad._sync_db -        sync_enc_pool = self._soledad._sync_enc_pool -        dbsyncer = self._soledad._dbsyncer  # Soledad.sync uses the dbsyncer - -        target = soledad_sync_target( -            self, self.db2._dbname, -            source_replica_uid=replica_uid, -            sync_db=sync_db, -            sync_enc_pool=sync_enc_pool) -        self.addCleanup(target.close) -        return sync.SoledadSynchronizer( -            dbsyncer, -            target).sync(defer_decryption=True) - -    def wait_for_sync(self): -        """ -        Wait for sync to finish. -        """ -        wait = 0 -        syncer = self.syncer -        if syncer is not None: -            while syncer.syncing: -                time.sleep(WAIT_STEP) -                wait += WAIT_STEP -                if wait >= MAX_WAIT: -                    raise SyncTimeoutError - -    @defer.inlineCallbacks -    def test_db_sync(self): -        """ -        Test sync. - -        Adapted to check for encrypted content. -        """ -        doc1 = self.db1.create_doc_from_json(tests.simple_doc) -        doc2 = self.db2.create_doc_from_json(tests.nested_doc) -        local_gen_before_sync = yield self.do_sync() - -        gen, _, changes = self.db1.whats_changed(local_gen_before_sync) -        self.assertEqual(1, len(changes)) - -        self.assertEqual(doc2.doc_id, changes[0][0]) -        self.assertEqual(1, gen - local_gen_before_sync) - -        self.assertGetEncryptedDoc( -            self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) -        self.assertGetEncryptedDoc( -            self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py index 2626ab2a..432a3cd2 100644 --- a/testing/tests/sync/test_sync_mutex.py +++ b/testing/tests/sync/test_sync_mutex.py @@ -47,7 +47,7 @@ from test_soledad.util import soledad_sync_target  _old_sync = SoledadSynchronizer.sync -def _timed_sync(self, defer_decryption=True): +def _timed_sync(self):      t = time.time()      sync_id = uuid.uuid4() @@ -62,10 +62,11 @@ def _timed_sync(self, defer_decryption=True):          self.source.sync_times[sync_id]['end'] = t          return passthrough -    d = _old_sync(self, defer_decryption=defer_decryption) +    d = _old_sync(self)      d.addBoth(_store_finish_time)      return d +  SoledadSynchronizer.sync = _timed_sync  # -- end of monkey-patching diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 964468ce..6ce9a5c5 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -30,10 +30,11 @@ from testscenarios import TestWithScenarios  from twisted.internet import defer  from leap.soledad.client import http_target as target -from leap.soledad.client import crypto +from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver  from leap.soledad.client.sqlcipher import SQLCipherU1DBSync  from leap.soledad.client.sqlcipher import SQLCipherOptions  from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client import _crypto  from leap.soledad.common import l2db @@ -44,6 +45,7 @@ from test_soledad.util import make_soledad_app  from test_soledad.util import make_token_soledad_app  from test_soledad.util import make_soledad_document_for_test  from test_soledad.util import soledad_sync_target +from twisted.trial import unittest  from test_soledad.util import SoledadWithCouchServerMixin  from test_soledad.util import ADDRESS  from test_soledad.util import SQLCIPHER_SCENARIOS @@ -53,92 +55,69 @@ from test_soledad.util import SQLCIPHER_SCENARIOS  # The following tests come from `u1db.tests.test_remote_sync_target`.  # ----------------------------------------------------------------------------- -class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): +class TestSoledadParseReceivedDocResponse(unittest.TestCase):      """      Some tests had to be copied to this class so we can instantiate our own      target.      """ -    def setUp(self): -        SoledadWithCouchServerMixin.setUp(self) -        creds = {'token': { -            'uuid': 'user-uuid', -            'token': 'auth-token', -        }} -        self.target = target.SoledadHTTPSyncTarget( -            self.couch_url, -            uuid4().hex, -            creds, -            self._soledad._crypto, -            None) - -    def tearDown(self): -        self.target.close() -        SoledadWithCouchServerMixin.tearDown(self) +    def parse(self, stream): +        parser = DocStreamReceiver(None, None, lambda *_: defer.succeed(42)) +        parser.dataReceived(stream) +        parser.finish()      def test_extra_comma(self): -        """ -        Test adapted to use encrypted content. -        """          doc = SoledadDocument('i', rev='r') -        doc.content = {} -        _crypto = self._soledad._crypto -        key = _crypto.doc_passphrase(doc.doc_id) -        secret = _crypto.secret +        doc.content = {'a': 'b'} -        enc_json = crypto.encrypt_docstr( -            doc.get_json(), doc.doc_id, doc.rev, -            key, secret) +        encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc)          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("[\r\n{},\r\n]") +            self.parse("[\r\n{},\r\n]")          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response( +            self.parse(                  ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + -                 '"content": %s, "gen": 3, "trans_id": "T-sid"}' + -                 ',\r\n]') % json.dumps(enc_json)) +                 '"gen": 3, "trans_id": "T-sid"},\r\n' + +                 '%s,\r\n]') % encrypted_docstr)      def test_wrong_start(self):          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("{}\r\n]") - -        with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("\r\n{}\r\n]") +            self.parse("{}\r\n]")          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("") +            self.parse("\r\n{}\r\n]")      def test_wrong_end(self):          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("[\r\n{}") +            self.parse("[\r\n{}")          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("[\r\n") +            self.parse("[\r\n")      def test_missing_comma(self):          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response( +            self.parse(                  '[\r\n{}\r\n{"id": "i", "rev": "r", '                  '"content": "c", "gen": 3}\r\n]')      def test_no_entries(self):          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response("[\r\n]") +            self.parse("[\r\n]")      def test_error_in_stream(self):          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response( +            self.parse(                  '[\r\n{"new_generation": 0},'                  '\r\n{"error": "unavailable"}\r\n')          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response( +            self.parse(                  '[\r\n{"error": "unavailable"}\r\n')          with self.assertRaises(l2db.errors.BrokenSyncStream): -            self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') +            self.parse('[\r\n{"error": "?"}\r\n')  #  # functions for TestRemoteSyncTargets @@ -151,13 +130,9 @@ def make_local_db_and_soledad_target(      test.startTwistedServer()      replica_uid = os.path.basename(path)      db = test.request_state._create_database(replica_uid) -    sync_db = test._soledad._sync_db -    sync_enc_pool = test._soledad._sync_enc_pool      st = soledad_sync_target(          test, db._dbname, -        source_replica_uid=source_replica_uid, -        sync_db=sync_db, -        sync_enc_pool=sync_enc_pool) +        source_replica_uid=source_replica_uid)      return db, st @@ -188,16 +163,11 @@ class TestSoledadSyncTarget(      def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex):          if self.port is None:              self.startTwistedServer() -        sync_db = self._soledad._sync_db -        sync_enc_pool = self._soledad._sync_enc_pool          if path is None:              path = self.db2._dbname          target = self.sync_target(              self, path, -            source_replica_uid=source_replica_uid, -            sync_db=sync_db, -            sync_enc_pool=sync_enc_pool) -        self.addCleanup(target.close) +            source_replica_uid=source_replica_uid)          return target      def setUp(self): @@ -229,10 +199,10 @@ class TestSoledadSyncTarget(              other_docs.append((doc.doc_id, doc.rev, doc.get_json()))          doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') +        get_doc = (lambda _: doc, (1,), {})          new_gen, trans_id = yield remote_target.sync_exchange( -            [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, -            last_known_trans_id=None, insert_doc_cb=receive_doc, -            defer_decryption=False) +            [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, +            last_known_trans_id=None, insert_doc_cb=receive_doc)          self.assertEqual(1, new_gen)          self.assertGetEncryptedDoc(              db, 'doc-here', 'replica:1', '{"value": "here"}', False) @@ -278,15 +248,16 @@ class TestSoledadSyncTarget(          doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}')          doc2 = self.make_document('doc-here2', 'replica:1',                                    '{"value": "here2"}') +        get_doc1 = (lambda _: doc1, (1,), {}) +        get_doc2 = (lambda _: doc2, (2,), {})          with self.assertRaises(l2db.errors.U1DBError):              yield remote_target.sync_exchange( -                [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], +                [(get_doc1, 10, 'T-sid'), (get_doc2, 11, 'T-sud')],                  'replica',                  last_known_generation=0,                  last_known_trans_id=None, -                insert_doc_cb=receive_doc, -                defer_decryption=False) +                insert_doc_cb=receive_doc)          self.assertGetEncryptedDoc(              db, 'doc-here', 'replica:1', '{"value": "here"}', @@ -297,9 +268,8 @@ class TestSoledadSyncTarget(          # retry          trigger_ids = []          new_gen, trans_id = yield remote_target.sync_exchange( -            [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, -            last_known_trans_id=None, insert_doc_cb=receive_doc, -            defer_decryption=False) +            [(get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, +            last_known_trans_id=None, insert_doc_cb=receive_doc)          self.assertGetEncryptedDoc(              db, 'doc-here2', 'replica:1', '{"value": "here2"}',              False) @@ -328,10 +298,11 @@ class TestSoledadSyncTarget(              replica_uid_box.append(replica_uid)          doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') +        get_doc = (lambda _: doc, (1,), {})          new_gen, trans_id = yield remote_target.sync_exchange( -            [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, +            [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0,              last_known_trans_id=None, insert_doc_cb=receive_doc, -            ensure_callback=ensure_cb, defer_decryption=False) +            ensure_callback=ensure_cb)          self.assertEqual(1, new_gen)          db = self.db2          self.assertEqual(1, len(replica_uid_box)) @@ -339,6 +310,37 @@ class TestSoledadSyncTarget(          self.assertGetEncryptedDoc(              db, 'doc-here', 'replica:1', '{"value": "here"}', False) +    @defer.inlineCallbacks +    def test_sync_exchange_send_events(self): +        """ +        Test for sync exchange's SOLEDAD_SYNC_SEND_STATUS event. +        """ +        remote_target = self.getSyncTarget() +        uuid = remote_target.uuid +        events = [] + +        def mocked_events(*args): +            events.append((args)) +        self.patch( +            target.send, '_emit_send_status', mocked_events) + +        doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') +        doc2 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') +        doc3 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') +        get_doc = (lambda _: doc, (1,), {}) +        get_doc2 = (lambda _: doc2, (1,), {}) +        get_doc3 = (lambda _: doc3, (1,), {}) +        docs = [(get_doc, 10, 'T-sid'), +                (get_doc2, 11, 'T-sid2'), (get_doc3, 12, 'T-sid3')] +        new_gen, trans_id = yield remote_target.sync_exchange( +            docs, 'replica', last_known_generation=0, +            last_known_trans_id=None, insert_doc_cb=lambda _: 1, +            ensure_callback=lambda _: 1) +        self.assertEqual(1, new_gen) +        self.assertEqual(4, len(events)) +        self.assertEquals([(uuid, 0, 3), (uuid, 1, 3), (uuid, 2, 3), +                           (uuid, 3, 3)], events) +      def test_sync_exchange_in_stream_error(self):          self.skipTest("bypass this test because our sync_exchange process "                        "does not return u1db error 503 \"unavailable\" for " @@ -421,7 +423,6 @@ class SoledadDatabaseSyncTargetTests(      def tearDown(self):          self.db.close() -        self.st.close()          tests.TestCaseWithServer.tearDown(self)          SoledadWithCouchServerMixin.tearDown(self) @@ -442,12 +443,12 @@ class SoledadDatabaseSyncTargetTests(          This test was adapted to decrypt remote content before assert.          """          docs_by_gen = [ -            (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, -             'T-sid')] +            ((self.make_document, +              ('doc-id', 'replica:1', tests.simple_doc,), {}), +                10, 'T-sid')]          new_gen, trans_id = yield self.st.sync_exchange(              docs_by_gen, 'replica', last_known_generation=0, -            last_known_trans_id=None, insert_doc_cb=self.receive_doc, -            defer_decryption=False) +            last_known_trans_id=None, insert_doc_cb=self.receive_doc)          self.assertGetEncryptedDoc(              self.db, 'doc-id', 'replica:1', tests.simple_doc, False)          self.assertTransactionLog(['doc-id'], self.db) @@ -465,14 +466,13 @@ class SoledadDatabaseSyncTargetTests(          This test was adapted to decrypt remote content before assert.          """          docs_by_gen = [ -            (self.make_document( -                'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), -            (self.make_document( -                'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] +            ((self.make_document, +                ('doc-id', 'replica:1', tests.simple_doc), {}), 10, 'T-1'), +            ((self.make_document, +                ('doc-id2', 'replica:1', tests.nested_doc), {}), 11, 'T-2')]          new_gen, trans_id = yield self.st.sync_exchange(              docs_by_gen, 'replica', last_known_generation=0, -            last_known_trans_id=None, insert_doc_cb=self.receive_doc, -            defer_decryption=False) +            last_known_trans_id=None, insert_doc_cb=self.receive_doc)          self.assertGetEncryptedDoc(              self.db, 'doc-id', 'replica:1', tests.simple_doc, False)          self.assertGetEncryptedDoc( @@ -498,8 +498,7 @@ class SoledadDatabaseSyncTargetTests(          self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)          new_gen, _ = yield self.st.sync_exchange(              [], 'other-replica', last_known_generation=0, -            last_known_trans_id=None, insert_doc_cb=self.receive_doc, -            defer_decryption=False) +            last_known_trans_id=None, insert_doc_cb=self.receive_doc)          self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)          self.assertEqual(2, new_gen)          self.assertEqual( @@ -552,7 +551,8 @@ class SoledadDatabaseSyncTargetTests(          doc = self.db.create_doc_from_json('{}')          edit_rev = 'replica:1|' + doc.rev          docs_by_gen = [ -            (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] +            ((self.make_document, (doc.doc_id, edit_rev, None), {}), +             10, 'T-sid')]          new_gen, trans_id = yield self.st.sync_exchange(              docs_by_gen, 'replica', last_known_generation=0,              last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -571,7 +571,7 @@ class SoledadDatabaseSyncTargetTests(          self.assertTransactionLog([doc.doc_id], self.db)          new_doc = '{"key": "altval"}'          docs_by_gen = [ -            (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, +            ((self.make_document, (doc.doc_id, 'replica:1', new_doc), {}), 10,               'T-sid')]          new_gen, _ = yield self.st.sync_exchange(              docs_by_gen, 'replica', last_known_generation=0, @@ -591,7 +591,7 @@ class SoledadDatabaseSyncTargetTests(          self.assertTransactionLog([doc.doc_id], self.db)          gen, txid = self.db._get_generation_info()          docs_by_gen = [ -            (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), +            ((self.make_document, (doc.doc_id, doc.rev, tests.simple_doc), {}),               10, 'T-sid')]          new_gen, _ = yield self.st.sync_exchange(              docs_by_gen, 'replica', last_known_generation=gen, @@ -624,9 +624,9 @@ class SoledadDatabaseSyncTargetTests(              [], 'other-replica', last_known_generation=0,              last_known_trans_id=None, insert_doc_cb=self.receive_doc)          self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) +        self.assertEqual(2, new_gen)          self.assertEqual(              (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) -        self.assertEqual(2, new_gen)          if self.whitebox:              self.assertEqual(self.db._last_exchange_log['return'],                               {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) @@ -637,7 +637,7 @@ class SoledadDatabaseSyncTargetTests(          self.assertTransactionLog([doc.doc_id], self.db)          new_doc = '{"key": "altval"}'          docs_by_gen = [ -            (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, +            ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10,               'T-sid')]          new_gen, _ = yield self.st.sync_exchange(              docs_by_gen, 'other-replica', last_known_generation=0, @@ -662,7 +662,7 @@ class SoledadDatabaseSyncTargetTests(          self.assertTransactionLog([doc.doc_id], self.db)          new_doc = '{"key": "altval"}'          docs_by_gen = [ -            (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, +            ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10,               'T-sid')]          new_gen, _ = yield self.st.sync_exchange(              docs_by_gen, 'other-replica', last_known_generation=0, @@ -683,7 +683,7 @@ class SoledadDatabaseSyncTargetTests(          self.assertTransactionLog([doc.doc_id], self.db)          new_doc = '{"key": "altval"}'          docs_by_gen = [ -            (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, +            ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10,               'T-sid')]          new_gen, _ = yield self.st.sync_exchange(              docs_by_gen, 'other-replica', last_known_generation=0, @@ -694,9 +694,9 @@ class SoledadDatabaseSyncTargetTests(      def test_sync_exchange_converged_handling(self):          doc = self.db.create_doc_from_json(tests.simple_doc)          docs_by_gen = [ -            (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), -            (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, -             'T-bar')] +            ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), +            ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), +                5, 'T-bar')]          new_gen, _ = yield self.st.sync_exchange(              docs_by_gen, 'other-replica', last_known_generation=0,              last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -780,9 +780,6 @@ class SoledadDatabaseSyncTargetTests(          self.assertEqual(expected, called) -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = False -  WAIT_STEP = 1  MAX_WAIT = 10  DBPASS = "pass" @@ -842,12 +839,10 @@ class TestSoledadDbSync(          import binascii          tohex = binascii.b2a_hex          key = tohex(self._soledad.secrets.get_local_storage_key()) -        sync_db_key = tohex(self._soledad.secrets.get_sync_db_key())          dbpath = self._soledad._local_db_path          self.opts = SQLCipherOptions( -            dbpath, key, is_raw_key=True, create=False, -            defer_encryption=True, sync_db_key=sync_db_key) +            dbpath, key, is_raw_key=True, create=False)          self.db1 = SQLCipherDatabase(self.opts)          self.db2 = self.request_state._create_database(replica_uid='test') @@ -886,12 +881,10 @@ class TestSoledadDbSync(                  self.opts,                  crypto,                  replica_uid, -                None, -                defer_encryption=True) +                None)              self.dbsyncer = dbsyncer              return dbsyncer.sync(target_url, -                                 creds=creds, -                                 defer_decryption=DEFER_DECRYPTION) +                                 creds=creds)          else:              return self._do_sync(self, target_name) | 
