From 1e3de25ce10156655bcb1bc879f5340baa889ead Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 16 Sep 2016 04:37:20 -0300 Subject: [bug] disable decpool Temporary fix for server streaming --- testing/tests/perf/test_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 0b48a0b9..4d42395b 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -23,7 +23,8 @@ def create_upload(uploads, size): def setup(): return load_up(client, uploads, payload(size)) - yield txbenchmark_with_setup(setup, client.sync) + yield txbenchmark_with_setup(setup, client.sync, + defer_decryption=False) return test -- cgit v1.2.3 From 35563cb74fcfd7f6ae969ed3af3a74d3c18cbf5b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 18 Sep 2016 00:26:45 -0300 Subject: [refactor] remove decpool It's not being used --- testing/tests/perf/test_encdecpool.py | 41 ----- testing/tests/perf/test_sync.py | 3 +- testing/tests/sync/test_encdecpool.py | 258 ------------------------------- testing/tests/sync/test_sync.py | 2 +- testing/tests/sync/test_sync_deferred.py | 15 +- testing/tests/sync/test_sync_mutex.py | 4 +- testing/tests/sync/test_sync_target.py | 27 +--- 7 files changed, 16 insertions(+), 334 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py index 77091a41..8e820b9c 100644 --- a/testing/tests/perf/test_encdecpool.py +++ b/testing/tests/perf/test_encdecpool.py @@ -3,7 +3,6 @@ import json from uuid import uuid4 from twisted.internet.defer import gatherResults from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool from leap.soledad.common.document import SoledadDocument # FIXME: test load is low due issue #7370, higher values will get out of memory @@ -36,43 +35,3 @@ def create_encrypt(amount, size): test_encdecpool_encrypt_100_10k = create_encrypt(100, 10*1000) test_encdecpool_encrypt_100_100k = create_encrypt(100, 100*1000) test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000) - - -def create_decrypt(amount, size): - @pytest.mark.benchmark(group="test_pool_decrypt") - @pytest.inlineCallbacks - def test(soledad_client, txbenchmark_with_setup, request, payload): - DOC_CONTENT = {'payload': payload(size)} - client = soledad_client() - - def setup(): - pool = SyncDecrypterPool( - client._crypto, - client._sync_db, - source_replica_uid=client._dbpool.replica_uid, - insert_doc_cb=lambda x, y, z: False) # ignored - pool.start(amount) - request.addfinalizer(pool.stop) - crypto = client._crypto - docs = [] - for _ in xrange(amount): - doc = SoledadDocument( - doc_id=uuid4().hex, rev='rev', - json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc.doc_id, encrypted_content)) - return pool, docs - - def put_and_wait(pool, docs): - deferreds = [] # fires on completion - for idx, (doc_id, content) in enumerate(docs, 1): - deferreds.append(pool.insert_encrypted_received_doc( - doc_id, 'rev', content, idx, "trans_id", idx)) - return gatherResults(deferreds) - - yield txbenchmark_with_setup(setup, put_and_wait) - return test - -test_encdecpool_decrypt_100_10k = create_decrypt(100, 10*1000) -test_encdecpool_decrypt_100_100k = create_decrypt(100, 100*1000) -test_encdecpool_decrypt_100_500k = create_decrypt(100, 500*1000) diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 4d42395b..0b48a0b9 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -23,8 +23,7 @@ def create_upload(uploads, size): def setup(): return load_up(client, uploads, payload(size)) - yield txbenchmark_with_setup(setup, client.sync, - defer_decryption=False) + yield txbenchmark_with_setup(setup, client.sync) return test diff --git a/testing/tests/sync/test_encdecpool.py b/testing/tests/sync/test_encdecpool.py index 4a32885e..7055a765 100644 --- a/testing/tests/sync/test_encdecpool.py +++ b/testing/tests/sync/test_encdecpool.py @@ -1,34 +1,11 @@ # -*- coding: utf-8 -*- -# test_encdecpool.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for encryption and decryption pool. -""" import json -from random import shuffle - -from mock import MagicMock from twisted.internet.defer import inlineCallbacks from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool from leap.soledad.common.document import SoledadDocument from test_soledad.util import BaseSoledadTest -from twisted.internet import defer DOC_ID = "mydoc" DOC_REV = "rev" @@ -69,238 +46,3 @@ class TestSyncEncrypterPool(BaseSoledadTest): encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) self.assertIsNotNone(encrypted) - - -class TestSyncDecrypterPool(BaseSoledadTest): - - def _insert_doc_cb(self, doc, gen, trans_id): - """ - Method used to mock the sync's return_doc_cb callback. - """ - self._inserted_docs.append((doc, gen, trans_id)) - - def _setup_pool(self, sync_db=None): - sync_db = sync_db or self._soledad._sync_db - return SyncDecrypterPool( - self._soledad._crypto, - sync_db, - source_replica_uid=self._soledad._dbpool.replica_uid, - insert_doc_cb=self._insert_doc_cb) - - def setUp(self): - BaseSoledadTest.setUp(self) - # setup the pool - self._pool = self._setup_pool() - # reset the inserted docs mock - self._inserted_docs = [] - - def tearDown(self): - if self._pool.running: - self._pool.stop() - BaseSoledadTest.tearDown(self) - - def test_insert_received_doc(self): - """ - Test that one document added to the pool is inserted using the - callback. - """ - self._pool.start(1) - self._pool.insert_received_doc( - DOC_ID, DOC_REV, "{}", 1, "trans_id", 1) - - def _assert_doc_was_inserted(_): - self.assertEqual( - self._inserted_docs, - [(SoledadDocument(DOC_ID, DOC_REV, "{}"), 1, u"trans_id")]) - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_looping_control(self): - """ - Start and stop cleanly. - """ - self._pool.start(10) - self.assertTrue(self._pool.running) - self._pool.stop() - self.assertFalse(self._pool.running) - self.assertTrue(self._pool.deferred.called) - - def test_sync_id_col_is_created_if_non_existing_in_docs_recvd_table(self): - """ - Test that docs_received table is migrated, and has the sync_id column - """ - mock_run_query = MagicMock(return_value=defer.succeed(None)) - mock_sync_db = MagicMock() - mock_sync_db.runQuery = mock_run_query - pool = self._setup_pool(mock_sync_db) - d = pool.start(10) - pool.stop() - - def assert_trial_to_create_sync_id_column(_): - mock_run_query.assert_called_once_with( - "ALTER TABLE docs_received ADD COLUMN sync_id") - - d.addCallback(assert_trial_to_create_sync_id_column) - return d - - def test_insert_received_doc_many(self): - """ - Test that many documents added to the pool are inserted using the - callback. - """ - many = 100 - self._pool.start(many) - - # insert many docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - self._pool.insert_received_doc( - doc_id, rev, content, gen, trans_id, idx) - - def _assert_doc_was_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_insert_encrypted_received_doc(self): - """ - Test that one encrypted document added to the pool is decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - - # insert the encrypted document in the pool - self._pool.start(1) - self._pool.insert_encrypted_received_doc( - DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1) - - def _assert_doc_was_decrypted_and_inserted(_): - self.assertEqual(1, len(self._inserted_docs)) - self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")]) - - self._pool.deferred.addCallback( - _assert_doc_was_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_processing_order(self): - """ - This test ensures that processing of documents only occur if there is - a sequence in place. - """ - crypto = self._soledad._crypto - - docs = [] - for i in xrange(1, 10): - i = str(i) - doc = SoledadDocument( - doc_id=DOC_ID + i, rev=DOC_REV + i, - json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc, encrypted_content)) - - # insert the encrypted document in the pool - yield self._pool.start(10) # pool is expecting to process 10 docs - self._pool._loop.stop() # we are processing manually - # first three arrives, forming a sequence - for i, (doc, encrypted_content) in enumerate(docs[:3]): - gen = idx = i + 1 - yield self._pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx) - - # last one arrives alone, so it can't be processed - doc, encrypted_content = docs[-1] - yield self._pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) - - yield self._pool._decrypt_and_recurse() - - self.assertEqual(3, self._pool._processed_docs) - - def test_insert_encrypted_received_doc_many(self, many=100): - """ - Test that many encrypted documents added to the pool are decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - self._pool.start(many) - docs = [] - - # insert many encrypted docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - - doc = SoledadDocument( - doc_id=doc_id, rev=rev, json=json.dumps(content)) - - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc_id, rev, encrypted_content, gen, - trans_id, idx)) - shuffle(docs) - - for doc in docs: - self._pool.insert_encrypted_received_doc(*doc) - - def _assert_docs_were_decrypted_and_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback( - _assert_docs_were_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_pool_reuse(self): - """ - The pool is reused between syncs, this test verifies that - reusing is fine. - """ - for i in xrange(3): - yield self.test_insert_encrypted_received_doc_many(5) - self._inserted_docs = [] - decrypted_docs = yield self._pool._get_docs(encrypted=False) - # check that decrypted docs staging is clean - self.assertEquals([], decrypted_docs) - self._pool.stop() diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index 5290003e..a7d0a92b 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -187,7 +187,7 @@ class TestSoledadDbSync( self.addCleanup(target.close) return sync.SoledadSynchronizer( self.db, - target).sync(defer_decryption=False) + target).sync() @defer.inlineCallbacks def test_db_sync(self): diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py index 4948aaf8..482b150c 100644 --- a/testing/tests/sync/test_sync_deferred.py +++ b/testing/tests/sync/test_sync_deferred.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Test Leap backend bits: sync with deferred encryption/decryption. +Test Leap backend bits: sync with deferred encryption. """ import time import os @@ -22,12 +22,8 @@ import random import string import shutil -from urlparse import urljoin - from twisted.internet import defer -from leap.soledad.common import couch - from leap.soledad.client import sync from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client.sqlcipher import SQLCipherDatabase @@ -41,9 +37,6 @@ from test_soledad.util import make_soledad_app from test_soledad.util import soledad_sync_target -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = True - WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" @@ -52,7 +45,7 @@ DBPASS = "pass" class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): """ - Another base class for testing the deferred encryption/decryption during + Another base class for testing the deferred encryption during the syncs, using the intermediate database. """ defer_sync_encryption = True @@ -109,7 +102,7 @@ class TestSoledadDbSyncDeferredEncDecr( """ Test db.sync remote sync shortcut. - Case with deferred encryption and decryption: using the intermediate + Case with deferred encryption: using the intermediate syncdb. """ @@ -158,7 +151,7 @@ class TestSoledadDbSyncDeferredEncDecr( self.addCleanup(target.close) return sync.SoledadSynchronizer( dbsyncer, - target).sync(defer_decryption=True) + target).sync() def wait_for_sync(self): """ diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py index 2626ab2a..2bcb3aec 100644 --- a/testing/tests/sync/test_sync_mutex.py +++ b/testing/tests/sync/test_sync_mutex.py @@ -47,7 +47,7 @@ from test_soledad.util import soledad_sync_target _old_sync = SoledadSynchronizer.sync -def _timed_sync(self, defer_decryption=True): +def _timed_sync(self): t = time.time() sync_id = uuid.uuid4() @@ -62,7 +62,7 @@ def _timed_sync(self, defer_decryption=True): self.source.sync_times[sync_id]['end'] = t return passthrough - d = _old_sync(self, defer_decryption=defer_decryption) + d = _old_sync(self) d.addBoth(_store_finish_time) return d diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 964468ce..a2935539 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -231,8 +231,7 @@ class TestSoledadSyncTarget( doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') new_gen, trans_id = yield remote_target.sync_exchange( [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertEqual(1, new_gen) self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) @@ -285,8 +284,7 @@ class TestSoledadSyncTarget( 'replica', last_known_generation=0, last_known_trans_id=None, - insert_doc_cb=receive_doc, - defer_decryption=False) + insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', @@ -298,8 +296,7 @@ class TestSoledadSyncTarget( trigger_ids = [] new_gen, trans_id = yield remote_target.sync_exchange( [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here2', 'replica:1', '{"value": "here2"}', False) @@ -331,7 +328,7 @@ class TestSoledadSyncTarget( new_gen, trans_id = yield remote_target.sync_exchange( [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc, - ensure_callback=ensure_cb, defer_decryption=False) + ensure_callback=ensure_cb) self.assertEqual(1, new_gen) db = self.db2 self.assertEqual(1, len(replica_uid_box)) @@ -446,8 +443,7 @@ class SoledadDatabaseSyncTargetTests( 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertGetEncryptedDoc( self.db, 'doc-id', 'replica:1', tests.simple_doc, False) self.assertTransactionLog(['doc-id'], self.db) @@ -471,8 +467,7 @@ class SoledadDatabaseSyncTargetTests( 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertGetEncryptedDoc( self.db, 'doc-id', 'replica:1', tests.simple_doc, False) self.assertGetEncryptedDoc( @@ -498,8 +493,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) new_gen, _ = yield self.st.sync_exchange( [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) self.assertEqual(2, new_gen) self.assertEqual( @@ -779,10 +773,6 @@ class SoledadDatabaseSyncTargetTests( yield self.st.record_sync_info('replica', 0, 'T-sid') self.assertEqual(expected, called) - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = False - WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" @@ -890,8 +880,7 @@ class TestSoledadDbSync( defer_encryption=True) self.dbsyncer = dbsyncer return dbsyncer.sync(target_url, - creds=creds, - defer_decryption=DEFER_DECRYPTION) + creds=creds) else: return self._do_sync(self, target_name) -- cgit v1.2.3 From 07dcb2ae5240f20a26903f53a432fcd49c7f1ec9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 19 Sep 2016 03:56:44 -0300 Subject: [feature] streaming download protocol This commit finishes reversion into u1db original streaming protocol for downloads. --- testing/tests/sync/test_sync_target.py | 65 ++++++++++++---------------------- 1 file changed, 22 insertions(+), 43 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index a2935539..997dcdcd 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -30,6 +30,7 @@ from testscenarios import TestWithScenarios from twisted.internet import defer from leap.soledad.client import http_target as target +from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver from leap.soledad.client import crypto from leap.soledad.client.sqlcipher import SQLCipherU1DBSync from leap.soledad.client.sqlcipher import SQLCipherOptions @@ -44,6 +45,7 @@ from test_soledad.util import make_soledad_app from test_soledad.util import make_token_soledad_app from test_soledad.util import make_soledad_document_for_test from test_soledad.util import soledad_sync_target +from twisted.trial import unittest from test_soledad.util import SoledadWithCouchServerMixin from test_soledad.util import ADDRESS from test_soledad.util import SQLCIPHER_SCENARIOS @@ -53,92 +55,69 @@ from test_soledad.util import SQLCIPHER_SCENARIOS # The following tests come from `u1db.tests.test_remote_sync_target`. # ----------------------------------------------------------------------------- -class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): +class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ Some tests had to be copied to this class so we can instantiate our own target. """ - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - self.target = target.SoledadHTTPSyncTarget( - self.couch_url, - uuid4().hex, - creds, - self._soledad._crypto, - None) - - def tearDown(self): - self.target.close() - SoledadWithCouchServerMixin.tearDown(self) + def parse(self, stream): + parser = DocStreamReceiver(None, None, lambda x, y: 42) + parser.dataReceived(stream) + parser.finish() def test_extra_comma(self): - """ - Test adapted to use encrypted content. - """ doc = SoledadDocument('i', rev='r') - doc.content = {} - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc.doc_id) - secret = _crypto.secret + doc.content = {'a': 'b'} - enc_json = crypto.encrypt_docstr( - doc.get_json(), doc.doc_id, doc.rev, - key, secret) + encrypted_docstr = crypto.SoledadCrypto('').encrypt_doc(doc) with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{},\r\n]") + self.parse("[\r\n{},\r\n]") with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + - '"content": %s, "gen": 3, "trans_id": "T-sid"}' + - ',\r\n]') % json.dumps(enc_json)) + '"gen": 3, "trans_id": "T-sid"},\r\n' + + '%s,\r\n]') % encrypted_docstr) def test_wrong_start(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("{}\r\n]") - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("\r\n{}\r\n]") + self.parse("{}\r\n]") with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("") + self.parse("\r\n{}\r\n]") def test_wrong_end(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{}") + self.parse("[\r\n{}") with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n") + self.parse("[\r\n") def test_missing_comma(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( '[\r\n{}\r\n{"id": "i", "rev": "r", ' '"content": "c", "gen": 3}\r\n]') def test_no_entries(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n]") + self.parse("[\r\n]") def test_error_in_stream(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( '[\r\n{"new_generation": 0},' '\r\n{"error": "unavailable"}\r\n') with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( '[\r\n{"error": "unavailable"}\r\n') with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') + self.parse('[\r\n{"error": "?"}\r\n') # # functions for TestRemoteSyncTargets -- cgit v1.2.3 From a8182bb4f954c02d53d699bfe2a645667d770269 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 19 Sep 2016 21:48:56 -0300 Subject: [feature] upload streaming 1) enable HTTP 1.1 chunked upload on server 2) make the client sync.py generate a list of function calls instead of a list of full docs 3) disable encryption pool 4) make the doc encryption a list of function calls 5) create a twisted protocol for sending 6) make a producer that calls the doc generation as necessary --- testing/tests/perf/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py index 6fa6b2c0..09567b88 100644 --- a/testing/tests/perf/conftest.py +++ b/testing/tests/perf/conftest.py @@ -243,7 +243,7 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): server_url=server_url, cert_file=None, auth_token=token, - defer_encryption=True) + defer_encryption=False) request.addfinalizer(soledad_client.close) return soledad_client return create -- cgit v1.2.3 From fcf3b3046dd2005992638ebf993d53897af8ed3a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 14 Sep 2016 01:52:36 -0400 Subject: [refactor] remove encryption pool --- testing/tests/client/test_crypto2.py | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 testing/tests/client/test_crypto2.py (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto2.py b/testing/tests/client/test_crypto2.py new file mode 100644 index 00000000..ae280020 --- /dev/null +++ b/testing/tests/client/test_crypto2.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# test_crypto2.py +# Copyright (C) 2016 LEAP Encryption Access Project +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Tests for the _crypto module +""" + +import StringIO + + +import leap.soledad.client +from leap.soledad.client import _crypto + + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend + + +def _aes_encrypt(key, iv, data): + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + encryptor = cipher.encryptor() + return encryptor.update(data) + encryptor.finalize() + + +def test_chunked_encryption(): + key = 'A' * 32 + iv = 'A' * 16 + data = ( + "You can't come up against " + "the world's most powerful intelligence " + "agencies and not accept the risk. " + "If they want to get you, over time " + "they will.") + + fd = StringIO.StringIO() + aes = _crypto.AESWriter(key, fd, iv) + + block = 16 + + for i in range(len(data)/block): + chunk = data[i * block:(i+1)*block] + aes.write(chunk) + aes.end() + + ciphertext_chunked = fd.getvalue() + ciphertext = _aes_encrypt(key, iv, data) + + assert ciphertext_chunked == ciphertext -- cgit v1.2.3 From 510c0ba3a0c0ade334090a1c36dab9ccae0ba1b4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Sep 2016 19:24:55 -0400 Subject: [feature] blob encryptor / decryptor --- testing/tests/client/test_crypto2.py | 126 ++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 9 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto2.py b/testing/tests/client/test_crypto2.py index ae280020..f0f6c4af 100644 --- a/testing/tests/client/test_crypto2.py +++ b/testing/tests/client/test_crypto2.py @@ -19,16 +19,28 @@ Tests for the _crypto module """ +import base64 +import binascii +import time +import struct import StringIO - import leap.soledad.client from leap.soledad.client import _crypto - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend +from twisted.trial import unittest + + +snowden1 = ( + "You can't come up against " + "the world's most powerful intelligence " + "agencies and not accept the risk. " + "If they want to get you, over time " + "they will.") + def _aes_encrypt(key, iv, data): backend = default_backend() @@ -36,20 +48,21 @@ def _aes_encrypt(key, iv, data): encryptor = cipher.encryptor() return encryptor.update(data) + encryptor.finalize() +def _aes_decrypt(key, iv, data): + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + decryptor = cipher.decryptor() + return decryptor.update(data) + decryptor.finalize() + def test_chunked_encryption(): key = 'A' * 32 iv = 'A' * 16 - data = ( - "You can't come up against " - "the world's most powerful intelligence " - "agencies and not accept the risk. " - "If they want to get you, over time " - "they will.") fd = StringIO.StringIO() - aes = _crypto.AESWriter(key, fd, iv) + aes = _crypto.AESEncryptor(key, iv, fd) + data = snowden1 block = 16 for i in range(len(data)/block): @@ -61,3 +74,98 @@ def test_chunked_encryption(): ciphertext = _aes_encrypt(key, iv, data) assert ciphertext_chunked == ciphertext + + +def test_decrypt(): + key = 'A' * 32 + iv = 'A' * 16 + + data = snowden1 + block = 16 + + ciphertext = _aes_encrypt(key, iv, data) + + fd = StringIO.StringIO() + aes = _crypto.AESDecryptor(key, iv, fd) + + for i in range(len(ciphertext)/block): + chunk = ciphertext[i * block:(i+1)*block] + aes.write(chunk) + aes.end() + + cleartext_chunked = fd.getvalue() + assert cleartext_chunked == data + + + +class BlobTestCase(unittest.TestCase): + + class doc_info: + doc_id = 'D-deadbeef' + rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + + def test_blob_encryptor(self): + + inf = StringIO.StringIO() + inf.write(snowden1) + inf.seek(0) + outf = StringIO.StringIO() + + blob = _crypto.BlobEncryptor( + self.doc_info, inf, result=outf, + secret='A' * 96, iv='B'*16) + + d = blob.encrypt() + d.addCallback(self._test_blob_encryptor_cb, outf) + return d + + def _test_blob_encryptor_cb(self, _, outf): + encrypted = outf.getvalue() + data = base64.urlsafe_b64decode(encrypted) + + assert data[0] == '\x80' + ts, sch, meth = struct.unpack( + 'Qbb', data[1:11]) + assert sch == 1 + assert meth == 1 + iv = data[11:27] + assert iv == 'B' * 16 + doc_id = data[27:37] + assert doc_id == 'D-deadbeef' + + rev = data[37:71] + assert rev == self.doc_info.rev + + ciphertext = data[71:-64] + aes_key = _crypto._get_sym_key_for_doc( + self.doc_info.doc_id, 'A'*96) + assert ciphertext == _aes_encrypt(aes_key, 'B'*16, snowden1) + + decrypted = _aes_decrypt(aes_key, 'B'*16, ciphertext) + assert str(decrypted) == snowden1 + + def test_blob_decryptor(self): + + inf = StringIO.StringIO() + inf.write(snowden1) + inf.seek(0) + outf = StringIO.StringIO() + + blob = _crypto.BlobEncryptor( + self.doc_info, inf, result=outf, + secret='A' * 96, iv='B' * 16) + + def do_decrypt(_, outf): + decryptor = _crypto.BlobDecryptor( + self.doc_info, outf, + secret='A' * 96) + d = decryptor.decrypt() + return d + + d = blob.encrypt() + d.addCallback(do_decrypt, outf) + d.addCallback(self._test_blob_decryptor_cb) + return d + + def _test_blob_decryptor_cb(self, decrypted): + assert decrypted.getvalue() == snowden1 -- cgit v1.2.3 From 0098849ffd6d9d7514a2eff7b6ced9403a9062ca Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 19 Sep 2016 22:00:04 -0400 Subject: [refactor] add SoledadCrypto interface --- testing/tests/perf/conftest.py | 3 +- testing/tests/perf/test_crypto.py | 79 +++++++++++++++++++++-------------- testing/tests/perf/test_encdecpool.py | 37 ---------------- 3 files changed, 48 insertions(+), 71 deletions(-) delete mode 100644 testing/tests/perf/test_encdecpool.py (limited to 'testing/tests') diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py index 09567b88..2964936b 100644 --- a/testing/tests/perf/conftest.py +++ b/testing/tests/perf/conftest.py @@ -242,8 +242,7 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): local_db_path=local_db_path, server_url=server_url, cert_file=None, - auth_token=token, - defer_encryption=False) + auth_token=token) request.addfinalizer(soledad_client.close) return soledad_client return create diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py index be00560b..9ce418ba 100644 --- a/testing/tests/perf/test_crypto.py +++ b/testing/tests/perf/test_crypto.py @@ -1,9 +1,21 @@ +""" +Benchmarks for crypto operations. +If you don't want to stress your local machine too much, you can pass the +SIZE_LIMT environment variable. + +For instance, to keep the maximum payload at 1MB: + +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +""" import pytest +import os import json from uuid import uuid4 + from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.crypto import encrypt_sym -from leap.soledad.client.crypto import decrypt_sym +from leap.soledad.client import _crypto + +LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) def create_doc_encryption(size): @@ -20,7 +32,11 @@ def create_doc_encryption(size): return test_doc_encryption +# TODO this test is really bullshit, because it's still including +# the json serialization. + def create_doc_decryption(size): + @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_crypto_decrypt_doc") def test_doc_decryption(soledad_client, benchmark, payload): crypto = soledad_client()._crypto @@ -29,32 +45,19 @@ def create_doc_decryption(size): doc = SoledadDocument( doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) - encrypted_doc = crypto.encrypt_doc(doc) + + encrypted_doc = yield crypto.encrypt_doc(doc) doc.set_json(encrypted_doc) benchmark(crypto.decrypt_doc, doc) return test_doc_decryption -test_encrypt_doc_10k = create_doc_encryption(10*1000) -test_encrypt_doc_100k = create_doc_encryption(100*1000) -test_encrypt_doc_500k = create_doc_encryption(500*1000) -test_encrypt_doc_1M = create_doc_encryption(1000*1000) -test_encrypt_doc_10M = create_doc_encryption(10*1000*1000) -test_encrypt_doc_50M = create_doc_encryption(50*1000*1000) -test_decrypt_doc_10k = create_doc_decryption(10*1000) -test_decrypt_doc_100k = create_doc_decryption(100*1000) -test_decrypt_doc_500k = create_doc_decryption(500*1000) -test_decrypt_doc_1M = create_doc_decryption(1000*1000) -test_decrypt_doc_10M = create_doc_decryption(10*1000*1000) -test_decrypt_doc_50M = create_doc_decryption(50*1000*1000) - - def create_raw_encryption(size): @pytest.mark.benchmark(group="test_crypto_raw_encrypt") def test_raw_encrypt(benchmark, payload): key = payload(32) - benchmark(encrypt_sym, payload(size), key) + benchmark(_crypto.encrypt_sym, payload(size), key) return test_raw_encrypt @@ -62,20 +65,32 @@ def create_raw_decryption(size): @pytest.mark.benchmark(group="test_crypto_raw_decrypt") def test_raw_decrypt(benchmark, payload): key = payload(32) - iv, ciphertext = encrypt_sym(payload(size), key) - benchmark(decrypt_sym, ciphertext, key, iv) + iv, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv) return test_raw_decrypt -test_encrypt_raw_10k = create_raw_encryption(10*1000) -test_encrypt_raw_100k = create_raw_encryption(100*1000) -test_encrypt_raw_500k = create_raw_encryption(500*1000) -test_encrypt_raw_1M = create_raw_encryption(1000*1000) -test_encrypt_raw_10M = create_raw_encryption(10*1000*1000) -test_encrypt_raw_50M = create_raw_encryption(50*1000*1000) -test_decrypt_raw_10k = create_raw_decryption(10*1000) -test_decrypt_raw_100k = create_raw_decryption(100*1000) -test_decrypt_raw_500k = create_raw_decryption(500*1000) -test_decrypt_raw_1M = create_raw_decryption(1000*1000) -test_decrypt_raw_10M = create_raw_decryption(10*1000*1000) -test_decrypt_raw_50M = create_raw_decryption(50*1000*1000) +# Create the TESTS in the global namespace, they'll be picked by the benchmark +# plugin. + +encryption_tests = [ + ('10k', 1E4), + ('100k', 1E5), + ('500k', 5E5), + ('1M', 1E6), + ('10M', 1E7), + ('50M', 5E7), +] + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) + globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) + + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz) + globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz) diff --git a/testing/tests/perf/test_encdecpool.py b/testing/tests/perf/test_encdecpool.py deleted file mode 100644 index 8e820b9c..00000000 --- a/testing/tests/perf/test_encdecpool.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest -import json -from uuid import uuid4 -from twisted.internet.defer import gatherResults -from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.common.document import SoledadDocument -# FIXME: test load is low due issue #7370, higher values will get out of memory - - -def create_encrypt(amount, size): - @pytest.mark.benchmark(group="test_pool_encrypt") - @pytest.inlineCallbacks - def test(soledad_client, txbenchmark_with_setup, request, payload): - DOC_CONTENT = {'payload': payload(size)} - - def setup(): - client = soledad_client() - pool = SyncEncrypterPool(client._crypto, client._sync_db) - pool.start() - request.addfinalizer(pool.stop) - docs = [ - SoledadDocument(doc_id=uuid4().hex, rev='rev', - json=json.dumps(DOC_CONTENT)) - for _ in xrange(amount) - ] - return pool, docs - - @pytest.inlineCallbacks - def put_and_wait(pool, docs): - yield gatherResults([pool.encrypt_doc(doc) for doc in docs]) - - yield txbenchmark_with_setup(setup, put_and_wait) - return test - -test_encdecpool_encrypt_100_10k = create_encrypt(100, 10*1000) -test_encdecpool_encrypt_100_100k = create_encrypt(100, 100*1000) -test_encdecpool_encrypt_100_500k = create_encrypt(100, 500*1000) -- cgit v1.2.3 From 75208477a2f1634664b80b8501818e5a905aa0f3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Sep 2016 01:42:26 -0400 Subject: [tests] adapt tests --- testing/tests/client/test_crypto.py | 263 +++++++++++++++++++++--------- testing/tests/client/test_crypto2.py | 171 ------------------- testing/tests/sync/test_encdecpool.py | 48 ------ testing/tests/sync/test_sqlcipher_sync.py | 14 +- 4 files changed, 193 insertions(+), 303 deletions(-) delete mode 100644 testing/tests/client/test_crypto2.py delete mode 100644 testing/tests/sync/test_encdecpool.py (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 77252b46..dc3054f2 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -17,47 +17,184 @@ """ Tests for cryptographic related stuff. """ -import os -import hashlib import binascii +import base64 +import hashlib +import json +import os +import struct + +from io import BytesIO + +import pytest + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend -from leap.soledad.client import crypto from leap.soledad.common.document import SoledadDocument from test_soledad.util import BaseSoledadTest -from leap.soledad.common.crypto import WrongMacError -from leap.soledad.common.crypto import UnknownMacMethodError -from leap.soledad.common.crypto import ENC_JSON_KEY -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.crypto import MAC_KEY -from leap.soledad.common.crypto import MAC_METHOD_KEY +from leap.soledad.client import _crypto + +from twisted.trial import unittest +from twisted.internet import defer + + +snowden1 = ( + "You can't come up against " + "the world's most powerful intelligence " + "agencies and not accept the risk. " + "If they want to get you, over time " + "they will.") + + +class AESTest(unittest.TestCase): + + def test_chunked_encryption(self): + key = 'A' * 32 + iv = 'A' * 16 + + fd = BytesIO() + aes = _crypto.AESEncryptor(key, iv, fd) + + data = snowden1 + block = 16 + + for i in range(len(data)/block): + chunk = data[i * block:(i+1)*block] + aes.write(chunk) + aes.end() + + ciphertext_chunked = fd.getvalue() + ciphertext = _aes_encrypt(key, iv, data) + + assert ciphertext_chunked == ciphertext + + + def test_decrypt(self): + key = 'A' * 32 + iv = 'A' * 16 + + data = snowden1 + block = 16 + + ciphertext = _aes_encrypt(key, iv, data) + + fd = BytesIO() + aes = _crypto.AESDecryptor(key, iv, fd) + + for i in range(len(ciphertext)/block): + chunk = ciphertext[i * block:(i+1)*block] + aes.write(chunk) + aes.end() + + cleartext_chunked = fd.getvalue() + assert cleartext_chunked == data + + + +class BlobTestCase(unittest.TestCase): + + class doc_info: + doc_id = 'D-deadbeef' + rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + + @defer.inlineCallbacks + def test_blob_encryptor(self): + + inf = BytesIO() + inf.write(snowden1) + inf.seek(0) + outf = BytesIO() + + blob = _crypto.BlobEncryptor( + self.doc_info, inf, result=outf, + secret='A' * 96, iv='B'*16) + + encrypted = yield blob.encrypt() + data = base64.urlsafe_b64decode(encrypted.getvalue()) + assert data[0] == '\x80' + ts, sch, meth = struct.unpack( + 'Qbb', data[1:11]) + assert sch == 1 + assert meth == 1 + iv = data[11:27] + assert iv == 'B' * 16 + doc_id = data[27:37] + assert doc_id == 'D-deadbeef' -class EncryptedSyncTestCase(BaseSoledadTest): + rev = data[37:71] + assert rev == self.doc_info.rev - """ - Tests that guarantee that data will always be encrypted when syncing. - """ + ciphertext = data[71:-64] + aes_key = _crypto._get_sym_key_for_doc( + self.doc_info.doc_id, 'A'*96) + assert ciphertext == _aes_encrypt(aes_key, 'B'*16, snowden1) - def test_encrypt_decrypt_json(self): + decrypted = _aes_decrypt(aes_key, 'B'*16, ciphertext) + assert str(decrypted) == snowden1 + + + @defer.inlineCallbacks + def test_blob_decryptor(self): + + inf = BytesIO() + inf.write(snowden1) + inf.seek(0) + outf = BytesIO() + + blob = _crypto.BlobEncryptor( + self.doc_info, inf, result=outf, + secret='A' * 96, iv='B' * 16) + yield blob.encrypt() + + decryptor = _crypto.BlobDecryptor( + self.doc_info, outf, + secret='A' * 96) + decrypted = yield decryptor.decrypt() + assert decrypted.getvalue() == snowden1 + + + @defer.inlineCallbacks + def test_encrypt_and_decrypt(self): + """ + Check that encrypting and decrypting gives same doc. """ - Test encrypting and decrypting documents. + crypto = _crypto.SoledadCrypto('A' * 96) + payload = {'key': 'someval'} + doc1 = SoledadDocument('id1', '1', json.dumps(payload)) + + encrypted = yield crypto.encrypt_doc(doc1) + assert encrypted != payload + assert 'raw' in encrypted + doc2 = SoledadDocument('id1', '1') + doc2.set_json(encrypted) + decrypted = yield crypto.decrypt_doc(doc2) + assert len(decrypted) != 0 + assert json.loads(decrypted) == payload + + + @defer.inlineCallbacks + def test_decrypt_with_wrong_mac_raises(self): """ - simpledoc = {'key': 'val'} - doc1 = SoledadDocument(doc_id='id') - doc1.content = simpledoc - - # encrypt doc - doc1.set_json(self._soledad._crypto.encrypt_doc(doc1)) - # assert content is different and includes keys - self.assertNotEqual( - simpledoc, doc1.content, - 'incorrect document encryption') - self.assertTrue(ENC_JSON_KEY in doc1.content) - self.assertTrue(ENC_SCHEME_KEY in doc1.content) - # decrypt doc - doc1.set_json(self._soledad._crypto.decrypt_doc(doc1)) - self.assertEqual( - simpledoc, doc1.content, 'incorrect document encryption') + Trying to decrypt a document with wrong MAC should raise. + """ + crypto = _crypto.SoledadCrypto('A' * 96) + payload = {'key': 'someval'} + doc1 = SoledadDocument('id1', '1', json.dumps(payload)) + + encrypted = yield crypto.encrypt_doc(doc1) + encdict = json.loads(encrypted) + raw = base64.urlsafe_b64decode(str(encdict['raw'])) + # mess with MAC + messed = raw[:-64] + '0' * 64 + newraw = base64.urlsafe_b64encode(str(messed)) + doc2 = SoledadDocument('id1', '1') + doc2.set_json(json.dumps({"raw": str(newraw)})) + + with pytest.raises(_crypto.InvalidBlob): + decrypted = yield crypto.decrypt_doc(doc2) + class RecoveryDocumentTestCase(BaseSoledadTest): @@ -146,60 +283,22 @@ class SoledadSecretsTestCase(BaseSoledadTest): "Should have a secret at this point") -class MacAuthTestCase(BaseSoledadTest): - - def test_decrypt_with_wrong_mac_raises(self): - """ - Trying to decrypt a document with wrong MAC should raise. - """ - simpledoc = {'key': 'val'} - doc = SoledadDocument(doc_id='id') - doc.content = simpledoc - # encrypt doc - doc.set_json(self._soledad._crypto.encrypt_doc(doc)) - self.assertTrue(MAC_KEY in doc.content) - self.assertTrue(MAC_METHOD_KEY in doc.content) - # mess with MAC - doc.content[MAC_KEY] = '1234567890ABCDEF' - # try to decrypt doc - self.assertRaises( - WrongMacError, - self._soledad._crypto.decrypt_doc, doc) - - def test_decrypt_with_unknown_mac_method_raises(self): - """ - Trying to decrypt a document with unknown MAC method should raise. - """ - simpledoc = {'key': 'val'} - doc = SoledadDocument(doc_id='id') - doc.content = simpledoc - # encrypt doc - doc.set_json(self._soledad._crypto.encrypt_doc(doc)) - self.assertTrue(MAC_KEY in doc.content) - self.assertTrue(MAC_METHOD_KEY in doc.content) - # mess with MAC method - doc.content[MAC_METHOD_KEY] = 'mymac' - # try to decrypt doc - self.assertRaises( - UnknownMacMethodError, - self._soledad._crypto.decrypt_doc, doc) - class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): # generate 256-bit key key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) + iv, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') - plaintext = crypto.decrypt_sym(cyphertext, key, iv) + plaintext = _crypto.decrypt_sym(cyphertext, key, iv) self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) + iv, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -208,13 +307,13 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongiv = rawiv while wrongiv == rawiv: wrongiv = os.urandom(1) + rawiv[1:] - plaintext = crypto.decrypt_sym( + plaintext = _crypto.decrypt_sym( cyphertext, key, iv=binascii.b2a_base64(wrongiv)) self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) + iv, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -222,5 +321,19 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): # ensure keys are different in case we are extremely lucky while wrongkey == key: wrongkey = os.urandom(32) - plaintext = crypto.decrypt_sym(cyphertext, wrongkey, iv) + plaintext = _crypto.decrypt_sym(cyphertext, wrongkey, iv) self.assertNotEqual('data', plaintext) + + +def _aes_encrypt(key, iv, data): + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + encryptor = cipher.encryptor() + return encryptor.update(data) + encryptor.finalize() + + +def _aes_decrypt(key, iv, data): + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + decryptor = cipher.decryptor() + return decryptor.update(data) + decryptor.finalize() diff --git a/testing/tests/client/test_crypto2.py b/testing/tests/client/test_crypto2.py deleted file mode 100644 index f0f6c4af..00000000 --- a/testing/tests/client/test_crypto2.py +++ /dev/null @@ -1,171 +0,0 @@ -# -*- coding: utf-8 -*- -# test_crypto2.py -# Copyright (C) 2016 LEAP Encryption Access Project -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -Tests for the _crypto module -""" - -import base64 -import binascii -import time -import struct -import StringIO - -import leap.soledad.client -from leap.soledad.client import _crypto - -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.backends import default_backend - -from twisted.trial import unittest - - -snowden1 = ( - "You can't come up against " - "the world's most powerful intelligence " - "agencies and not accept the risk. " - "If they want to get you, over time " - "they will.") - - -def _aes_encrypt(key, iv, data): - backend = default_backend() - cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) - encryptor = cipher.encryptor() - return encryptor.update(data) + encryptor.finalize() - -def _aes_decrypt(key, iv, data): - backend = default_backend() - cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) - decryptor = cipher.decryptor() - return decryptor.update(data) + decryptor.finalize() - - -def test_chunked_encryption(): - key = 'A' * 32 - iv = 'A' * 16 - - fd = StringIO.StringIO() - aes = _crypto.AESEncryptor(key, iv, fd) - - data = snowden1 - block = 16 - - for i in range(len(data)/block): - chunk = data[i * block:(i+1)*block] - aes.write(chunk) - aes.end() - - ciphertext_chunked = fd.getvalue() - ciphertext = _aes_encrypt(key, iv, data) - - assert ciphertext_chunked == ciphertext - - -def test_decrypt(): - key = 'A' * 32 - iv = 'A' * 16 - - data = snowden1 - block = 16 - - ciphertext = _aes_encrypt(key, iv, data) - - fd = StringIO.StringIO() - aes = _crypto.AESDecryptor(key, iv, fd) - - for i in range(len(ciphertext)/block): - chunk = ciphertext[i * block:(i+1)*block] - aes.write(chunk) - aes.end() - - cleartext_chunked = fd.getvalue() - assert cleartext_chunked == data - - - -class BlobTestCase(unittest.TestCase): - - class doc_info: - doc_id = 'D-deadbeef' - rev = '397932e0c77f45fcb7c3732930e7e9b2:1' - - def test_blob_encryptor(self): - - inf = StringIO.StringIO() - inf.write(snowden1) - inf.seek(0) - outf = StringIO.StringIO() - - blob = _crypto.BlobEncryptor( - self.doc_info, inf, result=outf, - secret='A' * 96, iv='B'*16) - - d = blob.encrypt() - d.addCallback(self._test_blob_encryptor_cb, outf) - return d - - def _test_blob_encryptor_cb(self, _, outf): - encrypted = outf.getvalue() - data = base64.urlsafe_b64decode(encrypted) - - assert data[0] == '\x80' - ts, sch, meth = struct.unpack( - 'Qbb', data[1:11]) - assert sch == 1 - assert meth == 1 - iv = data[11:27] - assert iv == 'B' * 16 - doc_id = data[27:37] - assert doc_id == 'D-deadbeef' - - rev = data[37:71] - assert rev == self.doc_info.rev - - ciphertext = data[71:-64] - aes_key = _crypto._get_sym_key_for_doc( - self.doc_info.doc_id, 'A'*96) - assert ciphertext == _aes_encrypt(aes_key, 'B'*16, snowden1) - - decrypted = _aes_decrypt(aes_key, 'B'*16, ciphertext) - assert str(decrypted) == snowden1 - - def test_blob_decryptor(self): - - inf = StringIO.StringIO() - inf.write(snowden1) - inf.seek(0) - outf = StringIO.StringIO() - - blob = _crypto.BlobEncryptor( - self.doc_info, inf, result=outf, - secret='A' * 96, iv='B' * 16) - - def do_decrypt(_, outf): - decryptor = _crypto.BlobDecryptor( - self.doc_info, outf, - secret='A' * 96) - d = decryptor.decrypt() - return d - - d = blob.encrypt() - d.addCallback(do_decrypt, outf) - d.addCallback(self._test_blob_decryptor_cb) - return d - - def _test_blob_decryptor_cb(self, decrypted): - assert decrypted.getvalue() == snowden1 diff --git a/testing/tests/sync/test_encdecpool.py b/testing/tests/sync/test_encdecpool.py deleted file mode 100644 index 7055a765..00000000 --- a/testing/tests/sync/test_encdecpool.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from twisted.internet.defer import inlineCallbacks - -from leap.soledad.client.encdecpool import SyncEncrypterPool - -from leap.soledad.common.document import SoledadDocument -from test_soledad.util import BaseSoledadTest - -DOC_ID = "mydoc" -DOC_REV = "rev" -DOC_CONTENT = {'simple': 'document'} - - -class TestSyncEncrypterPool(BaseSoledadTest): - - def setUp(self): - BaseSoledadTest.setUp(self) - crypto = self._soledad._crypto - sync_db = self._soledad._sync_db - self._pool = SyncEncrypterPool(crypto, sync_db) - self._pool.start() - - def tearDown(self): - self._pool.stop() - BaseSoledadTest.tearDown(self) - - @inlineCallbacks - def test_get_encrypted_doc_returns_none(self): - """ - Test that trying to get an encrypted doc from the pool returns None if - the document was never added for encryption. - """ - doc = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - self.assertIsNone(doc) - - @inlineCallbacks - def test_encrypt_doc_and_get_it_back(self): - """ - Test that the pool actually encrypts a document added to the queue. - """ - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - - yield self._pool.encrypt_doc(doc) - encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - - self.assertIsNotNone(encrypted) diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 3cbefc8b..2528600d 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -27,8 +27,6 @@ from leap.soledad.common.l2db import sync from leap.soledad.common.l2db import vectorclock from leap.soledad.common.l2db import errors -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict from leap.soledad.client.http_target import SoledadHTTPSyncTarget from test_soledad import u1db_tests as tests @@ -545,13 +543,11 @@ class SQLCipherDatabaseSyncTests( self.assertFalse(doc2.has_conflicts) self.sync(self.db2, db3) doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) + + _crypto = self._soledad._crypto + decrypted = _crypto.decrypt_doc(doc3) + doc3.set_json(decrypted) + self.assertEqual(doc4.get_json(), doc3.get_json()) self.assertFalse(doc3.has_conflicts) self.db1.close() -- cgit v1.2.3 From 69958161af00958b1d92ab6153a591aa89199e35 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Sep 2016 03:08:20 -0400 Subject: [refactor] remove lingering refs to syncdb --- testing/tests/sync/test_sqlcipher_sync.py | 6 +----- testing/tests/sync/test_sync_deferred.py | 10 ++-------- testing/tests/sync/test_sync_target.py | 19 ++++--------------- 3 files changed, 7 insertions(+), 28 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 2528600d..c3cd8444 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -709,13 +709,9 @@ def make_local_db_and_soledad_target( test.startTwistedServer() replica_uid = os.path.basename(path) db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool st = soledad_sync_target( test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=source_replica_uid) return db, st target_scenarios = [ diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py index 482b150c..eb71ea73 100644 --- a/testing/tests/sync/test_sync_deferred.py +++ b/testing/tests/sync/test_sync_deferred.py @@ -71,12 +71,10 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): import binascii tohex = binascii.b2a_hex key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) + dbpath, key, is_raw_key=True, create=False) self.db1 = SQLCipherDatabase(self.opts) self.db2 = self.request_state._create_database('test') @@ -139,15 +137,11 @@ class TestSoledadDbSyncDeferredEncDecr( and Token auth. """ replica_uid = self._soledad._dbpool.replica_uid - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer target = soledad_sync_target( self, self.db2._dbname, - source_replica_uid=replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=replica_uid) self.addCleanup(target.close) return sync.SoledadSynchronizer( dbsyncer, diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 997dcdcd..b301c48b 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -130,13 +130,9 @@ def make_local_db_and_soledad_target( test.startTwistedServer() replica_uid = os.path.basename(path) db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool st = soledad_sync_target( test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=source_replica_uid) return db, st @@ -167,15 +163,11 @@ class TestSoledadSyncTarget( def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex): if self.port is None: self.startTwistedServer() - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool if path is None: path = self.db2._dbname target = self.sync_target( self, path, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=source_replica_uid) self.addCleanup(target.close) return target @@ -811,12 +803,10 @@ class TestSoledadDbSync( import binascii tohex = binascii.b2a_hex key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) + dbpath, key, is_raw_key=True, create=False) self.db1 = SQLCipherDatabase(self.opts) self.db2 = self.request_state._create_database(replica_uid='test') @@ -855,8 +845,7 @@ class TestSoledadDbSync( self.opts, crypto, replica_uid, - None, - defer_encryption=True) + None) self.dbsyncer = dbsyncer return dbsyncer.sync(target_url, creds=creds) -- cgit v1.2.3 From 40742021a8beeb68b159456b423e4c3674f7926d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Sep 2016 16:12:33 -0400 Subject: [bug] fix import --- testing/tests/sync/test_sync_target.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index b301c48b..7c93cd7c 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -31,10 +31,10 @@ from twisted.internet import defer from leap.soledad.client import http_target as target from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver -from leap.soledad.client import crypto from leap.soledad.client.sqlcipher import SQLCipherU1DBSync from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client import _crypto from leap.soledad.common import l2db @@ -71,7 +71,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): doc = SoledadDocument('i', rev='r') doc.content = {'a': 'b'} - encrypted_docstr = crypto.SoledadCrypto('').encrypt_doc(doc) + encrypted_docstr = _crypto.SoledadCrypto('').encrypt_doc(doc) with self.assertRaises(l2db.errors.BrokenSyncStream): self.parse("[\r\n{},\r\n]") -- cgit v1.2.3 From 781984f3485b1fd479d09278a665f599c1bd10dc Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 22 Sep 2016 20:37:23 -0300 Subject: [test] fix test and remove leftovers defer_encryption --- testing/tests/server/test_server.py | 11 +++-------- testing/tests/sync/test_sqlcipher_sync.py | 4 ---- testing/tests/sync/test_sync_target.py | 4 ++-- 3 files changed, 5 insertions(+), 14 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 6bbcf002..a7cc97d4 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -41,7 +41,7 @@ from test_soledad.util import ( BaseSoledadTest, ) -from leap.soledad.common import crypto +from leap.soledad.client import _crypto from leap.soledad.client import Soledad from leap.soledad.server.config import load_configuration from leap.soledad.server.config import CONFIG_DEFAULTS @@ -412,13 +412,8 @@ class EncryptedSyncTestCase( self.assertEqual(soldoc.doc_id, couchdoc.doc_id) self.assertEqual(soldoc.rev, couchdoc.rev) couch_content = couchdoc.content.keys() - self.assertEqual(6, len(couch_content)) - self.assertTrue(crypto.ENC_JSON_KEY in couch_content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couch_content) - self.assertTrue(crypto.ENC_METHOD_KEY in couch_content) - self.assertTrue(crypto.ENC_IV_KEY in couch_content) - self.assertTrue(crypto.MAC_KEY in couch_content) - self.assertTrue(crypto.MAC_METHOD_KEY in couch_content) + self.assertEqual(['raw'], couch_content) + self.assertTrue(_crypto.is_symmetrically_encrypted(couchdoc.get_json())) d = sol1.get_all_docs() d.addCallback(_db1AssertEmptyDocList) diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index c3cd8444..029164eb 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -544,10 +544,6 @@ class SQLCipherDatabaseSyncTests( self.sync(self.db2, db3) doc3 = db3.get_doc('the-doc') - _crypto = self._soledad._crypto - decrypted = _crypto.decrypt_doc(doc3) - doc3.set_json(decrypted) - self.assertEqual(doc4.get_json(), doc3.get_json()) self.assertFalse(doc3.has_conflicts) self.db1.close() diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 7c93cd7c..ef034142 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -71,7 +71,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): doc = SoledadDocument('i', rev='r') doc.content = {'a': 'b'} - encrypted_docstr = _crypto.SoledadCrypto('').encrypt_doc(doc) + encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc) with self.assertRaises(l2db.errors.BrokenSyncStream): self.parse("[\r\n{},\r\n]") @@ -589,9 +589,9 @@ class SoledadDatabaseSyncTargetTests( [], 'other-replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(2, new_gen) self.assertEqual( (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) if self.whitebox: self.assertEqual(self.db._last_exchange_log['return'], {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) -- cgit v1.2.3 From 288434178a4e89f86b9740cfe77a4dc0ce9e45f7 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 19 Oct 2016 02:14:46 -0300 Subject: [bug] total number of documents Parsing from metadata we can store the total of docs and handle it for the doc parser in order to be able to keep consistent events info. --- testing/tests/sync/test_sync_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index ef034142..17223606 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -63,7 +63,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ def parse(self, stream): - parser = DocStreamReceiver(None, None, lambda x, y: 42) + parser = DocStreamReceiver(None, None, lambda *_: 42) parser.dataReceived(stream) parser.finish() -- cgit v1.2.3 From 2505f61f7374cd0afeb9392c03589607d7b63b64 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 19 Oct 2016 14:05:27 -0300 Subject: [refactor] stop using leap.common.http We aren't using leap.common.http implementation and we need specific features from original Twisted Web Agent. This commit implements it on HTTP Targer. --- testing/tests/sync/test_sync.py | 1 - testing/tests/sync/test_sync_deferred.py | 1 - testing/tests/sync/test_sync_target.py | 2 -- 3 files changed, 4 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index a7d0a92b..a434e944 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -184,7 +184,6 @@ class TestSoledadDbSync( target = soledad_sync_target( self, self.db2._dbname, source_replica_uid=self._soledad._dbpool.replica_uid) - self.addCleanup(target.close) return sync.SoledadSynchronizer( self.db, target).sync() diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py index eb71ea73..001612a6 100644 --- a/testing/tests/sync/test_sync_deferred.py +++ b/testing/tests/sync/test_sync_deferred.py @@ -142,7 +142,6 @@ class TestSoledadDbSyncDeferredEncDecr( target = soledad_sync_target( self, self.db2._dbname, source_replica_uid=replica_uid) - self.addCleanup(target.close) return sync.SoledadSynchronizer( dbsyncer, target).sync() diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 17223606..fd1d413e 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -168,7 +168,6 @@ class TestSoledadSyncTarget( target = self.sync_target( self, path, source_replica_uid=source_replica_uid) - self.addCleanup(target.close) return target def setUp(self): @@ -389,7 +388,6 @@ class SoledadDatabaseSyncTargetTests( def tearDown(self): self.db.close() - self.st.close() tests.TestCaseWithServer.tearDown(self) SoledadWithCouchServerMixin.tearDown(self) -- cgit v1.2.3 From a45084e4beb3fa16962735d7cebfa9fdac73dc6c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Oct 2016 17:42:26 -0300 Subject: [tests] fall back to TestCase classes This test was using pytest, but it was making other trial based tests fail. I couldn't figure out why, but falling back to TestCase solved it. --- testing/tests/couch/conftest.py | 31 ------------------------------- testing/tests/couch/test_state.py | 33 ++++++++++++++++++++------------- 2 files changed, 20 insertions(+), 44 deletions(-) delete mode 100644 testing/tests/couch/conftest.py (limited to 'testing/tests') diff --git a/testing/tests/couch/conftest.py b/testing/tests/couch/conftest.py deleted file mode 100644 index 1074f091..00000000 --- a/testing/tests/couch/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -import couchdb -import pytest -import random -import string - - -@pytest.fixture -def random_name(): - return 'user-' + ''.join( - random.choice( - string.ascii_lowercase) for _ in range(10)) - - -class RandomDatabase(object): - - def __init__(self, couch_url, name): - self.couch_url = couch_url - self.name = name - self.server = couchdb.client.Server(couch_url) - self.database = self.server.create(name) - - def teardown(self): - self.server.delete(self.name) - - -@pytest.fixture -def db(random_name, request): - couch_url = request.config.getoption('--couch-url') - db = RandomDatabase(couch_url, random_name) - request.addfinalizer(db.teardown) - return db diff --git a/testing/tests/couch/test_state.py b/testing/tests/couch/test_state.py index e293b5b8..e5ac3704 100644 --- a/testing/tests/couch/test_state.py +++ b/testing/tests/couch/test_state.py @@ -1,25 +1,32 @@ import pytest - from leap.soledad.common.couch import CONFIG_DOC_ID from leap.soledad.common.couch import SCHEMA_VERSION from leap.soledad.common.couch import SCHEMA_VERSION_KEY from leap.soledad.common.couch.state import CouchServerState +from uuid import uuid4 from leap.soledad.common.errors import WrongCouchSchemaVersionError from leap.soledad.common.errors import MissingCouchConfigDocumentError +from test_soledad.util import CouchDBTestCase + +class CouchDesignDocsTests(CouchDBTestCase): -def test_wrong_couch_version_raises(db): - wrong_schema_version = SCHEMA_VERSION + 1 - db.database.create( - {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version}) - with pytest.raises(WrongCouchSchemaVersionError): - CouchServerState(db.couch_url, create_cmd='/bin/echo', - check_schema_versions=True) + def setUp(self): + CouchDBTestCase.setUp(self) + self.db = self.couch_server.create('user-' + uuid4().hex) + self.addCleanup(self.delete_db, self.db.name) + def test_wrong_couch_version_raises(self): + wrong_schema_version = SCHEMA_VERSION + 1 + self.db.create( + {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version}) + with pytest.raises(WrongCouchSchemaVersionError): + CouchServerState(self.couch_url, create_cmd='/bin/echo', + check_schema_versions=True) -def test_missing_config_doc_raises(db): - db.database.create({}) - with pytest.raises(MissingCouchConfigDocumentError): - CouchServerState(db.couch_url, create_cmd='/bin/echo', - check_schema_versions=True) + def test_missing_config_doc_raises(self): + self.db.create({}) + with pytest.raises(MissingCouchConfigDocumentError): + CouchServerState(self.couch_url, create_cmd='/bin/echo', + check_schema_versions=True) -- cgit v1.2.3 From 349e42d73225282935b2d4677e778821db25634b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Oct 2016 21:37:56 -0300 Subject: [tests] improve doc creation on benchmarks If we create all at once we cant test higher loads because it will try to hold all in memory at the same time. Also, this code is smaller and more readable. --- testing/tests/perf/test_sync.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 0b48a0b9..7b3c4bf0 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -1,17 +1,11 @@ import pytest -from twisted.internet.defer import gatherResults - +@pytest.inlineCallbacks def load_up(client, amount, payload): - deferreds = [] # create a bunch of local documents for i in xrange(amount): - d = client.create_doc({'content': payload}) - deferreds.append(d) - d = gatherResults(deferreds) - d.addCallback(lambda _: None) - return d + yield client.create_doc({'content': payload}) def create_upload(uploads, size): -- cgit v1.2.3 From e14a50f15f34898a00f4afa1eb288ef708b4fbd4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Oct 2016 21:40:05 -0300 Subject: [tests] remove test_sync_very_large_files We have benchmarks now to test sync limits and 100mb is too far from current needs. --- testing/tests/server/test_server.py | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index a7cc97d4..2a3cb751 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -468,16 +468,6 @@ class EncryptedSyncTestCase( """ return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëíìïóòöõúùüñç') - def test_sync_very_large_files(self): - """ - Test if Soledad can sync very large files. - """ - self.skipTest( - "Work in progress. For reference, see: " - "https://leap.se/code/issues/7370") - length = 100 * (10 ** 6) # 100 MB - return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) - def test_sync_many_small_files(self): """ Test if Soledad can sync many smallfiles. -- cgit v1.2.3 From cc81bb2a8ed0e989159f17061a567230a5059c21 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 11 Nov 2016 22:47:56 -0300 Subject: [feature] Adds back support to deprecated crypto Will be removed when we have the proper tool to migrate data. --- testing/tests/server/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 2a3cb751..2f958b29 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -413,7 +413,7 @@ class EncryptedSyncTestCase( self.assertEqual(soldoc.rev, couchdoc.rev) couch_content = couchdoc.content.keys() self.assertEqual(['raw'], couch_content) - self.assertTrue(_crypto.is_symmetrically_encrypted(couchdoc.get_json())) + self.assertTrue(_crypto.is_symmetrically_encrypted(couchdoc)) d = sol1.get_all_docs() d.addCallback(_db1AssertEmptyDocList) -- cgit v1.2.3 From 45763ed191296df9d0c5a3b9b37a07ebee3293ad Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 16 Nov 2016 22:14:01 -0300 Subject: [tests] remove test_sync_deferred Deferred encryption option is gone. --- testing/tests/sync/test_sync_deferred.py | 182 ------------------------------- 1 file changed, 182 deletions(-) delete mode 100644 testing/tests/sync/test_sync_deferred.py (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py deleted file mode 100644 index 001612a6..00000000 --- a/testing/tests/sync/test_sync_deferred.py +++ /dev/null @@ -1,182 +0,0 @@ -# test_sync_deferred.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: sync with deferred encryption. -""" -import time -import os -import random -import string -import shutil - -from twisted.internet import defer - -from leap.soledad.client import sync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from testscenarios import TestWithScenarios - -from test_soledad import u1db_tests as tests -from test_soledad.util import ADDRESS -from test_soledad.util import SoledadWithCouchServerMixin -from test_soledad.util import make_soledad_app -from test_soledad.util import soledad_sync_target - - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): - - """ - Another base class for testing the deferred encryption during - the syncs, using the intermediate database. - """ - defer_sync_encryption = True - - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - os.unlink(self.db1_file) - self.db_pass = DBPASS - self.email = ADDRESS - - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # open test dbs: db1 will be the local sqlcipher db (which - # instantiates a syncdb). We use the self._soledad instance that was - # already created on some setUp method. - import binascii - tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) - dbpath = self._soledad._local_db_path - - self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False) - self.db1 = SQLCipherDatabase(self.opts) - - self.db2 = self.request_state._create_database('test') - - def tearDown(self): - # XXX should not access "private" attrs - shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - SoledadWithCouchServerMixin.tearDown(self) - - -class SyncTimeoutError(Exception): - - """ - Dummy exception to notify timeout during sync. - """ - pass - - -class TestSoledadDbSyncDeferredEncDecr( - TestWithScenarios, - BaseSoledadDeferredEncTest, - tests.TestCaseWithServer): - - """ - Test db.sync remote sync shortcut. - Case with deferred encryption: using the intermediate - syncdb. - """ - - scenarios = [ - ('http', { - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - }), - ] - - oauth = False - token = True - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - BaseSoledadDeferredEncTest.setUp(self) - self.server = self.server_thread = None - self.syncer = None - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - dbsyncer = getattr(self, 'dbsyncer', None) - if dbsyncer: - dbsyncer.close() - BaseSoledadDeferredEncTest.tearDown(self) - - def do_sync(self): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - replica_uid = self._soledad._dbpool.replica_uid - dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer - - target = soledad_sync_target( - self, self.db2._dbname, - source_replica_uid=replica_uid) - return sync.SoledadSynchronizer( - dbsyncer, - target).sync() - - def wait_for_sync(self): - """ - Wait for sync to finish. - """ - wait = 0 - syncer = self.syncer - if syncer is not None: - while syncer.syncing: - time.sleep(WAIT_STEP) - wait += WAIT_STEP - if wait >= MAX_WAIT: - raise SyncTimeoutError - - @defer.inlineCallbacks - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = yield self.do_sync() - - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) -- cgit v1.2.3 From 632cd2414a47bc7f471d8f501ab6250293aedf51 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 17 Nov 2016 00:38:55 -0300 Subject: [bug] include_deleted=True on sync Also refactored tests and code to stop relying on old parameters which included docs instead of get_doc calls. --- testing/tests/sync/test_sync.py | 19 ++++++++++++++++ testing/tests/sync/test_sync_target.py | 40 +++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 18 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index a434e944..76757c5b 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -19,6 +19,7 @@ import threading import time from urlparse import urljoin +from mock import Mock from twisted.internet import defer from testscenarios import TestWithScenarios @@ -210,3 +211,21 @@ class TestSoledadDbSync( self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration + + +class TestSoledadSynchronizer(BaseSoledadTest): + + def setUp(self): + BaseSoledadTest.setUp(self) + self.db = Mock() + self.target = Mock() + self.synchronizer = sync.SoledadSynchronizer( + self.db, + self.target) + + def test_docs_by_gen_includes_deleted(self): + changes = [('id', 'gen', 'trans')] + docs_by_gen = self.synchronizer._docs_by_gen_from_changes(changes) + f, args, kwargs = docs_by_gen[0][0] + self.assertIn('include_deleted', kwargs) + self.assertTrue(kwargs['include_deleted']) diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index fd1d413e..e32f08b3 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -199,8 +199,9 @@ class TestSoledadSyncTarget( other_docs.append((doc.doc_id, doc.rev, doc.get_json())) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertEqual(1, new_gen) self.assertGetEncryptedDoc( @@ -247,10 +248,12 @@ class TestSoledadSyncTarget( doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') doc2 = self.make_document('doc-here2', 'replica:1', '{"value": "here2"}') + get_doc1 = (lambda _: doc1, (1,), {}) + get_doc2 = (lambda _: doc2, (2,), {}) with self.assertRaises(l2db.errors.U1DBError): yield remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], + [(get_doc1, 10, 'T-sid'), (get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, @@ -265,7 +268,7 @@ class TestSoledadSyncTarget( # retry trigger_ids = [] new_gen, trans_id = yield remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, + [(get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here2', 'replica:1', '{"value": "here2"}', @@ -295,8 +298,9 @@ class TestSoledadSyncTarget( replica_uid_box.append(replica_uid) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc, ensure_callback=ensure_cb) self.assertEqual(1, new_gen) @@ -408,8 +412,8 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] + ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}), + 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -430,10 +434,10 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc), {}), 10, 'T-1'), + ((self.make_document, + ('doc-id2', 'replica:1', tests.nested_doc), {}), 11, 'T-2')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -515,7 +519,7 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json('{}') edit_rev = 'replica:1|' + doc.rev docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -534,7 +538,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + ((self.make_document, (doc.doc_id, 'replica:1', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, @@ -554,7 +558,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) gen, txid = self.db._get_generation_info() docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), + ((self.make_document, (doc.doc_id, doc.rev, tests.simple_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=gen, @@ -600,7 +604,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -625,7 +629,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -646,7 +650,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -657,8 +661,8 @@ class SoledadDatabaseSyncTargetTests( def test_sync_exchange_converged_handling(self): doc = self.db.create_doc_from_json(tests.simple_doc) docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5, 'T-bar')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, -- cgit v1.2.3 From 529dbdf27804f12da80907d25c412d10e9fa3763 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 17 Nov 2016 01:33:04 -0300 Subject: [style] fix pep8 and confs Fixes setup.cfg, adding current exclude rules, simplified tox.ini to use setup.cfg and fixed all. --- testing/tests/client/test_crypto.py | 27 ++++++++++----------------- testing/tests/couch/test_command.py | 3 ++- testing/tests/perf/test_crypto.py | 14 +++++++------- testing/tests/perf/test_sqlcipher.py | 12 ++++++------ testing/tests/perf/test_sync.py | 12 ++++++------ testing/tests/sync/test_sqlcipher_sync.py | 1 + testing/tests/sync/test_sync_mutex.py | 1 + testing/tests/sync/test_sync_target.py | 11 +++++++---- 8 files changed, 40 insertions(+), 41 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index dc3054f2..483c7803 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -59,8 +59,8 @@ class AESTest(unittest.TestCase): data = snowden1 block = 16 - for i in range(len(data)/block): - chunk = data[i * block:(i+1)*block] + for i in range(len(data) / block): + chunk = data[i * block:(i + 1) * block] aes.write(chunk) aes.end() @@ -69,7 +69,6 @@ class AESTest(unittest.TestCase): assert ciphertext_chunked == ciphertext - def test_decrypt(self): key = 'A' * 32 iv = 'A' * 16 @@ -82,8 +81,8 @@ class AESTest(unittest.TestCase): fd = BytesIO() aes = _crypto.AESDecryptor(key, iv, fd) - for i in range(len(ciphertext)/block): - chunk = ciphertext[i * block:(i+1)*block] + for i in range(len(ciphertext) / block): + chunk = ciphertext[i * block:(i + 1) * block] aes.write(chunk) aes.end() @@ -91,7 +90,6 @@ class AESTest(unittest.TestCase): assert cleartext_chunked == data - class BlobTestCase(unittest.TestCase): class doc_info: @@ -108,13 +106,13 @@ class BlobTestCase(unittest.TestCase): blob = _crypto.BlobEncryptor( self.doc_info, inf, result=outf, - secret='A' * 96, iv='B'*16) + secret='A' * 96, iv='B' * 16) encrypted = yield blob.encrypt() data = base64.urlsafe_b64decode(encrypted.getvalue()) assert data[0] == '\x80' - ts, sch, meth = struct.unpack( + ts, sch, meth = struct.unpack( 'Qbb', data[1:11]) assert sch == 1 assert meth == 1 @@ -128,13 +126,12 @@ class BlobTestCase(unittest.TestCase): ciphertext = data[71:-64] aes_key = _crypto._get_sym_key_for_doc( - self.doc_info.doc_id, 'A'*96) - assert ciphertext == _aes_encrypt(aes_key, 'B'*16, snowden1) + self.doc_info.doc_id, 'A' * 96) + assert ciphertext == _aes_encrypt(aes_key, 'B' * 16, snowden1) - decrypted = _aes_decrypt(aes_key, 'B'*16, ciphertext) + decrypted = _aes_decrypt(aes_key, 'B' * 16, ciphertext) assert str(decrypted) == snowden1 - @defer.inlineCallbacks def test_blob_decryptor(self): @@ -154,7 +151,6 @@ class BlobTestCase(unittest.TestCase): decrypted = yield decryptor.decrypt() assert decrypted.getvalue() == snowden1 - @defer.inlineCallbacks def test_encrypt_and_decrypt(self): """ @@ -173,7 +169,6 @@ class BlobTestCase(unittest.TestCase): assert len(decrypted) != 0 assert json.loads(decrypted) == payload - @defer.inlineCallbacks def test_decrypt_with_wrong_mac_raises(self): """ @@ -193,8 +188,7 @@ class BlobTestCase(unittest.TestCase): doc2.set_json(json.dumps({"raw": str(newraw)})) with pytest.raises(_crypto.InvalidBlob): - decrypted = yield crypto.decrypt_doc(doc2) - + yield crypto.decrypt_doc(doc2) class RecoveryDocumentTestCase(BaseSoledadTest): @@ -283,7 +277,6 @@ class SoledadSecretsTestCase(BaseSoledadTest): "Should have a secret at this point") - class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): diff --git a/testing/tests/couch/test_command.py b/testing/tests/couch/test_command.py index 68097fb1..9fb2c153 100644 --- a/testing/tests/couch/test_command.py +++ b/testing/tests/couch/test_command.py @@ -25,6 +25,7 @@ class CommandBasedDBCreationTest(unittest.TestCase): state.ensure_database, "user-1337") def test_raises_unauthorized_by_default(self): - state = couch_state.CouchServerState("url", check_schema_versions=False) + state = couch_state.CouchServerState("url", + check_schema_versions=False) self.assertRaises(u1db_errors.Unauthorized, state.ensure_database, "user-1337") diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py index 9ce418ba..367c3b5b 100644 --- a/testing/tests/perf/test_crypto.py +++ b/testing/tests/perf/test_crypto.py @@ -5,7 +5,7 @@ SIZE_LIMT environment variable. For instance, to keep the maximum payload at 1MB: -SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py """ import pytest import os @@ -45,7 +45,7 @@ def create_doc_decryption(size): doc = SoledadDocument( doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) - + encrypted_doc = yield crypto.encrypt_doc(doc) doc.set_json(encrypted_doc) @@ -74,16 +74,16 @@ def create_raw_decryption(size): # plugin. encryption_tests = [ - ('10k', 1E4), + ('10k', 1E4), ('100k', 1E5), ('500k', 5E5), - ('1M', 1E6), - ('10M', 1E7), - ('50M', 5E7), + ('1M', 1E6), + ('10M', 1E7), + ('50M', 5E7), ] for name, size in encryption_tests: - if size < LIMIT: + if size < LIMIT: sz = int(size) globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py index e7a54228..39c9e3ad 100644 --- a/testing/tests/perf/test_sqlcipher.py +++ b/testing/tests/perf/test_sqlcipher.py @@ -29,10 +29,10 @@ def build_test_sqlcipher_create(amount, size): return test -test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500*1000) -test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100*1000) -test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10*1000) +test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500 * 1000) +test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) +test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000) # synchronous -test_create_20_500k = build_test_sqlcipher_create(20, 500*1000) -test_create_100_100k = build_test_sqlcipher_create(100, 100*1000) -test_create_1000_10k = build_test_sqlcipher_create(1000, 10*1000) +test_create_20_500k = build_test_sqlcipher_create(20, 500 * 1000) +test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) +test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 7b3c4bf0..1bf6cc21 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -21,9 +21,9 @@ def create_upload(uploads, size): return test -test_upload_20_500k = create_upload(20, 500*1000) -test_upload_100_100k = create_upload(100, 100*1000) -test_upload_1000_10k = create_upload(1000, 10*1000) +test_upload_20_500k = create_upload(20, 500 * 1000) +test_upload_100_100k = create_upload(100, 100 * 1000) +test_upload_1000_10k = create_upload(1000, 10 * 1000) def create_download(downloads, size): @@ -46,9 +46,9 @@ def create_download(downloads, size): return test -test_download_20_500k = create_download(20, 500*1000) -test_download_100_100k = create_download(100, 100*1000) -test_download_1000_10k = create_download(1000, 10*1000) +test_download_20_500k = create_download(20, 500 * 1000) +test_download_100_100k = create_download(100, 100 * 1000) +test_download_1000_10k = create_download(1000, 10 * 1000) @pytest.inlineCallbacks diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 029164eb..26f63a40 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -710,6 +710,7 @@ def make_local_db_and_soledad_target( source_replica_uid=source_replica_uid) return db, st + target_scenarios = [ ('leap', { 'create_db_and_target': make_local_db_and_soledad_target, diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py index 2bcb3aec..432a3cd2 100644 --- a/testing/tests/sync/test_sync_mutex.py +++ b/testing/tests/sync/test_sync_mutex.py @@ -66,6 +66,7 @@ def _timed_sync(self): d.addBoth(_store_finish_time) return d + SoledadSynchronizer.sync = _timed_sync # -- end of monkey-patching diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index e32f08b3..dd69ffa1 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -412,7 +412,8 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}), + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc,), {}), 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, @@ -519,7 +520,8 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json('{}') edit_rev = 'replica:1|' + doc.rev docs_by_gen = [ - ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')] + ((self.make_document, (doc.doc_id, edit_rev, None), {}), + 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -662,8 +664,8 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json(tests.simple_doc) docs_by_gen = [ ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), - ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5, - 'T-bar')] + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), + 5, 'T-bar')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -746,6 +748,7 @@ class SoledadDatabaseSyncTargetTests( yield self.st.record_sync_info('replica', 0, 'T-sid') self.assertEqual(expected, called) + WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" -- cgit v1.2.3 From 5294f5b9ae01429e6d0ee75c8dc98441ba760845 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 17 Nov 2016 22:14:51 -0200 Subject: [test] use tags for selecting benchmark tests --- testing/tests/perf/conftest.py | 3 --- testing/tests/perf/test_crypto.py | 3 +++ testing/tests/perf/test_misc.py | 2 ++ testing/tests/perf/test_sqlcipher.py | 2 ++ testing/tests/perf/test_sync.py | 2 ++ 5 files changed, 9 insertions(+), 3 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py index 2964936b..d08ea61d 100644 --- a/testing/tests/perf/conftest.py +++ b/testing/tests/perf/conftest.py @@ -25,9 +25,6 @@ server.ensure_server() def pytest_addoption(parser): - parser.addoption( - "--couch-url", type="string", default="http://127.0.0.1:5984", - help="the url for the couch server to be used during tests") parser.addoption( "--num-docs", type="int", default=100, help="the number of documents to use in performance tests") diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py index 367c3b5b..a438ee49 100644 --- a/testing/tests/perf/test_crypto.py +++ b/testing/tests/perf/test_crypto.py @@ -18,6 +18,9 @@ from leap.soledad.client import _crypto LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) +pytestmark = pytest.mark.perf + + def create_doc_encryption(size): @pytest.mark.benchmark(group="test_crypto_encrypt_doc") def test_doc_encryption(soledad_client, benchmark, payload): diff --git a/testing/tests/perf/test_misc.py b/testing/tests/perf/test_misc.py index ead48adf..b45dc04e 100644 --- a/testing/tests/perf/test_misc.py +++ b/testing/tests/perf/test_misc.py @@ -1,5 +1,7 @@ import pytest +pytestmark = pytest.mark.perf + @pytest.mark.benchmark(group="test_instance") def test_initialization(soledad_client, benchmark): diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py index 39c9e3ad..807af6e9 100644 --- a/testing/tests/perf/test_sqlcipher.py +++ b/testing/tests/perf/test_sqlcipher.py @@ -5,6 +5,8 @@ import pytest from twisted.internet.defer import gatherResults +pytestmark = pytest.mark.perf + def load_up(client, amount, payload, defer=True): results = [client.create_doc({'content': payload}) for _ in xrange(amount)] diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 1bf6cc21..9bb20389 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -1,5 +1,7 @@ import pytest +pytestmark = pytest.mark.perf + @pytest.inlineCallbacks def load_up(client, amount, payload): -- cgit v1.2.3 From eadc36dbc23368f8d11a8cf7c9bd5571641d5b36 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 17 Nov 2016 22:15:32 -0200 Subject: [test] move fixtures one level up --- testing/tests/conftest.py | 189 +++++++++++++++++++++++++++++++++++++++++ testing/tests/perf/conftest.py | 188 ---------------------------------------- 2 files changed, 189 insertions(+), 188 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 9e4319ac..5698c8a9 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -1,4 +1,29 @@ +import json +import os import pytest +import requests +import signal +import time + +from hashlib import sha512 +from subprocess import call +from urlparse import urljoin +from uuid import uuid4 + +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.client import Soledad + + +# +# default options for all tests +# + +DEFAULT_PASSPHRASE = '123' + +DEFAULT_URL = 'http://127.0.0.1:2424' +DEFAULT_PRIVKEY = 'soledad_privkey.pem' +DEFAULT_CERTKEY = 'soledad_certkey.pem' +DEFAULT_TOKEN = 'an-auth-token' def pytest_addoption(parser): @@ -16,3 +41,167 @@ def couch_url(request): @pytest.fixture def method_tmpdir(request, tmpdir): request.instance.tempdir = tmpdir.strpath + + +# +# remote_db fixture: provides an empty database for a given user in a per +# function scope. +# + +class UserDatabase(object): + + def __init__(self, url, uuid): + self._remote_db_url = urljoin(url, 'user-%s' % uuid) + + def setup(self): + return CouchDatabase.open_database( + url=self._remote_db_url, create=True, replica_uid=None) + + def teardown(self): + requests.delete(self._remote_db_url) + + +@pytest.fixture() +def remote_db(request): + couch_url = request.config.option.couch_url + + def create(uuid): + db = UserDatabase(couch_url, uuid) + request.addfinalizer(db.teardown) + return db.setup() + return create + + +def get_pid(pidfile): + if not os.path.isfile(pidfile): + return 0 + try: + with open(pidfile) as f: + return int(f.read()) + except IOError: + return 0 + + +# +# soledad_server fixture: provides a running soledad server in a per module +# context (same soledad server for all tests in this module). +# + +class SoledadServer(object): + + def __init__(self, tmpdir_factory, couch_url): + tmpdir = tmpdir_factory.mktemp('soledad-server') + self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid') + self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log') + self._couch_url = couch_url + + def start(self): + self._create_conf_file() + # start the server + call([ + 'twistd', + '--logfile=%s' % self._logfile, + '--pidfile=%s' % self._pidfile, + 'web', + '--wsgi=leap.soledad.server.application', + '--port=2424' + ]) + + def _create_conf_file(self): + if not os.access('/etc', os.W_OK): + return + if not os.path.isdir('/etc/soledad'): + os.mkdir('/etc/soledad') + with open('/etc/soledad/soledad-server.conf', 'w') as f: + content = '[soledad-server]\ncouch_url = %s' % self._couch_url + f.write(content) + + def stop(self): + pid = get_pid(self._pidfile) + os.kill(pid, signal.SIGKILL) + + +@pytest.fixture(scope='module') +def soledad_server(tmpdir_factory, request): + couch_url = request.config.option.couch_url + server = SoledadServer(tmpdir_factory, couch_url) + server.start() + request.addfinalizer(server.stop) + return server + + +# +# soledad_dbs fixture: provides all databases needed by soledad server in a per +# module scope (same databases for all tests in this module). +# + +def _token_dbname(): + dbname = 'tokens_' + \ + str(int(time.time() / (30 * 24 * 3600))) + return dbname + + +class SoledadDatabases(object): + + def __init__(self, url): + self._token_db_url = urljoin(url, _token_dbname()) + self._shared_db_url = urljoin(url, 'shared') + + def setup(self, uuid): + self._create_dbs() + self._add_token(uuid) + + def _create_dbs(self): + requests.put(self._token_db_url) + requests.put(self._shared_db_url) + + def _add_token(self, uuid): + token = sha512(DEFAULT_TOKEN).hexdigest() + content = {'type': 'Token', 'user_id': uuid} + requests.put( + self._token_db_url + '/' + token, data=json.dumps(content)) + + def teardown(self): + requests.delete(self._token_db_url) + requests.delete(self._shared_db_url) + + +@pytest.fixture() +def soledad_dbs(request): + couch_url = request.config.option.couch_url + + def create(uuid): + db = SoledadDatabases(couch_url) + request.addfinalizer(db.teardown) + return db.setup(uuid) + return create + + +# +# soledad_client fixture: provides a clean soledad client for a test function. +# + +@pytest.fixture() +def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): + passphrase = DEFAULT_PASSPHRASE + server_url = DEFAULT_URL + token = DEFAULT_TOKEN + default_uuid = uuid4().hex + remote_db(default_uuid) + soledad_dbs(default_uuid) + + # get a soledad instance + def create(): + secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % default_uuid) + local_db_path = os.path.join(tmpdir.strpath, '%s.db' % default_uuid) + soledad_client = Soledad( + default_uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=None, + auth_token=token) + request.addfinalizer(soledad_client.close) + return soledad_client + return create diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py index d08ea61d..a9cc3464 100644 --- a/testing/tests/perf/conftest.py +++ b/testing/tests/perf/conftest.py @@ -1,21 +1,9 @@ -import json -import os import pytest -import requests import random import base64 -import signal -import time -from hashlib import sha512 -from uuid import uuid4 -from subprocess import call -from urlparse import urljoin from twisted.internet import threads, reactor -from leap.soledad.client import Soledad -from leap.soledad.common.couch import CouchDatabase - # we have to manually setup the events server in order to be able to signal # events. This is usually done by the enclosing application using soledad @@ -30,18 +18,6 @@ def pytest_addoption(parser): help="the number of documents to use in performance tests") -# -# default options for all tests -# - -DEFAULT_PASSPHRASE = '123' - -DEFAULT_URL = 'http://127.0.0.1:2424' -DEFAULT_PRIVKEY = 'soledad_privkey.pem' -DEFAULT_CERTKEY = 'soledad_certkey.pem' -DEFAULT_TOKEN = 'an-auth-token' - - @pytest.fixture() def payload(): def generate(size): @@ -52,140 +28,6 @@ def payload(): return generate -# -# soledad_dbs fixture: provides all databases needed by soledad server in a per -# module scope (same databases for all tests in this module). -# - -def _token_dbname(): - dbname = 'tokens_' + \ - str(int(time.time() / (30 * 24 * 3600))) - return dbname - - -class SoledadDatabases(object): - - def __init__(self, url): - self._token_db_url = urljoin(url, _token_dbname()) - self._shared_db_url = urljoin(url, 'shared') - - def setup(self, uuid): - self._create_dbs() - self._add_token(uuid) - - def _create_dbs(self): - requests.put(self._token_db_url) - requests.put(self._shared_db_url) - - def _add_token(self, uuid): - token = sha512(DEFAULT_TOKEN).hexdigest() - content = {'type': 'Token', 'user_id': uuid} - requests.put( - self._token_db_url + '/' + token, data=json.dumps(content)) - - def teardown(self): - requests.delete(self._token_db_url) - requests.delete(self._shared_db_url) - - -@pytest.fixture() -def soledad_dbs(request): - couch_url = request.config.option.couch_url - - def create(uuid): - db = SoledadDatabases(couch_url) - request.addfinalizer(db.teardown) - return db.setup(uuid) - return create - - -# -# remote_db fixture: provides an empty database for a given user in a per -# function scope. -# - -class UserDatabase(object): - - def __init__(self, url, uuid): - self._remote_db_url = urljoin(url, 'user-%s' % uuid) - - def setup(self): - return CouchDatabase.open_database( - url=self._remote_db_url, create=True, replica_uid=None) - - def teardown(self): - requests.delete(self._remote_db_url) - - -@pytest.fixture() -def remote_db(request): - couch_url = request.config.option.couch_url - - def create(uuid): - db = UserDatabase(couch_url, uuid) - request.addfinalizer(db.teardown) - return db.setup() - return create - - -def get_pid(pidfile): - if not os.path.isfile(pidfile): - return 0 - try: - with open(pidfile) as f: - return int(f.read()) - except IOError: - return 0 - - -# -# soledad_server fixture: provides a running soledad server in a per module -# context (same soledad server for all tests in this module). -# - -class SoledadServer(object): - - def __init__(self, tmpdir_factory, couch_url): - tmpdir = tmpdir_factory.mktemp('soledad-server') - self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid') - self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log') - self._couch_url = couch_url - - def start(self): - self._create_conf_file() - # start the server - call([ - 'twistd', - '--logfile=%s' % self._logfile, - '--pidfile=%s' % self._pidfile, - 'web', - '--wsgi=leap.soledad.server.application.wsgi_application', - '--port=2424' - ]) - - def _create_conf_file(self): - if not os.access('/etc', os.W_OK): - return - if not os.path.isdir('/etc/soledad'): - os.mkdir('/etc/soledad') - with open('/etc/soledad/soledad-server.conf', 'w') as f: - content = '[soledad-server]\ncouch_url = %s' % self._couch_url - f.write(content) - - def stop(self): - pid = get_pid(self._pidfile) - os.kill(pid, signal.SIGKILL) - - -@pytest.fixture(scope='module') -def soledad_server(tmpdir_factory, request): - couch_url = request.config.option.couch_url - server = SoledadServer(tmpdir_factory, couch_url) - server.start() - request.addfinalizer(server.stop) - return server - - @pytest.fixture() def txbenchmark(benchmark): def blockOnThread(*args, **kwargs): @@ -213,33 +55,3 @@ def txbenchmark_with_setup(benchmark): rounds=4, warmup_rounds=1) return threads.deferToThread(bench) return blockOnThreadWithSetup - - -# -# soledad_client fixture: provides a clean soledad client for a test function. -# - -@pytest.fixture() -def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): - passphrase = DEFAULT_PASSPHRASE - server_url = DEFAULT_URL - token = DEFAULT_TOKEN - default_uuid = uuid4().hex - remote_db(default_uuid) - soledad_dbs(default_uuid) - - # get a soledad instance - def create(): - secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % uuid4().hex) - local_db_path = os.path.join(tmpdir.strpath, '%s.db' % uuid4().hex) - soledad_client = Soledad( - default_uuid, - unicode(passphrase), - secrets_path=secrets_path, - local_db_path=local_db_path, - server_url=server_url, - cert_file=None, - auth_token=token) - request.addfinalizer(soledad_client.close) - return soledad_client - return create -- cgit v1.2.3 From 87259d4210e3488b00876d7ec83a8cc21e341712 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 17 Nov 2016 22:16:28 -0200 Subject: [test] add test for deprecated crypto format update --- testing/tests/client/test_deprecated_crypto.py | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 testing/tests/client/test_deprecated_crypto.py (limited to 'testing/tests') diff --git a/testing/tests/client/test_deprecated_crypto.py b/testing/tests/client/test_deprecated_crypto.py new file mode 100644 index 00000000..ca1b1558 --- /dev/null +++ b/testing/tests/client/test_deprecated_crypto.py @@ -0,0 +1,67 @@ +import json +import pytest + +from leap.soledad.client import crypto as old_crypto +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common import crypto as common_crypto + +from test_soledad.u1db_tests import simple_doc + + +def deprecate_client_crypto(client): + secret = client._crypto.secret + _crypto = old_crypto.SoledadCrypto(secret) + setattr(client._dbsyncer, '_crypto', _crypto) + return client + + +def couch_database(couch_url, uuid): + db = CouchDatabase(couch_url, "user-%s" % (uuid,)) + return db + + +@pytest.inlineCallbacks +def test_touch_updates_remote_representation( + soledad_client, request): + + client = soledad_client() + deprecated_client = deprecate_client_crypto(soledad_client()) + + couch_url = request.config.option.couch_url + remote = couch_database(couch_url, client._uuid) + + # ensure remote db is empty + gen, docs = remote.get_all_docs() + assert gen == 0 + assert len(docs) == 0 + + # create a doc with deprecated client and sync + yield deprecated_client.create_doc(json.loads(simple_doc)) + yield deprecated_client.sync() + + # check for doc in remote db + gen, docs = remote.get_all_docs() + assert gen == 1 + assert len(docs) == 1 + doc = docs.pop() + content = doc.content + assert common_crypto.ENC_JSON_KEY in content + assert common_crypto.ENC_SCHEME_KEY in content + assert common_crypto.ENC_METHOD_KEY in content + assert common_crypto.ENC_IV_KEY in content + assert common_crypto.MAC_KEY in content + assert common_crypto.MAC_METHOD_KEY in content + + # "touch" the document with a newer client and synx + _, docs = yield client.get_all_docs() + yield client.put_doc(doc) + yield client.sync() + + # check for newer representation of doc in remote db + gen, docs = remote.get_all_docs() + assert gen == 2 + assert len(docs) == 1 + doc = docs.pop() + content = doc.content + assert len(content) == 1 + assert 'raw' in content -- cgit v1.2.3 From 378a07113a713a7c25f0fb8510d18ecdae2198bd Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 17 Nov 2016 22:35:21 -0200 Subject: [test] rename benchmark tests directory and tag --- testing/tests/benchmarks/assets/cert_default.conf | 15 ++++ testing/tests/benchmarks/conftest.py | 57 +++++++++++++ testing/tests/benchmarks/pytest.ini | 2 + testing/tests/benchmarks/test_crypto.py | 99 +++++++++++++++++++++++ testing/tests/benchmarks/test_misc.py | 8 ++ testing/tests/benchmarks/test_sqlcipher.py | 40 +++++++++ testing/tests/benchmarks/test_sync.py | 64 +++++++++++++++ testing/tests/perf/assets/cert_default.conf | 15 ---- testing/tests/perf/conftest.py | 57 ------------- testing/tests/perf/pytest.ini | 2 - testing/tests/perf/test_crypto.py | 99 ----------------------- testing/tests/perf/test_misc.py | 8 -- testing/tests/perf/test_sqlcipher.py | 40 --------- testing/tests/perf/test_sync.py | 64 --------------- 14 files changed, 285 insertions(+), 285 deletions(-) create mode 100644 testing/tests/benchmarks/assets/cert_default.conf create mode 100644 testing/tests/benchmarks/conftest.py create mode 100644 testing/tests/benchmarks/pytest.ini create mode 100644 testing/tests/benchmarks/test_crypto.py create mode 100644 testing/tests/benchmarks/test_misc.py create mode 100644 testing/tests/benchmarks/test_sqlcipher.py create mode 100644 testing/tests/benchmarks/test_sync.py delete mode 100644 testing/tests/perf/assets/cert_default.conf delete mode 100644 testing/tests/perf/conftest.py delete mode 100644 testing/tests/perf/pytest.ini delete mode 100644 testing/tests/perf/test_crypto.py delete mode 100644 testing/tests/perf/test_misc.py delete mode 100644 testing/tests/perf/test_sqlcipher.py delete mode 100644 testing/tests/perf/test_sync.py (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/assets/cert_default.conf b/testing/tests/benchmarks/assets/cert_default.conf new file mode 100644 index 00000000..8043cea3 --- /dev/null +++ b/testing/tests/benchmarks/assets/cert_default.conf @@ -0,0 +1,15 @@ +[ req ] +default_bits = 1024 +default_keyfile = keyfile.pem +distinguished_name = req_distinguished_name +prompt = no +output_password = mypass + +[ req_distinguished_name ] +C = GB +ST = Test State or Province +L = Test Locality +O = Organization Name +OU = Organizational Unit Name +CN = localhost +emailAddress = test@email.address diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py new file mode 100644 index 00000000..a9cc3464 --- /dev/null +++ b/testing/tests/benchmarks/conftest.py @@ -0,0 +1,57 @@ +import pytest +import random +import base64 + +from twisted.internet import threads, reactor + + +# we have to manually setup the events server in order to be able to signal +# events. This is usually done by the enclosing application using soledad +# client (i.e. bitmask client). +from leap.common.events import server +server.ensure_server() + + +def pytest_addoption(parser): + parser.addoption( + "--num-docs", type="int", default=100, + help="the number of documents to use in performance tests") + + +@pytest.fixture() +def payload(): + def generate(size): + random.seed(1337) # same seed to avoid different bench results + payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) + # encode as base64 to avoid ascii encode/decode errors + return base64.b64encode(payload_bytes)[:size] # remove b64 overhead + return generate + + +@pytest.fixture() +def txbenchmark(benchmark): + def blockOnThread(*args, **kwargs): + return threads.deferToThread( + benchmark, threads.blockingCallFromThread, + reactor, *args, **kwargs) + return blockOnThread + + +@pytest.fixture() +def txbenchmark_with_setup(benchmark): + def blockOnThreadWithSetup(setup, f): + def blocking_runner(*args, **kwargs): + return threads.blockingCallFromThread(reactor, f, *args, **kwargs) + + def blocking_setup(): + args = threads.blockingCallFromThread(reactor, setup) + try: + return tuple(arg for arg in args), {} + except TypeError: + return ((args,), {}) if args else None + + def bench(): + return benchmark.pedantic(blocking_runner, setup=blocking_setup, + rounds=4, warmup_rounds=1) + return threads.deferToThread(bench) + return blockOnThreadWithSetup diff --git a/testing/tests/benchmarks/pytest.ini b/testing/tests/benchmarks/pytest.ini new file mode 100644 index 00000000..7a0508ce --- /dev/null +++ b/testing/tests/benchmarks/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +twisted = yes diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py new file mode 100644 index 00000000..ab586bea --- /dev/null +++ b/testing/tests/benchmarks/test_crypto.py @@ -0,0 +1,99 @@ +""" +Benchmarks for crypto operations. +If you don't want to stress your local machine too much, you can pass the +SIZE_LIMT environment variable. + +For instance, to keep the maximum payload at 1MB: + +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +""" +import pytest +import os +import json +from uuid import uuid4 + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import _crypto + +LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) + + +pytestmark = pytest.mark.benchmark + + +def create_doc_encryption(size): + @pytest.mark.benchmark(group="test_crypto_encrypt_doc") + def test_doc_encryption(soledad_client, benchmark, payload): + crypto = soledad_client()._crypto + + DOC_CONTENT = {'payload': payload(size)} + doc = SoledadDocument( + doc_id=uuid4().hex, rev='rev', + json=json.dumps(DOC_CONTENT)) + + benchmark(crypto.encrypt_doc, doc) + return test_doc_encryption + + +# TODO this test is really bullshit, because it's still including +# the json serialization. + +def create_doc_decryption(size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_crypto_decrypt_doc") + def test_doc_decryption(soledad_client, benchmark, payload): + crypto = soledad_client()._crypto + + DOC_CONTENT = {'payload': payload(size)} + doc = SoledadDocument( + doc_id=uuid4().hex, rev='rev', + json=json.dumps(DOC_CONTENT)) + + encrypted_doc = yield crypto.encrypt_doc(doc) + doc.set_json(encrypted_doc) + + benchmark(crypto.decrypt_doc, doc) + return test_doc_decryption + + +def create_raw_encryption(size): + @pytest.mark.benchmark(group="test_crypto_raw_encrypt") + def test_raw_encrypt(benchmark, payload): + key = payload(32) + benchmark(_crypto.encrypt_sym, payload(size), key) + return test_raw_encrypt + + +def create_raw_decryption(size): + @pytest.mark.benchmark(group="test_crypto_raw_decrypt") + def test_raw_decrypt(benchmark, payload): + key = payload(32) + iv, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv) + return test_raw_decrypt + + +# Create the TESTS in the global namespace, they'll be picked by the benchmark +# plugin. + +encryption_tests = [ + ('10k', 1E4), + ('100k', 1E5), + ('500k', 5E5), + ('1M', 1E6), + ('10M', 1E7), + ('50M', 5E7), +] + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) + globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) + + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz) + globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz) diff --git a/testing/tests/benchmarks/test_misc.py b/testing/tests/benchmarks/test_misc.py new file mode 100644 index 00000000..2f32ad7c --- /dev/null +++ b/testing/tests/benchmarks/test_misc.py @@ -0,0 +1,8 @@ +import pytest + +pytestmark = pytest.mark.benchmark + + +@pytest.mark.benchmark(group="test_instance") +def test_initialization(soledad_client, benchmark): + benchmark(soledad_client) diff --git a/testing/tests/benchmarks/test_sqlcipher.py b/testing/tests/benchmarks/test_sqlcipher.py new file mode 100644 index 00000000..7f8842bd --- /dev/null +++ b/testing/tests/benchmarks/test_sqlcipher.py @@ -0,0 +1,40 @@ +''' +Tests SoledadClient/SQLCipher interaction +''' +import pytest + +from twisted.internet.defer import gatherResults + +pytestmark = pytest.mark.benchmark + + +def load_up(client, amount, payload, defer=True): + results = [client.create_doc({'content': payload}) for _ in xrange(amount)] + if defer: + return gatherResults(results) + + +def build_test_sqlcipher_async_create(amount, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_sqlcipher_async_create") + def test(soledad_client, txbenchmark, payload): + client = soledad_client() + yield txbenchmark(load_up, client, amount, payload(size)) + return test + + +def build_test_sqlcipher_create(amount, size): + @pytest.mark.benchmark(group="test_sqlcipher_create") + def test(soledad_client, benchmark, payload): + client = soledad_client()._dbsyncer + benchmark(load_up, client, amount, payload(size), defer=False) + return test + + +test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500 * 1000) +test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) +test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000) +# synchronous +test_create_20_500k = build_test_sqlcipher_create(20, 500 * 1000) +test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) +test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py new file mode 100644 index 00000000..88afe9f8 --- /dev/null +++ b/testing/tests/benchmarks/test_sync.py @@ -0,0 +1,64 @@ +import pytest + +pytestmark = pytest.mark.benchmark + + +@pytest.inlineCallbacks +def load_up(client, amount, payload): + # create a bunch of local documents + for i in xrange(amount): + yield client.create_doc({'content': payload}) + + +def create_upload(uploads, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_upload") + def test(soledad_client, txbenchmark_with_setup, payload): + client = soledad_client() + + def setup(): + return load_up(client, uploads, payload(size)) + + yield txbenchmark_with_setup(setup, client.sync) + return test + + +test_upload_20_500k = create_upload(20, 500 * 1000) +test_upload_100_100k = create_upload(100, 100 * 1000) +test_upload_1000_10k = create_upload(1000, 10 * 1000) + + +def create_download(downloads, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_download") + def test(soledad_client, txbenchmark_with_setup, payload): + client = soledad_client() + + yield load_up(client, downloads, payload(size)) + yield client.sync() + # We could create them directly on couch, but sending them + # ensures we are dealing with properly encrypted docs + + def setup(): + return soledad_client() + + def sync(clean_client): + return clean_client.sync() + yield txbenchmark_with_setup(setup, sync) + return test + + +test_download_20_500k = create_download(20, 500 * 1000) +test_download_100_100k = create_download(100, 100 * 1000) +test_download_1000_10k = create_download(1000, 10 * 1000) + + +@pytest.inlineCallbacks +@pytest.mark.benchmark(group="test_nothing_to_sync") +def test_nothing_to_sync(soledad_client, txbenchmark_with_setup): + def setup(): + return soledad_client() + + def sync(clean_client): + return clean_client.sync() + yield txbenchmark_with_setup(setup, sync) diff --git a/testing/tests/perf/assets/cert_default.conf b/testing/tests/perf/assets/cert_default.conf deleted file mode 100644 index 8043cea3..00000000 --- a/testing/tests/perf/assets/cert_default.conf +++ /dev/null @@ -1,15 +0,0 @@ -[ req ] -default_bits = 1024 -default_keyfile = keyfile.pem -distinguished_name = req_distinguished_name -prompt = no -output_password = mypass - -[ req_distinguished_name ] -C = GB -ST = Test State or Province -L = Test Locality -O = Organization Name -OU = Organizational Unit Name -CN = localhost -emailAddress = test@email.address diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py deleted file mode 100644 index a9cc3464..00000000 --- a/testing/tests/perf/conftest.py +++ /dev/null @@ -1,57 +0,0 @@ -import pytest -import random -import base64 - -from twisted.internet import threads, reactor - - -# we have to manually setup the events server in order to be able to signal -# events. This is usually done by the enclosing application using soledad -# client (i.e. bitmask client). -from leap.common.events import server -server.ensure_server() - - -def pytest_addoption(parser): - parser.addoption( - "--num-docs", type="int", default=100, - help="the number of documents to use in performance tests") - - -@pytest.fixture() -def payload(): - def generate(size): - random.seed(1337) # same seed to avoid different bench results - payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) - # encode as base64 to avoid ascii encode/decode errors - return base64.b64encode(payload_bytes)[:size] # remove b64 overhead - return generate - - -@pytest.fixture() -def txbenchmark(benchmark): - def blockOnThread(*args, **kwargs): - return threads.deferToThread( - benchmark, threads.blockingCallFromThread, - reactor, *args, **kwargs) - return blockOnThread - - -@pytest.fixture() -def txbenchmark_with_setup(benchmark): - def blockOnThreadWithSetup(setup, f): - def blocking_runner(*args, **kwargs): - return threads.blockingCallFromThread(reactor, f, *args, **kwargs) - - def blocking_setup(): - args = threads.blockingCallFromThread(reactor, setup) - try: - return tuple(arg for arg in args), {} - except TypeError: - return ((args,), {}) if args else None - - def bench(): - return benchmark.pedantic(blocking_runner, setup=blocking_setup, - rounds=4, warmup_rounds=1) - return threads.deferToThread(bench) - return blockOnThreadWithSetup diff --git a/testing/tests/perf/pytest.ini b/testing/tests/perf/pytest.ini deleted file mode 100644 index 7a0508ce..00000000 --- a/testing/tests/perf/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -twisted = yes diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py deleted file mode 100644 index a438ee49..00000000 --- a/testing/tests/perf/test_crypto.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Benchmarks for crypto operations. -If you don't want to stress your local machine too much, you can pass the -SIZE_LIMT environment variable. - -For instance, to keep the maximum payload at 1MB: - -SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py -""" -import pytest -import os -import json -from uuid import uuid4 - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client import _crypto - -LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) - - -pytestmark = pytest.mark.perf - - -def create_doc_encryption(size): - @pytest.mark.benchmark(group="test_crypto_encrypt_doc") - def test_doc_encryption(soledad_client, benchmark, payload): - crypto = soledad_client()._crypto - - DOC_CONTENT = {'payload': payload(size)} - doc = SoledadDocument( - doc_id=uuid4().hex, rev='rev', - json=json.dumps(DOC_CONTENT)) - - benchmark(crypto.encrypt_doc, doc) - return test_doc_encryption - - -# TODO this test is really bullshit, because it's still including -# the json serialization. - -def create_doc_decryption(size): - @pytest.inlineCallbacks - @pytest.mark.benchmark(group="test_crypto_decrypt_doc") - def test_doc_decryption(soledad_client, benchmark, payload): - crypto = soledad_client()._crypto - - DOC_CONTENT = {'payload': payload(size)} - doc = SoledadDocument( - doc_id=uuid4().hex, rev='rev', - json=json.dumps(DOC_CONTENT)) - - encrypted_doc = yield crypto.encrypt_doc(doc) - doc.set_json(encrypted_doc) - - benchmark(crypto.decrypt_doc, doc) - return test_doc_decryption - - -def create_raw_encryption(size): - @pytest.mark.benchmark(group="test_crypto_raw_encrypt") - def test_raw_encrypt(benchmark, payload): - key = payload(32) - benchmark(_crypto.encrypt_sym, payload(size), key) - return test_raw_encrypt - - -def create_raw_decryption(size): - @pytest.mark.benchmark(group="test_crypto_raw_decrypt") - def test_raw_decrypt(benchmark, payload): - key = payload(32) - iv, ciphertext = _crypto.encrypt_sym(payload(size), key) - benchmark(_crypto.decrypt_sym, ciphertext, key, iv) - return test_raw_decrypt - - -# Create the TESTS in the global namespace, they'll be picked by the benchmark -# plugin. - -encryption_tests = [ - ('10k', 1E4), - ('100k', 1E5), - ('500k', 5E5), - ('1M', 1E6), - ('10M', 1E7), - ('50M', 5E7), -] - -for name, size in encryption_tests: - if size < LIMIT: - sz = int(size) - globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) - globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) - - -for name, size in encryption_tests: - if size < LIMIT: - sz = int(size) - globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz) - globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz) diff --git a/testing/tests/perf/test_misc.py b/testing/tests/perf/test_misc.py deleted file mode 100644 index b45dc04e..00000000 --- a/testing/tests/perf/test_misc.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -pytestmark = pytest.mark.perf - - -@pytest.mark.benchmark(group="test_instance") -def test_initialization(soledad_client, benchmark): - benchmark(soledad_client) diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py deleted file mode 100644 index 807af6e9..00000000 --- a/testing/tests/perf/test_sqlcipher.py +++ /dev/null @@ -1,40 +0,0 @@ -''' -Tests SoledadClient/SQLCipher interaction -''' -import pytest - -from twisted.internet.defer import gatherResults - -pytestmark = pytest.mark.perf - - -def load_up(client, amount, payload, defer=True): - results = [client.create_doc({'content': payload}) for _ in xrange(amount)] - if defer: - return gatherResults(results) - - -def build_test_sqlcipher_async_create(amount, size): - @pytest.inlineCallbacks - @pytest.mark.benchmark(group="test_sqlcipher_async_create") - def test(soledad_client, txbenchmark, payload): - client = soledad_client() - yield txbenchmark(load_up, client, amount, payload(size)) - return test - - -def build_test_sqlcipher_create(amount, size): - @pytest.mark.benchmark(group="test_sqlcipher_create") - def test(soledad_client, benchmark, payload): - client = soledad_client()._dbsyncer - benchmark(load_up, client, amount, payload(size), defer=False) - return test - - -test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500 * 1000) -test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) -test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000) -# synchronous -test_create_20_500k = build_test_sqlcipher_create(20, 500 * 1000) -test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) -test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py deleted file mode 100644 index 9bb20389..00000000 --- a/testing/tests/perf/test_sync.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytest - -pytestmark = pytest.mark.perf - - -@pytest.inlineCallbacks -def load_up(client, amount, payload): - # create a bunch of local documents - for i in xrange(amount): - yield client.create_doc({'content': payload}) - - -def create_upload(uploads, size): - @pytest.inlineCallbacks - @pytest.mark.benchmark(group="test_upload") - def test(soledad_client, txbenchmark_with_setup, payload): - client = soledad_client() - - def setup(): - return load_up(client, uploads, payload(size)) - - yield txbenchmark_with_setup(setup, client.sync) - return test - - -test_upload_20_500k = create_upload(20, 500 * 1000) -test_upload_100_100k = create_upload(100, 100 * 1000) -test_upload_1000_10k = create_upload(1000, 10 * 1000) - - -def create_download(downloads, size): - @pytest.inlineCallbacks - @pytest.mark.benchmark(group="test_download") - def test(soledad_client, txbenchmark_with_setup, payload): - client = soledad_client() - - yield load_up(client, downloads, payload(size)) - yield client.sync() - # We could create them directly on couch, but sending them - # ensures we are dealing with properly encrypted docs - - def setup(): - return soledad_client() - - def sync(clean_client): - return clean_client.sync() - yield txbenchmark_with_setup(setup, sync) - return test - - -test_download_20_500k = create_download(20, 500 * 1000) -test_download_100_100k = create_download(100, 100 * 1000) -test_download_1000_10k = create_download(1000, 10 * 1000) - - -@pytest.inlineCallbacks -@pytest.mark.benchmark(group="test_nothing_to_sync") -def test_nothing_to_sync(soledad_client, txbenchmark_with_setup): - def setup(): - return soledad_client() - - def sync(clean_client): - return clean_client.sync() - yield txbenchmark_with_setup(setup, sync) -- cgit v1.2.3 From 1804b5f74a4efa6b25c06fe353ac960fd42e4fb6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 19 Nov 2016 04:13:52 -0300 Subject: [tests] migrate pytest to trial test_deprecated_crypto was using pytest, which unfortunately doesnt work when mixed with trial. Migrated back. Also added norecursedirs option back, as it is necessary for parallel testing mode. --- testing/tests/client/test_deprecated_crypto.py | 116 +++++++++++++++---------- 1 file changed, 70 insertions(+), 46 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_deprecated_crypto.py b/testing/tests/client/test_deprecated_crypto.py index ca1b1558..8ee3735c 100644 --- a/testing/tests/client/test_deprecated_crypto.py +++ b/testing/tests/client/test_deprecated_crypto.py @@ -1,11 +1,16 @@ import json -import pytest +from twisted.internet import defer +from uuid import uuid4 +from urlparse import urljoin from leap.soledad.client import crypto as old_crypto from leap.soledad.common.couch import CouchDatabase from leap.soledad.common import crypto as common_crypto from test_soledad.u1db_tests import simple_doc +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import make_token_soledad_app +from test_soledad.u1db_tests import TestCaseWithServer def deprecate_client_crypto(client): @@ -20,48 +25,67 @@ def couch_database(couch_url, uuid): return db -@pytest.inlineCallbacks -def test_touch_updates_remote_representation( - soledad_client, request): - - client = soledad_client() - deprecated_client = deprecate_client_crypto(soledad_client()) - - couch_url = request.config.option.couch_url - remote = couch_database(couch_url, client._uuid) - - # ensure remote db is empty - gen, docs = remote.get_all_docs() - assert gen == 0 - assert len(docs) == 0 - - # create a doc with deprecated client and sync - yield deprecated_client.create_doc(json.loads(simple_doc)) - yield deprecated_client.sync() - - # check for doc in remote db - gen, docs = remote.get_all_docs() - assert gen == 1 - assert len(docs) == 1 - doc = docs.pop() - content = doc.content - assert common_crypto.ENC_JSON_KEY in content - assert common_crypto.ENC_SCHEME_KEY in content - assert common_crypto.ENC_METHOD_KEY in content - assert common_crypto.ENC_IV_KEY in content - assert common_crypto.MAC_KEY in content - assert common_crypto.MAC_METHOD_KEY in content - - # "touch" the document with a newer client and synx - _, docs = yield client.get_all_docs() - yield client.put_doc(doc) - yield client.sync() - - # check for newer representation of doc in remote db - gen, docs = remote.get_all_docs() - assert gen == 2 - assert len(docs) == 1 - doc = docs.pop() - content = doc.content - assert len(content) == 1 - assert 'raw' in content +class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): + + def setUp(self): + SoledadWithCouchServerMixin.setUp(self) + TestCaseWithServer.setUp(self) + + def tearDown(self): + SoledadWithCouchServerMixin.tearDown(self) + TestCaseWithServer.tearDown(self) + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + @defer.inlineCallbacks + def test_touch_updates_remote_representation(self): + self.startTwistedServer() + user = 'user-' + uuid4().hex + server_url = 'http://%s:%d' % (self.server_address) + client = self._soledad_instance(user=user, server_url=server_url) + deprecated_client = deprecate_client_crypto( + self._soledad_instance(user=user, server_url=server_url)) + + self.make_app() + remote = self.request_state._create_database(replica_uid=client._uuid) + remote = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + user), + create=True) + + # ensure remote db is empty + gen, docs = remote.get_all_docs() + assert gen == 0 + assert len(docs) == 0 + + # create a doc with deprecated client and sync + yield deprecated_client.create_doc(json.loads(simple_doc)) + yield deprecated_client.sync() + + # check for doc in remote db + gen, docs = remote.get_all_docs() + assert gen == 1 + assert len(docs) == 1 + doc = docs.pop() + content = doc.content + assert common_crypto.ENC_JSON_KEY in content + assert common_crypto.ENC_SCHEME_KEY in content + assert common_crypto.ENC_METHOD_KEY in content + assert common_crypto.ENC_IV_KEY in content + assert common_crypto.MAC_KEY in content + assert common_crypto.MAC_METHOD_KEY in content + + # "touch" the document with a newer client and synx + _, docs = yield client.get_all_docs() + yield client.put_doc(doc) + yield client.sync() + + # check for newer representation of doc in remote db + gen, docs = remote.get_all_docs() + assert gen == 2 + assert len(docs) == 1 + doc = docs.pop() + content = doc.content + assert len(content) == 1 + assert 'raw' in content -- cgit v1.2.3 From 63c33b1b20e013571fb870205302bbc9e4a06e23 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 19 Nov 2016 21:52:30 -0300 Subject: [tests] use options instead of marks When we use marks the new pytests from benchmarks folder are collected and ignored, but this causes trial to fail sometimes. Using --ignore avoids it from being loaded while --benchmark-only will properly select the benchmarks for tox, as intended. --- testing/tests/benchmarks/test_crypto.py | 3 --- testing/tests/benchmarks/test_misc.py | 2 -- testing/tests/benchmarks/test_sqlcipher.py | 2 -- testing/tests/benchmarks/test_sync.py | 2 -- 4 files changed, 9 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index ab586bea..367c3b5b 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -18,9 +18,6 @@ from leap.soledad.client import _crypto LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) -pytestmark = pytest.mark.benchmark - - def create_doc_encryption(size): @pytest.mark.benchmark(group="test_crypto_encrypt_doc") def test_doc_encryption(soledad_client, benchmark, payload): diff --git a/testing/tests/benchmarks/test_misc.py b/testing/tests/benchmarks/test_misc.py index 2f32ad7c..ead48adf 100644 --- a/testing/tests/benchmarks/test_misc.py +++ b/testing/tests/benchmarks/test_misc.py @@ -1,7 +1,5 @@ import pytest -pytestmark = pytest.mark.benchmark - @pytest.mark.benchmark(group="test_instance") def test_initialization(soledad_client, benchmark): diff --git a/testing/tests/benchmarks/test_sqlcipher.py b/testing/tests/benchmarks/test_sqlcipher.py index 7f8842bd..39c9e3ad 100644 --- a/testing/tests/benchmarks/test_sqlcipher.py +++ b/testing/tests/benchmarks/test_sqlcipher.py @@ -5,8 +5,6 @@ import pytest from twisted.internet.defer import gatherResults -pytestmark = pytest.mark.benchmark - def load_up(client, amount, payload, defer=True): results = [client.create_doc({'content': payload}) for _ in xrange(amount)] diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index 88afe9f8..1bf6cc21 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -1,7 +1,5 @@ import pytest -pytestmark = pytest.mark.benchmark - @pytest.inlineCallbacks def load_up(client, amount, payload): -- cgit v1.2.3 From bfe330a7eaad1c51640dbbb91be233a65d2a4bd7 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 19 Nov 2016 21:54:13 -0300 Subject: [tests] fixes test_crypto bench encrypt returns a deferred and needs the adapted benchmark runner. --- testing/tests/benchmarks/test_crypto.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 367c3b5b..75ad9a30 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -20,7 +20,8 @@ LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) def create_doc_encryption(size): @pytest.mark.benchmark(group="test_crypto_encrypt_doc") - def test_doc_encryption(soledad_client, benchmark, payload): + @pytest.inlineCallbacks + def test_doc_encryption(soledad_client, txbenchmark, payload): crypto = soledad_client()._crypto DOC_CONTENT = {'payload': payload(size)} @@ -28,7 +29,7 @@ def create_doc_encryption(size): doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) - benchmark(crypto.encrypt_doc, doc) + yield txbenchmark(crypto.encrypt_doc, doc) return test_doc_encryption -- cgit v1.2.3 From b17eecaa35240333d0270c1a3437e67510fe4f20 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 20 Nov 2016 15:41:21 -0300 Subject: [bug] emit last sent doc event Document sending happens after encryption, so the last sent document needs to be signalled after request end. --- testing/tests/sync/test_sync_target.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index dd69ffa1..d02aba68 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -310,6 +310,37 @@ class TestSoledadSyncTarget( self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) + @defer.inlineCallbacks + def test_sync_exchange_send_events(self): + """ + Test for sync exchange's SOLEDAD_SYNC_SEND_STATUS event. + """ + remote_target = self.getSyncTarget() + uuid = remote_target.uuid + events = [] + + def mocked_events(*args): + events.append((args)) + self.patch( + target.send, '_emit_send_status', mocked_events) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc2 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc3 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) + get_doc2 = (lambda _: doc2, (1,), {}) + get_doc3 = (lambda _: doc3, (1,), {}) + docs = [(get_doc, 10, 'T-sid'), + (get_doc2, 11, 'T-sid2'), (get_doc3, 12, 'T-sid3')] + new_gen, trans_id = yield remote_target.sync_exchange( + docs, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=lambda _: 1, + ensure_callback=lambda _: 1) + self.assertEqual(1, new_gen) + self.assertEqual(4, len(events)) + self.assertEquals([(uuid, 0, 3), (uuid, 1, 3), (uuid, 2, 3), + (uuid, 3, 3)], events) + def test_sync_exchange_in_stream_error(self): self.skipTest("bypass this test because our sync_exchange process " "does not return u1db error 503 \"unavailable\" for " -- cgit v1.2.3 From 89d5c898527c32baec8454d6e3c749935d00a313 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 21 Nov 2016 16:49:41 -0200 Subject: [refactor] separate server application into another file --- testing/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 5698c8a9..1ff1cbb7 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -103,7 +103,7 @@ class SoledadServer(object): '--logfile=%s' % self._logfile, '--pidfile=%s' % self._pidfile, 'web', - '--wsgi=leap.soledad.server.application', + '--wsgi=leap.soledad.server.application.wsgi_application', '--port=2424' ]) -- cgit v1.2.3 From 1c8e3359734831562fca76b529c0b1f95af565d5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 25 Nov 2016 19:55:36 -0300 Subject: [refactor] Hide IV, simplify some calls IV was being set during tests and this required some defensive coding to avoid IV being set in production. This commits makes the test use the generated IV and "hides" it using a read-only property to let it clear this should never happen. Also refactored out some parameters that are generated automatically to reduce some lines of code and enhance readability. --- testing/tests/client/test_crypto.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 483c7803..6d896604 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -51,10 +51,10 @@ class AESTest(unittest.TestCase): def test_chunked_encryption(self): key = 'A' * 32 - iv = 'A' * 16 fd = BytesIO() - aes = _crypto.AESEncryptor(key, iv, fd) + aes = _crypto.AESEncryptor(key, fd) + iv = aes.iv data = snowden1 block = 16 @@ -99,14 +99,11 @@ class BlobTestCase(unittest.TestCase): @defer.inlineCallbacks def test_blob_encryptor(self): - inf = BytesIO() - inf.write(snowden1) - inf.seek(0) - outf = BytesIO() + inf = BytesIO(snowden1) blob = _crypto.BlobEncryptor( - self.doc_info, inf, result=outf, - secret='A' * 96, iv='B' * 16) + self.doc_info, inf, + secret='A' * 96) encrypted = yield blob.encrypt() data = base64.urlsafe_b64decode(encrypted.getvalue()) @@ -117,7 +114,7 @@ class BlobTestCase(unittest.TestCase): assert sch == 1 assert meth == 1 iv = data[11:27] - assert iv == 'B' * 16 + assert iv == blob.iv doc_id = data[27:37] assert doc_id == 'D-deadbeef' @@ -127,26 +124,23 @@ class BlobTestCase(unittest.TestCase): ciphertext = data[71:-64] aes_key = _crypto._get_sym_key_for_doc( self.doc_info.doc_id, 'A' * 96) - assert ciphertext == _aes_encrypt(aes_key, 'B' * 16, snowden1) + assert ciphertext == _aes_encrypt(aes_key, blob.iv, snowden1) - decrypted = _aes_decrypt(aes_key, 'B' * 16, ciphertext) + decrypted = _aes_decrypt(aes_key, blob.iv, ciphertext) assert str(decrypted) == snowden1 @defer.inlineCallbacks def test_blob_decryptor(self): - inf = BytesIO() - inf.write(snowden1) - inf.seek(0) - outf = BytesIO() + inf = BytesIO(snowden1) blob = _crypto.BlobEncryptor( - self.doc_info, inf, result=outf, - secret='A' * 96, iv='B' * 16) - yield blob.encrypt() + self.doc_info, inf, + secret='A' * 96) + ciphertext = yield blob.encrypt() decryptor = _crypto.BlobDecryptor( - self.doc_info, outf, + self.doc_info, ciphertext, secret='A' * 96) decrypted = yield decryptor.decrypt() assert decrypted.getvalue() == snowden1 -- cgit v1.2.3 From e65cb7bfecd530252e86878dfec117c2793aa04b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 01:11:28 -0300 Subject: [feature] delimit preamble from ciphertext We now encode preamble and ciphertext+hmac in two distinct payloads separated by a space. This allows metadata to be extracted and used before decoding the whole document. It also introduces a single packer for packing and unpacking of data instead of reads and writes. Downside: doc_id and rev are limited to 255 chars now. --- testing/tests/client/test_crypto.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 6d896604..78da8d24 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -22,7 +22,6 @@ import base64 import hashlib import json import os -import struct from io import BytesIO @@ -106,22 +105,19 @@ class BlobTestCase(unittest.TestCase): secret='A' * 96) encrypted = yield blob.encrypt() - data = base64.urlsafe_b64decode(encrypted.getvalue()) + preamble, ciphertext = _crypto._split(encrypted.getvalue()) + ciphertext = ciphertext[:-64] - assert data[0] == '\x80' - ts, sch, meth = struct.unpack( - 'Qbb', data[1:11]) + assert len(preamble) == _crypto.PACMAN.size + unpacked_data = _crypto.PACMAN.unpack(preamble) + pad, ts, sch, meth, iv, doc_id, rev = unpacked_data + assert pad == '\x80' assert sch == 1 assert meth == 1 - iv = data[11:27] assert iv == blob.iv - doc_id = data[27:37] assert doc_id == 'D-deadbeef' - - rev = data[37:71] assert rev == self.doc_info.rev - ciphertext = data[71:-64] aes_key = _crypto._get_sym_key_for_doc( self.doc_info.doc_id, 'A' * 96) assert ciphertext == _aes_encrypt(aes_key, blob.iv, snowden1) @@ -159,6 +155,7 @@ class BlobTestCase(unittest.TestCase): assert 'raw' in encrypted doc2 = SoledadDocument('id1', '1') doc2.set_json(encrypted) + assert _crypto.is_symmetrically_encrypted(doc2) decrypted = yield crypto.decrypt_doc(doc2) assert len(decrypted) != 0 assert json.loads(decrypted) == payload @@ -174,10 +171,12 @@ class BlobTestCase(unittest.TestCase): encrypted = yield crypto.encrypt_doc(doc1) encdict = json.loads(encrypted) - raw = base64.urlsafe_b64decode(str(encdict['raw'])) + preamble, raw = _crypto._split(str(encdict['raw'])) # mess with MAC messed = raw[:-64] + '0' * 64 - newraw = base64.urlsafe_b64encode(str(messed)) + + preamble = base64.urlsafe_b64encode(preamble) + newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed)) doc2 = SoledadDocument('id1', '1') doc2.set_json(json.dumps({"raw": str(newraw)})) -- cgit v1.2.3 From 42082cfa648ec10612823086e72dc2a70a0e773c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 18:09:26 -0300 Subject: [feature] make _crypto stream on decryption We are already doing this on encryption, now we can stream also from decryption. This unblocks the reactor and will be valuable for blobs-io. --- testing/tests/benchmarks/test_crypto.py | 4 ++-- testing/tests/client/test_crypto.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 75ad9a30..8ee9b899 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -39,7 +39,7 @@ def create_doc_encryption(size): def create_doc_decryption(size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_crypto_decrypt_doc") - def test_doc_decryption(soledad_client, benchmark, payload): + def test_doc_decryption(soledad_client, txbenchmark, payload): crypto = soledad_client()._crypto DOC_CONTENT = {'payload': payload(size)} @@ -50,7 +50,7 @@ def create_doc_decryption(size): encrypted_doc = yield crypto.encrypt_doc(doc) doc.set_json(encrypted_doc) - benchmark(crypto.decrypt_doc, doc) + yield txbenchmark(crypto.decrypt_doc, doc) return test_doc_decryption diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 78da8d24..863873f7 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -139,7 +139,7 @@ class BlobTestCase(unittest.TestCase): self.doc_info, ciphertext, secret='A' * 96) decrypted = yield decryptor.decrypt() - assert decrypted.getvalue() == snowden1 + assert decrypted == snowden1 @defer.inlineCallbacks def test_encrypt_and_decrypt(self): -- cgit v1.2.3 From d72e3763538d1156bcf72b643626c2111a5a02cf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 18:11:11 -0300 Subject: [bug] make the semaphore cover all parsing Unfortunately, if a doc finishes decryption before the previous one we will still have an issue while inserting. This commits solves it by adding the parse and decrypt inside of the semaphore. --- testing/tests/sync/test_sync_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index d02aba68..6ce9a5c5 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -63,7 +63,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ def parse(self, stream): - parser = DocStreamReceiver(None, None, lambda *_: 42) + parser = DocStreamReceiver(None, None, lambda *_: defer.succeed(42)) parser.dataReceived(stream) parser.finish() -- cgit v1.2.3 From bae95c183e68481db0fe36f066cd14c97bff3013 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 21:26:23 -0300 Subject: [refactor] simplify _crypto After adding the streaming decrypt, some classes were doing almost the same thing. Unified them. Also fixed some module level variables to upper case and some class name to camel case. --- testing/tests/client/test_crypto.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 863873f7..7643f75d 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -52,7 +52,7 @@ class AESTest(unittest.TestCase): key = 'A' * 32 fd = BytesIO() - aes = _crypto.AESEncryptor(key, fd) + aes = _crypto.AESConsumer(key, _buffer=fd) iv = aes.iv data = snowden1 @@ -78,7 +78,8 @@ class AESTest(unittest.TestCase): ciphertext = _aes_encrypt(key, iv, data) fd = BytesIO() - aes = _crypto.AESDecryptor(key, iv, fd) + operation = _crypto.AESConsumer.decrypt + aes = _crypto.AESConsumer(key, iv, fd, operation) for i in range(len(ciphertext) / block): chunk = ciphertext[i * block:(i + 1) * block] -- cgit v1.2.3 From dc80d2b59edd14ab463dc74e5fa19d1a04c27ca1 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 27 Nov 2016 02:25:07 -0300 Subject: [refactor] introduces a GenericWriter AESWriter and HMACWriter are just applying hmac or aes into a flow of data. Abstracted the application of those operations into a super class and highlighted just the difference on each implementation. --- testing/tests/client/test_crypto.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 7643f75d..aad588c0 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -52,7 +52,7 @@ class AESTest(unittest.TestCase): key = 'A' * 32 fd = BytesIO() - aes = _crypto.AESConsumer(key, _buffer=fd) + aes = _crypto.AESWriter(key, _buffer=fd) iv = aes.iv data = snowden1 @@ -78,8 +78,7 @@ class AESTest(unittest.TestCase): ciphertext = _aes_encrypt(key, iv, data) fd = BytesIO() - operation = _crypto.AESConsumer.decrypt - aes = _crypto.AESConsumer(key, iv, fd, operation) + aes = _crypto.AESWriter(key, iv, fd, encrypt=False) for i in range(len(ciphertext) / block): chunk = ciphertext[i * block:(i + 1) * block] -- cgit v1.2.3 From 67917656589d08a84f98ff675f6aeade809e1faf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 29 Nov 2016 02:04:57 -0300 Subject: [feature] speed up sync benchmark setup code We aren't testing huge payloads on CI, so it doesn't make sense to insert docs one by one. 'gatherResults' can speed up bench setup. --- testing/tests/benchmarks/test_sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index 1bf6cc21..1501d74b 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -1,11 +1,14 @@ import pytest +from twisted.internet.defer import gatherResults @pytest.inlineCallbacks def load_up(client, amount, payload): # create a bunch of local documents + deferreds = [] for i in xrange(amount): - yield client.create_doc({'content': payload}) + deferreds.append(client.create_doc({'content': payload})) + yield gatherResults(deferreds) def create_upload(uploads, size): -- cgit v1.2.3 From 694e5670da53e923cf809948e400cd546154162b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 30 Nov 2016 00:07:24 -0300 Subject: [refactor] improve blob signature magic usage Our magic value wasn't being used and were represented as a string. Refactored it to a constant, increased it's size to 2 bytes and optimzed is_symmetrically_encrypted to look for the magic and symmetrically encrypted flag under base64 encoding. Most file types will use this feature to help identifying themselves, so it got refactored to serve the purpose it was created. --- testing/tests/client/test_crypto.py | 6 +++--- testing/tests/server/test_server.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index aad588c0..33a660c9 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -110,8 +110,8 @@ class BlobTestCase(unittest.TestCase): assert len(preamble) == _crypto.PACMAN.size unpacked_data = _crypto.PACMAN.unpack(preamble) - pad, ts, sch, meth, iv, doc_id, rev = unpacked_data - assert pad == '\x80' + magic, sch, meth, ts, iv, doc_id, rev = unpacked_data + assert magic == _crypto.BLOB_SIGNATURE_MAGIC assert sch == 1 assert meth == 1 assert iv == blob.iv @@ -155,7 +155,7 @@ class BlobTestCase(unittest.TestCase): assert 'raw' in encrypted doc2 = SoledadDocument('id1', '1') doc2.set_json(encrypted) - assert _crypto.is_symmetrically_encrypted(doc2) + assert _crypto.is_symmetrically_encrypted(encrypted) decrypted = yield crypto.decrypt_doc(doc2) assert len(decrypted) != 0 assert json.loads(decrypted) == payload diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 2f958b29..6710caaf 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -413,7 +413,8 @@ class EncryptedSyncTestCase( self.assertEqual(soldoc.rev, couchdoc.rev) couch_content = couchdoc.content.keys() self.assertEqual(['raw'], couch_content) - self.assertTrue(_crypto.is_symmetrically_encrypted(couchdoc)) + content = couchdoc.get_json() + self.assertTrue(_crypto.is_symmetrically_encrypted(content)) d = sol1.get_all_docs() d.addCallback(_db1AssertEmptyDocList) -- cgit v1.2.3 From 349a49d2be011a428023a4ece14001fda57e65c4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 6 Dec 2016 23:16:28 -0300 Subject: [feature] use GCM instead of CTR+HMAC Resolves: #8668 - client: substitute usage of CTR mode + HMAC by GCM cipher mode Signed-off-by: Victor Shyba --- testing/tests/benchmarks/test_crypto.py | 4 +-- testing/tests/client/test_crypto.py | 48 ++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 24 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 8ee9b899..631ac041 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -66,8 +66,8 @@ def create_raw_decryption(size): @pytest.mark.benchmark(group="test_crypto_raw_decrypt") def test_raw_decrypt(benchmark, payload): key = payload(32) - iv, ciphertext = _crypto.encrypt_sym(payload(size), key) - benchmark(_crypto.decrypt_sym, ciphertext, key, iv) + iv, tag, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv, tag) return test_raw_decrypt diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 33a660c9..10acba56 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -29,6 +29,7 @@ import pytest from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend +from cryptography.exceptions import InvalidTag from leap.soledad.common.document import SoledadDocument from test_soledad.util import BaseSoledadTest @@ -64,7 +65,7 @@ class AESTest(unittest.TestCase): aes.end() ciphertext_chunked = fd.getvalue() - ciphertext = _aes_encrypt(key, iv, data) + ciphertext, tag = _aes_encrypt(key, iv, data) assert ciphertext_chunked == ciphertext @@ -75,10 +76,10 @@ class AESTest(unittest.TestCase): data = snowden1 block = 16 - ciphertext = _aes_encrypt(key, iv, data) + ciphertext, tag = _aes_encrypt(key, iv, data) fd = BytesIO() - aes = _crypto.AESWriter(key, iv, fd, encrypt=False) + aes = _crypto.AESWriter(key, iv, fd, tag=tag) for i in range(len(ciphertext) / block): chunk = ciphertext[i * block:(i + 1) * block] @@ -106,7 +107,7 @@ class BlobTestCase(unittest.TestCase): encrypted = yield blob.encrypt() preamble, ciphertext = _crypto._split(encrypted.getvalue()) - ciphertext = ciphertext[:-64] + ciphertext = ciphertext[:-16] assert len(preamble) == _crypto.PACMAN.size unpacked_data = _crypto.PACMAN.unpack(preamble) @@ -120,9 +121,10 @@ class BlobTestCase(unittest.TestCase): aes_key = _crypto._get_sym_key_for_doc( self.doc_info.doc_id, 'A' * 96) - assert ciphertext == _aes_encrypt(aes_key, blob.iv, snowden1) + assert ciphertext == _aes_encrypt(aes_key, blob.iv, snowden1)[0] - decrypted = _aes_decrypt(aes_key, blob.iv, ciphertext) + decrypted = _aes_decrypt(aes_key, blob.iv, blob.tag, ciphertext, + preamble) assert str(decrypted) == snowden1 @defer.inlineCallbacks @@ -173,7 +175,7 @@ class BlobTestCase(unittest.TestCase): encdict = json.loads(encrypted) preamble, raw = _crypto._split(str(encdict['raw'])) # mess with MAC - messed = raw[:-64] + '0' * 64 + messed = raw[:-16] + '0' * 16 preamble = base64.urlsafe_b64encode(preamble) newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed)) @@ -275,16 +277,16 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): # generate 256-bit key key = os.urandom(32) - iv, cyphertext = _crypto.encrypt_sym('data', key) + iv, tag, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') - plaintext = _crypto.decrypt_sym(cyphertext, key, iv) + plaintext = _crypto.decrypt_sym(cyphertext, key, iv, tag) self.assertEqual('data', plaintext) - def test_decrypt_with_wrong_iv_fails(self): + def test_decrypt_with_wrong_iv_raises(self): key = os.urandom(32) - iv, cyphertext = _crypto.encrypt_sym('data', key) + iv, tag, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -293,13 +295,13 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongiv = rawiv while wrongiv == rawiv: wrongiv = os.urandom(1) + rawiv[1:] - plaintext = _crypto.decrypt_sym( - cyphertext, key, iv=binascii.b2a_base64(wrongiv)) - self.assertNotEqual('data', plaintext) + with pytest.raises(InvalidTag): + _crypto.decrypt_sym( + cyphertext, key, iv=binascii.b2a_base64(wrongiv), tag=tag) - def test_decrypt_with_wrong_key_fails(self): + def test_decrypt_with_wrong_key_raises(self): key = os.urandom(32) - iv, cyphertext = _crypto.encrypt_sym('data', key) + iv, tag, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -307,19 +309,21 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): # ensure keys are different in case we are extremely lucky while wrongkey == key: wrongkey = os.urandom(32) - plaintext = _crypto.decrypt_sym(cyphertext, wrongkey, iv) - self.assertNotEqual('data', plaintext) + with pytest.raises(InvalidTag): + _crypto.decrypt_sym(cyphertext, wrongkey, iv, tag) def _aes_encrypt(key, iv, data): backend = default_backend() - cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend) encryptor = cipher.encryptor() - return encryptor.update(data) + encryptor.finalize() + return encryptor.update(data) + encryptor.finalize(), encryptor.tag -def _aes_decrypt(key, iv, data): +def _aes_decrypt(key, iv, tag, data, aead=''): backend = default_backend() - cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + cipher = Cipher(algorithms.AES(key), modes.GCM(iv, tag), backend=backend) decryptor = cipher.decryptor() + if aead: + decryptor.authenticate_additional_data(aead) return decryptor.update(data) + decryptor.finalize() -- cgit v1.2.3 From b3fcc5c5bddc73475596c4fe74e3402f0d5c021a Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 7 Dec 2016 01:24:53 -0300 Subject: [feature] Add retro compat on secrets.py ciphers Integrated the secrets's JSON key that specifies ciphers into _crypto and added optional GCM. Also added a test to check if both cipher types can be imported. Resolves: #8680 Signed-off-by: Victor Shyba --- testing/tests/benchmarks/test_crypto.py | 4 ++-- testing/tests/client/test_crypto.py | 30 +++++++++++++++++++----------- 2 files changed, 21 insertions(+), 13 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py index 631ac041..8ee9b899 100644 --- a/testing/tests/benchmarks/test_crypto.py +++ b/testing/tests/benchmarks/test_crypto.py @@ -66,8 +66,8 @@ def create_raw_decryption(size): @pytest.mark.benchmark(group="test_crypto_raw_decrypt") def test_raw_decrypt(benchmark, payload): key = payload(32) - iv, tag, ciphertext = _crypto.encrypt_sym(payload(size), key) - benchmark(_crypto.decrypt_sym, ciphertext, key, iv, tag) + iv, ciphertext = _crypto.encrypt_sym(payload(size), key) + benchmark(_crypto.decrypt_sym, ciphertext, key, iv) return test_raw_decrypt diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 10acba56..277d5430 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -114,7 +114,7 @@ class BlobTestCase(unittest.TestCase): magic, sch, meth, ts, iv, doc_id, rev = unpacked_data assert magic == _crypto.BLOB_SIGNATURE_MAGIC assert sch == 1 - assert meth == 1 + assert meth == _crypto.ENC_METHOD.aes_256_gcm assert iv == blob.iv assert doc_id == 'D-deadbeef' assert rev == self.doc_info.rev @@ -163,7 +163,7 @@ class BlobTestCase(unittest.TestCase): assert json.loads(decrypted) == payload @defer.inlineCallbacks - def test_decrypt_with_wrong_mac_raises(self): + def test_decrypt_with_wrong_tag_raises(self): """ Trying to decrypt a document with wrong MAC should raise. """ @@ -174,7 +174,7 @@ class BlobTestCase(unittest.TestCase): encrypted = yield crypto.encrypt_doc(doc1) encdict = json.loads(encrypted) preamble, raw = _crypto._split(str(encdict['raw'])) - # mess with MAC + # mess with tag messed = raw[:-16] + '0' * 16 preamble = base64.urlsafe_b64encode(preamble) @@ -205,8 +205,8 @@ class RecoveryDocumentTestCase(BaseSoledadTest): self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) - def test_import_recovery_document(self): - rd = self._soledad.secrets._export_recovery_document() + def test_import_recovery_document(self, cipher='aes256'): + rd = self._soledad.secrets._export_recovery_document(cipher) s = self._soledad_instance() s.secrets._import_recovery_document(rd) s.secrets.set_secret_id(self._soledad.secrets._secret_id) @@ -215,6 +215,14 @@ class RecoveryDocumentTestCase(BaseSoledadTest): 'Failed settinng secret for symmetric encryption.') s.close() + def test_import_GCM_recovery_document(self): + cipher = self._soledad.secrets.CIPHER_AES256_GCM + self.test_import_recovery_document(cipher) + + def test_import_legacy_CTR_recovery_document(self): + cipher = self._soledad.secrets.CIPHER_AES256 + self.test_import_recovery_document(cipher) + class SoledadSecretsTestCase(BaseSoledadTest): @@ -277,16 +285,16 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): # generate 256-bit key key = os.urandom(32) - iv, tag, cyphertext = _crypto.encrypt_sym('data', key) + iv, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') - plaintext = _crypto.decrypt_sym(cyphertext, key, iv, tag) + plaintext = _crypto.decrypt_sym(cyphertext, key, iv) self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_raises(self): key = os.urandom(32) - iv, tag, cyphertext = _crypto.encrypt_sym('data', key) + iv, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -297,11 +305,11 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongiv = os.urandom(1) + rawiv[1:] with pytest.raises(InvalidTag): _crypto.decrypt_sym( - cyphertext, key, iv=binascii.b2a_base64(wrongiv), tag=tag) + cyphertext, key, iv=binascii.b2a_base64(wrongiv)) def test_decrypt_with_wrong_key_raises(self): key = os.urandom(32) - iv, tag, cyphertext = _crypto.encrypt_sym('data', key) + iv, cyphertext = _crypto.encrypt_sym('data', key) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -310,7 +318,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): while wrongkey == key: wrongkey = os.urandom(32) with pytest.raises(InvalidTag): - _crypto.decrypt_sym(cyphertext, wrongkey, iv, tag) + _crypto.decrypt_sym(cyphertext, wrongkey, iv) def _aes_encrypt(key, iv, data): -- cgit v1.2.3 From 7877527fe64eaee1f7f107913a4a3dc78767a338 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 7 Dec 2016 02:03:58 -0300 Subject: [feature] Change CTR to GCM on secrets.py Current implementation can allow tampering and the CTR->GCM exchange can help to avoid it. This commits also alters a behaviour where we moved ahead after failing to decrypt a recovery document. IMHO we can't move ahead as this is a fatal error. Signed-off-by: Victor Shyba --- testing/tests/client/test_aux_methods.py | 4 ++-- testing/tests/client/test_crypto.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'testing/tests') diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py index c25ff8ca..9b4a175f 100644 --- a/testing/tests/client/test_aux_methods.py +++ b/testing/tests/client/test_aux_methods.py @@ -21,10 +21,10 @@ import os from twisted.internet import defer -from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client import Soledad from leap.soledad.client.adbapi import U1DBConnectionPool from leap.soledad.client.secrets import PassphraseTooShort +from leap.soledad.client.secrets import SecretsException from test_soledad.util import BaseSoledadTest @@ -108,7 +108,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol.change_passphrase(u'654321') sol.close() - with self.assertRaises(DatabaseAccessError): + with self.assertRaises(SecretsException): self._soledad_instance( 'leap@leap.se', passphrase=u'123', diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 277d5430..49a61438 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -200,8 +200,9 @@ class RecoveryDocumentTestCase(BaseSoledadTest): encrypted_secret = rd[ self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) - self.assertTrue( - encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') + self.assertEquals( + _crypto.ENC_METHOD.aes_256_gcm, + encrypted_secret[self._soledad.secrets.CIPHER_KEY]) self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) -- cgit v1.2.3