From 35563cb74fcfd7f6ae969ed3af3a74d3c18cbf5b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 18 Sep 2016 00:26:45 -0300 Subject: [refactor] remove decpool It's not being used --- testing/tests/sync/test_encdecpool.py | 258 ------------------------------- testing/tests/sync/test_sync.py | 2 +- testing/tests/sync/test_sync_deferred.py | 15 +- testing/tests/sync/test_sync_mutex.py | 4 +- testing/tests/sync/test_sync_target.py | 27 +--- 5 files changed, 15 insertions(+), 291 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_encdecpool.py b/testing/tests/sync/test_encdecpool.py index 4a32885e..7055a765 100644 --- a/testing/tests/sync/test_encdecpool.py +++ b/testing/tests/sync/test_encdecpool.py @@ -1,34 +1,11 @@ # -*- coding: utf-8 -*- -# test_encdecpool.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for encryption and decryption pool. -""" import json -from random import shuffle - -from mock import MagicMock from twisted.internet.defer import inlineCallbacks from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool from leap.soledad.common.document import SoledadDocument from test_soledad.util import BaseSoledadTest -from twisted.internet import defer DOC_ID = "mydoc" DOC_REV = "rev" @@ -69,238 +46,3 @@ class TestSyncEncrypterPool(BaseSoledadTest): encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) self.assertIsNotNone(encrypted) - - -class TestSyncDecrypterPool(BaseSoledadTest): - - def _insert_doc_cb(self, doc, gen, trans_id): - """ - Method used to mock the sync's return_doc_cb callback. - """ - self._inserted_docs.append((doc, gen, trans_id)) - - def _setup_pool(self, sync_db=None): - sync_db = sync_db or self._soledad._sync_db - return SyncDecrypterPool( - self._soledad._crypto, - sync_db, - source_replica_uid=self._soledad._dbpool.replica_uid, - insert_doc_cb=self._insert_doc_cb) - - def setUp(self): - BaseSoledadTest.setUp(self) - # setup the pool - self._pool = self._setup_pool() - # reset the inserted docs mock - self._inserted_docs = [] - - def tearDown(self): - if self._pool.running: - self._pool.stop() - BaseSoledadTest.tearDown(self) - - def test_insert_received_doc(self): - """ - Test that one document added to the pool is inserted using the - callback. - """ - self._pool.start(1) - self._pool.insert_received_doc( - DOC_ID, DOC_REV, "{}", 1, "trans_id", 1) - - def _assert_doc_was_inserted(_): - self.assertEqual( - self._inserted_docs, - [(SoledadDocument(DOC_ID, DOC_REV, "{}"), 1, u"trans_id")]) - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_looping_control(self): - """ - Start and stop cleanly. - """ - self._pool.start(10) - self.assertTrue(self._pool.running) - self._pool.stop() - self.assertFalse(self._pool.running) - self.assertTrue(self._pool.deferred.called) - - def test_sync_id_col_is_created_if_non_existing_in_docs_recvd_table(self): - """ - Test that docs_received table is migrated, and has the sync_id column - """ - mock_run_query = MagicMock(return_value=defer.succeed(None)) - mock_sync_db = MagicMock() - mock_sync_db.runQuery = mock_run_query - pool = self._setup_pool(mock_sync_db) - d = pool.start(10) - pool.stop() - - def assert_trial_to_create_sync_id_column(_): - mock_run_query.assert_called_once_with( - "ALTER TABLE docs_received ADD COLUMN sync_id") - - d.addCallback(assert_trial_to_create_sync_id_column) - return d - - def test_insert_received_doc_many(self): - """ - Test that many documents added to the pool are inserted using the - callback. - """ - many = 100 - self._pool.start(many) - - # insert many docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - self._pool.insert_received_doc( - doc_id, rev, content, gen, trans_id, idx) - - def _assert_doc_was_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_insert_encrypted_received_doc(self): - """ - Test that one encrypted document added to the pool is decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - - # insert the encrypted document in the pool - self._pool.start(1) - self._pool.insert_encrypted_received_doc( - DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1) - - def _assert_doc_was_decrypted_and_inserted(_): - self.assertEqual(1, len(self._inserted_docs)) - self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")]) - - self._pool.deferred.addCallback( - _assert_doc_was_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_processing_order(self): - """ - This test ensures that processing of documents only occur if there is - a sequence in place. - """ - crypto = self._soledad._crypto - - docs = [] - for i in xrange(1, 10): - i = str(i) - doc = SoledadDocument( - doc_id=DOC_ID + i, rev=DOC_REV + i, - json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc, encrypted_content)) - - # insert the encrypted document in the pool - yield self._pool.start(10) # pool is expecting to process 10 docs - self._pool._loop.stop() # we are processing manually - # first three arrives, forming a sequence - for i, (doc, encrypted_content) in enumerate(docs[:3]): - gen = idx = i + 1 - yield self._pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx) - - # last one arrives alone, so it can't be processed - doc, encrypted_content = docs[-1] - yield self._pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) - - yield self._pool._decrypt_and_recurse() - - self.assertEqual(3, self._pool._processed_docs) - - def test_insert_encrypted_received_doc_many(self, many=100): - """ - Test that many encrypted documents added to the pool are decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - self._pool.start(many) - docs = [] - - # insert many encrypted docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - - doc = SoledadDocument( - doc_id=doc_id, rev=rev, json=json.dumps(content)) - - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc_id, rev, encrypted_content, gen, - trans_id, idx)) - shuffle(docs) - - for doc in docs: - self._pool.insert_encrypted_received_doc(*doc) - - def _assert_docs_were_decrypted_and_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback( - _assert_docs_were_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_pool_reuse(self): - """ - The pool is reused between syncs, this test verifies that - reusing is fine. - """ - for i in xrange(3): - yield self.test_insert_encrypted_received_doc_many(5) - self._inserted_docs = [] - decrypted_docs = yield self._pool._get_docs(encrypted=False) - # check that decrypted docs staging is clean - self.assertEquals([], decrypted_docs) - self._pool.stop() diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index 5290003e..a7d0a92b 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -187,7 +187,7 @@ class TestSoledadDbSync( self.addCleanup(target.close) return sync.SoledadSynchronizer( self.db, - target).sync(defer_decryption=False) + target).sync() @defer.inlineCallbacks def test_db_sync(self): diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py index 4948aaf8..482b150c 100644 --- a/testing/tests/sync/test_sync_deferred.py +++ b/testing/tests/sync/test_sync_deferred.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Test Leap backend bits: sync with deferred encryption/decryption. +Test Leap backend bits: sync with deferred encryption. """ import time import os @@ -22,12 +22,8 @@ import random import string import shutil -from urlparse import urljoin - from twisted.internet import defer -from leap.soledad.common import couch - from leap.soledad.client import sync from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client.sqlcipher import SQLCipherDatabase @@ -41,9 +37,6 @@ from test_soledad.util import make_soledad_app from test_soledad.util import soledad_sync_target -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = True - WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" @@ -52,7 +45,7 @@ DBPASS = "pass" class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): """ - Another base class for testing the deferred encryption/decryption during + Another base class for testing the deferred encryption during the syncs, using the intermediate database. """ defer_sync_encryption = True @@ -109,7 +102,7 @@ class TestSoledadDbSyncDeferredEncDecr( """ Test db.sync remote sync shortcut. - Case with deferred encryption and decryption: using the intermediate + Case with deferred encryption: using the intermediate syncdb. """ @@ -158,7 +151,7 @@ class TestSoledadDbSyncDeferredEncDecr( self.addCleanup(target.close) return sync.SoledadSynchronizer( dbsyncer, - target).sync(defer_decryption=True) + target).sync() def wait_for_sync(self): """ diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py index 2626ab2a..2bcb3aec 100644 --- a/testing/tests/sync/test_sync_mutex.py +++ b/testing/tests/sync/test_sync_mutex.py @@ -47,7 +47,7 @@ from test_soledad.util import soledad_sync_target _old_sync = SoledadSynchronizer.sync -def _timed_sync(self, defer_decryption=True): +def _timed_sync(self): t = time.time() sync_id = uuid.uuid4() @@ -62,7 +62,7 @@ def _timed_sync(self, defer_decryption=True): self.source.sync_times[sync_id]['end'] = t return passthrough - d = _old_sync(self, defer_decryption=defer_decryption) + d = _old_sync(self) d.addBoth(_store_finish_time) return d diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 964468ce..a2935539 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -231,8 +231,7 @@ class TestSoledadSyncTarget( doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') new_gen, trans_id = yield remote_target.sync_exchange( [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertEqual(1, new_gen) self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) @@ -285,8 +284,7 @@ class TestSoledadSyncTarget( 'replica', last_known_generation=0, last_known_trans_id=None, - insert_doc_cb=receive_doc, - defer_decryption=False) + insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', @@ -298,8 +296,7 @@ class TestSoledadSyncTarget( trigger_ids = [] new_gen, trans_id = yield remote_target.sync_exchange( [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here2', 'replica:1', '{"value": "here2"}', False) @@ -331,7 +328,7 @@ class TestSoledadSyncTarget( new_gen, trans_id = yield remote_target.sync_exchange( [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc, - ensure_callback=ensure_cb, defer_decryption=False) + ensure_callback=ensure_cb) self.assertEqual(1, new_gen) db = self.db2 self.assertEqual(1, len(replica_uid_box)) @@ -446,8 +443,7 @@ class SoledadDatabaseSyncTargetTests( 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertGetEncryptedDoc( self.db, 'doc-id', 'replica:1', tests.simple_doc, False) self.assertTransactionLog(['doc-id'], self.db) @@ -471,8 +467,7 @@ class SoledadDatabaseSyncTargetTests( 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertGetEncryptedDoc( self.db, 'doc-id', 'replica:1', tests.simple_doc, False) self.assertGetEncryptedDoc( @@ -498,8 +493,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) new_gen, _ = yield self.st.sync_exchange( [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) + last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) self.assertEqual(2, new_gen) self.assertEqual( @@ -779,10 +773,6 @@ class SoledadDatabaseSyncTargetTests( yield self.st.record_sync_info('replica', 0, 'T-sid') self.assertEqual(expected, called) - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = False - WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" @@ -890,8 +880,7 @@ class TestSoledadDbSync( defer_encryption=True) self.dbsyncer = dbsyncer return dbsyncer.sync(target_url, - creds=creds, - defer_decryption=DEFER_DECRYPTION) + creds=creds) else: return self._do_sync(self, target_name) -- cgit v1.2.3 From 07dcb2ae5240f20a26903f53a432fcd49c7f1ec9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 19 Sep 2016 03:56:44 -0300 Subject: [feature] streaming download protocol This commit finishes reversion into u1db original streaming protocol for downloads. --- testing/tests/sync/test_sync_target.py | 65 ++++++++++++---------------------- 1 file changed, 22 insertions(+), 43 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index a2935539..997dcdcd 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -30,6 +30,7 @@ from testscenarios import TestWithScenarios from twisted.internet import defer from leap.soledad.client import http_target as target +from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver from leap.soledad.client import crypto from leap.soledad.client.sqlcipher import SQLCipherU1DBSync from leap.soledad.client.sqlcipher import SQLCipherOptions @@ -44,6 +45,7 @@ from test_soledad.util import make_soledad_app from test_soledad.util import make_token_soledad_app from test_soledad.util import make_soledad_document_for_test from test_soledad.util import soledad_sync_target +from twisted.trial import unittest from test_soledad.util import SoledadWithCouchServerMixin from test_soledad.util import ADDRESS from test_soledad.util import SQLCIPHER_SCENARIOS @@ -53,92 +55,69 @@ from test_soledad.util import SQLCIPHER_SCENARIOS # The following tests come from `u1db.tests.test_remote_sync_target`. # ----------------------------------------------------------------------------- -class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): +class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ Some tests had to be copied to this class so we can instantiate our own target. """ - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - self.target = target.SoledadHTTPSyncTarget( - self.couch_url, - uuid4().hex, - creds, - self._soledad._crypto, - None) - - def tearDown(self): - self.target.close() - SoledadWithCouchServerMixin.tearDown(self) + def parse(self, stream): + parser = DocStreamReceiver(None, None, lambda x, y: 42) + parser.dataReceived(stream) + parser.finish() def test_extra_comma(self): - """ - Test adapted to use encrypted content. - """ doc = SoledadDocument('i', rev='r') - doc.content = {} - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc.doc_id) - secret = _crypto.secret + doc.content = {'a': 'b'} - enc_json = crypto.encrypt_docstr( - doc.get_json(), doc.doc_id, doc.rev, - key, secret) + encrypted_docstr = crypto.SoledadCrypto('').encrypt_doc(doc) with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{},\r\n]") + self.parse("[\r\n{},\r\n]") with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + - '"content": %s, "gen": 3, "trans_id": "T-sid"}' + - ',\r\n]') % json.dumps(enc_json)) + '"gen": 3, "trans_id": "T-sid"},\r\n' + + '%s,\r\n]') % encrypted_docstr) def test_wrong_start(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("{}\r\n]") - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("\r\n{}\r\n]") + self.parse("{}\r\n]") with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("") + self.parse("\r\n{}\r\n]") def test_wrong_end(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{}") + self.parse("[\r\n{}") with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n") + self.parse("[\r\n") def test_missing_comma(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( '[\r\n{}\r\n{"id": "i", "rev": "r", ' '"content": "c", "gen": 3}\r\n]') def test_no_entries(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n]") + self.parse("[\r\n]") def test_error_in_stream(self): with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( '[\r\n{"new_generation": 0},' '\r\n{"error": "unavailable"}\r\n') with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( + self.parse( '[\r\n{"error": "unavailable"}\r\n') with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') + self.parse('[\r\n{"error": "?"}\r\n') # # functions for TestRemoteSyncTargets -- cgit v1.2.3 From 75208477a2f1634664b80b8501818e5a905aa0f3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Sep 2016 01:42:26 -0400 Subject: [tests] adapt tests --- testing/tests/sync/test_encdecpool.py | 48 ------------------------------- testing/tests/sync/test_sqlcipher_sync.py | 14 ++++----- 2 files changed, 5 insertions(+), 57 deletions(-) delete mode 100644 testing/tests/sync/test_encdecpool.py (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_encdecpool.py b/testing/tests/sync/test_encdecpool.py deleted file mode 100644 index 7055a765..00000000 --- a/testing/tests/sync/test_encdecpool.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from twisted.internet.defer import inlineCallbacks - -from leap.soledad.client.encdecpool import SyncEncrypterPool - -from leap.soledad.common.document import SoledadDocument -from test_soledad.util import BaseSoledadTest - -DOC_ID = "mydoc" -DOC_REV = "rev" -DOC_CONTENT = {'simple': 'document'} - - -class TestSyncEncrypterPool(BaseSoledadTest): - - def setUp(self): - BaseSoledadTest.setUp(self) - crypto = self._soledad._crypto - sync_db = self._soledad._sync_db - self._pool = SyncEncrypterPool(crypto, sync_db) - self._pool.start() - - def tearDown(self): - self._pool.stop() - BaseSoledadTest.tearDown(self) - - @inlineCallbacks - def test_get_encrypted_doc_returns_none(self): - """ - Test that trying to get an encrypted doc from the pool returns None if - the document was never added for encryption. - """ - doc = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - self.assertIsNone(doc) - - @inlineCallbacks - def test_encrypt_doc_and_get_it_back(self): - """ - Test that the pool actually encrypts a document added to the queue. - """ - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - - yield self._pool.encrypt_doc(doc) - encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - - self.assertIsNotNone(encrypted) diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 3cbefc8b..2528600d 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -27,8 +27,6 @@ from leap.soledad.common.l2db import sync from leap.soledad.common.l2db import vectorclock from leap.soledad.common.l2db import errors -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict from leap.soledad.client.http_target import SoledadHTTPSyncTarget from test_soledad import u1db_tests as tests @@ -545,13 +543,11 @@ class SQLCipherDatabaseSyncTests( self.assertFalse(doc2.has_conflicts) self.sync(self.db2, db3) doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) + + _crypto = self._soledad._crypto + decrypted = _crypto.decrypt_doc(doc3) + doc3.set_json(decrypted) + self.assertEqual(doc4.get_json(), doc3.get_json()) self.assertFalse(doc3.has_conflicts) self.db1.close() -- cgit v1.2.3 From 69958161af00958b1d92ab6153a591aa89199e35 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Sep 2016 03:08:20 -0400 Subject: [refactor] remove lingering refs to syncdb --- testing/tests/sync/test_sqlcipher_sync.py | 6 +----- testing/tests/sync/test_sync_deferred.py | 10 ++-------- testing/tests/sync/test_sync_target.py | 19 ++++--------------- 3 files changed, 7 insertions(+), 28 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 2528600d..c3cd8444 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -709,13 +709,9 @@ def make_local_db_and_soledad_target( test.startTwistedServer() replica_uid = os.path.basename(path) db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool st = soledad_sync_target( test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=source_replica_uid) return db, st target_scenarios = [ diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py index 482b150c..eb71ea73 100644 --- a/testing/tests/sync/test_sync_deferred.py +++ b/testing/tests/sync/test_sync_deferred.py @@ -71,12 +71,10 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): import binascii tohex = binascii.b2a_hex key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) + dbpath, key, is_raw_key=True, create=False) self.db1 = SQLCipherDatabase(self.opts) self.db2 = self.request_state._create_database('test') @@ -139,15 +137,11 @@ class TestSoledadDbSyncDeferredEncDecr( and Token auth. """ replica_uid = self._soledad._dbpool.replica_uid - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer target = soledad_sync_target( self, self.db2._dbname, - source_replica_uid=replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=replica_uid) self.addCleanup(target.close) return sync.SoledadSynchronizer( dbsyncer, diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 997dcdcd..b301c48b 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -130,13 +130,9 @@ def make_local_db_and_soledad_target( test.startTwistedServer() replica_uid = os.path.basename(path) db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool st = soledad_sync_target( test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=source_replica_uid) return db, st @@ -167,15 +163,11 @@ class TestSoledadSyncTarget( def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex): if self.port is None: self.startTwistedServer() - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool if path is None: path = self.db2._dbname target = self.sync_target( self, path, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) + source_replica_uid=source_replica_uid) self.addCleanup(target.close) return target @@ -811,12 +803,10 @@ class TestSoledadDbSync( import binascii tohex = binascii.b2a_hex key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) + dbpath, key, is_raw_key=True, create=False) self.db1 = SQLCipherDatabase(self.opts) self.db2 = self.request_state._create_database(replica_uid='test') @@ -855,8 +845,7 @@ class TestSoledadDbSync( self.opts, crypto, replica_uid, - None, - defer_encryption=True) + None) self.dbsyncer = dbsyncer return dbsyncer.sync(target_url, creds=creds) -- cgit v1.2.3 From 40742021a8beeb68b159456b423e4c3674f7926d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Sep 2016 16:12:33 -0400 Subject: [bug] fix import --- testing/tests/sync/test_sync_target.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index b301c48b..7c93cd7c 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -31,10 +31,10 @@ from twisted.internet import defer from leap.soledad.client import http_target as target from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver -from leap.soledad.client import crypto from leap.soledad.client.sqlcipher import SQLCipherU1DBSync from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client import _crypto from leap.soledad.common import l2db @@ -71,7 +71,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): doc = SoledadDocument('i', rev='r') doc.content = {'a': 'b'} - encrypted_docstr = crypto.SoledadCrypto('').encrypt_doc(doc) + encrypted_docstr = _crypto.SoledadCrypto('').encrypt_doc(doc) with self.assertRaises(l2db.errors.BrokenSyncStream): self.parse("[\r\n{},\r\n]") -- cgit v1.2.3 From 781984f3485b1fd479d09278a665f599c1bd10dc Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 22 Sep 2016 20:37:23 -0300 Subject: [test] fix test and remove leftovers defer_encryption --- testing/tests/sync/test_sqlcipher_sync.py | 4 ---- testing/tests/sync/test_sync_target.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index c3cd8444..029164eb 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -544,10 +544,6 @@ class SQLCipherDatabaseSyncTests( self.sync(self.db2, db3) doc3 = db3.get_doc('the-doc') - _crypto = self._soledad._crypto - decrypted = _crypto.decrypt_doc(doc3) - doc3.set_json(decrypted) - self.assertEqual(doc4.get_json(), doc3.get_json()) self.assertFalse(doc3.has_conflicts) self.db1.close() diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 7c93cd7c..ef034142 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -71,7 +71,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): doc = SoledadDocument('i', rev='r') doc.content = {'a': 'b'} - encrypted_docstr = _crypto.SoledadCrypto('').encrypt_doc(doc) + encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc) with self.assertRaises(l2db.errors.BrokenSyncStream): self.parse("[\r\n{},\r\n]") @@ -589,9 +589,9 @@ class SoledadDatabaseSyncTargetTests( [], 'other-replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(2, new_gen) self.assertEqual( (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) if self.whitebox: self.assertEqual(self.db._last_exchange_log['return'], {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) -- cgit v1.2.3 From 288434178a4e89f86b9740cfe77a4dc0ce9e45f7 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 19 Oct 2016 02:14:46 -0300 Subject: [bug] total number of documents Parsing from metadata we can store the total of docs and handle it for the doc parser in order to be able to keep consistent events info. --- testing/tests/sync/test_sync_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index ef034142..17223606 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -63,7 +63,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ def parse(self, stream): - parser = DocStreamReceiver(None, None, lambda x, y: 42) + parser = DocStreamReceiver(None, None, lambda *_: 42) parser.dataReceived(stream) parser.finish() -- cgit v1.2.3 From 2505f61f7374cd0afeb9392c03589607d7b63b64 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 19 Oct 2016 14:05:27 -0300 Subject: [refactor] stop using leap.common.http We aren't using leap.common.http implementation and we need specific features from original Twisted Web Agent. This commit implements it on HTTP Targer. --- testing/tests/sync/test_sync.py | 1 - testing/tests/sync/test_sync_deferred.py | 1 - testing/tests/sync/test_sync_target.py | 2 -- 3 files changed, 4 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index a7d0a92b..a434e944 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -184,7 +184,6 @@ class TestSoledadDbSync( target = soledad_sync_target( self, self.db2._dbname, source_replica_uid=self._soledad._dbpool.replica_uid) - self.addCleanup(target.close) return sync.SoledadSynchronizer( self.db, target).sync() diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py index eb71ea73..001612a6 100644 --- a/testing/tests/sync/test_sync_deferred.py +++ b/testing/tests/sync/test_sync_deferred.py @@ -142,7 +142,6 @@ class TestSoledadDbSyncDeferredEncDecr( target = soledad_sync_target( self, self.db2._dbname, source_replica_uid=replica_uid) - self.addCleanup(target.close) return sync.SoledadSynchronizer( dbsyncer, target).sync() diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 17223606..fd1d413e 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -168,7 +168,6 @@ class TestSoledadSyncTarget( target = self.sync_target( self, path, source_replica_uid=source_replica_uid) - self.addCleanup(target.close) return target def setUp(self): @@ -389,7 +388,6 @@ class SoledadDatabaseSyncTargetTests( def tearDown(self): self.db.close() - self.st.close() tests.TestCaseWithServer.tearDown(self) SoledadWithCouchServerMixin.tearDown(self) -- cgit v1.2.3 From 45763ed191296df9d0c5a3b9b37a07ebee3293ad Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 16 Nov 2016 22:14:01 -0300 Subject: [tests] remove test_sync_deferred Deferred encryption option is gone. --- testing/tests/sync/test_sync_deferred.py | 182 ------------------------------- 1 file changed, 182 deletions(-) delete mode 100644 testing/tests/sync/test_sync_deferred.py (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py deleted file mode 100644 index 001612a6..00000000 --- a/testing/tests/sync/test_sync_deferred.py +++ /dev/null @@ -1,182 +0,0 @@ -# test_sync_deferred.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: sync with deferred encryption. -""" -import time -import os -import random -import string -import shutil - -from twisted.internet import defer - -from leap.soledad.client import sync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from testscenarios import TestWithScenarios - -from test_soledad import u1db_tests as tests -from test_soledad.util import ADDRESS -from test_soledad.util import SoledadWithCouchServerMixin -from test_soledad.util import make_soledad_app -from test_soledad.util import soledad_sync_target - - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): - - """ - Another base class for testing the deferred encryption during - the syncs, using the intermediate database. - """ - defer_sync_encryption = True - - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - os.unlink(self.db1_file) - self.db_pass = DBPASS - self.email = ADDRESS - - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # open test dbs: db1 will be the local sqlcipher db (which - # instantiates a syncdb). We use the self._soledad instance that was - # already created on some setUp method. - import binascii - tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) - dbpath = self._soledad._local_db_path - - self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False) - self.db1 = SQLCipherDatabase(self.opts) - - self.db2 = self.request_state._create_database('test') - - def tearDown(self): - # XXX should not access "private" attrs - shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - SoledadWithCouchServerMixin.tearDown(self) - - -class SyncTimeoutError(Exception): - - """ - Dummy exception to notify timeout during sync. - """ - pass - - -class TestSoledadDbSyncDeferredEncDecr( - TestWithScenarios, - BaseSoledadDeferredEncTest, - tests.TestCaseWithServer): - - """ - Test db.sync remote sync shortcut. - Case with deferred encryption: using the intermediate - syncdb. - """ - - scenarios = [ - ('http', { - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - }), - ] - - oauth = False - token = True - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - BaseSoledadDeferredEncTest.setUp(self) - self.server = self.server_thread = None - self.syncer = None - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - dbsyncer = getattr(self, 'dbsyncer', None) - if dbsyncer: - dbsyncer.close() - BaseSoledadDeferredEncTest.tearDown(self) - - def do_sync(self): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - replica_uid = self._soledad._dbpool.replica_uid - dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer - - target = soledad_sync_target( - self, self.db2._dbname, - source_replica_uid=replica_uid) - return sync.SoledadSynchronizer( - dbsyncer, - target).sync() - - def wait_for_sync(self): - """ - Wait for sync to finish. - """ - wait = 0 - syncer = self.syncer - if syncer is not None: - while syncer.syncing: - time.sleep(WAIT_STEP) - wait += WAIT_STEP - if wait >= MAX_WAIT: - raise SyncTimeoutError - - @defer.inlineCallbacks - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = yield self.do_sync() - - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) -- cgit v1.2.3 From 632cd2414a47bc7f471d8f501ab6250293aedf51 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 17 Nov 2016 00:38:55 -0300 Subject: [bug] include_deleted=True on sync Also refactored tests and code to stop relying on old parameters which included docs instead of get_doc calls. --- testing/tests/sync/test_sync.py | 19 ++++++++++++++++ testing/tests/sync/test_sync_target.py | 40 +++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 18 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index a434e944..76757c5b 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -19,6 +19,7 @@ import threading import time from urlparse import urljoin +from mock import Mock from twisted.internet import defer from testscenarios import TestWithScenarios @@ -210,3 +211,21 @@ class TestSoledadDbSync( self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration + + +class TestSoledadSynchronizer(BaseSoledadTest): + + def setUp(self): + BaseSoledadTest.setUp(self) + self.db = Mock() + self.target = Mock() + self.synchronizer = sync.SoledadSynchronizer( + self.db, + self.target) + + def test_docs_by_gen_includes_deleted(self): + changes = [('id', 'gen', 'trans')] + docs_by_gen = self.synchronizer._docs_by_gen_from_changes(changes) + f, args, kwargs = docs_by_gen[0][0] + self.assertIn('include_deleted', kwargs) + self.assertTrue(kwargs['include_deleted']) diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index fd1d413e..e32f08b3 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -199,8 +199,9 @@ class TestSoledadSyncTarget( other_docs.append((doc.doc_id, doc.rev, doc.get_json())) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertEqual(1, new_gen) self.assertGetEncryptedDoc( @@ -247,10 +248,12 @@ class TestSoledadSyncTarget( doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') doc2 = self.make_document('doc-here2', 'replica:1', '{"value": "here2"}') + get_doc1 = (lambda _: doc1, (1,), {}) + get_doc2 = (lambda _: doc2, (2,), {}) with self.assertRaises(l2db.errors.U1DBError): yield remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], + [(get_doc1, 10, 'T-sid'), (get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, @@ -265,7 +268,7 @@ class TestSoledadSyncTarget( # retry trigger_ids = [] new_gen, trans_id = yield remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, + [(get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here2', 'replica:1', '{"value": "here2"}', @@ -295,8 +298,9 @@ class TestSoledadSyncTarget( replica_uid_box.append(replica_uid) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc, ensure_callback=ensure_cb) self.assertEqual(1, new_gen) @@ -408,8 +412,8 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] + ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}), + 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -430,10 +434,10 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc), {}), 10, 'T-1'), + ((self.make_document, + ('doc-id2', 'replica:1', tests.nested_doc), {}), 11, 'T-2')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -515,7 +519,7 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json('{}') edit_rev = 'replica:1|' + doc.rev docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -534,7 +538,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + ((self.make_document, (doc.doc_id, 'replica:1', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, @@ -554,7 +558,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) gen, txid = self.db._get_generation_info() docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), + ((self.make_document, (doc.doc_id, doc.rev, tests.simple_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=gen, @@ -600,7 +604,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -625,7 +629,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -646,7 +650,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -657,8 +661,8 @@ class SoledadDatabaseSyncTargetTests( def test_sync_exchange_converged_handling(self): doc = self.db.create_doc_from_json(tests.simple_doc) docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5, 'T-bar')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, -- cgit v1.2.3 From 529dbdf27804f12da80907d25c412d10e9fa3763 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 17 Nov 2016 01:33:04 -0300 Subject: [style] fix pep8 and confs Fixes setup.cfg, adding current exclude rules, simplified tox.ini to use setup.cfg and fixed all. --- testing/tests/sync/test_sqlcipher_sync.py | 1 + testing/tests/sync/test_sync_mutex.py | 1 + testing/tests/sync/test_sync_target.py | 11 +++++++---- 3 files changed, 9 insertions(+), 4 deletions(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 029164eb..26f63a40 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -710,6 +710,7 @@ def make_local_db_and_soledad_target( source_replica_uid=source_replica_uid) return db, st + target_scenarios = [ ('leap', { 'create_db_and_target': make_local_db_and_soledad_target, diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py index 2bcb3aec..432a3cd2 100644 --- a/testing/tests/sync/test_sync_mutex.py +++ b/testing/tests/sync/test_sync_mutex.py @@ -66,6 +66,7 @@ def _timed_sync(self): d.addBoth(_store_finish_time) return d + SoledadSynchronizer.sync = _timed_sync # -- end of monkey-patching diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index e32f08b3..dd69ffa1 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -412,7 +412,8 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}), + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc,), {}), 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, @@ -519,7 +520,8 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json('{}') edit_rev = 'replica:1|' + doc.rev docs_by_gen = [ - ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')] + ((self.make_document, (doc.doc_id, edit_rev, None), {}), + 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -662,8 +664,8 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json(tests.simple_doc) docs_by_gen = [ ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), - ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5, - 'T-bar')] + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), + 5, 'T-bar')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -746,6 +748,7 @@ class SoledadDatabaseSyncTargetTests( yield self.st.record_sync_info('replica', 0, 'T-sid') self.assertEqual(expected, called) + WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" -- cgit v1.2.3 From b17eecaa35240333d0270c1a3437e67510fe4f20 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 20 Nov 2016 15:41:21 -0300 Subject: [bug] emit last sent doc event Document sending happens after encryption, so the last sent document needs to be signalled after request end. --- testing/tests/sync/test_sync_target.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index dd69ffa1..d02aba68 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -310,6 +310,37 @@ class TestSoledadSyncTarget( self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) + @defer.inlineCallbacks + def test_sync_exchange_send_events(self): + """ + Test for sync exchange's SOLEDAD_SYNC_SEND_STATUS event. + """ + remote_target = self.getSyncTarget() + uuid = remote_target.uuid + events = [] + + def mocked_events(*args): + events.append((args)) + self.patch( + target.send, '_emit_send_status', mocked_events) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc2 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc3 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) + get_doc2 = (lambda _: doc2, (1,), {}) + get_doc3 = (lambda _: doc3, (1,), {}) + docs = [(get_doc, 10, 'T-sid'), + (get_doc2, 11, 'T-sid2'), (get_doc3, 12, 'T-sid3')] + new_gen, trans_id = yield remote_target.sync_exchange( + docs, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=lambda _: 1, + ensure_callback=lambda _: 1) + self.assertEqual(1, new_gen) + self.assertEqual(4, len(events)) + self.assertEquals([(uuid, 0, 3), (uuid, 1, 3), (uuid, 2, 3), + (uuid, 3, 3)], events) + def test_sync_exchange_in_stream_error(self): self.skipTest("bypass this test because our sync_exchange process " "does not return u1db error 503 \"unavailable\" for " -- cgit v1.2.3 From d72e3763538d1156bcf72b643626c2111a5a02cf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 18:11:11 -0300 Subject: [bug] make the semaphore cover all parsing Unfortunately, if a doc finishes decryption before the previous one we will still have an issue while inserting. This commits solves it by adding the parse and decrypt inside of the semaphore. --- testing/tests/sync/test_sync_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testing/tests/sync') diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index d02aba68..6ce9a5c5 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -63,7 +63,7 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ def parse(self, stream): - parser = DocStreamReceiver(None, None, lambda *_: 42) + parser = DocStreamReceiver(None, None, lambda *_: defer.succeed(42)) parser.dataReceived(stream) parser.finish() -- cgit v1.2.3