From 632cd2414a47bc7f471d8f501ab6250293aedf51 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 17 Nov 2016 00:38:55 -0300 Subject: [bug] include_deleted=True on sync Also refactored tests and code to stop relying on old parameters which included docs instead of get_doc calls. --- client/src/leap/soledad/client/http_target/send.py | 14 +++----- client/src/leap/soledad/client/sync.py | 15 +++++--- testing/tests/sync/test_sync.py | 19 ++++++++++ testing/tests/sync/test_sync_target.py | 40 ++++++++++++---------- 4 files changed, 56 insertions(+), 32 deletions(-) diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index ae0b126f..c15488a0 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -86,21 +86,17 @@ class HTTPDocSender(object): @defer.inlineCallbacks def _prepare_one_doc(self, entry, body, idx, total): - get_doc, gen, trans_id = entry - doc, content = yield self._encrypt_doc(get_doc) + get_doc_call, gen, trans_id = entry + doc, content = yield self._encrypt_doc(get_doc_call) body.insert_info( id=doc.doc_id, rev=doc.rev, content=content, gen=gen, trans_id=trans_id, number_of_docs=total, doc_idx=idx) @defer.inlineCallbacks - def _encrypt_doc(self, get_doc): - if type(get_doc) == tuple: - f, args = get_doc - doc = yield f(args) - else: - # tests - doc = get_doc + def _encrypt_doc(self, get_doc_call): + f, args, kwargs = get_doc_call + doc = yield f(*args, **kwargs) if doc.is_tombstone(): defer.returnValue((doc, None)) else: diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 6bd58f8f..70c841d6 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -141,11 +141,7 @@ class SoledadSynchronizer(Synchronizer): self.sync_phase[0] += 1 # -------------------------------------------------------------------- - ids_sent = [doc_id for doc_id, _, _ in changes] - docs_by_generation = [] - for doc_id, gen, trans in changes: - get_doc = (self.source.get_doc, doc_id) - docs_by_generation.append((get_doc, gen, trans)) + docs_by_generation = self._docs_by_gen_from_changes(changes) # exchange documents and try to insert the returned ones with # the target, return target synced-up-to gen. @@ -153,6 +149,7 @@ class SoledadSynchronizer(Synchronizer): docs_by_generation, self.source._replica_uid, target_last_known_gen, target_last_known_trans_id, self._insert_doc_from_target, ensure_callback=ensure_callback) + ids_sent = [doc_id for doc_id, _, _ in changes] logger.debug("target gen after sync: %d" % new_gen) logger.debug("target trans_id after sync: %s" % new_trans_id) if hasattr(self.source, 'commit'): # sqlcipher backend speed up @@ -185,6 +182,14 @@ class SoledadSynchronizer(Synchronizer): defer.returnValue(my_gen) + def _docs_by_gen_from_changes(self, changes): + docs_by_generation = [] + kwargs = {'include_deleted': True} + for doc_id, gen, trans in changes: + get_doc = (self.source.get_doc, (doc_id,), kwargs) + docs_by_generation.append((get_doc, gen, trans)) + return docs_by_generation + def complete_sync(self): """ Last stage of the synchronization: diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py index a434e944..76757c5b 100644 --- a/testing/tests/sync/test_sync.py +++ b/testing/tests/sync/test_sync.py @@ -19,6 +19,7 @@ import threading import time from urlparse import urljoin +from mock import Mock from twisted.internet import defer from testscenarios import TestWithScenarios @@ -210,3 +211,21 @@ class TestSoledadDbSync( self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration + + +class TestSoledadSynchronizer(BaseSoledadTest): + + def setUp(self): + BaseSoledadTest.setUp(self) + self.db = Mock() + self.target = Mock() + self.synchronizer = sync.SoledadSynchronizer( + self.db, + self.target) + + def test_docs_by_gen_includes_deleted(self): + changes = [('id', 'gen', 'trans')] + docs_by_gen = self.synchronizer._docs_by_gen_from_changes(changes) + f, args, kwargs = docs_by_gen[0][0] + self.assertIn('include_deleted', kwargs) + self.assertTrue(kwargs['include_deleted']) diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index fd1d413e..e32f08b3 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -199,8 +199,9 @@ class TestSoledadSyncTarget( other_docs.append((doc.doc_id, doc.rev, doc.get_json())) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertEqual(1, new_gen) self.assertGetEncryptedDoc( @@ -247,10 +248,12 @@ class TestSoledadSyncTarget( doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') doc2 = self.make_document('doc-here2', 'replica:1', '{"value": "here2"}') + get_doc1 = (lambda _: doc1, (1,), {}) + get_doc2 = (lambda _: doc2, (2,), {}) with self.assertRaises(l2db.errors.U1DBError): yield remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], + [(get_doc1, 10, 'T-sid'), (get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, @@ -265,7 +268,7 @@ class TestSoledadSyncTarget( # retry trigger_ids = [] new_gen, trans_id = yield remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, + [(get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc) self.assertGetEncryptedDoc( db, 'doc-here2', 'replica:1', '{"value": "here2"}', @@ -295,8 +298,9 @@ class TestSoledadSyncTarget( replica_uid_box.append(replica_uid) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=receive_doc, ensure_callback=ensure_cb) self.assertEqual(1, new_gen) @@ -408,8 +412,8 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] + ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}), + 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -430,10 +434,10 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc), {}), 10, 'T-1'), + ((self.make_document, + ('doc-id2', 'replica:1', tests.nested_doc), {}), 11, 'T-2')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -515,7 +519,7 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json('{}') edit_rev = 'replica:1|' + doc.rev docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -534,7 +538,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + ((self.make_document, (doc.doc_id, 'replica:1', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, @@ -554,7 +558,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) gen, txid = self.db._get_generation_info() docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), + ((self.make_document, (doc.doc_id, doc.rev, tests.simple_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=gen, @@ -600,7 +604,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -625,7 +629,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -646,7 +650,7 @@ class SoledadDatabaseSyncTargetTests( self.assertTransactionLog([doc.doc_id], self.db) new_doc = '{"key": "altval"}' docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, 'T-sid')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, @@ -657,8 +661,8 @@ class SoledadDatabaseSyncTargetTests( def test_sync_exchange_converged_handling(self): doc = self.db.create_doc_from_json(tests.simple_doc) docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5, 'T-bar')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, -- cgit v1.2.3