[bug] include_deleted=True on sync

Also refactored tests and code to stop relying on old parameters which
included docs instead of get_doc calls.

4 files changed, 56 insertions, 32 deletions

diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index ae0b126f..c15488a0 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -86,21 +86,17 @@ class HTTPDocSender(object):
 
     @defer.inlineCallbacks
     def _prepare_one_doc(self, entry, body, idx, total):
-        get_doc, gen, trans_id = entry
-        doc, content = yield self._encrypt_doc(get_doc)
+        get_doc_call, gen, trans_id = entry
+        doc, content = yield self._encrypt_doc(get_doc_call)
         body.insert_info(
             id=doc.doc_id, rev=doc.rev, content=content, gen=gen,
             trans_id=trans_id, number_of_docs=total,
             doc_idx=idx)
 
     @defer.inlineCallbacks
-    def _encrypt_doc(self, get_doc):
-        if type(get_doc) == tuple:
-            f, args = get_doc
-            doc = yield f(args)
-        else:
-            # tests
-            doc = get_doc
+    def _encrypt_doc(self, get_doc_call):
+        f, args, kwargs = get_doc_call
+        doc = yield f(*args, **kwargs)
         if doc.is_tombstone():
             defer.returnValue((doc, None))
         else:
diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 6bd58f8f..70c841d6 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -141,11 +141,7 @@ class SoledadSynchronizer(Synchronizer):
             self.sync_phase[0] += 1
         # --------------------------------------------------------------------
 
-        ids_sent = [doc_id for doc_id, _, _ in changes]
-        docs_by_generation = []
-        for doc_id, gen, trans in changes:
-            get_doc = (self.source.get_doc, doc_id)
-            docs_by_generation.append((get_doc, gen, trans))
+        docs_by_generation = self._docs_by_gen_from_changes(changes)
 
         # exchange documents and try to insert the returned ones with
         # the target, return target synced-up-to gen.
@@ -153,6 +149,7 @@ class SoledadSynchronizer(Synchronizer):
             docs_by_generation, self.source._replica_uid,
             target_last_known_gen, target_last_known_trans_id,
             self._insert_doc_from_target, ensure_callback=ensure_callback)
+        ids_sent = [doc_id for doc_id, _, _ in changes]
         logger.debug("target gen after sync: %d" % new_gen)
         logger.debug("target trans_id after sync: %s" % new_trans_id)
         if hasattr(self.source, 'commit'):  # sqlcipher backend speed up
@@ -185,6 +182,14 @@ class SoledadSynchronizer(Synchronizer):
 
         defer.returnValue(my_gen)
 
+    def _docs_by_gen_from_changes(self, changes):
+        docs_by_generation = []
+        kwargs = {'include_deleted': True}
+        for doc_id, gen, trans in changes:
+            get_doc = (self.source.get_doc, (doc_id,), kwargs)
+            docs_by_generation.append((get_doc, gen, trans))
+        return docs_by_generation
+
     def complete_sync(self):
         """
         Last stage of the synchronization:
diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py
index a434e944..76757c5b 100644
--- a/testing/tests/sync/test_sync.py
+++ b/testing/tests/sync/test_sync.py
@@ -19,6 +19,7 @@ import threading
 import time
 
 from urlparse import urljoin
+from mock import Mock
 from twisted.internet import defer
 
 from testscenarios import TestWithScenarios
@@ -210,3 +211,21 @@ class TestSoledadDbSync(
             self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False)
 
     # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration
+
+
+class TestSoledadSynchronizer(BaseSoledadTest):
+
+    def setUp(self):
+        BaseSoledadTest.setUp(self)
+        self.db = Mock()
+        self.target = Mock()
+        self.synchronizer = sync.SoledadSynchronizer(
+            self.db,
+            self.target)
+
+    def test_docs_by_gen_includes_deleted(self):
+        changes = [('id', 'gen', 'trans')]
+        docs_by_gen = self.synchronizer._docs_by_gen_from_changes(changes)
+        f, args, kwargs = docs_by_gen[0][0]
+        self.assertIn('include_deleted', kwargs)
+        self.assertTrue(kwargs['include_deleted'])
diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py
index fd1d413e..e32f08b3 100644
--- a/testing/tests/sync/test_sync_target.py
+++ b/testing/tests/sync/test_sync_target.py
@@ -199,8 +199,9 @@ class TestSoledadSyncTarget(
             other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
 
         doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
+        get_doc = (lambda _: doc, (1,), {})
         new_gen, trans_id = yield remote_target.sync_exchange(
-            [(doc, 10, 'T-sid')], 'replica', last_known_generation=0,
+            [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0,
             last_known_trans_id=None, insert_doc_cb=receive_doc)
         self.assertEqual(1, new_gen)
         self.assertGetEncryptedDoc(
@@ -247,10 +248,12 @@ class TestSoledadSyncTarget(
         doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
         doc2 = self.make_document('doc-here2', 'replica:1',
                                   '{"value": "here2"}')
+        get_doc1 = (lambda _: doc1, (1,), {})
+        get_doc2 = (lambda _: doc2, (2,), {})
 
         with self.assertRaises(l2db.errors.U1DBError):
             yield remote_target.sync_exchange(
-                [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')],
+                [(get_doc1, 10, 'T-sid'), (get_doc2, 11, 'T-sud')],
                 'replica',
                 last_known_generation=0,
                 last_known_trans_id=None,
@@ -265,7 +268,7 @@ class TestSoledadSyncTarget(
         # retry
         trigger_ids = []
         new_gen, trans_id = yield remote_target.sync_exchange(
-            [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0,
+            [(get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0,
             last_known_trans_id=None, insert_doc_cb=receive_doc)
         self.assertGetEncryptedDoc(
             db, 'doc-here2', 'replica:1', '{"value": "here2"}',
@@ -295,8 +298,9 @@ class TestSoledadSyncTarget(
             replica_uid_box.append(replica_uid)
 
         doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
+        get_doc = (lambda _: doc, (1,), {})
         new_gen, trans_id = yield remote_target.sync_exchange(
-            [(doc, 10, 'T-sid')], 'replica', last_known_generation=0,
+            [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0,
             last_known_trans_id=None, insert_doc_cb=receive_doc,
             ensure_callback=ensure_cb)
         self.assertEqual(1, new_gen)
@@ -408,8 +412,8 @@ class SoledadDatabaseSyncTargetTests(
         This test was adapted to decrypt remote content before assert.
         """
         docs_by_gen = [
-            (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
-             'T-sid')]
+            ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}),
+                10, 'T-sid')]
         new_gen, trans_id = yield self.st.sync_exchange(
             docs_by_gen, 'replica', last_known_generation=0,
             last_known_trans_id=None, insert_doc_cb=self.receive_doc)
@@ -430,10 +434,10 @@ class SoledadDatabaseSyncTargetTests(
         This test was adapted to decrypt remote content before assert.
         """
         docs_by_gen = [
-            (self.make_document(
-                'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'),
-            (self.make_document(
-                'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')]
+            ((self.make_document,
+                ('doc-id', 'replica:1', tests.simple_doc), {}), 10, 'T-1'),
+            ((self.make_document,
+                ('doc-id2', 'replica:1', tests.nested_doc), {}), 11, 'T-2')]
         new_gen, trans_id = yield self.st.sync_exchange(
             docs_by_gen, 'replica', last_known_generation=0,
             last_known_trans_id=None, insert_doc_cb=self.receive_doc)
@@ -515,7 +519,7 @@ class SoledadDatabaseSyncTargetTests(
         doc = self.db.create_doc_from_json('{}')
         edit_rev = 'replica:1|' + doc.rev
         docs_by_gen = [
-            (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')]
+            ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')]
         new_gen, trans_id = yield self.st.sync_exchange(
             docs_by_gen, 'replica', last_known_generation=0,
             last_known_trans_id=None, insert_doc_cb=self.receive_doc)
@@ -534,7 +538,7 @@ class SoledadDatabaseSyncTargetTests(
         self.assertTransactionLog([doc.doc_id], self.db)
         new_doc = '{"key": "altval"}'
         docs_by_gen = [
-            (self.make_document(doc.doc_id, 'replica:1', new_doc), 10,
+            ((self.make_document, (doc.doc_id, 'replica:1', new_doc), {}), 10,
              'T-sid')]
         new_gen, _ = yield self.st.sync_exchange(
             docs_by_gen, 'replica', last_known_generation=0,
@@ -554,7 +558,7 @@ class SoledadDatabaseSyncTargetTests(
         self.assertTransactionLog([doc.doc_id], self.db)
         gen, txid = self.db._get_generation_info()
         docs_by_gen = [
-            (self.make_document(doc.doc_id, doc.rev, tests.simple_doc),
+            ((self.make_document, (doc.doc_id, doc.rev, tests.simple_doc), {}),
              10, 'T-sid')]
         new_gen, _ = yield self.st.sync_exchange(
             docs_by_gen, 'replica', last_known_generation=gen,
@@ -600,7 +604,7 @@ class SoledadDatabaseSyncTargetTests(
         self.assertTransactionLog([doc.doc_id], self.db)
         new_doc = '{"key": "altval"}'
         docs_by_gen = [
-            (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10,
+            ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10,
              'T-sid')]
         new_gen, _ = yield self.st.sync_exchange(
             docs_by_gen, 'other-replica', last_known_generation=0,
@@ -625,7 +629,7 @@ class SoledadDatabaseSyncTargetTests(
         self.assertTransactionLog([doc.doc_id], self.db)
         new_doc = '{"key": "altval"}'
         docs_by_gen = [
-            (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10,
+            ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10,
              'T-sid')]
         new_gen, _ = yield self.st.sync_exchange(
             docs_by_gen, 'other-replica', last_known_generation=0,
@@ -646,7 +650,7 @@ class SoledadDatabaseSyncTargetTests(
         self.assertTransactionLog([doc.doc_id], self.db)
         new_doc = '{"key": "altval"}'
         docs_by_gen = [
-            (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10,
+            ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10,
              'T-sid')]
         new_gen, _ = yield self.st.sync_exchange(
             docs_by_gen, 'other-replica', last_known_generation=0,
@@ -657,8 +661,8 @@ class SoledadDatabaseSyncTargetTests(
     def test_sync_exchange_converged_handling(self):
         doc = self.db.create_doc_from_json(tests.simple_doc)
         docs_by_gen = [
-            (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'),
-            (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5,
+            ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'),
+            ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5,
              'T-bar')]
         new_gen, _ = yield self.st.sync_exchange(
             docs_by_gen, 'other-replica', last_known_generation=0,