Fix symmetric encryption when syncing.

Also does:

  * Remove all crypto methods from LeapDocument.
  * Encode 'encryption_scheme' inside the document JSON.
  * Add functions for encrypting and decrypting.
  * Fix LeapSyncTarget so its connection actually returns a LeapSyncTarget.
  * Fix symmetric encryption when syncing:
    - don't try to encrypt tombstone documents.
  * Fix symmetric decryption when syncing:
    - Only try do decrypt if the incoming document has an '_encryption_scheme'
      entry with value equal to EncryptionSchemes.PUBKEY.
  * Fix doc skipping for non-syncable docs.
  * Fix tests that compared raw content with encrypted content.

1 files changed, 294 insertions, 30 deletions

diff --git a/src/leap/soledad/tests/test_sqlcipher.py b/src/leap/soledad/tests/test_sqlcipher.py
index 337ab4ee..73388202 100644
--- a/src/leap/soledad/tests/test_sqlcipher.py
+++ b/src/leap/soledad/tests/test_sqlcipher.py
@@ -2,16 +2,21 @@
 
 import os
 import time
-from pysqlcipher import dbapi2
 import unittest
-from StringIO import StringIO
+try:
+    import simplejson as json
+except ImportError:
+    import json  # noqa
 import threading
+from pysqlcipher import dbapi2
+from StringIO import StringIO
 
 # u1db stuff.
 from u1db import (
     errors,
     query_parser,
     sync,
+    vectorclock,
 )
 from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase
 
@@ -24,10 +29,12 @@ from leap.soledad.backends.sqlcipher import open as u1db_open
 from leap.soledad.backends.leap_backend import (
     LeapDocument,
     EncryptionSchemes,
+    decrypt_doc_json,
+    encrypt_doc_json,
 )
 
 # u1db tests stuff.
-from leap.soledad.tests import u1db_tests as tests
+from leap.soledad.tests import u1db_tests as tests, BaseSoledadTest
 from leap.soledad.tests.u1db_tests import test_sqlite_backend
 from leap.soledad.tests.u1db_tests import test_backends
 from leap.soledad.tests.u1db_tests import test_open
@@ -345,29 +352,6 @@ class TestSQLCipherPartialExpandDatabase(
         self.db.put_doc(doc)
         self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable)
 
-    def test_store_encryption_scheme(self):
-        doc = self.db.create_doc_from_json(tests.simple_doc)
-        # assert that docs have no encryption_scheme by default
-        self.assertEqual(EncryptionSchemes.NONE, doc.encryption_scheme)
-        # assert that we can store a different value
-        doc.encryption_scheme = EncryptionSchemes.PUBKEY
-        self.db.put_doc(doc)
-        self.assertEqual(
-            EncryptionSchemes.PUBKEY,
-            self.db.get_doc(doc.doc_id).encryption_scheme)
-        # assert that we can store another different value
-        doc.encryption_scheme = EncryptionSchemes.SYMKEY
-        self.db.put_doc(doc)
-        self.assertEqual(
-            EncryptionSchemes.SYMKEY,
-            self.db.get_doc(doc.doc_id).encryption_scheme)
-        # assert that we can store the default value
-        doc.encryption_scheme = EncryptionSchemes.NONE
-        self.db.put_doc(doc)
-        self.assertEqual(
-            EncryptionSchemes.NONE,
-            self.db.get_doc(doc.doc_id).encryption_scheme)
-
 
 #-----------------------------------------------------------------------------
 # The following tests come from `u1db.tests.test_open`.
@@ -430,7 +414,7 @@ def sync_via_synchronizer_and_leap(test, db_source, db_target,
     if trace_hook:
         test.skipTest("full trace hook unsupported over http")
     path = test._http_at[db_target]
-    target = LeapSyncTarget.connect(test.getURL(path))
+    target = LeapSyncTarget.connect(test.getURL(path), test._soledad._crypto)
     if trace_hook_shallow:
         target._set_trace_hook_shallow(trace_hook_shallow)
     return sync.Synchronizer(db_source, target).sync()
@@ -445,15 +429,192 @@ sync_scenarios.append(('pyleap', {
 }))
 
 
-class SQLCipherDatabaseSyncTests(test_sync.DatabaseSyncTests):
+class SQLCipherDatabaseSyncTests(
+        test_sync.DatabaseSyncTests, BaseSoledadTest):
 
     scenarios = sync_scenarios
 
+    def setUp(self):
+        test_sync.DatabaseSyncTests.setUp(self)
+        BaseSoledadTest.setUp(self)
+
+    def tearDown(self):
+        test_sync.DatabaseSyncTests.tearDown(self)
+        BaseSoledadTest.tearDown(self)
+
+    def test_sync_autoresolves(self):
+        # The remote database can't autoresolve conflicts based on magic
+        # content convergence, so we modify this test to leave the possibility
+        # of the remode document ending up in conflicted state.
+        self.db1 = self.create_database('test1', 'source')
+        self.db2 = self.create_database('test2', 'target')
+        doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc')
+        rev1 = doc1.rev
+        doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc')
+        rev2 = doc2.rev
+        self.sync(self.db1, self.db2)
+        doc = self.db1.get_doc('doc')
+        self.assertFalse(doc.has_conflicts)
+        # if remote content is in conflicted state, then document revisions
+        # will be different.
+        #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev)
+        v = vectorclock.VectorClockRev(doc.rev)
+        self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1)))
+        self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2)))
+
+    def test_sync_autoresolves_moar(self):
+        # here we test that when a database that has a conflicted document is
+        # the source of a sync, and the target database has a revision of the
+        # conflicted document that is newer than the source database's, and
+        # that target's database's document's content is the same as the
+        # source's document's conflict's, the source's document's conflict gets
+        # autoresolved, and the source's document's revision bumped.
+        #
+        # idea is as follows:
+        # A          B
+        # a1         -
+        #   `------->
+        # a1         a1
+        # v          v
+        # a2         a1b1
+        #   `------->
+        # a1b1+a2    a1b1
+        #            v
+        # a1b1+a2    a1b2 (a1b2 has same content as a2)
+        #   `------->
+        # a3b2       a1b2 (autoresolved)
+        #   `------->
+        # a3b2       a3b2
+        self.db1 = self.create_database('test1', 'source')
+        self.db2 = self.create_database('test2', 'target')
+        self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc')
+        self.sync(self.db1, self.db2)
+        for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]:
+            doc = db.get_doc('doc')
+            doc.set_json(content)
+            db.put_doc(doc)
+        self.sync(self.db1, self.db2)
+        # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict
+        doc = self.db1.get_doc('doc')
+        rev1 = doc.rev
+        self.assertTrue(doc.has_conflicts)
+        # set db2 to have a doc of {} (same as db1 before the conflict)
+        doc = self.db2.get_doc('doc')
+        doc.set_json('{}')
+        self.db2.put_doc(doc)
+        rev2 = doc.rev
+        # sync it across
+        self.sync(self.db1, self.db2)
+        # tadaa!
+        doc = self.db1.get_doc('doc')
+        self.assertFalse(doc.has_conflicts)
+        vec1 = vectorclock.VectorClockRev(rev1)
+        vec2 = vectorclock.VectorClockRev(rev2)
+        vec3 = vectorclock.VectorClockRev(doc.rev)
+        self.assertTrue(vec3.is_newer(vec1))
+        self.assertTrue(vec3.is_newer(vec2))
+        # because the conflict is on the source, sync it another time
+        self.sync(self.db1, self.db2)
+        # make sure db2 now has the exact same thing
+        doc1 = self.db1.get_doc('doc')
+        doc2 = self.db1.get_doc('doc')
+        if '_encryption_scheme' in doc2.content:
+            doc2.set_json(
+                decrypt_doc_json(
+                    self._soledad._crypto, doc2, doc2.get_json()))
+        self.assertEqual(doc1, doc2)
+
+    def test_sync_autoresolves_moar_backwards(self):
+        # here we would test that when a database that has a conflicted
+        # document is the target of a sync, and the source database has a
+        # revision of the conflicted document that is newer than the target
+        # database's, and that source's database's document's content is the
+        # same as the target's document's conflict's, the target's document's
+        # conflict gets autoresolved, and the document's revision bumped.
+        #
+        # Despite that, in Soledad we suppose that the server never syncs, so
+        # it never has conflicted documents. Also, if it had, convergence
+        # would not be possible in terms of document's contents.
+        #
+        # Therefore we suppress this test.
+        pass
+
+    def test_sync_autoresolves_moar_backwards_three(self):
+        # We use the same reasoning from the last test to suppress this one.
+        pass
+
+    def test_sync_propagates_resolution(self):
+        self.db1 = self.create_database('test1', 'both')
+        self.db2 = self.create_database('test2', 'both')
+        doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc')
+        db3 = self.create_database('test3', 'both')
+        self.sync(self.db2, self.db1)
+        self.assertEqual(
+            self.db1._get_generation_info(),
+            self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid))
+        self.assertEqual(
+            self.db2._get_generation_info(),
+            self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid))
+        self.sync(db3, self.db1)
+        # update on 2
+        doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}')
+        self.db2.put_doc(doc2)
+        self.sync(self.db2, db3)
+        self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev)
+        # update on 1
+        doc1.set_json('{"a": 3}')
+        self.db1.put_doc(doc1)
+        # conflicts
+        self.sync(self.db2, self.db1)
+        self.sync(db3, self.db1)
+        self.assertTrue(self.db2.get_doc('the-doc').has_conflicts)
+        self.assertTrue(db3.get_doc('the-doc').has_conflicts)
+        # resolve
+        conflicts = self.db2.get_doc_conflicts('the-doc')
+        doc4 = self.make_document('the-doc', None, '{"a": 4}')
+        revs = [doc.rev for doc in conflicts]
+        self.db2.resolve_doc(doc4, revs)
+        doc2 = self.db2.get_doc('the-doc')
+        self.assertEqual(doc4.get_json(), doc2.get_json())
+        self.assertFalse(doc2.has_conflicts)
+        self.sync(self.db2, db3)
+        doc3 = db3.get_doc('the-doc')
+        if '_encryption_scheme' in doc3.content:
+            doc3.set_json(
+                decrypt_doc_json(
+                    self._soledad._crypto, doc3.doc_id, doc3.get_json()))
+        self.assertEqual(doc4.get_json(), doc3.get_json())
+        self.assertFalse(doc3.has_conflicts)
+
+    def test_sync_puts_changes(self):
+        self.db1 = self.create_database('test1', 'source')
+        self.db2 = self.create_database('test2', 'target')
+        doc = self.db1.create_doc_from_json(tests.simple_doc)
+        self.assertEqual(1, self.sync(self.db1, self.db2))
+        exp_doc = self.make_document(
+            doc.doc_id, doc.rev, tests.simple_doc, False)
+        doc2 = self.db2.get_doc(doc.doc_id)
+         # decrypt to compare it it is the case
+        if '_encryption_scheme' in doc2.content:
+            doc2 = self.db2.get_doc(doc.doc_id)
+            doc2.set_json(
+                decrypt_doc_json(
+                    self._soledad._crypto, doc.doc_id, doc2.get_json()))
+        self.assertEqual(exp_doc, doc2)
+        self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0])
+        self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0])
+        self.assertLastExchangeLog(
+            self.db2,
+            {'receive': {'docs': [(doc.doc_id, doc.rev)],
+                         'source_uid': 'test1',
+                         'source_gen': 1, 'last_known_gen': 0},
+             'return': {'docs': [], 'last_gen': 1}})
+
 
 def _make_local_db_and_leap_target(test, path='test'):
     test.startServer()
     db = test.request_state._create_database(os.path.basename(path))
-    st = LeapSyncTarget.connect(test.getURL(path))
+    st = LeapSyncTarget.connect(test.getURL(path), test._soledad._crypto)
     return db, st
 
 
@@ -464,11 +625,114 @@ target_scenarios = [
 ]
 
 
-class SQLCipherSyncTargetTests(test_sync.DatabaseSyncTargetTests):
+class SQLCipherSyncTargetTests(
+        test_sync.DatabaseSyncTargetTests, BaseSoledadTest):
 
     scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS,
                                           target_scenarios))
 
+    def setUp(self):
+        test_sync.DatabaseSyncTargetTests.setUp(self)
+        BaseSoledadTest.setUp(self)
+
+    def tearDown(self):
+        test_sync.DatabaseSyncTargetTests.tearDown(self)
+        BaseSoledadTest.tearDown(self)
+
+    def test_sync_exchange(self):
+        """
+        Modified to account for possibly receiving encrypted documents from
+        sever-side.
+        """
+        docs_by_gen = [
+            (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
+             'T-sid')]
+        new_gen, trans_id = self.st.sync_exchange(
+            docs_by_gen, 'replica', last_known_generation=0,
+            last_known_trans_id=None, return_doc_cb=self.receive_doc)
+        # decrypt doc1 for comparison if needed
+        tmpdoc = self.db.get_doc('doc-id')
+        if '_encryption_scheme' in tmpdoc.content:
+            tmpdoc.set_json(
+                decrypt_doc_json(
+                    self._soledad._crypto, tmpdoc.doc_id,
+                    tmpdoc.get_json()))
+        self.assertEqual(tmpdoc.rev, 'replica:1')
+        self.assertEqual(tmpdoc.content, json.loads(tests.simple_doc))
+        self.assertFalse(tmpdoc.has_conflicts)
+        self.assertTransactionLog(['doc-id'], self.db)
+        last_trans_id = self.getLastTransId(self.db)
+        self.assertEqual(([], 1, last_trans_id),
+                         (self.other_changes, new_gen, last_trans_id))
+        self.assertEqual(10, self.st.get_sync_info('replica')[3])
+
+    def test_sync_exchange_push_many(self):
+        """
+        Modified to account for possibly receiving encrypted documents from
+        sever-side.
+        """
+        docs_by_gen = [
+            (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
+             'T-1'),
+            (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11,
+             'T-2')]
+        new_gen, trans_id = self.st.sync_exchange(
+            docs_by_gen, 'replica', last_known_generation=0,
+            last_known_trans_id=None, return_doc_cb=self.receive_doc)
+        # decrypt doc1 for comparison if needed
+        tmpdoc1 = self.db.get_doc('doc-id')
+        if '_encryption_scheme' in tmpdoc1.content:
+            tmpdoc1.set_json(
+                decrypt_doc_json(
+                    self._soledad._crypto, tmpdoc1.doc_id,
+                    tmpdoc1.get_json()))
+        self.assertEqual(tmpdoc1.rev, 'replica:1')
+        self.assertEqual(tmpdoc1.content, json.loads(tests.simple_doc))
+        self.assertFalse(tmpdoc1.has_conflicts)
+        # decrypt doc2 for comparison if needed
+        tmpdoc2 = self.db.get_doc('doc-id2')
+        if '_encryption_scheme' in tmpdoc2.content:
+            tmpdoc2.set_json(
+                decrypt_doc_json(
+                    self._soledad._crypto, tmpdoc2.doc_id,
+                    tmpdoc2.get_json()))
+        self.assertEqual(tmpdoc2.rev, 'replica:1')
+        self.assertEqual(tmpdoc2.content, json.loads(tests.nested_doc))
+        self.assertFalse(tmpdoc2.has_conflicts)
+        self.assertTransactionLog(['doc-id', 'doc-id2'], self.db)
+        last_trans_id = self.getLastTransId(self.db)
+        self.assertEqual(([], 2, last_trans_id),
+                         (self.other_changes, new_gen, trans_id))
+        self.assertEqual(11, self.st.get_sync_info('replica')[3])
+
+    def test_sync_exchange_returns_many_new_docs(self):
+        """
+        Modified to account for JSON serialization differences.
+        """
+        doc = self.db.create_doc_from_json(tests.simple_doc)
+        doc2 = self.db.create_doc_from_json(tests.nested_doc)
+        self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
+        new_gen, _ = self.st.sync_exchange(
+            [], 'other-replica', last_known_generation=0,
+            last_known_trans_id=None, return_doc_cb=self.receive_doc)
+        self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
+        self.assertEqual(2, new_gen)
+        self.assertEqual(
+            [(doc.doc_id, doc.rev, 1),
+             (doc2.doc_id, doc2.rev, 2)],
+            [c[:2]+c[3:4] for c in self.other_changes])
+        self.assertEqual(
+            json.dumps(tests.simple_doc),
+            json.dumps(self.other_changes[0][2]))
+        self.assertEqual(
+            json.loads(tests.nested_doc),
+            json.loads(self.other_changes[1][2]))
+        if self.whitebox:
+            self.assertEqual(
+                self.db._last_exchange_log['return'],
+                {'last_gen': 2, 'docs':
+                 [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
+
 
 #-----------------------------------------------------------------------------
 # Tests for actual encryption of the database