From c8bda65e0029999e1c2587b74d490aee2d05137e Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 14 Jul 2014 19:22:44 -0300
Subject: Add changes file (#5895).

---
 client/changes/feature_5895-store-all-incoming-documents-in-sync-db | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 client/changes/feature_5895-store-all-incoming-documents-in-sync-db

(limited to 'client')

diff --git a/client/changes/feature_5895-store-all-incoming-documents-in-sync-db b/client/changes/feature_5895-store-all-incoming-documents-in-sync-db
new file mode 100644
index 00000000..71d5a91f
--- /dev/null
+++ b/client/changes/feature_5895-store-all-incoming-documents-in-sync-db
@@ -0,0 +1 @@
+  o Store all incoming documents in the sync db (#5895).
-- 
cgit v1.2.3


From 69f5087c718cc534a969fcba0fcb35812c88ad8b Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 14 Jul 2014 20:01:01 -0300
Subject: Add encrypted field to sync db (#5895).

---
 client/src/leap/soledad/client/crypto.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 7133f804..89220860 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -691,7 +691,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
     """
     # TODO implement throttling to reduce cpu usage??
     TABLE_NAME = "docs_received"
-    FIELD_NAMES = "doc_id, rev, content, gen, trans_id"
+    FIELD_NAMES = "doc_id, rev, content, gen, trans_id, encrypted"
 
     write_encrypted_lock = threading.Lock()
 
@@ -733,13 +733,15 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :type trans_id: str
         """
         docstr = json.dumps(content)
-        sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % (
+        sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % (
             self.TABLE_NAME,)
 
         con = self._sync_db
         with self._sync_db_write_lock:
             with con:
-                con.execute(sql_ins, (doc_id, doc_rev, docstr, gen, trans_id))
+                con.execute(
+                    sql_ins,
+                    (doc_id, doc_rev, docstr, gen, trans_id, 1))
 
     def insert_marker_for_received_doc(self, doc_id, doc_rev, gen):
         """
@@ -757,12 +759,12 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :param gen: the Document Generation
         :type gen: int
         """
-        sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % (
+        sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % (
             self.TABLE_NAME,)
         con = self._sync_db
         with self._sync_db_write_lock:
             with con:
-                con.execute(sql_ins, (doc_id, doc_rev, '', gen, ''))
+                con.execute(sql_ins, (doc_id, doc_rev, '', gen, '', 0))
 
     def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id):
         """
-- 
cgit v1.2.3


From 95f34ccab21e36ea48e0d01a4b9ee00e6094d1ec Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 14 Jul 2014 20:05:00 -0300
Subject: Store non-encrypted docs in the sync db (#5895).

---
 client/src/leap/soledad/client/crypto.py | 40 +++++++-------------------------
 1 file changed, 8 insertions(+), 32 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 89220860..128e40d7 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -743,29 +743,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
                     sql_ins,
                     (doc_id, doc_rev, docstr, gen, trans_id, 1))
 
-    def insert_marker_for_received_doc(self, doc_id, doc_rev, gen):
-        """
-        Insert a marker with the document id, revision and generation on the
-        sync db. This document does not have an encrypted payload, so the
-        content has already been inserted into the decrypted_docs dictionary
-        from where it can be picked following generation order.
-        We need to leave here the marker to be able to calculate the expected
-        insertion order for a synchronization batch.
-
-        :param doc_id: The Document ID.
-        :type doc_id: str
-        :param doc_rev: The Document Revision
-        :param doc_rev: str
-        :param gen: the Document Generation
-        :type gen: int
-        """
-        sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % (
-            self.TABLE_NAME,)
-        con = self._sync_db
-        with self._sync_db_write_lock:
-            with con:
-                con.execute(sql_ins, (doc_id, doc_rev, '', gen, '', 0))
-
     def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id):
         """
         Insert a document that is not symmetrically encrypted.
@@ -783,16 +760,15 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :param trans_id: Transaction ID
         :type trans_id: str
         """
-        # XXX this need a deeper review / testing.
-        # I believe that what I'm doing here is prone to problems
-        # if the sync is interrupted (ie, client crash) in the worst possible
-        # moment. We would need a recover strategy in that case
-        # (or, insert the document in the table all the same, but with a flag
-        # saying if the document is sym-encrypted or not),
         content = json.dumps(content)
-        result = doc_id, doc_rev, content, gen, trans_id
-        self.decrypted_docs[gen] = result
-        self.insert_marker_for_received_doc(doc_id, doc_rev, gen)
+        sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % (
+            self.TABLE_NAME,)
+        con = self._sync_db
+        with self._sync_db_write_lock:
+            with con:
+                con.execute(
+                    sql_ins,
+                    (doc_id, doc_rev, content, gen, trans_id, 0))
 
     def delete_encrypted_received_doc(self, doc_id, doc_rev):
         """
-- 
cgit v1.2.3


From 51e0bf7f79a444661b10fe418af85b0a60f41afb Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 14 Jul 2014 20:09:01 -0300
Subject: Insert received docs in sync db after decryption (#5895).

---
 client/src/leap/soledad/client/crypto.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 128e40d7..d0a5a693 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -876,7 +876,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         """
         doc_id, rev, content, gen, trans_id = result
         logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" % (doc_id, rev, gen))
-        self.decrypted_docs[gen] = result
+        self.insert_received_doc(doc_id, rev, content, gen, trans_id)
 
     def get_docs_by_generation(self):
         """
-- 
cgit v1.2.3


From 54a69eb14189e06556af15dcdf5d5ed424778fc2 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Tue, 15 Jul 2014 13:46:42 -0300
Subject: Store all received docs in sync db (#5895).

---
 client/src/leap/soledad/client/crypto.py | 156 ++++++++++++++++---------------
 client/src/leap/soledad/client/target.py |  12 ++-
 2 files changed, 89 insertions(+), 79 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index d0a5a693..4a73a910 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -256,11 +256,11 @@ class SoledadCrypto(object):
     secret = property(
         _get_secret, doc='The secret used for symmetric encryption')
 
+
 #
 # Crypto utilities for a SoledadDocument.
 #
 
-
 def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret):
     """
     Calculate a MAC for C{doc} using C{ciphertext}.
@@ -657,26 +657,6 @@ def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret):
     return doc_id, doc_rev, decrypted_content, gen, trans_id
 
 
-def get_insertable_docs_by_gen(expected, got):
-    """
-    Return a list of documents ready to be inserted. This list is computed
-    by aligning the expected list with the already gotten docs, and returning
-    the maximum number of docs that can be processed in the expected order
-    before finding a gap.
-
-    :param expected: A list of generations to be inserted.
-    :type expected: list
-
-    :param got: A dictionary whose values are the docs to be inserted.
-    :type got: dict
-    """
-    ordered = [got.get(i) for i in expected]
-    if None in ordered:
-        return ordered[:ordered.index(None)]
-    else:
-        return ordered
-
-
 class SyncDecrypterPool(SyncEncryptDecryptPool):
     """
     Pool of workers that spawn subprocesses to execute the symmetric decryption
@@ -700,10 +680,18 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         Initialize the decrypter pool, and setup a dict for putting the
         results of the decrypted docs until they are picked by the insert
         routine that gets them in order.
+
+        :param insert_doc_cb: A callback for inserting received documents from
+                              target. If not overriden, this will call u1db
+                              insert_doc_from_target in synchronizer, which
+                              implements the TAKE OTHER semantics.
+        :type insert_doc_cb: function
+        :param last_known_generation: Target's last known generation.
+        :type last_known_generation: int
         """
         self._insert_doc_cb = kwargs.pop("insert_doc_cb")
+        self._last_known_generation = kwargs.pop("last_known_generation")
         SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
-        self.decrypted_docs = {}
         self.source_replica_uid = None
 
     def set_source_replica_uid(self, source_replica_uid):
@@ -733,12 +721,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :type trans_id: str
         """
         docstr = json.dumps(content)
+        sql_del = "DELETE FROM '%s' WHERE doc_id=?" % (self.TABLE_NAME,)
         sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % (
             self.TABLE_NAME,)
 
         con = self._sync_db
         with self._sync_db_write_lock:
             with con:
+                con.execute(sql_del, (doc_id, ))
                 con.execute(
                     sql_ins,
                     (doc_id, doc_rev, docstr, gen, trans_id, 1))
@@ -760,20 +750,23 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :param trans_id: Transaction ID
         :type trans_id: str
         """
-        content = json.dumps(content)
+        if not isinstance(content, str):
+            content = json.dumps(content)
+        sql_del = "DELETE FROM '%s' WHERE doc_id=?" % (
+            self.TABLE_NAME,)
         sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % (
             self.TABLE_NAME,)
         con = self._sync_db
         with self._sync_db_write_lock:
             with con:
+                con.execute(sql_del, (doc_id,))
                 con.execute(
                     sql_ins,
                     (doc_id, doc_rev, content, gen, trans_id, 0))
 
-    def delete_encrypted_received_doc(self, doc_id, doc_rev):
+    def delete_received_doc(self, doc_id, doc_rev):
         """
-        Delete a encrypted received doc after it was inserted into the local
-        db.
+        Delete a received doc after it was inserted into the local db.
 
         :param doc_id: Document ID.
         :type doc_id: str
@@ -787,7 +780,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             with con:
                 con.execute(sql_del, (doc_id, doc_rev))
 
-    def decrypt_doc(self, doc_id, rev, source_replica_uid, workers=True):
+    def decrypt_doc(self, doc_id, rev, content, gen, trans_id,
+                    source_replica_uid, workers=True):
         """
         Symmetrically decrypt a document.
 
@@ -795,6 +789,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :type doc: str
         :param rev: The revision of the document.
         :type rev: str
+        :param content: The serialized content of the document.
+        :type content: str
+        :param gen: The generation corresponding to the modification of that
+                    document.
+        :type gen: int
+        :param trans_id: The transaction id corresponding to the modification
+                         of that document.
+        :type trans_id: str
         :param source_replica_uid:
         :type source_replica_uid: str
 
@@ -813,33 +815,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             logger.debug("Sync decrypter pool: no insert_doc_cb() yet.")
             return
 
-        # XXX move to get_doc function...
-        c = self._sync_db.cursor()
-        sql = "SELECT * FROM '%s' WHERE doc_id=? AND rev=?" % (
-            self.TABLE_NAME,)
-        try:
-            c.execute(sql, (doc_id, rev))
-            res = c.fetchone()
-        except Exception as exc:
-            logger.warning("Error getting docs from syncdb: %r" % (exc,))
-            return
-        if res is None:
-            logger.debug("Doc %s:%s does not exist in sync db" % (doc_id, rev))
-            return
-
         soledad_assert(self._crypto is not None, "need a crypto object")
-        try:
-            doc_id, rev, docstr, gen, trans_id = res
-        except ValueError:
-            logger.warning("Wrong entry in sync db")
-            return
 
-        if len(docstr) == 0:
+        if len(content) == 0:
             # not encrypted payload
             return
 
         try:
-            content = json.loads(docstr)
+            content = json.loads(content)
         except TypeError:
             logger.warning("Wrong type while decoding json: %s" % repr(docstr))
             return
@@ -867,34 +850,61 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
 
     def decrypt_doc_cb(self, result):
         """
-        Temporarily store the decryption result in a dictionary where it will
-        be picked by process_decrypted.
+        Store the decryption result in the sync db from where it will later be
+        picked by process_decrypted.
 
         :param result: A tuple containing the doc id, revision and encrypted
         content.
         :type result: tuple(str, str, str)
         """
         doc_id, rev, content, gen, trans_id = result
-        logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" % (doc_id, rev, gen))
+        logger.debug("Sync decrypter pool: decrypted doc %s: %s %s"
+                     % (doc_id, rev, gen))
         self.insert_received_doc(doc_id, rev, content, gen, trans_id)
 
-    def get_docs_by_generation(self):
+    def get_docs_by_generation(self, encrypted=None):
         """
         Get all documents in the received table from the sync db,
         ordered by generation.
 
-        :return: list of doc_id, rev, generation
+        :param encrypted: If not None, only return documents with encrypted
+                          field equal to given parameter.
+        :type encrypted: bool
+
+        :return: list of doc_id, rev, generation, gen, trans_id
+        :rtype: list
         """
+        sql = "SELECT doc_id, rev, content, gen, trans_id, encrypted FROM %s" \
+              % self.TABLE_NAME
+        if encrypted is not None:
+            sql += " WHERE encrypted = %d" % int(encrypted)
+        sql += " ORDER BY gen"
         c = self._sync_db.cursor()
-        sql = "SELECT doc_id, rev, gen FROM %s ORDER BY gen" % (
-            self.TABLE_NAME,)
         c.execute(sql)
-        return c.fetchall()
+        # TODO: due to unknown reasons, the fetchall() method may return empty
+        # values, so we filter them out here. We have to perform some tests to
+        # understand why and when this happens.
+        docs = filter(lambda entry: len(entry) > 0, c.fetchall())
+        return docs
+
+    def get_insertable_docs_by_gen(self):
+        """
+        Return a list of documents ready to be inserted.
+        """
+        docs = self.get_docs_by_generation(encrypted=False)
+        insertable = []
+        if docs:
+            last_gen = self._last_known_generation
+            for doc_id, rev, content, gen, trans_id, _ in docs:
+                if gen != (last_gen + 1):
+                    break
+                insertable.append((doc_id, rev, content, gen, trans_id))
+                last_gen = gen
+        return insertable
 
-    def count_received_encrypted_docs(self):
+    def count_docs_in_sync_db(self):
         """
-        Count how many documents we have in the table for received and
-        encrypted docs.
+        Count how many documents we have in the table for received docs.
 
         :return: The count of documents.
         :rtype: int
@@ -916,11 +926,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         Get all the encrypted documents from the sync database and dispatch a
         decrypt worker to decrypt each one of them.
         """
-        docs_by_generation = self.get_docs_by_generation()
+        docs_by_generation = self.get_docs_by_generation(encrypted=True)
         logger.debug("Sync decrypter pool: There are %d documents to " \
                      "decrypt." % len(docs_by_generation))
-        for doc_id, rev, gen in filter(None, docs_by_generation):
-            self.decrypt_doc(doc_id, rev, self.source_replica_uid)
+        for doc_id, rev, content, gen, trans_id, _ \
+                in filter(None, docs_by_generation):
+            self.decrypt_doc(
+                doc_id, rev, content, gen, trans_id, self.source_replica_uid)
 
     def process_decrypted(self):
         """
@@ -934,15 +946,9 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         # getting data from the syncing stream, to avoid InvalidGeneration
         # problems.
         with self.write_encrypted_lock:
-            already_decrypted = self.decrypted_docs
-            docs = self.get_docs_by_generation()
-            docs = filter(lambda entry: len(entry) > 0, docs)
-            expected = [gen for doc_id, rev, gen in docs]
-            docs_to_insert = get_insertable_docs_by_gen(
-                expected, already_decrypted)
-            for doc_fields in docs_to_insert:
+            for doc_fields in self.get_insertable_docs_by_gen():
                 self.insert_decrypted_local_doc(*doc_fields)
-        remaining = self.count_received_encrypted_docs()
+        remaining = self.count_docs_in_sync_db()
         return remaining == 0
 
     def insert_decrypted_local_doc(self, doc_id, doc_rev, content,
@@ -974,14 +980,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             if content == 'null':
                 content = None
             doc = SoledadDocument(doc_id, doc_rev, content)
-            insert_fun(doc, int(gen), trans_id)
+            gen = int(gen)
+            insert_fun(doc, gen, trans_id)
+            self._last_known_generation = gen
         except Exception as exc:
             logger.error("Sync decrypter pool: error while inserting "
                          "decrypted doc into local db.")
             logger.exception(exc)
 
         else:
-            # If no errors found, remove it from the local temporary dict
-            # and from the received database.
-            self.decrypted_docs.pop(gen)
-            self.delete_encrypted_received_doc(doc_id, doc_rev)
+            # If no errors found, remove it from the received database.
+            self.delete_received_doc(doc_id, doc_rev)
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 70e4d3a2..089a48a0 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -804,16 +804,20 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             self._sync_db = sync_db
             self._sync_db_write_lock = sync_db_write_lock
 
-    def _setup_sync_decr_pool(self):
+    def _setup_sync_decr_pool(self, last_known_generation):
         """
         Set up the SyncDecrypterPool for deferred decryption.
+
+        :param last_known_generation: Target's last known generation.
+        :type last_known_generation: int
         """
         if self._sync_decr_pool is None:
             # initialize syncing queue decryption pool
             self._sync_decr_pool = SyncDecrypterPool(
                 self._crypto, self._sync_db,
                 self._sync_db_write_lock,
-                insert_doc_cb=self._insert_doc_cb)
+                insert_doc_cb=self._insert_doc_cb,
+                last_known_generation=last_known_generation)
             self._sync_decr_pool.set_source_replica_uid(
                 self.source_replica_uid)
 
@@ -1127,7 +1131,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
 
         if defer_decryption:
             self._sync_exchange_lock.acquire()
-            self._setup_sync_decr_pool()
+            self._setup_sync_decr_pool(last_known_generation)
             self._setup_sync_watcher()
             self._defer_decryption = True
 
@@ -1402,7 +1406,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         :rtype: bool
         """
         if self._sync_decr_pool is not None:
-            return self._sync_decr_pool.count_received_encrypted_docs() == 0
+            return self._sync_decr_pool.count_docs_in_sync_db() == 0
         else:
             return True
 
-- 
cgit v1.2.3


From 5e4dae3427f40879156ddfaaaa8f878ab2504ee3 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Wed, 23 Jul 2014 10:29:09 -0300
Subject: On sync, fetch all docs before decrypting.

---
 client/src/leap/soledad/client/crypto.py | 30 +++++++++++++++++-------------
 client/src/leap/soledad/client/target.py | 32 +++++++++++++++++++++++---------
 2 files changed, 40 insertions(+), 22 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 4a73a910..5ae5937f 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -690,7 +690,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :type last_known_generation: int
         """
         self._insert_doc_cb = kwargs.pop("insert_doc_cb")
-        self._last_known_generation = kwargs.pop("last_known_generation")
         SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
         self.source_replica_uid = None
 
@@ -858,8 +857,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         :type result: tuple(str, str, str)
         """
         doc_id, rev, content, gen, trans_id = result
-        logger.debug("Sync decrypter pool: decrypted doc %s: %s %s"
-                     % (doc_id, rev, gen))
+        logger.debug("Sync decrypter pool: decrypted doc %s: %s %s %s"
+                     % (doc_id, rev, gen, trans_id))
         self.insert_received_doc(doc_id, rev, content, gen, trans_id)
 
     def get_docs_by_generation(self, encrypted=None):
@@ -878,7 +877,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
               % self.TABLE_NAME
         if encrypted is not None:
             sql += " WHERE encrypted = %d" % int(encrypted)
-        sql += " ORDER BY gen"
+        sql += " ORDER BY gen ASC"
         c = self._sync_db.cursor()
         c.execute(sql)
         # TODO: due to unknown reasons, the fetchall() method may return empty
@@ -891,21 +890,25 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         """
         Return a list of documents ready to be inserted.
         """
-        docs = self.get_docs_by_generation(encrypted=False)
+        all_docs = self.get_docs_by_generation()
+        decrypted_docs = self.get_docs_by_generation(encrypted=False)
         insertable = []
-        if docs:
-            last_gen = self._last_known_generation
-            for doc_id, rev, content, gen, trans_id, _ in docs:
-                if gen != (last_gen + 1):
-                    break
+        for doc_id, rev, content, gen, trans_id, encrypted in all_docs:
+            next_decrypted = decrypted_docs.pop(0)
+            if doc_id == next_decrypted[0]:
                 insertable.append((doc_id, rev, content, gen, trans_id))
-                last_gen = gen
+            else:
+                break
         return insertable
 
-    def count_docs_in_sync_db(self):
+    def count_docs_in_sync_db(self, encrypted=None):
         """
         Count how many documents we have in the table for received docs.
 
+        :param encrypted: If not None, return count of documents with
+                          encrypted field equal to given parameter.
+        :type encrypted: bool
+
         :return: The count of documents.
         :rtype: int
         """
@@ -914,6 +917,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             return
         c = self._sync_db.cursor()
         sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,)
+        if encrypted is not None:
+            sql += " WHERE encrypted = %d" % int(encrypted)
         c.execute(sql)
         res = c.fetchone()
         if res is not None:
@@ -982,7 +987,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             doc = SoledadDocument(doc_id, doc_rev, content)
             gen = int(gen)
             insert_fun(doc, gen, trans_id)
-            self._last_known_generation = gen
         except Exception as exc:
             logger.error("Sync decrypter pool: error while inserting "
                          "decrypted doc into local db.")
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 089a48a0..032134ec 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -816,8 +816,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             self._sync_decr_pool = SyncDecrypterPool(
                 self._crypto, self._sync_db,
                 self._sync_db_write_lock,
-                insert_doc_cb=self._insert_doc_cb,
-                last_known_generation=last_known_generation)
+                insert_doc_cb=self._insert_doc_cb)
             self._sync_decr_pool.set_source_replica_uid(
                 self.source_replica_uid)
 
@@ -1251,15 +1250,26 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             sent += 1
 
         # make sure all threads finished and we have up-to-date info
+        last_successful_thread = None
         while threads:
             # check if there are failures
             t, doc = threads.pop(0)
             t.join()
             if t.success:
                 synced.append((doc.doc_id, doc.rev))
+                last_successful_thread = t
 
-        if defer_decryption:
-            self._sync_watcher.start()
+        # delete documents from the sync database
+        if defer_encryption:
+            self.delete_encrypted_docs_from_db(synced)
+
+        # get target gen and trans_id after docs
+        gen_after_send = None
+        trans_id_after_send = None
+        if last_successful_thread is not None:
+            response_dict = json.loads(last_successful_thread.response[0])[0]
+            gen_after_send = response_dict['new_generation']
+            trans_id_after_send  = response_dict['new_transaction_id']
 
         # get docs from target
         if self.stopped is False:
@@ -1268,20 +1278,24 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
                 last_known_generation, last_known_trans_id, headers,
                 return_doc_cb, ensure_callback, sync_id, syncer_pool,
                 defer_decryption=defer_decryption)
-        syncer_pool.cleanup()
 
-        # delete documents from the sync database
-        if defer_encryption:
-            self.delete_encrypted_docs_from_db(synced)
+        syncer_pool.cleanup()
 
-        # wait for deferred decryption to finish
+        # decrypt docs in case of deferred decryption
         if defer_decryption:
+            self._sync_watcher.start()
             while self.clear_to_sync() is False:
                 sleep(self.DECRYPT_TASK_PERIOD)
             self._teardown_sync_watcher()
             self._teardown_sync_decr_pool()
             self._sync_exchange_lock.release()
 
+        # update gen and trans id info in case we just sent and did not
+        # receive docs.
+        if gen_after_send is not None and gen_after_send > cur_target_gen:
+            cur_target_gen = gen_after_send
+            cur_target_trans_id = trans_id_after_send
+
         self.stop()
         return cur_target_gen, cur_target_trans_id
 
-- 
cgit v1.2.3


From 8afb79c4d2171b03270143639296cbb6d9d0fdfa Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Wed, 23 Jul 2014 10:29:39 -0300
Subject: Allow deferred decryption without deferred encryption.

---
 client/src/leap/soledad/client/sqlcipher.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index 2df9606e..5a30b125 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -243,19 +243,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             self._ensure_schema()
             self._crypto = crypto
 
+        # define sync-db attrs
         self._sync_db = None
         self._sync_db_write_lock = None
         self._sync_enc_pool = None
+        self._init_sync_db(sqlcipher_file)
 
         if self.defer_encryption:
-            if sqlcipher_file != ":memory:":
-                self._sync_db_path = "%s-sync" % sqlcipher_file
-            else:
-                self._sync_db_path = ":memory:"
-
             # initialize sync db
-            self._init_sync_db()
-
             # initialize syncing queue encryption pool
             self._sync_enc_pool = SyncEncrypterPool(
                 self._crypto, self._sync_db, self._sync_db_write_lock)
@@ -449,7 +444,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         # the following context manager blocks until the syncing lock can be
         # acquired.
         with self.syncer(url, creds=creds) as syncer:
-
             # XXX could mark the critical section here...
             try:
                 res = syncer.sync(autocreate=autocreate,
@@ -542,14 +536,21 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             'ALTER TABLE document '
             'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
 
-    def _init_sync_db(self):
+    def _init_sync_db(self, sqlcipher_file):
         """
         Initialize the Symmetrically-Encrypted document to be synced database,
         and the queue to communicate with subprocess workers.
+
+        :param sqlcipher_file: The path for the SQLCipher file.
+        :type sqlcipher_file: str
         """
-        self._sync_db = sqlite3.connect(self._sync_db_path,
+        sync_db_path = None
+        if sqlcipher_file != ":memory:":
+            sync_db_path = "%s-sync" % sqlcipher_file
+        else:
+            sync_db_path = ":memory:"
+        self._sync_db = sqlite3.connect(sync_db_path,
                                         check_same_thread=False)
-
         self._sync_db_write_lock = threading.Lock()
         self._create_sync_db_tables()
         self.sync_queue = multiprocessing.Queue()
-- 
cgit v1.2.3


From 609669077b2f7223c31feed3679c8fcd74ab9ba7 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Wed, 23 Jul 2014 10:49:44 -0300
Subject: Avoid deadlocks when cancelling failed sync threads.

---
 client/src/leap/soledad/client/target.py | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 032134ec..5fe55216 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -376,6 +376,12 @@ class DocumentSyncerPool(object):
             t.request_lock.release()
             t.callback_lock.acquire(False)  # just in case
             t.callback_lock.release()
+        # release any blocking semaphores
+        for i in xrange(DocumentSyncerPool.POOL_SIZE):
+            try:
+                self._semaphore_pool.release()
+            except ValueError:
+                break
         logger.warning("Soledad sync: cancelled sync threads.")
 
     def cleanup(self):
-- 
cgit v1.2.3


From 622708945d51a1e22dde95424a6214e8e67be180 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Wed, 23 Jul 2014 16:26:24 -0300
Subject: Make sync database multiprocessing-safe.

---
 client/src/leap/soledad/client/crypto.py     |  46 +++++-------
 client/src/leap/soledad/client/mp_safe_db.py | 101 +++++++++++++++++++++++++++
 client/src/leap/soledad/client/sqlcipher.py  |  22 ++++--
 client/src/leap/soledad/client/target.py     |  17 ++---
 4 files changed, 143 insertions(+), 43 deletions(-)
 create mode 100644 client/src/leap/soledad/client/mp_safe_db.py

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 5ae5937f..eb5a4f64 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -224,7 +224,7 @@ class SoledadCrypto(object):
 
         The password is derived using HMAC having sha256 as underlying hash
         function. The key used for HMAC are the first
-        C{soledad.REMOTE_STORAGE_SECRET_KENGTH} bytes of Soledad's storage
+        C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage
         secret stripped from the first MAC_KEY_LENGTH characters. The HMAC
         message is C{doc_id}.
 
@@ -623,9 +623,8 @@ class SyncEncrypterPool(SyncEncryptDecryptPool):
 
         con = self._sync_db
         with self._sync_db_write_lock:
-            with con:
-                con.execute(sql_del, (doc_id, ))
-                con.execute(sql_ins, (doc_id, doc_rev, content))
+            con.execute(sql_del, (doc_id, ))
+            con.execute(sql_ins, (doc_id, doc_rev, content))
 
 
 def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret):
@@ -726,11 +725,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
 
         con = self._sync_db
         with self._sync_db_write_lock:
-            with con:
-                con.execute(sql_del, (doc_id, ))
-                con.execute(
-                    sql_ins,
-                    (doc_id, doc_rev, docstr, gen, trans_id, 1))
+            con.execute(sql_del, (doc_id, ))
+            con.execute(
+                sql_ins,
+                (doc_id, doc_rev, docstr, gen, trans_id, 1))
 
     def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id):
         """
@@ -757,11 +755,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             self.TABLE_NAME,)
         con = self._sync_db
         with self._sync_db_write_lock:
-            with con:
-                con.execute(sql_del, (doc_id,))
-                con.execute(
-                    sql_ins,
-                    (doc_id, doc_rev, content, gen, trans_id, 0))
+            con.execute(sql_del, (doc_id,))
+            con.execute(
+                sql_ins,
+                (doc_id, doc_rev, content, gen, trans_id, 0))
 
     def delete_received_doc(self, doc_id, doc_rev):
         """
@@ -776,8 +773,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             self.TABLE_NAME,)
         con = self._sync_db
         with self._sync_db_write_lock:
-            with con:
-                con.execute(sql_del, (doc_id, doc_rev))
+            con.execute(sql_del, (doc_id, doc_rev))
 
     def decrypt_doc(self, doc_id, rev, content, gen, trans_id,
                     source_replica_uid, workers=True):
@@ -878,12 +874,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         if encrypted is not None:
             sql += " WHERE encrypted = %d" % int(encrypted)
         sql += " ORDER BY gen ASC"
-        c = self._sync_db.cursor()
-        c.execute(sql)
-        # TODO: due to unknown reasons, the fetchall() method may return empty
-        # values, so we filter them out here. We have to perform some tests to
-        # understand why and when this happens.
-        docs = filter(lambda entry: len(entry) > 0, c.fetchall())
+        docs = self._sync_db.select(sql)
         return docs
 
     def get_insertable_docs_by_gen(self):
@@ -894,7 +885,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         decrypted_docs = self.get_docs_by_generation(encrypted=False)
         insertable = []
         for doc_id, rev, content, gen, trans_id, encrypted in all_docs:
-            next_decrypted = decrypted_docs.pop(0)
+            next_decrypted = decrypted_docs.next()
             if doc_id == next_decrypted[0]:
                 insertable.append((doc_id, rev, content, gen, trans_id))
             else:
@@ -915,14 +906,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         if self._sync_db is None:
             logger.warning("cannot return count with null sync_db")
             return
-        c = self._sync_db.cursor()
         sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,)
         if encrypted is not None:
             sql += " WHERE encrypted = %d" % int(encrypted)
-        c.execute(sql)
-        res = c.fetchone()
+        res = self._sync_db.select(sql)
         if res is not None:
-            return res[0]
+            val = res.next()
+            return val[0]
         else:
             return 0
 
@@ -932,8 +922,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         decrypt worker to decrypt each one of them.
         """
         docs_by_generation = self.get_docs_by_generation(encrypted=True)
-        logger.debug("Sync decrypter pool: There are %d documents to " \
-                     "decrypt." % len(docs_by_generation))
         for doc_id, rev, content, gen, trans_id, _ \
                 in filter(None, docs_by_generation):
             self.decrypt_doc(
diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py
new file mode 100644
index 00000000..a9ab5649
--- /dev/null
+++ b/client/src/leap/soledad/client/mp_safe_db.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# crypto.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Multiprocessing-safe SQLite database.
+"""
+
+
+from threading import Thread
+from Queue import Queue
+from sqlite3 import connect as sqlite3_connect
+
+
+# Thanks to http://code.activestate.com/recipes/526618/
+
+class MPSafeSQLiteDB(Thread):
+    """
+    A multiprocessing-safe SQLite database accessor.
+    """
+
+    CLOSE = "--close--"
+    NO_MORE = "--no more--"
+
+    def __init__(self, db_path):
+        """
+        Initialize the process
+        """
+        Thread.__init__(self)
+        self._db_path = db_path
+        self._requests = Queue()
+        self.start()
+
+    def run(self):
+        """
+        Run the multiprocessing-safe database accessor.
+        """
+        conn = sqlite3_connect(self._db_path)
+        while True:
+            req, arg, res = self._requests.get()
+            if req == self.CLOSE:
+                break
+            with conn:
+                cursor = conn.cursor()
+                cursor.execute(req, arg)
+                if res:
+                    for rec in cursor.fetchall():
+                        res.put(rec)
+                    res.put(self.NO_MORE)
+        conn.close()
+
+    def execute(self, req, arg=None, res=None):
+        """
+        Execute a request on the database.
+
+        :param req: The request to be executed.
+        :type req: str
+        :param arg: The arguments for the request.
+        :type arg: tuple
+        :param res: A queue to write request results.
+        :type res: multiprocessing.Queue
+        """
+        self._requests.put((req, arg or tuple(), res))
+
+    def select(self, req, arg=None):
+        """
+        Run a select query on the database and yield results.
+
+        :param req: The request to be executed.
+        :type req: str
+        :param arg: The arguments for the request.
+        :type arg: tuple
+        """
+        res = Queue()
+        self.execute(req, arg, res)
+        while True:
+            rec=res.get()
+            if rec == self.NO_MORE:
+                break
+            yield rec
+
+    def close(self):
+        """
+        Close the database connection.
+        """
+        self.execute(self.CLOSE)
+        self.join()
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index 5a30b125..85b0391b 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -44,7 +44,6 @@ handled by Soledad should be created by SQLCipher >= 2.0.
 import logging
 import multiprocessing
 import os
-import sqlite3
 import string
 import threading
 import time
@@ -63,6 +62,7 @@ from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool
 from leap.soledad.client.target import SoledadSyncTarget
 from leap.soledad.client.target import PendingReceivedDocsSyncError
 from leap.soledad.client.sync import SoledadSynchronizer
+from leap.soledad.client.mp_safe_db import MPSafeSQLiteDB
 from leap.soledad.common.document import SoledadDocument
 
 
@@ -549,8 +549,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             sync_db_path = "%s-sync" % sqlcipher_file
         else:
             sync_db_path = ":memory:"
-        self._sync_db = sqlite3.connect(sync_db_path,
-                                        check_same_thread=False)
+        self._sync_db = MPSafeSQLiteDB(sync_db_path)
         self._sync_db_write_lock = threading.Lock()
         self._create_sync_db_tables()
         self.sync_queue = multiprocessing.Queue()
@@ -567,9 +566,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             decr.TABLE_NAME, decr.FIELD_NAMES))
 
         with self._sync_db_write_lock:
-            with self._sync_db:
-                self._sync_db.execute(sql_encr)
-                self._sync_db.execute(sql_decr)
+            self._sync_db.execute(sql_encr)
+            self._sync_db.execute(sql_decr)
 
     #
     # Symmetric encryption of syncing docs
@@ -1076,16 +1074,28 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         Close db_handle and close syncer.
         """
         logger.debug("Sqlcipher backend: closing")
+        # stop the sync watcher for deferred encryption
         if self._sync_watcher is not None:
             self._sync_watcher.stop()
             self._sync_watcher.shutdown()
+        # close all open syncers
         for url in self._syncers:
             _, syncer = self._syncers[url]
             syncer.close()
+        # stop the encryption pool
         if self._sync_enc_pool is not None:
             self._sync_enc_pool.close()
+        # close the actual database
         if self._db_handle is not None:
             self._db_handle.close()
+        # close the sync database
+        if self._sync_db is not None:
+            self._sync_db.close()
+        # close the sync queue
+        if self.sync_queue is not None:
+            self.sync_queue.close()
+            del self.sync_queue
+            self.sync_queue = None
 
     @property
     def replica_uid(self):
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 5fe55216..01e1231a 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -1346,13 +1346,16 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         :type doc_rev: str
         """
         encr = SyncEncrypterPool
-        c = self._sync_db.cursor()
         sql = ("SELECT content FROM %s WHERE doc_id=? and rev=?" % (
             encr.TABLE_NAME,))
-        c.execute(sql, (doc_id, doc_rev))
-        res = c.fetchall()
-        if len(res) != 0:
-            return res[0][0]
+        res = self._sync_db.select(sql, (doc_id, doc_rev))
+        try:
+            val = res.next()
+            return val[0]
+        except StopIteration:
+            # no doc found
+            return None
+
 
     def delete_encrypted_docs_from_db(self, docs_ids):
         """
@@ -1365,12 +1368,10 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         """
         if docs_ids:
             encr = SyncEncrypterPool
-            c = self._sync_db.cursor()
             for doc_id, doc_rev in docs_ids:
                 sql = ("DELETE FROM %s WHERE doc_id=? and rev=?" % (
                     encr.TABLE_NAME,))
-                c.execute(sql, (doc_id, doc_rev))
-            self._sync_db.commit()
+                self._sync_db.execute(sql, (doc_id, doc_rev))
 
     def _save_encrypted_received_doc(self, doc, gen, trans_id, idx, total):
         """
-- 
cgit v1.2.3


From 074848f78bdac78328eb4de7fe72d85830da561d Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Fri, 1 Aug 2014 12:47:43 -0300
Subject: Refactor secrets out of main soledad client class.

---
 client/src/leap/soledad/client/__init__.py | 659 +++-------------------------
 client/src/leap/soledad/client/crypto.py   |  21 +-
 client/src/leap/soledad/client/secrets.py  | 681 +++++++++++++++++++++++++++++
 3 files changed, 757 insertions(+), 604 deletions(-)
 create mode 100644 client/src/leap/soledad/client/secrets.py

(limited to 'client')

diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 586e3389..0fd6672a 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -31,9 +31,7 @@ import os
 import socket
 import ssl
 import urlparse
-import hmac
 
-from hashlib import sha256
 
 try:
     import cchardet as chardet
@@ -43,41 +41,20 @@ except ImportError:
 from u1db.remote import http_client
 from u1db.remote.ssl_match_hostname import match_hostname
 
-import scrypt
-import simplejson as json
-
 from leap.common.config import get_path_prefix
 from leap.soledad.common import (
     SHARED_DB_NAME,
     soledad_assert,
     soledad_assert_type
 )
-from leap.soledad.common.errors import (
-    InvalidTokenError,
-    NotLockedError,
-    AlreadyLockedError,
-    LockTimedOutError,
-)
-from leap.soledad.common.crypto import (
-    MacMethods,
-    UnknownMacMethod,
-    WrongMac,
-    MAC_KEY,
-    MAC_METHOD_KEY,
-)
 from leap.soledad.client.events import (
-    SOLEDAD_CREATING_KEYS,
-    SOLEDAD_DONE_CREATING_KEYS,
-    SOLEDAD_DOWNLOADING_KEYS,
-    SOLEDAD_DONE_DOWNLOADING_KEYS,
-    SOLEDAD_UPLOADING_KEYS,
-    SOLEDAD_DONE_UPLOADING_KEYS,
     SOLEDAD_NEW_DATA_TO_SYNC,
     SOLEDAD_DONE_DATA_SYNC,
     signal,
 )
 from leap.soledad.common.document import SoledadDocument
 from leap.soledad.client.crypto import SoledadCrypto
+from leap.soledad.client.secrets import SoledadSecrets
 from leap.soledad.client.shared_db import SoledadSharedDatabase
 from leap.soledad.client.sqlcipher import open as sqlcipher_open
 from leap.soledad.client.sqlcipher import SQLCipherDatabase
@@ -102,27 +79,6 @@ Soledad client and server.
 # Soledad: local encrypted storage and remote encrypted sync.
 #
 
-class NoStorageSecret(Exception):
-    """
-    Raised when trying to use a storage secret but none is available.
-    """
-    pass
-
-
-class PassphraseTooShort(Exception):
-    """
-    Raised when trying to change the passphrase but the provided passphrase is
-    too short.
-    """
-
-
-class BootstrapSequenceError(Exception):
-    """
-    Raised when an attempt to generate a secret and store it in a recovery
-    documents on server failed.
-    """
-
-
 class Soledad(object):
     """
     Soledad provides encrypted data storage and sync.
@@ -166,57 +122,6 @@ class Soledad(object):
     The name of the file where the storage secrets will be stored.
     """
 
-    GENERATED_SECRET_LENGTH = 1024
-    """
-    The length of the generated secret used to derive keys for symmetric
-    encryption for local and remote storage.
-    """
-
-    LOCAL_STORAGE_SECRET_LENGTH = 512
-    """
-    The length of the secret used to derive a passphrase for the SQLCipher
-    database.
-    """
-
-    REMOTE_STORAGE_SECRET_LENGTH = \
-        GENERATED_SECRET_LENGTH - LOCAL_STORAGE_SECRET_LENGTH
-    """
-    The length of the secret used to derive an encryption key and a MAC auth
-    key for remote storage.
-    """
-
-    SALT_LENGTH = 64
-    """
-    The length of the salt used to derive the key for the storage secret
-    encryption.
-    """
-
-    MINIMUM_PASSPHRASE_LENGTH = 6
-    """
-    The minimum length for a passphrase. The passphrase length is only checked
-    when the user changes her passphrase, not when she instantiates Soledad.
-    """
-
-    IV_SEPARATOR = ":"
-    """
-    A separator used for storing the encryption initial value prepended to the
-    ciphertext.
-    """
-
-    UUID_KEY = 'uuid'
-    STORAGE_SECRETS_KEY = 'storage_secrets'
-    SECRET_KEY = 'secret'
-    CIPHER_KEY = 'cipher'
-    LENGTH_KEY = 'length'
-    KDF_KEY = 'kdf'
-    KDF_SALT_KEY = 'kdf_salt'
-    KDF_LENGTH_KEY = 'kdf_length'
-    KDF_SCRYPT = 'scrypt'
-    CIPHER_AES256 = 'aes256'
-    """
-    Keys used to access storage secrets in recovery documents.
-    """
-
     DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad')
     """
     Prefix for default values for path.
@@ -266,41 +171,49 @@ class Soledad(object):
                                        storage on server sequence has failed
                                        for some reason.
         """
-        # get config params
+        # store config params
         self._uuid = uuid
-        soledad_assert_type(passphrase, unicode)
         self._passphrase = passphrase
-        # init crypto variables
-        self._secrets = {}
-        self._secret_id = secret_id
+        self._secrets_path = secrets_path
+        self._local_db_path = local_db_path
+        self._server_url = server_url
+        # configure SSL certificate
+        global SOLEDAD_CERT
+        SOLEDAD_CERT = cert_file
+        self._set_token(auth_token)
         self._defer_encryption = defer_encryption
 
-        self._init_config(secrets_path, local_db_path, server_url)
+        self._init_config()
+        self._init_dirs()
 
-        self._set_token(auth_token)
+        # init crypto variables
         self._shared_db_instance = None
-        # configure SSL certificate
-        global SOLEDAD_CERT
-        SOLEDAD_CERT = cert_file
+        self._crypto = SoledadCrypto(self)
+        self._secrets = SoledadSecrets(
+            self._uuid,
+            self._passphrase,
+            self._secrets_path,
+            self._shared_db,
+            self._crypto,
+            secret_id=secret_id)
+
         # initiate bootstrap sequence
         self._bootstrap()  # might raise BootstrapSequenceError()
 
-    def _init_config(self, secrets_path, local_db_path, server_url):
+    def _init_config(self):
         """
         Initialize configuration using default values for missing params.
         """
+        soledad_assert_type(self._passphrase, unicode)
         # initialize secrets_path
-        self._secrets_path = secrets_path
         if self._secrets_path is None:
             self._secrets_path = os.path.join(
                 self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME)
         # initialize local_db_path
-        self._local_db_path = local_db_path
         if self._local_db_path is None:
             self._local_db_path = os.path.join(
                 self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME)
         # initialize server_url
-        self._server_url = server_url
         soledad_assert(
             self._server_url is not None,
             'Missing URL for Soledad server.')
@@ -309,129 +222,18 @@ class Soledad(object):
     # initialization/destruction methods
     #
 
-    def _get_or_gen_crypto_secrets(self):
-        """
-        Retrieves or generates the crypto secrets.
-
-        Might raise BootstrapSequenceError
-        """
-        doc = self._get_secrets_from_shared_db()
-
-        if doc:
-            logger.info(
-                'Found cryptographic secrets in shared recovery '
-                'database.')
-            _, mac = self.import_recovery_document(doc.content)
-            if mac is False:
-                self.put_secrets_in_shared_db()
-            self._store_secrets()  # save new secrets in local file
-            if self._secret_id is None:
-                self._set_secret_id(self._secrets.items()[0][0])
-        else:
-            # STAGE 3 - there are no secrets in server also, so
-            # generate a secret and store it in remote db.
-            logger.info(
-                'No cryptographic secrets found, creating new '
-                ' secrets...')
-            self._set_secret_id(self._gen_secret())
-            try:
-                self._put_secrets_in_shared_db()
-            except Exception as ex:
-                # storing generated secret in shared db failed for
-                # some reason, so we erase the generated secret and
-                # raise.
-                try:
-                    os.unlink(self._secrets_path)
-                except OSError as e:
-                    if e.errno != errno.ENOENT:  # no such file or directory
-                        logger.exception(e)
-                logger.exception(ex)
-                raise BootstrapSequenceError(
-                    'Could not store generated secret in the shared '
-                    'database, bailing out...')
-
     def _bootstrap(self):
         """
         Bootstrap local Soledad instance.
 
-        Soledad Client bootstrap is the following sequence of stages:
-
-        * stage 0 - local environment setup.
-            - directory initialization.
-            - crypto submodule initialization
-        * stage 1 - local secret loading:
-            - if secrets exist locally, load them.
-        * stage 2 - remote secret loading:
-            - else, if secrets exist in server, download them.
-        * stage 3 - secret generation:
-            - else, generate a new secret and store in server.
-        * stage 4 - database initialization.
-
-        This method decides which bootstrap stages have already been performed
-        and performs the missing ones in order.
-
         :raise BootstrapSequenceError: Raised when the secret generation and
             storage on server sequence has failed for some reason.
         """
-        # STAGE 0  - local environment setup
-        self._init_dirs()
-        self._crypto = SoledadCrypto(self)
-
-        secrets_problem = None
-
-        # STAGE 1 - verify if secrets exist locally
-        if not self._has_secret():  # try to load from local storage.
-
-            # STAGE 2 - there are no secrets in local storage, so try to fetch
-            # encrypted secrets from server.
-            logger.info(
-                'Trying to fetch cryptographic secrets from shared recovery '
-                'database...')
-
-            # --- start of atomic operation in shared db ---
-
-            # obtain lock on shared db
-            token = timeout = None
-            try:
-                token, timeout = self._shared_db.lock()
-            except AlreadyLockedError:
-                raise BootstrapSequenceError('Database is already locked.')
-            except LockTimedOutError:
-                raise BootstrapSequenceError('Lock operation timed out.')
-
-            try:
-                self._get_or_gen_crypto_secrets()
-            except Exception as e:
-                secrets_problem = e
-
-            # release the lock on shared db
-            try:
-                self._shared_db.unlock(token)
-                self._shared_db.close()
-            except NotLockedError:
-                # for some reason the lock expired. Despite that, secret
-                # loading or generation/storage must have been executed
-                # successfully, so we pass.
-                pass
-            except InvalidTokenError:
-                # here, our lock has not only expired but also some other
-                # client application has obtained a new lock and is currently
-                # doing its thing in the shared database. Using the same
-                # reasoning as above, we assume everything went smooth and
-                # pass.
-                pass
-            except Exception as e:
-                logger.error("Unhandled exception when unlocking shared "
-                             "database.")
-                logger.exception(e)
-
-            # --- end of atomic operation in shared db ---
-
-        # STAGE 4 - local database initialization
-        if secrets_problem is None:
+        try:
+            self._secrets.bootstrap()
             self._init_db()
-        else:
-            raise secrets_problem
+        except:
+            raise
 
     def _init_dirs(self):
         """
@@ -460,27 +262,8 @@ class Soledad(object):
         Currently, Soledad uses the default SQLCipher cipher, i.e.
         'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and
         uses the 'raw PRAGMA key' format to handle the key to SQLCipher.
-
-        The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage
-        secret are used for remote storage encryption. We use the next
-        C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage.
-        From these bytes, the first C{self.SALT_LENGTH} are used as the salt
-        and the rest as the password for the scrypt hashing.
-        """
-        # salt indexes
-        salt_start = self.REMOTE_STORAGE_SECRET_LENGTH
-        salt_end = salt_start + self.SALT_LENGTH
-        # password indexes
-        pwd_start = salt_end
-        pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH
-        # calculate the key for local encryption
-        secret = self._get_storage_secret()
-        key = scrypt.hash(
-            secret[pwd_start:pwd_end],  # the password
-            secret[salt_start:salt_end],  # the salt
-            buflen=32,  # we need a key with 256 bits (32 bytes)
-        )
-
+        """
+        key = self._secrets.get_local_storage_key()
         self._db = sqlcipher_open(
             self._local_db_path,
             binascii.b2a_hex(key),  # sqlcipher only accepts the hex version
@@ -501,186 +284,6 @@ class Soledad(object):
             self._db.stop_sync()
             self._db.close()
 
-    #
-    # Management of secret for symmetric encryption.
-    #
-
-    def _get_storage_secret(self):
-        """
-        Return the storage secret.
-
-        Storage secret is encrypted before being stored. This method decrypts
-        and returns the stored secret.
-
-        :return: The storage secret.
-        :rtype: str
-        """
-        # calculate the encryption key
-        key = scrypt.hash(
-            self._passphrase_as_string(),
-            # the salt is stored base64 encoded
-            binascii.a2b_base64(
-                self._secrets[self._secret_id][self.KDF_SALT_KEY]),
-            buflen=32,  # we need a key with 256 bits (32 bytes).
-        )
-        # recover the initial value and ciphertext
-        iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split(
-            self.IV_SEPARATOR, 1)
-        ciphertext = binascii.a2b_base64(ciphertext)
-        return self._crypto.decrypt_sym(ciphertext, key, iv=iv)
-
-    def _set_secret_id(self, secret_id):
-        """
-        Define the id of the storage secret to be used.
-
-        This method will also replace the secret in the crypto object.
-
-        :param secret_id: The id of the storage secret to be used.
-        :type secret_id: str
-        """
-        self._secret_id = secret_id
-
-    def _load_secrets(self):
-        """
-        Load storage secrets from local file.
-        """
-        # does the file exist in disk?
-        if not os.path.isfile(self._secrets_path):
-            raise IOError('File does not exist: %s' % self._secrets_path)
-        # read storage secrets from file
-        content = None
-        with open(self._secrets_path, 'r') as f:
-            content = json.loads(f.read())
-        _, mac = self.import_recovery_document(content)
-        if mac is False:
-            self._store_secrets()
-            self._put_secrets_in_shared_db()
-        # choose first secret if no secret_id was given
-        if self._secret_id is None:
-            self._set_secret_id(self._secrets.items()[0][0])
-
-    def _has_secret(self):
-        """
-        Return whether there is a storage secret available for use or not.
-
-        :return: Whether there's a storage secret for symmetric encryption.
-        :rtype: bool
-        """
-        if self._secret_id is None or self._secret_id not in self._secrets:
-            try:
-                self._load_secrets()  # try to load from disk
-            except IOError, e:
-                logger.warning('IOError: %s' % str(e))
-        try:
-            self._get_storage_secret()
-            return True
-        except Exception:
-            return False
-
-    def _gen_secret(self):
-        """
-        Generate a secret for symmetric encryption and store in a local
-        encrypted file.
-
-        This method emits the following signals:
-
-            * SOLEDAD_CREATING_KEYS
-            * SOLEDAD_DONE_CREATING_KEYS
-
-        A secret has the following structure:
-
-            {
-                '<secret_id>': {
-                        'kdf': 'scrypt',
-                        'kdf_salt': '<b64 repr of salt>'
-                        'kdf_length': <key length>
-                        'cipher': 'aes256',
-                        'length': <secret length>,
-                        'secret': '<encrypted b64 repr of storage_secret>',
-                }
-            }
-
-        :return: The id of the generated secret.
-        :rtype: str
-        """
-        signal(SOLEDAD_CREATING_KEYS, self._uuid)
-        # generate random secret
-        secret = os.urandom(self.GENERATED_SECRET_LENGTH)
-        secret_id = sha256(secret).hexdigest()
-        # generate random salt
-        salt = os.urandom(self.SALT_LENGTH)
-        # get a 256-bit key
-        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
-        iv, ciphertext = self._crypto.encrypt_sym(secret, key)
-        self._secrets[secret_id] = {
-            # leap.soledad.crypto submodule uses AES256 for symmetric
-            # encryption.
-            self.KDF_KEY: self.KDF_SCRYPT,
-            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
-            self.KDF_LENGTH_KEY: len(key),
-            self.CIPHER_KEY: self.CIPHER_AES256,
-            self.LENGTH_KEY: len(secret),
-            self.SECRET_KEY: '%s%s%s' % (
-                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
-        }
-        self._store_secrets()
-        signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid)
-        return secret_id
-
-    def _store_secrets(self):
-        """
-        Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
-        """
-        with open(self._secrets_path, 'w') as f:
-            f.write(
-                json.dumps(
-                    self.export_recovery_document()))
-
-    def change_passphrase(self, new_passphrase):
-        """
-        Change the passphrase that encrypts the storage secret.
-
-        :param new_passphrase: The new passphrase.
-        :type new_passphrase: unicode
-
-        :raise NoStorageSecret: Raised if there's no storage secret available.
-        """
-        # maybe we want to add more checks to guarantee passphrase is
-        # reasonable?
-        soledad_assert_type(new_passphrase, unicode)
-        if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH:
-            raise PassphraseTooShort(
-                'Passphrase must be at least %d characters long!' %
-                self.MINIMUM_PASSPHRASE_LENGTH)
-        # ensure there's a secret for which the passphrase will be changed.
-        if not self._has_secret():
-            raise NoStorageSecret()
-        secret = self._get_storage_secret()
-        # generate random salt
-        new_salt = os.urandom(self.SALT_LENGTH)
-        # get a 256-bit key
-        key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32)
-        iv, ciphertext = self._crypto.encrypt_sym(secret, key)
-        # XXX update all secrets in the dict
-        self._secrets[self._secret_id] = {
-            # leap.soledad.crypto submodule uses AES256 for symmetric
-            # encryption.
-            self.KDF_KEY: self.KDF_SCRYPT,  # TODO: remove hard coded kdf
-            self.KDF_SALT_KEY: binascii.b2a_base64(new_salt),
-            self.KDF_LENGTH_KEY: len(key),
-            self.CIPHER_KEY: self.CIPHER_AES256,
-            self.LENGTH_KEY: len(secret),
-            self.SECRET_KEY: '%s%s%s' % (
-                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
-        }
-        self._passphrase = new_passphrase
-        self._store_secrets()
-        self._put_secrets_in_shared_db()
-
-    #
-    # General crypto utility methods.
-    #
-
     @property
     def _shared_db(self):
         """
@@ -697,63 +300,6 @@ class Soledad(object):
                 creds=self._creds)
         return self._shared_db_instance
 
-    def _shared_db_doc_id(self):
-        """
-        Calculate the doc_id of the document in the shared db that stores key
-        material.
-
-        :return: the hash
-        :rtype: str
-        """
-        return sha256(
-            '%s%s' %
-            (self._passphrase_as_string(), self.uuid)).hexdigest()
-
-    def _get_secrets_from_shared_db(self):
-        """
-        Retrieve the document with encrypted key material from the shared
-        database.
-
-        :return: a document with encrypted key material in its contents
-        :rtype: SoledadDocument
-        """
-        signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid)
-        db = self._shared_db
-        if not db:
-            logger.warning('No shared db found')
-            return
-        doc = db.get_doc(self._shared_db_doc_id())
-        signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
-        return doc
-
-    def _put_secrets_in_shared_db(self):
-        """
-        Assert local keys are the same as shared db's ones.
-
-        Try to fetch keys from shared recovery database. If they already exist
-        in the remote db, assert that that data is the same as local data.
-        Otherwise, upload keys to shared recovery database.
-        """
-        soledad_assert(
-            self._has_secret(),
-            'Tried to send keys to server but they don\'t exist in local '
-            'storage.')
-        # try to get secrets doc from server, otherwise create it
-        doc = self._get_secrets_from_shared_db()
-        if doc is None:
-            doc = SoledadDocument(
-                doc_id=self._shared_db_doc_id())
-        # fill doc with encrypted secrets
-        doc.content = self.export_recovery_document()
-        # upload secrets to server
-        signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
-        db = self._shared_db
-        if not db:
-            logger.warning('No shared db found')
-            return
-        db.put_doc(doc)
-        signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
-
     #
     # Document storage, retrieval and sync.
     #
@@ -1152,104 +698,6 @@ class Soledad(object):
 
     token = property(_get_token, _set_token, doc='The authentication Token.')
 
-    #
-    # Recovery document export and import methods
-    #
-
-    def export_recovery_document(self):
-        """
-        Export the storage secrets.
-
-        A recovery document has the following structure:
-
-            {
-                'storage_secrets': {
-                    '<storage_secret id>': {
-                        'kdf': 'scrypt',
-                        'kdf_salt': '<b64 repr of salt>'
-                        'kdf_length': <key length>
-                        'cipher': 'aes256',
-                        'length': <secret length>,
-                        'secret': '<encrypted storage_secret>',
-                    },
-                },
-                'kdf': 'scrypt',
-                'kdf_salt': '<b64 repr of salt>',
-                'kdf_length: <key length>,
-                '_mac_method': 'hmac',
-                '_mac': '<mac>'
-            }
-
-        Note that multiple storage secrets might be stored in one recovery
-        document. This method will also calculate a MAC of a string
-        representation of the secrets dictionary.
-
-        :return: The recovery document.
-        :rtype: dict
-        """
-        # create salt and key for calculating MAC
-        salt = os.urandom(self.SALT_LENGTH)
-        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
-        data = {
-            self.STORAGE_SECRETS_KEY: self._secrets,
-            self.KDF_KEY: self.KDF_SCRYPT,
-            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
-            self.KDF_LENGTH_KEY: len(key),
-            MAC_METHOD_KEY: MacMethods.HMAC,
-            MAC_KEY: hmac.new(
-                key,
-                json.dumps(self._secrets),
-                sha256).hexdigest(),
-        }
-        return data
-
-    def import_recovery_document(self, data):
-        """
-        Import storage secrets for symmetric encryption and uuid (if present)
-        from a recovery document.
-
-        Note that this method does not store the imported data on disk. For
-        that, use C{self._store_secrets()}.
-
-        :param data: The recovery document.
-        :type data: dict
-
-        :return: A tuple containing the number of imported secrets and whether
-                 there was MAC informationa available for authenticating.
-        :rtype: (int, bool)
-        """
-        soledad_assert(self.STORAGE_SECRETS_KEY in data)
-        # check mac of the recovery document
-        mac = None
-        if MAC_KEY in data:
-            soledad_assert(data[MAC_KEY] is not None)
-            soledad_assert(MAC_METHOD_KEY in data)
-            soledad_assert(self.KDF_KEY in data)
-            soledad_assert(self.KDF_SALT_KEY in data)
-            soledad_assert(self.KDF_LENGTH_KEY in data)
-            if data[MAC_METHOD_KEY] == MacMethods.HMAC:
-                key = scrypt.hash(
-                    self._passphrase_as_string(),
-                    binascii.a2b_base64(data[self.KDF_SALT_KEY]),
-                    buflen=32)
-                mac = hmac.new(
-                    key,
-                    json.dumps(data[self.STORAGE_SECRETS_KEY]),
-                    sha256).hexdigest()
-            else:
-                raise UnknownMacMethod('Unknown MAC method: %s.' %
-                                       data[MAC_METHOD_KEY])
-            if mac != data[MAC_KEY]:
-                raise WrongMac('Could not authenticate recovery document\'s '
-                               'contents.')
-        # include secrets in the secret pool.
-        secrets = 0
-        for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items():
-            if secret_id not in self._secrets:
-                secrets += 1
-                self._secrets[secret_id] = secret_data
-        return secrets, mac
-
     #
     # Setters/getters
     #
@@ -1259,18 +707,26 @@ class Soledad(object):
 
     uuid = property(_get_uuid, doc='The user uuid.')
 
-    def _get_secret_id(self):
-        return self._secret_id
+    def get_secret_id(self):
+        return self._secrets.secret_id
+
+    def set_secret_id(self, secret_id):
+        self._secrets.set_secret_id(secret_id)
 
     secret_id = property(
-        _get_secret_id,
+        get_secret_id,
+        set_secret_id,
         doc='The active secret id.')
 
+    def _set_secrets_path(self, secrets_path):
+        self._secrets.secrets_path = secrets_path
+
     def _get_secrets_path(self):
-        return self._secrets_path
+        return self._secrets.secrets_path
 
     secrets_path = property(
         _get_secrets_path,
+        _set_secrets_path,
         doc='The path for the file containing the encrypted symmetric secret.')
 
     def _get_local_db_path(self):
@@ -1287,20 +743,31 @@ class Soledad(object):
         _get_server_url,
         doc='The URL of the Soledad server.')
 
-    storage_secret = property(
-        _get_storage_secret,
-        doc='The secret used for symmetric encryption.')
+    @property
+    def storage_secret(self):
+        """
+        Return the secret used for symmetric encryption.
+        """
+        return self._secrets.storage_secret
+
+    @property
+    def secrets(self):
+        return self._secrets
 
-    def _get_passphrase(self):
-        return self._passphrase
+    @property
+    def passphrase(self):
+        return self._secrets.passphrase
 
-    passphrase = property(
-        _get_passphrase,
-        doc='The passphrase for locking and unlocking encryption secrets for '
-            'local and remote storage.')
+    def change_passphrase(self, new_passphrase):
+        """
+        Change the passphrase that encrypts the storage secret.
 
-    def _passphrase_as_string(self):
-        return self._passphrase.encode('utf-8')
+        :param new_passphrase: The new passphrase.
+        :type new_passphrase: unicode
+
+        :raise NoStorageSecret: Raised if there's no storage secret available.
+        """
+        self._secrets.change_passphrase(new_passphrase)
 
 
 # ----------------------------------------------------------------------------
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index eb5a4f64..4a64b5a8 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -242,7 +242,7 @@ class SoledadCrypto(object):
         return hmac.new(
             self.secret[
                 MAC_KEY_LENGTH:
-                self._soledad.REMOTE_STORAGE_SECRET_LENGTH],
+                self._soledad.secrets.REMOTE_STORAGE_SECRET_LENGTH],
             doc_id,
             hashlib.sha256).digest()
 
@@ -819,7 +819,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         try:
             content = json.loads(content)
         except TypeError:
-            logger.warning("Wrong type while decoding json: %s" % repr(docstr))
+            logger.warning("Wrong type while decoding json: %s"
+                           % repr(content))
             return
 
         key = self._crypto.doc_passphrase(doc_id)
@@ -884,11 +885,15 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         all_docs = self.get_docs_by_generation()
         decrypted_docs = self.get_docs_by_generation(encrypted=False)
         insertable = []
-        for doc_id, rev, content, gen, trans_id, encrypted in all_docs:
-            next_decrypted = decrypted_docs.next()
-            if doc_id == next_decrypted[0]:
-                insertable.append((doc_id, rev, content, gen, trans_id))
-            else:
+        for doc_id, rev, _, gen, trans_id, encrypted in all_docs:
+            try:
+                next_decrypted = decrypted_docs.next()
+                if doc_id == next_decrypted[0]:
+                    content = next_decrypted[2]
+                    insertable.append((doc_id, rev, content, gen, trans_id))
+                else:
+                    break
+            except StopIteration:
                 break
         return insertable
 
@@ -966,7 +971,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         """
         # could pass source_replica in params for callback chain
         insert_fun = self._insert_doc_cb[self.source_replica_uid]
-        logger.debug("Sync decrypter pool: inserting doc in local db: " \
+        logger.debug("Sync decrypter pool: inserting doc in local db: "
                      "%s:%s %s" % (doc_id, doc_rev, gen))
         try:
             # convert deleted documents to avoid error on document creation
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
new file mode 100644
index 00000000..3c6fc569
--- /dev/null
+++ b/client/src/leap/soledad/client/secrets.py
@@ -0,0 +1,681 @@
+# -*- coding: utf-8 -*-
+# secrets.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Soledad secrets handling.
+"""
+
+
+import os
+import scrypt
+import hmac
+import logging
+import binascii
+import errno
+
+
+from hashlib import sha256
+import simplejson as json
+
+
+from leap.soledad.common import (
+    soledad_assert,
+    soledad_assert_type
+)
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.crypto import (
+    MacMethods,
+    UnknownMacMethod,
+    WrongMac,
+    MAC_KEY,
+    MAC_METHOD_KEY,
+)
+from leap.soledad.common.errors import (
+    InvalidTokenError,
+    NotLockedError,
+    AlreadyLockedError,
+    LockTimedOutError,
+)
+from leap.soledad.client.events import (
+    SOLEDAD_CREATING_KEYS,
+    SOLEDAD_DONE_CREATING_KEYS,
+    SOLEDAD_DOWNLOADING_KEYS,
+    SOLEDAD_DONE_DOWNLOADING_KEYS,
+    SOLEDAD_UPLOADING_KEYS,
+    SOLEDAD_DONE_UPLOADING_KEYS,
+    signal,
+)
+
+
+logger = logging.getLogger(name=__name__)
+
+
+#
+# Exceptions
+#
+
+class NoStorageSecret(Exception):
+    """
+    Raised when trying to use a storage secret but none is available.
+    """
+    pass
+
+
+class PassphraseTooShort(Exception):
+    """
+    Raised when trying to change the passphrase but the provided passphrase is
+    too short.
+    """
+
+
+class BootstrapSequenceError(Exception):
+    """
+    Raised when an attempt to generate a secret and store it in a recovery
+    document on server failed.
+    """
+
+
+#
+# Secrets handler
+#
+
+class SoledadSecrets(object):
+    """
+    Soledad secrets handler.
+
+    The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage
+    secret are used for remote storage encryption. We use the next
+    C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage.
+    From these bytes, the first C{self.SALT_LENGTH} bytes are used as the
+    salt and the rest as the password for the scrypt hashing.
+    """
+
+    LOCAL_STORAGE_SECRET_LENGTH = 512
+    """
+    The length of the secret used to derive a passphrase for the SQLCipher
+    database.
+    """
+
+    REMOTE_STORAGE_SECRET_LENGTH = 512
+    """
+    The length of the secret used to derive an encryption key and a MAC auth
+    key for remote storage.
+    """
+
+    SALT_LENGTH = 64
+    """
+    The length of the salt used to derive the key for the storage secret
+    encryption.
+    """
+
+    MINIMUM_PASSPHRASE_LENGTH = 6
+    """
+    The minimum length for a passphrase. The passphrase length is only checked
+    when the user changes her passphrase, not when she instantiates Soledad.
+    """
+
+    IV_SEPARATOR = ":"
+    """
+    A separator used for storing the encryption initial value prepended to the
+    ciphertext.
+    """
+
+    UUID_KEY = 'uuid'
+    STORAGE_SECRETS_KEY = 'storage_secrets'
+    SECRET_KEY = 'secret'
+    CIPHER_KEY = 'cipher'
+    LENGTH_KEY = 'length'
+    KDF_KEY = 'kdf'
+    KDF_SALT_KEY = 'kdf_salt'
+    KDF_LENGTH_KEY = 'kdf_length'
+    KDF_SCRYPT = 'scrypt'
+    CIPHER_AES256 = 'aes256'
+    """
+    Keys used to access storage secrets in recovery documents.
+    """
+
+    def __init__(self, uuid, passphrase, secrets_path, shared_db, crypto,
+                 secret_id=None):
+        """
+        Initialize the secrets manager.
+
+        :param uuid: User's unique id.
+        :type uuid: str
+        :param passphrase: The passphrase for locking and unlocking encryption
+                           secrets for local and remote storage.
+        :type passphrase: unicode
+        :param secrets_path: Path for storing encrypted key used for
+                             symmetric encryption.
+        :type secrets_path: str
+        :param shared_db: The shared database that stores user secrets.
+        :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase
+        :param crypto: A soledad crypto object.
+        :type crypto: SoledadCrypto
+        :param secret_id: The id of the storage secret to be used.
+        :type secret_id: str
+        """
+        self._uuid = uuid
+        self._passphrase = passphrase
+        self._secrets_path = secrets_path
+        self._shared_db = shared_db
+        self._crypto = crypto
+        self._secret_id = secret_id
+        self._secrets = {}
+
+    def bootstrap(self):
+        """
+        Bootstrap secrets.
+
+        Soledad secrets bootstrap is the following sequence of stages:
+
+        * stage 1 - local secret loading:
+            - if secrets exist locally, load them.
+        * stage 2 - remote secret loading:
+            - else, if secrets exist in server, download them.
+        * stage 3 - secret generation:
+            - else, generate a new secret and store in server.
+
+        This method decides which bootstrap stages have already been performed
+        and performs the missing ones in order.
+
+        :raise BootstrapSequenceError: Raised when the secret generation and
+            storage on server sequence has failed for some reason.
+        """
+        # STAGE 1 - verify if secrets exist locally
+        if not self._has_secret():  # try to load from local storage.
+
+            # STAGE 2 - there are no secrets in local storage, so try to fetch
+            # encrypted secrets from server.
+            logger.info(
+                'Trying to fetch cryptographic secrets from shared recovery '
+                'database...')
+
+            # --- start of atomic operation in shared db ---
+
+            # obtain lock on shared db
+            token = timeout = None
+            try:
+                token, timeout = self._shared_db.lock()
+            except AlreadyLockedError:
+                raise BootstrapSequenceError('Database is already locked.')
+            except LockTimedOutError:
+                raise BootstrapSequenceError('Lock operation timed out.')
+
+            self._get_or_gen_crypto_secrets()
+
+            # release the lock on shared db
+            try:
+                self._shared_db.unlock(token)
+                self._shared_db.close()
+            except NotLockedError:
+                # for some reason the lock expired. Despite that, secret
+                # loading or generation/storage must have been executed
+                # successfully, so we pass.
+                pass
+            except InvalidTokenError:
+                # here, our lock has not only expired but also some other
+                # client application has obtained a new lock and is currently
+                # doing its thing in the shared database. Using the same
+                # reasoning as above, we assume everything went smooth and
+                # pass.
+                pass
+            except Exception as e:
+                logger.error("Unhandled exception when unlocking shared "
+                             "database.")
+                logger.exception(e)
+
+            # --- end of atomic operation in shared db ---
+
+    def _has_secret(self):
+        """
+        Return whether there is a storage secret available for use or not.
+
+        :return: Whether there's a storage secret for symmetric encryption.
+        :rtype: bool
+        """
+        if self._secret_id is None or self._secret_id not in self._secrets:
+            try:
+                self._load_secrets()  # try to load from disk
+            except IOError as e:
+                logger.warning('IOError: %s' % str(e))
+        try:
+            self.storage_secret
+            return True
+        except Exception as e:
+            logger.warning("Couldn't load storage secret: %s" % str(e))
+            return False
+
+    def _load_secrets(self):
+        """
+        Load storage secrets from local file.
+        """
+        # does the file exist in disk?
+        if not os.path.isfile(self._secrets_path):
+            raise IOError('File does not exist: %s' % self._secrets_path)
+        # read storage secrets from file
+        content = None
+        with open(self._secrets_path, 'r') as f:
+            content = json.loads(f.read())
+        _, mac = self._import_recovery_document(content)
+        if mac is False:
+            self._store_secrets()
+            self._put_secrets_in_shared_db()
+        # choose first secret if no secret_id was given
+        if self._secret_id is None:
+            self.set_secret_id(self._secrets.items()[0][0])
+
+    def _get_or_gen_crypto_secrets(self):
+        """
+        Retrieves or generates the crypto secrets.
+
+        :raises BootstrapSequenceError: Raised when unable to store secrets in
+                                        shared database.
+        """
+        doc = self._get_secrets_from_shared_db()
+
+        if doc:
+            logger.info(
+                'Found cryptographic secrets in shared recovery '
+                'database.')
+            _, mac = self._import_recovery_document(doc.content)
+            if mac is False:
+                self.put_secrets_in_shared_db()
+            self._store_secrets()  # save new secrets in local file
+            if self._secret_id is None:
+                self.set_secret_id(self._secrets.items()[0][0])
+        else:
+            # STAGE 3 - there are no secrets in server also, so
+            # generate a secret and store it in remote db.
+            logger.info(
+                'No cryptographic secrets found, creating new '
+                ' secrets...')
+            self.set_secret_id(self._gen_secret())
+            try:
+                self._put_secrets_in_shared_db()
+            except Exception as ex:
+                # storing generated secret in shared db failed for
+                # some reason, so we erase the generated secret and
+                # raise.
+                try:
+                    os.unlink(self._secrets_path)
+                except OSError as e:
+                    if e.errno != errno.ENOENT:  # no such file or directory
+                        logger.exception(e)
+                logger.exception(ex)
+                raise BootstrapSequenceError(
+                    'Could not store generated secret in the shared '
+                    'database, bailing out...')
+
+    #
+    # Shared DB related methods
+    #
+
+    def _shared_db_doc_id(self):
+        """
+        Calculate the doc_id of the document in the shared db that stores key
+        material.
+
+        :return: the hash
+        :rtype: str
+        """
+        return sha256(
+            '%s%s' %
+            (self._passphrase_as_string(), self._uuid)).hexdigest()
+
+    def _export_recovery_document(self):
+        """
+        Export the storage secrets.
+
+        A recovery document has the following structure:
+
+            {
+                'storage_secrets': {
+                    '<storage_secret id>': {
+                        'kdf': 'scrypt',
+                        'kdf_salt': '<b64 repr of salt>'
+                        'kdf_length': <key length>
+                        'cipher': 'aes256',
+                        'length': <secret length>,
+                        'secret': '<encrypted storage_secret>',
+                    },
+                },
+                'kdf': 'scrypt',
+                'kdf_salt': '<b64 repr of salt>',
+                'kdf_length: <key length>,
+                '_mac_method': 'hmac',
+                '_mac': '<mac>'
+            }
+
+        Note that multiple storage secrets might be stored in one recovery
+        document. This method will also calculate a MAC of a string
+        representation of the secrets dictionary.
+
+        :return: The recovery document.
+        :rtype: dict
+        """
+        # create salt and key for calculating MAC
+        salt = os.urandom(self.SALT_LENGTH)
+        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+        data = {
+            self.STORAGE_SECRETS_KEY: self._secrets,
+            self.KDF_KEY: self.KDF_SCRYPT,
+            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+            self.KDF_LENGTH_KEY: len(key),
+            MAC_METHOD_KEY: MacMethods.HMAC,
+            MAC_KEY: hmac.new(
+                key,
+                json.dumps(self._secrets),
+                sha256).hexdigest(),
+        }
+        return data
+
+    def _import_recovery_document(self, data):
+        """
+        Import storage secrets for symmetric encryption and uuid (if present)
+        from a recovery document.
+
+        Note that this method does not store the imported data on disk. For
+        that, use C{self._store_secrets()}.
+
+        :param data: The recovery document.
+        :type data: dict
+
+        :return: A tuple containing the number of imported secrets and whether
+                 there was MAC informationa available for authenticating.
+        :rtype: (int, bool)
+        """
+        soledad_assert(self.STORAGE_SECRETS_KEY in data)
+        # check mac of the recovery document
+        mac = None
+        if MAC_KEY in data:
+            soledad_assert(data[MAC_KEY] is not None)
+            soledad_assert(MAC_METHOD_KEY in data)
+            soledad_assert(self.KDF_KEY in data)
+            soledad_assert(self.KDF_SALT_KEY in data)
+            soledad_assert(self.KDF_LENGTH_KEY in data)
+            if data[MAC_METHOD_KEY] == MacMethods.HMAC:
+                key = scrypt.hash(
+                    self._passphrase_as_string(),
+                    binascii.a2b_base64(data[self.KDF_SALT_KEY]),
+                    buflen=32)
+                mac = hmac.new(
+                    key,
+                    json.dumps(data[self.STORAGE_SECRETS_KEY]),
+                    sha256).hexdigest()
+            else:
+                raise UnknownMacMethod('Unknown MAC method: %s.' %
+                                       data[MAC_METHOD_KEY])
+            if mac != data[MAC_KEY]:
+                raise WrongMac('Could not authenticate recovery document\'s '
+                               'contents.')
+        # include secrets in the secret pool.
+        secrets = 0
+        for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items():
+            if secret_id not in self._secrets:
+                secrets += 1
+                self._secrets[secret_id] = secret_data
+        return secrets, mac
+
+    def _get_secrets_from_shared_db(self):
+        """
+        Retrieve the document with encrypted key material from the shared
+        database.
+
+        :return: a document with encrypted key material in its contents
+        :rtype: SoledadDocument
+        """
+        signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid)
+        db = self._shared_db
+        if not db:
+            logger.warning('No shared db found')
+            return
+        doc = db.get_doc(self._shared_db_doc_id())
+        signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
+        return doc
+
+    def _put_secrets_in_shared_db(self):
+        """
+        Assert local keys are the same as shared db's ones.
+
+        Try to fetch keys from shared recovery database. If they already exist
+        in the remote db, assert that that data is the same as local data.
+        Otherwise, upload keys to shared recovery database.
+        """
+        soledad_assert(
+            self._has_secret(),
+            'Tried to send keys to server but they don\'t exist in local '
+            'storage.')
+        # try to get secrets doc from server, otherwise create it
+        doc = self._get_secrets_from_shared_db()
+        if doc is None:
+            doc = SoledadDocument(
+                doc_id=self._shared_db_doc_id())
+        # fill doc with encrypted secrets
+        doc.content = self._export_recovery_document()
+        # upload secrets to server
+        signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
+        db = self._shared_db
+        if not db:
+            logger.warning('No shared db found')
+            return
+        db.put_doc(doc)
+        signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
+
+    #
+    # Management of secret for symmetric encryption.
+    #
+
+    @property
+    def storage_secret(self):
+        """
+        Return the storage secret.
+
+        Storage secret is encrypted before being stored. This method decrypts
+        and returns the stored secret.
+
+        :return: The storage secret.
+        :rtype: str
+        """
+        # calculate the encryption key
+        key = scrypt.hash(
+            self._passphrase_as_string(),
+            # the salt is stored base64 encoded
+            binascii.a2b_base64(
+                self._secrets[self._secret_id][self.KDF_SALT_KEY]),
+            buflen=32,  # we need a key with 256 bits (32 bytes).
+        )
+        # recover the initial value and ciphertext
+        iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split(
+            self.IV_SEPARATOR, 1)
+        ciphertext = binascii.a2b_base64(ciphertext)
+        return self._crypto.decrypt_sym(ciphertext, key, iv=iv)
+
+    def set_secret_id(self, secret_id):
+        """
+        Define the id of the storage secret to be used.
+
+        This method will also replace the secret in the crypto object.
+
+        :param secret_id: The id of the storage secret to be used.
+        :type secret_id: str
+        """
+        self._secret_id = secret_id
+
+    def _gen_secret(self):
+        """
+        Generate a secret for symmetric encryption and store in a local
+        encrypted file.
+
+        This method emits the following signals:
+
+            * SOLEDAD_CREATING_KEYS
+            * SOLEDAD_DONE_CREATING_KEYS
+
+        A secret has the following structure:
+
+            {
+                '<secret_id>': {
+                        'kdf': 'scrypt',
+                        'kdf_salt': '<b64 repr of salt>'
+                        'kdf_length': <key length>
+                        'cipher': 'aes256',
+                        'length': <secret length>,
+                        'secret': '<encrypted b64 repr of storage_secret>',
+                }
+            }
+
+        :return: The id of the generated secret.
+        :rtype: str
+        """
+        signal(SOLEDAD_CREATING_KEYS, self._uuid)
+        # generate random secret
+        secret = os.urandom(
+            self.LOCAL_STORAGE_SECRET_LENGTH
+            + self.REMOTE_STORAGE_SECRET_LENGTH)
+        secret_id = sha256(secret).hexdigest()
+        # generate random salt
+        salt = os.urandom(self.SALT_LENGTH)
+        # get a 256-bit key
+        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+        iv, ciphertext = self._crypto.encrypt_sym(secret, key)
+        self._secrets[secret_id] = {
+            # leap.soledad.crypto submodule uses AES256 for symmetric
+            # encryption.
+            self.KDF_KEY: self.KDF_SCRYPT,
+            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+            self.KDF_LENGTH_KEY: len(key),
+            self.CIPHER_KEY: self.CIPHER_AES256,
+            self.LENGTH_KEY: len(secret),
+            self.SECRET_KEY: '%s%s%s' % (
+                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
+        }
+        self._store_secrets()
+        signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid)
+        return secret_id
+
+    def _store_secrets(self):
+        """
+        Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
+        """
+        with open(self._secrets_path, 'w') as f:
+            f.write(
+                json.dumps(
+                    self._export_recovery_document()))
+
+    def change_passphrase(self, new_passphrase):
+        """
+        Change the passphrase that encrypts the storage secret.
+
+        :param new_passphrase: The new passphrase.
+        :type new_passphrase: unicode
+
+        :raise NoStorageSecret: Raised if there's no storage secret available.
+        """
+        # TODO: maybe we want to add more checks to guarantee passphrase is
+        # reasonable?
+        soledad_assert_type(new_passphrase, unicode)
+        if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH:
+            raise PassphraseTooShort(
+                'Passphrase must be at least %d characters long!' %
+                self.MINIMUM_PASSPHRASE_LENGTH)
+        # ensure there's a secret for which the passphrase will be changed.
+        if not self._has_secret():
+            raise NoStorageSecret()
+        secret = self.storage_secret
+        # generate random salt
+        new_salt = os.urandom(self.SALT_LENGTH)
+        # get a 256-bit key
+        key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32)
+        iv, ciphertext = self._crypto.encrypt_sym(secret, key)
+        # XXX update all secrets in the dict
+        self._secrets[self._secret_id] = {
+            # leap.soledad.crypto submodule uses AES256 for symmetric
+            # encryption.
+            self.KDF_KEY: self.KDF_SCRYPT,  # TODO: remove hard coded kdf
+            self.KDF_SALT_KEY: binascii.b2a_base64(new_salt),
+            self.KDF_LENGTH_KEY: len(key),
+            self.CIPHER_KEY: self.CIPHER_AES256,
+            self.LENGTH_KEY: len(secret),
+            self.SECRET_KEY: '%s%s%s' % (
+                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
+        }
+        self._passphrase = new_passphrase
+        self._store_secrets()
+        self._put_secrets_in_shared_db()
+
+    #
+    # Setters and getters
+    #
+
+    @property
+    def secret_id(self):
+        return self._secret_id
+
+    def _get_secrets_path(self):
+        return self._secrets_path
+
+    def _set_secrets_path(self, secrets_path):
+        self._secrets_path = secrets_path
+
+    secrets_path = property(
+         _get_secrets_path,
+         _set_secrets_path,
+        doc='The path for the file containing the encrypted symmetric secret.')
+
+    @property
+    def passphrase(self):
+        """
+        Return the passphrase for locking and unlocking encryption secrets for
+        local and remote storage.
+        """
+        return self._passphrase
+
+    def _passphrase_as_string(self):
+        return self._passphrase.encode('utf-8')
+
+    def get_syncdb_secret(self):
+        """
+        Return the secret for sync db.
+        """
+        # TODO: implement.
+        pass
+
+    def get_remote_secret(self):
+        """
+        Return the secret for remote storage.
+        """
+        # TODO: implement
+        pass
+
+    def get_local_storage_key(self):
+        """
+        Return the local storage key derived from the local storage secret.
+        """
+        # salt indexes
+        salt_start = self.REMOTE_STORAGE_SECRET_LENGTH
+        salt_end = salt_start + self.SALT_LENGTH
+        # password indexes
+        pwd_start = salt_end
+        pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH
+        # calculate the key for local encryption
+        secret = self.storage_secret
+        return scrypt.hash(
+            secret[pwd_start:pwd_end],  # the password
+            secret[salt_start:salt_end],  # the salt
+            buflen=32,  # we need a key with 256 bits (32 bytes)
+        )
-- 
cgit v1.2.3


From 21a3f854c07c1d40d50da8c922e956d3247a08b2 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Fri, 1 Aug 2014 12:53:05 -0300
Subject: Fix tests after many changes in client.

---
 client/src/leap/soledad/client/sync.py   |  2 --
 client/src/leap/soledad/client/target.py | 34 +++++++++++++++-----------------
 2 files changed, 16 insertions(+), 20 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 5d545a77..c158f2a7 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -29,8 +29,6 @@ Extend u1db Synchronizer with the ability to:
 """
 
 
-import json
-
 import logging
 import traceback
 from threading import Lock
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 01e1231a..12175f48 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -28,12 +28,10 @@ import logging
 import re
 import urllib
 import threading
-import urlparse
 
 from collections import defaultdict
 from time import sleep
 from uuid import uuid4
-from contextlib import contextmanager
 
 import simplejson as json
 from taskthread import TimerTask
@@ -44,7 +42,6 @@ from u1db.remote.http_client import _encode_query_parameter, HTTPClientBase
 from zope.proxy import ProxyBase
 from zope.proxy import sameProxiedObjects, setProxiedObject
 
-from leap.soledad.common import soledad_assert
 from leap.soledad.common.document import SoledadDocument
 from leap.soledad.client.auth import TokenBasedAuth
 from leap.soledad.client.crypto import is_symmetrically_encrypted
@@ -87,7 +84,7 @@ class DocumentSyncerThread(threading.Thread):
     """
 
     def __init__(self, doc_syncer, release_method, failed_method,
-            idx, total, last_request_lock=None, last_callback_lock=None):
+                 idx, total, last_request_lock=None, last_callback_lock=None):
         """
         Initialize a new syncer thread.
 
@@ -246,7 +243,7 @@ class DocumentSyncerPool(object):
     """
 
     def __init__(self, raw_url, raw_creds, query_string, headers,
-            ensure_callback, stop_method):
+                 ensure_callback, stop_method):
         """
         Initialize the document syncer pool.
 
@@ -279,7 +276,7 @@ class DocumentSyncerPool(object):
         self._threads = []
 
     def new_syncer_thread(self, idx, total, last_request_lock=None,
-            last_callback_lock=None):
+                          last_callback_lock=None):
         """
         Yield a new document syncer thread.
 
@@ -619,7 +616,7 @@ class HTTPDocumentSyncer(HTTPClientBase, TokenBasedAuth):
         self._conn.endheaders()
 
     def _get_doc(self, received, sync_id, last_known_generation,
-            last_known_trans_id):
+                 last_known_trans_id):
         """
         Get a sync document from server by means of a POST request.
 
@@ -658,7 +655,7 @@ class HTTPDocumentSyncer(HTTPClientBase, TokenBasedAuth):
         return self._response()
 
     def _put_doc(self, sync_id, last_known_generation, last_known_trans_id,
-            id, rev, content, gen, trans_id, number_of_docs, doc_idx):
+                 id, rev, content, gen, trans_id, number_of_docs, doc_idx):
         """
         Put a sync document on server by means of a POST request.
 
@@ -765,7 +762,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
     #
 
     def __init__(self, url, source_replica_uid=None, creds=None, crypto=None,
-            sync_db=None, sync_db_write_lock=None):
+                 sync_db=None, sync_db_write_lock=None):
         """
         Initialize the SoledadSyncTarget.
 
@@ -925,7 +922,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         """
         new_generation, new_transaction_id, number_of_changes, doc_id, \
             rev, content, gen, trans_id = \
-                self._parse_received_doc_response(response)
+            self._parse_received_doc_response(response)
         if doc_id is not None:
             # decrypt incoming document and insert into local database
             # -------------------------------------------------------------
@@ -1134,11 +1131,14 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         """
         self._ensure_callback = ensure_callback
 
-        if defer_decryption:
+        if defer_decryption and self._sync_db is not None:
             self._sync_exchange_lock.acquire()
             self._setup_sync_decr_pool(last_known_generation)
             self._setup_sync_watcher()
             self._defer_decryption = True
+        else:
+            # fall back
+            defer_decryption = False
 
         self.start()
 
@@ -1149,7 +1149,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         setProxiedObject(self._insert_doc_cb[source_replica_uid],
                          return_doc_cb)
 
-        if not self.clear_to_sync():
+        if defer_decryption is True and not self.clear_to_sync():
             raise PendingReceivedDocsSyncError
 
         self._ensure_connection()
@@ -1171,7 +1171,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             self._raw_url, self._raw_creds, url, headers, ensure_callback,
             self.stop)
         threads = []
-        last_request_lock = None
         last_callback_lock = None
         sent = 0
         total = len(docs_by_generations)
@@ -1227,7 +1226,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             t.doc_syncer.set_request_method(
                 'put', sync_id, cur_target_gen, cur_target_trans_id,
                 id=doc.doc_id, rev=doc.rev, content=doc_json, gen=gen,
-                trans_id=trans_id, number_of_docs=number_of_docs, doc_idx=sent + 1)
+                trans_id=trans_id, number_of_docs=number_of_docs,
+                doc_idx=sent + 1)
             # set the success calback
 
             def _success_callback(idx, total, response):
@@ -1251,7 +1251,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             # save thread and append
             t.start()
             threads.append((t, doc))
-            last_request_lock = t.request_lock
             last_callback_lock = t.callback_lock
             sent += 1
 
@@ -1275,7 +1274,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         if last_successful_thread is not None:
             response_dict = json.loads(last_successful_thread.response[0])[0]
             gen_after_send = response_dict['new_generation']
-            trans_id_after_send  = response_dict['new_transaction_id']
+            trans_id_after_send = response_dict['new_transaction_id']
 
         # get docs from target
         if self.stopped is False:
@@ -1356,7 +1355,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             # no doc found
             return None
 
-
     def delete_encrypted_docs_from_db(self, docs_ids):
         """
         Delete several encrypted documents from the database of symmetrically
@@ -1467,7 +1465,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
 
         decrypter = self._sync_decr_pool
         decrypter.decrypt_received_docs()
-        done = decrypter.process_decrypted()
+        decrypter.process_decrypted()
 
     def _sign_request(self, method, url_query, params):
         """
-- 
cgit v1.2.3


From bb4ef28014b7846df8982f0008635f4d05b5a0b8 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 4 Aug 2014 10:34:20 -0300
Subject: Add instructions for closing SQLCipher db on docstrings.

---
 client/src/leap/soledad/client/sqlcipher.py | 37 ++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index 85b0391b..7823e235 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -92,7 +92,16 @@ SQLITE_ISOLATION_LEVEL = None
 def open(path, password, create=True, document_factory=None, crypto=None,
          raw_key=False, cipher='aes-256-cbc', kdf_iter=4000,
          cipher_page_size=1024, defer_encryption=False):
-    """Open a database at the given location.
+    """
+    Open a database at the given location.
+
+    *** IMPORTANT ***
+
+    Don't forget to close the database after use by calling the close()
+    method otherwise some resources might not be freed and you may experience
+    several kinds of leakages.
+
+    *** IMPORTANT ***
 
     Will raise u1db.errors.DatabaseDoesNotExist if create=False and the
     database does not already exist.
@@ -195,6 +204,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         Connect to an existing SQLCipher database, creating a new sqlcipher
         database file if needed.
 
+        *** IMPORTANT ***
+
+        Don't forget to close the database after use by calling the close()
+        method otherwise some resources might not be freed and you may
+        experience several kinds of leakages.
+
+        *** IMPORTANT ***
+
         :param sqlcipher_file: The path for the SQLCipher file.
         :type sqlcipher_file: str
         :param password: The password that protects the SQLCipher db.
@@ -356,6 +373,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         """
         Open a SQLCipher database.
 
+        *** IMPORTANT ***
+
+        Don't forget to close the database after use by calling the close()
+        method otherwise some resources might not be freed and you may
+        experience several kinds of leakages.
+
+        *** IMPORTANT ***
+
         :param sqlcipher_file: The path for the SQLCipher file.
         :type sqlcipher_file: str
 
@@ -1097,6 +1122,16 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             del self.sync_queue
             self.sync_queue = None
 
+    def __del__(self):
+        """
+        Free resources when deleting or garbage collecting the database.
+
+        This is only here to minimze problems if someone ever forgets to call
+        the close() method after using the database; you should not rely on
+        garbage collecting to free up the database resources.
+        """
+        self.close()
+
     @property
     def replica_uid(self):
         return self._get_replica_uid()
-- 
cgit v1.2.3


From 9f455ab44d8f229840a5c6a75e0e7b6a88b04f57 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 4 Aug 2014 11:40:37 -0300
Subject: Store decrypted storage secret in memory.

---
 client/src/leap/soledad/client/secrets.py | 182 +++++++++++++++++-------------
 1 file changed, 105 insertions(+), 77 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index 3c6fc569..621e2d99 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -252,13 +252,9 @@ class SoledadSecrets(object):
             try:
                 self._load_secrets()  # try to load from disk
             except IOError as e:
-                logger.warning('IOError: %s' % str(e))
-        try:
-            self.storage_secret
-            return True
-        except Exception as e:
-            logger.warning("Couldn't load storage secret: %s" % str(e))
-            return False
+                logger.warning('IOError while loading secrets from disk: %s' % str(e))
+                return False
+        return self.storage_secret is not None
 
     def _load_secrets(self):
         """
@@ -371,15 +367,21 @@ class SoledadSecrets(object):
         # create salt and key for calculating MAC
         salt = os.urandom(self.SALT_LENGTH)
         key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+        # encrypt secrets
+        encrypted_secrets = {}
+        for secret_id in self._secrets:
+            encrypted_secrets[secret_id] = self._encrypt_storage_secret(
+                self._secrets[secret_id])
+        # create the recovery document
         data = {
-            self.STORAGE_SECRETS_KEY: self._secrets,
+            self.STORAGE_SECRETS_KEY: encrypted_secrets,
             self.KDF_KEY: self.KDF_SCRYPT,
             self.KDF_SALT_KEY: binascii.b2a_base64(salt),
             self.KDF_LENGTH_KEY: len(key),
             MAC_METHOD_KEY: MacMethods.HMAC,
             MAC_KEY: hmac.new(
                 key,
-                json.dumps(self._secrets),
+                json.dumps(encrypted_secrets),
                 sha256).hexdigest(),
         }
         return data
@@ -425,10 +427,11 @@ class SoledadSecrets(object):
                                'contents.')
         # include secrets in the secret pool.
         secrets = 0
-        for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items():
+        for secret_id, encrypted_secret in data[self.STORAGE_SECRETS_KEY].items():
             if secret_id not in self._secrets:
                 secrets += 1
-                self._secrets[secret_id] = secret_data
+                self._secrets[secret_id] = \
+                    self._decrypt_storage_secret(encrypted_secret)
         return secrets, mac
 
     def _get_secrets_from_shared_db(self):
@@ -480,30 +483,92 @@ class SoledadSecrets(object):
     # Management of secret for symmetric encryption.
     #
 
-    @property
-    def storage_secret(self):
+    def _decrypt_storage_secret(self, encrypted_secret_dict):
         """
-        Return the storage secret.
+        Decrypt the storage secret.
 
         Storage secret is encrypted before being stored. This method decrypts
-        and returns the stored secret.
+        and returns the decrypted storage secret.
 
-        :return: The storage secret.
+        :param encrypted_secret_dict: The encrypted storage secret.
+        :type encrypted_secret_dict:  dict
+
+        :return: The decrypted storage secret.
         :rtype: str
         """
         # calculate the encryption key
+        if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT:
+            raise Exception("Unknown KDF in stored secret.")
         key = scrypt.hash(
             self._passphrase_as_string(),
             # the salt is stored base64 encoded
             binascii.a2b_base64(
-                self._secrets[self._secret_id][self.KDF_SALT_KEY]),
+                encrypted_secret_dict[self.KDF_SALT_KEY]),
             buflen=32,  # we need a key with 256 bits (32 bytes).
         )
+        if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key):
+            raise Exception("Wrong length of decryption key.")
+        if encrypted_secret_dict[self.CIPHER_KEY] != self.CIPHER_AES256:
+            raise Exception("Unknown cipher in stored secret.")
         # recover the initial value and ciphertext
-        iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split(
+        iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split(
             self.IV_SEPARATOR, 1)
         ciphertext = binascii.a2b_base64(ciphertext)
-        return self._crypto.decrypt_sym(ciphertext, key, iv=iv)
+        decrypted_secret = self._crypto.decrypt_sym(ciphertext, key, iv=iv)
+        if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret):
+            raise Exception("Wrong length of decrypted secret.")
+        return decrypted_secret
+
+    def _encrypt_storage_secret(self, decrypted_secret):
+        """
+        Encrypt the storage secret.
+
+        An encrypted secret has the following structure:
+
+            {
+                '<secret_id>': {
+                        'kdf': 'scrypt',
+                        'kdf_salt': '<b64 repr of salt>'
+                        'kdf_length': <key length>
+                        'cipher': 'aes256',
+                        'length': <secret length>,
+                        'secret': '<encrypted b64 repr of storage_secret>',
+                }
+            }
+
+        :param decrypted_secret: The decrypted storage secret.
+        :type decrypted_secret: str
+
+        :return: The encrypted storage secret.
+        :rtype: dict
+        """
+        # generate random salt
+        salt = os.urandom(self.SALT_LENGTH)
+        # get a 256-bit key
+        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+        iv, ciphertext = self._crypto.encrypt_sym(decrypted_secret, key)
+        encrypted_secret_dict = {
+            # leap.soledad.crypto submodule uses AES256 for symmetric
+            # encryption.
+            self.KDF_KEY: self.KDF_SCRYPT,
+            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+            self.KDF_LENGTH_KEY: len(key),
+            self.CIPHER_KEY: self.CIPHER_AES256,
+            self.LENGTH_KEY: len(decrypted_secret),
+            self.SECRET_KEY: '%s%s%s' % (
+                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
+        }
+        return encrypted_secret_dict
+
+    @property
+    def storage_secret(self):
+        """
+        Return the storage secret.
+
+        :return: The decrypted storage secret.
+        :rtype: str
+        """
+        return self._secrets.get(self._secret_id)
 
     def set_secret_id(self, secret_id):
         """
@@ -526,19 +591,6 @@ class SoledadSecrets(object):
             * SOLEDAD_CREATING_KEYS
             * SOLEDAD_DONE_CREATING_KEYS
 
-        A secret has the following structure:
-
-            {
-                '<secret_id>': {
-                        'kdf': 'scrypt',
-                        'kdf_salt': '<b64 repr of salt>'
-                        'kdf_length': <key length>
-                        'cipher': 'aes256',
-                        'length': <secret length>,
-                        'secret': '<encrypted b64 repr of storage_secret>',
-                }
-            }
-
         :return: The id of the generated secret.
         :rtype: str
         """
@@ -548,22 +600,7 @@ class SoledadSecrets(object):
             self.LOCAL_STORAGE_SECRET_LENGTH
             + self.REMOTE_STORAGE_SECRET_LENGTH)
         secret_id = sha256(secret).hexdigest()
-        # generate random salt
-        salt = os.urandom(self.SALT_LENGTH)
-        # get a 256-bit key
-        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
-        iv, ciphertext = self._crypto.encrypt_sym(secret, key)
-        self._secrets[secret_id] = {
-            # leap.soledad.crypto submodule uses AES256 for symmetric
-            # encryption.
-            self.KDF_KEY: self.KDF_SCRYPT,
-            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
-            self.KDF_LENGTH_KEY: len(key),
-            self.CIPHER_KEY: self.CIPHER_AES256,
-            self.LENGTH_KEY: len(secret),
-            self.SECRET_KEY: '%s%s%s' % (
-                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
-        }
+        self._secrets[secret_id] = secret
         self._store_secrets()
         signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid)
         return secret_id
@@ -596,24 +633,6 @@ class SoledadSecrets(object):
         # ensure there's a secret for which the passphrase will be changed.
         if not self._has_secret():
             raise NoStorageSecret()
-        secret = self.storage_secret
-        # generate random salt
-        new_salt = os.urandom(self.SALT_LENGTH)
-        # get a 256-bit key
-        key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32)
-        iv, ciphertext = self._crypto.encrypt_sym(secret, key)
-        # XXX update all secrets in the dict
-        self._secrets[self._secret_id] = {
-            # leap.soledad.crypto submodule uses AES256 for symmetric
-            # encryption.
-            self.KDF_KEY: self.KDF_SCRYPT,  # TODO: remove hard coded kdf
-            self.KDF_SALT_KEY: binascii.b2a_base64(new_salt),
-            self.KDF_LENGTH_KEY: len(key),
-            self.CIPHER_KEY: self.CIPHER_AES256,
-            self.LENGTH_KEY: len(secret),
-            self.SECRET_KEY: '%s%s%s' % (
-                str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
-        }
         self._passphrase = new_passphrase
         self._store_secrets()
         self._put_secrets_in_shared_db()
@@ -655,27 +674,36 @@ class SoledadSecrets(object):
         # TODO: implement.
         pass
 
-    def get_remote_secret(self):
+    def _get_remote_storage_secret(self):
         """
         Return the secret for remote storage.
         """
         # TODO: implement
         pass
 
-    def get_local_storage_key(self):
+
+    def _get_local_storage_secret(self):
         """
-        Return the local storage key derived from the local storage secret.
+        Return the local storage secret.
+        """
+        pwd_start = self.REMOTE_STORAGE_SECRET_LENGTH + self.SALT_LENGTH
+        pwd_end = self.REMOTE_STORAGE_SECRET_LENGTH + self.LOCAL_STORAGE_SECRET_LENGTH
+        return self.storage_secret[pwd_start:pwd_end]
+
+    def _get_local_storage_salt(self):
+        """
+        Return the local storage salt.
         """
-        # salt indexes
         salt_start = self.REMOTE_STORAGE_SECRET_LENGTH
         salt_end = salt_start + self.SALT_LENGTH
-        # password indexes
-        pwd_start = salt_end
-        pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH
-        # calculate the key for local encryption
-        secret = self.storage_secret
+        return self.storage_secret[salt_start:salt_end]
+
+    def get_local_storage_key(self):
+        """
+        Return the local storage key derived from the local storage secret.
+        """
         return scrypt.hash(
-            secret[pwd_start:pwd_end],  # the password
-            secret[salt_start:salt_end],  # the salt
+            self._get_local_storage_secret(),  # the password
+            self._get_local_storage_salt(),    # the salt
             buflen=32,  # we need a key with 256 bits (32 bytes)
         )
-- 
cgit v1.2.3


From aa8fcba828bc917eaf8e6b0dacb76f0de904bf59 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 4 Aug 2014 16:17:09 -0300
Subject: Add salt for sync sb key derivation.

---
 client/src/leap/soledad/client/__init__.py |  7 +++
 client/src/leap/soledad/client/crypto.py   |  6 +--
 client/src/leap/soledad/client/secrets.py  | 82 ++++++++++++++++++++++++------
 3 files changed, 76 insertions(+), 19 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 0fd6672a..e66055e0 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -750,6 +750,13 @@ class Soledad(object):
         """
         return self._secrets.storage_secret
 
+    @property
+    def remote_storage_secret(self):
+        """
+        Return the secret used for encryption of remotelly stored data.
+        """
+        return self._secrets.remote_storage_secret
+
     @property
     def secrets(self):
         return self._secrets
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 4a64b5a8..1b01913d 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -240,9 +240,7 @@ class SoledadCrypto(object):
         if self.secret is None:
             raise NoSymmetricSecret()
         return hmac.new(
-            self.secret[
-                MAC_KEY_LENGTH:
-                self._soledad.secrets.REMOTE_STORAGE_SECRET_LENGTH],
+            self.secret[MAC_KEY_LENGTH:],
             doc_id,
             hashlib.sha256).digest()
 
@@ -251,7 +249,7 @@ class SoledadCrypto(object):
     #
 
     def _get_secret(self):
-        return self._soledad.storage_secret
+        return self._soledad.secrets.remote_storage_secret
 
     secret = property(
         _get_secret, doc='The secret used for symmetric encryption')
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index 621e2d99..55580692 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -123,6 +123,14 @@ class SoledadSecrets(object):
     encryption.
     """
 
+    GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \
+        + REMOTE_STORAGE_SECRET_LENGTH \
+        + SALT_LENGTH  # for sync db
+    """
+    The length of the secret to be generated. This includes local and remote
+    secrets, and the salt for deriving the sync db secret.
+    """
+
     MINIMUM_PASSPHRASE_LENGTH = 6
     """
     The minimum length for a passphrase. The passphrase length is only checked
@@ -268,12 +276,21 @@ class SoledadSecrets(object):
         with open(self._secrets_path, 'r') as f:
             content = json.loads(f.read())
         _, mac = self._import_recovery_document(content)
-        if mac is False:
-            self._store_secrets()
-            self._put_secrets_in_shared_db()
         # choose first secret if no secret_id was given
         if self._secret_id is None:
             self.set_secret_id(self._secrets.items()[0][0])
+        # enlarge secret if needed
+        enlarged = False
+        if len(self._secrets[self._secret_id]) < self.GEN_SECRET_LENGTH:
+            gen_len = self.GEN_SECRET_LENGTH \
+                - len(self._secrets[self._secret_id])
+            new_piece = os.urandom(gen_len)
+            self._secrets[self._secret_id] += new_piece
+            enlarged = True
+        # store and save in shared db if needed
+        if mac is False or enlarged is True:
+            self._store_secrets()
+            self._put_secrets_in_shared_db()
 
     def _get_or_gen_crypto_secrets(self):
         """
@@ -596,9 +613,7 @@ class SoledadSecrets(object):
         """
         signal(SOLEDAD_CREATING_KEYS, self._uuid)
         # generate random secret
-        secret = os.urandom(
-            self.LOCAL_STORAGE_SECRET_LENGTH
-            + self.REMOTE_STORAGE_SECRET_LENGTH)
+        secret = os.urandom(self.GEN_SECRET_LENGTH)
         secret_id = sha256(secret).hexdigest()
         self._secrets[secret_id] = secret
         self._store_secrets()
@@ -667,24 +682,29 @@ class SoledadSecrets(object):
     def _passphrase_as_string(self):
         return self._passphrase.encode('utf-8')
 
-    def get_syncdb_secret(self):
-        """
-        Return the secret for sync db.
-        """
-        # TODO: implement.
-        pass
+    #
+    # remote storage secret
+    #
 
-    def _get_remote_storage_secret(self):
+    @property
+    def remote_storage_secret(self):
         """
         Return the secret for remote storage.
         """
-        # TODO: implement
-        pass
+        key_start = 0
+        key_end =  self.REMOTE_STORAGE_SECRET_LENGTH
+        return self.storage_secret[key_start:key_end]
 
+    #
+    # local storage key
+    #
 
     def _get_local_storage_secret(self):
         """
         Return the local storage secret.
+
+        :return: The local storage secret.
+        :rtype: str
         """
         pwd_start = self.REMOTE_STORAGE_SECRET_LENGTH + self.SALT_LENGTH
         pwd_end = self.REMOTE_STORAGE_SECRET_LENGTH + self.LOCAL_STORAGE_SECRET_LENGTH
@@ -693,6 +713,9 @@ class SoledadSecrets(object):
     def _get_local_storage_salt(self):
         """
         Return the local storage salt.
+
+        :return: The local storage salt.
+        :rtype: str
         """
         salt_start = self.REMOTE_STORAGE_SECRET_LENGTH
         salt_end = salt_start + self.SALT_LENGTH
@@ -701,9 +724,38 @@ class SoledadSecrets(object):
     def get_local_storage_key(self):
         """
         Return the local storage key derived from the local storage secret.
+
+        :return: The key for protecting the local database.
+        :rtype: str
         """
         return scrypt.hash(
             self._get_local_storage_secret(),  # the password
             self._get_local_storage_salt(),    # the salt
             buflen=32,  # we need a key with 256 bits (32 bytes)
         )
+
+   #
+   # sync db key
+   #
+
+    def _get_sync_db_salt(self):
+        """
+        Return the salt for sync db.
+        """
+        salt_start = self.LOCAL_STORAGE_SECRET_LENGTH \
+            + self.REMOTE_STORAGE_SECRET_LENGTH
+        salt_end = salt_start + self.SALT_LENGTH
+        return self.storage_secret[salt_start:salt_end]
+
+    def get_sync_db_key(self):
+        """
+        Return the key for protecting the sync database.
+
+        :return: The key for protecting the sync database.
+        :rtype: str
+        """
+        return scrypt.hash(
+            self._get_local_storage_secret(),  # the password
+            self._get_sync_db_salt(),    # the salt
+            buflen=32,  # we need a key with 256 bits (32 bytes)
+        )
-- 
cgit v1.2.3


From 30aa5c040c093aa82be09e94dd403c18597320e5 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 4 Aug 2014 16:42:56 -0300
Subject: Protect sync db with a password.

---
 client/src/leap/soledad/client/__init__.py   |  4 +++-
 client/src/leap/soledad/client/mp_safe_db.py | 15 +++++++++++++--
 client/src/leap/soledad/client/sqlcipher.py  | 28 ++++++++++++++++++----------
 3 files changed, 34 insertions(+), 13 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index e66055e0..0b72be27 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -264,6 +264,7 @@ class Soledad(object):
         uses the 'raw PRAGMA key' format to handle the key to SQLCipher.
         """
         key = self._secrets.get_local_storage_key()
+        sync_db_key = self._secrets.get_sync_db_key()
         self._db = sqlcipher_open(
             self._local_db_path,
             binascii.b2a_hex(key),  # sqlcipher only accepts the hex version
@@ -271,7 +272,8 @@ class Soledad(object):
             document_factory=SoledadDocument,
             crypto=self._crypto,
             raw_key=True,
-            defer_encryption=self._defer_encryption)
+            defer_encryption=self._defer_encryption,
+            sync_db_key=binascii.b2a_hex(sync_db_key))
 
     def close(self):
         """
diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py
index a9ab5649..2c6b7e24 100644
--- a/client/src/leap/soledad/client/mp_safe_db.py
+++ b/client/src/leap/soledad/client/mp_safe_db.py
@@ -23,7 +23,7 @@ Multiprocessing-safe SQLite database.
 
 from threading import Thread
 from Queue import Queue
-from sqlite3 import connect as sqlite3_connect
+from pysqlcipher import dbapi2
 
 
 # Thanks to http://code.activestate.com/recipes/526618/
@@ -49,7 +49,7 @@ class MPSafeSQLiteDB(Thread):
         """
         Run the multiprocessing-safe database accessor.
         """
-        conn = sqlite3_connect(self._db_path)
+        conn = dbapi2.connect(self._db_path)
         while True:
             req, arg, res = self._requests.get()
             if req == self.CLOSE:
@@ -99,3 +99,14 @@ class MPSafeSQLiteDB(Thread):
         """
         self.execute(self.CLOSE)
         self.join()
+
+    def cursor(self):
+        """
+        Return a fake cursor object.
+
+        Not really a cursor, but allows for calling db.cursor().execute().
+
+        :return: Self.
+        :rtype: MPSafeSQLiteDatabase
+        """
+        return self
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index 7823e235..a7ddab24 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -91,7 +91,7 @@ SQLITE_ISOLATION_LEVEL = None
 
 def open(path, password, create=True, document_factory=None, crypto=None,
          raw_key=False, cipher='aes-256-cbc', kdf_iter=4000,
-         cipher_page_size=1024, defer_encryption=False):
+         cipher_page_size=1024, defer_encryption=False, sync_db_key=None):
     """
     Open a database at the given location.
 
@@ -136,7 +136,8 @@ def open(path, password, create=True, document_factory=None, crypto=None,
     return SQLCipherDatabase.open_database(
         path, password, create=create, document_factory=document_factory,
         crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter,
-        cipher_page_size=cipher_page_size, defer_encryption=defer_encryption)
+        cipher_page_size=cipher_page_size, defer_encryption=defer_encryption,
+        sync_db_key=sync_db_key)
 
 
 #
@@ -199,7 +200,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
 
     def __init__(self, sqlcipher_file, password, document_factory=None,
                  crypto=None, raw_key=False, cipher='aes-256-cbc',
-                 kdf_iter=4000, cipher_page_size=1024):
+                 kdf_iter=4000, cipher_page_size=1024, sync_db_key=None):
         """
         Connect to an existing SQLCipher database, creating a new sqlcipher
         database file if needed.
@@ -264,7 +265,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         self._sync_db = None
         self._sync_db_write_lock = None
         self._sync_enc_pool = None
-        self._init_sync_db(sqlcipher_file)
+        self._init_sync_db(sqlcipher_file, sync_db_key)
 
         if self.defer_encryption:
             # initialize sync db
@@ -293,7 +294,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
     def _open_database(cls, sqlcipher_file, password, document_factory=None,
                        crypto=None, raw_key=False, cipher='aes-256-cbc',
                        kdf_iter=4000, cipher_page_size=1024,
-                       defer_encryption=False):
+                       defer_encryption=False, sync_db_key=None):
         """
         Open a SQLCipher database.
 
@@ -363,13 +364,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         return SQLCipherDatabase._sqlite_registry[v](
             sqlcipher_file, password, document_factory=document_factory,
             crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter,
-            cipher_page_size=cipher_page_size)
+            cipher_page_size=cipher_page_size, sync_db_key=sync_db_key)
 
     @classmethod
     def open_database(cls, sqlcipher_file, password, create, backend_cls=None,
                       document_factory=None, crypto=None, raw_key=False,
                       cipher='aes-256-cbc', kdf_iter=4000,
-                      cipher_page_size=1024, defer_encryption=False):
+                      cipher_page_size=1024, defer_encryption=False,
+                      sync_db_key=None):
         """
         Open a SQLCipher database.
 
@@ -429,7 +431,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
                 sqlcipher_file, password, document_factory=document_factory,
                 crypto=crypto, raw_key=raw_key, cipher=cipher,
                 kdf_iter=kdf_iter, cipher_page_size=cipher_page_size,
-                defer_encryption=defer_encryption)
+                defer_encryption=defer_encryption, sync_db_key=sync_db_key)
         except u1db_errors.DatabaseDoesNotExist:
             if not create:
                 raise
@@ -440,7 +442,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             return backend_cls(
                 sqlcipher_file, password, document_factory=document_factory,
                 crypto=crypto, raw_key=raw_key, cipher=cipher,
-                kdf_iter=kdf_iter, cipher_page_size=cipher_page_size)
+                kdf_iter=kdf_iter, cipher_page_size=cipher_page_size,
+                sync_db_key=sync_db_key)
 
     def sync(self, url, creds=None, autocreate=True, defer_decryption=True):
         """
@@ -561,7 +564,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             'ALTER TABLE document '
             'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
 
-    def _init_sync_db(self, sqlcipher_file):
+    def _init_sync_db(self, sqlcipher_file, sync_db_password):
         """
         Initialize the Symmetrically-Encrypted document to be synced database,
         and the queue to communicate with subprocess workers.
@@ -575,6 +578,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         else:
             sync_db_path = ":memory:"
         self._sync_db = MPSafeSQLiteDB(sync_db_path)
+        # protect the sync db with a password
+        if sync_db_password is not None:
+            self._set_crypto_pragmas(
+                self._sync_db, sync_db_password, True,
+                'aes-256-cbc', 4000, 1024)
         self._sync_db_write_lock = threading.Lock()
         self._create_sync_db_tables()
         self.sync_queue = multiprocessing.Queue()
-- 
cgit v1.2.3


From afdb1cefe605cabfe325df3124b9beb3174568ff Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 4 Aug 2014 16:48:21 -0300
Subject: Delete the received docs from sync db before starting a new sync.

---
 client/src/leap/soledad/client/crypto.py | 7 +++++++
 client/src/leap/soledad/client/target.py | 3 ++-
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 1b01913d..a24f2053 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -986,3 +986,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         else:
             # If no errors found, remove it from the received database.
             self.delete_received_doc(doc_id, doc_rev)
+
+    def empty(self):
+        """
+        Empty the received docs table of the sync database.
+        """
+        sql = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,)
+        res = self._sync_db.execute(sql)
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 12175f48..1cb02856 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -1149,8 +1149,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
         setProxiedObject(self._insert_doc_cb[source_replica_uid],
                          return_doc_cb)
 
+        # empty the database before starting a new sync
         if defer_decryption is True and not self.clear_to_sync():
-            raise PendingReceivedDocsSyncError
+            self._sync_decr_pool.empty()
 
         self._ensure_connection()
         if self._trace_hook:  # for tests
-- 
cgit v1.2.3


From ab7850bbdcded8b0e36cb27a2468f55d1910c218 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Tue, 12 Aug 2014 11:06:23 -0300
Subject: Fix bits from pullreq review.

---
 client/src/leap/soledad/client/__init__.py   |  2 +-
 client/src/leap/soledad/client/crypto.py     | 19 +++++---
 client/src/leap/soledad/client/mp_safe_db.py |  2 +-
 client/src/leap/soledad/client/secrets.py    | 69 +++++++++++++++++-----------
 client/src/leap/soledad/client/sqlcipher.py  | 51 ++++++++++++--------
 client/src/leap/soledad/client/target.py     |  7 +--
 6 files changed, 90 insertions(+), 60 deletions(-)

(limited to 'client')

diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 0b72be27..c76e4a4a 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -755,7 +755,7 @@ class Soledad(object):
     @property
     def remote_storage_secret(self):
         """
-        Return the secret used for encryption of remotelly stored data.
+        Return the secret used for encryption of remotely stored data.
         """
         return self._secrets.remote_storage_secret
 
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index a24f2053..5e3760b3 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -863,7 +863,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
 
         :param encrypted: If not None, only return documents with encrypted
                           field equal to given parameter.
-        :type encrypted: bool
+        :type encrypted: bool or None
 
         :return: list of doc_id, rev, generation, gen, trans_id
         :rtype: list
@@ -878,16 +878,23 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
 
     def get_insertable_docs_by_gen(self):
         """
-        Return a list of documents ready to be inserted.
+        Return a list of non-encrypted documents ready to be inserted.
         """
+        # here, we compare the list of all available docs with the list of
+        # decrypted docs and find the longest common prefix between these two
+        # lists. Note that the order of lists fetch matters: if instead we
+        # first fetch the list of decrypted docs and then the list of all
+        # docs, then some document might have been decrypted between these two
+        # calls, and if it is just the right doc then it might not be caught
+        # by the next loop.
         all_docs = self.get_docs_by_generation()
         decrypted_docs = self.get_docs_by_generation(encrypted=False)
         insertable = []
         for doc_id, rev, _, gen, trans_id, encrypted in all_docs:
             try:
-                next_decrypted = decrypted_docs.next()
-                if doc_id == next_decrypted[0]:
-                    content = next_decrypted[2]
+                next_doc_id, _, next_content, _, _, _ = decrypted_docs.next()
+                if doc_id == next_doc_id:
+                    content = next_content
                     insertable.append((doc_id, rev, content, gen, trans_id))
                 else:
                     break
@@ -901,7 +908,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
 
         :param encrypted: If not None, return count of documents with
                           encrypted field equal to given parameter.
-        :type encrypted: bool
+        :type encrypted: bool or None
 
         :return: The count of documents.
         :rtype: int
diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py
index 2c6b7e24..780b7153 100644
--- a/client/src/leap/soledad/client/mp_safe_db.py
+++ b/client/src/leap/soledad/client/mp_safe_db.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# crypto.py
+# mp_safe_db.py
 # Copyright (C) 2014 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index 55580692..b1c22371 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -69,21 +69,28 @@ logger = logging.getLogger(name=__name__)
 # Exceptions
 #
 
-class NoStorageSecret(Exception):
+
+class SecretsException(Exception):
+    """
+    Generic exception type raised by this module.
+    """
+
+
+class NoStorageSecret(SecretsException):
     """
     Raised when trying to use a storage secret but none is available.
     """
     pass
 
 
-class PassphraseTooShort(Exception):
+class PassphraseTooShort(SecretsException):
     """
     Raised when trying to change the passphrase but the provided passphrase is
     too short.
     """
 
 
-class BootstrapSequenceError(Exception):
+class BootstrapSequenceError(SecretsException):
     """
     Raised when an attempt to generate a secret and store it in a recovery
     document on server failed.
@@ -107,34 +114,35 @@ class SoledadSecrets(object):
 
     LOCAL_STORAGE_SECRET_LENGTH = 512
     """
-    The length of the secret used to derive a passphrase for the SQLCipher
-    database.
+    The length, in bytes, of the secret used to derive a passphrase for the
+    SQLCipher database.
     """
 
     REMOTE_STORAGE_SECRET_LENGTH = 512
     """
-    The length of the secret used to derive an encryption key and a MAC auth
-    key for remote storage.
+    The length, in bytes, of the secret used to derive an encryption key and a
+    MAC auth key for remote storage.
     """
 
     SALT_LENGTH = 64
     """
-    The length of the salt used to derive the key for the storage secret
-    encryption.
+    The length, in bytes, of the salt used to derive the key for the storage
+    secret encryption.
     """
 
     GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \
         + REMOTE_STORAGE_SECRET_LENGTH \
         + SALT_LENGTH  # for sync db
     """
-    The length of the secret to be generated. This includes local and remote
-    secrets, and the salt for deriving the sync db secret.
+    The length, in bytes, of the secret to be generated. This includes local
+    and remote secrets, and the salt for deriving the sync db secret.
     """
 
     MINIMUM_PASSPHRASE_LENGTH = 6
     """
-    The minimum length for a passphrase. The passphrase length is only checked
-    when the user changes her passphrase, not when she instantiates Soledad.
+    The minimum length, in bytes, for a passphrase. The passphrase length is
+    only checked when the user changes her passphrase, not when she
+    instantiates Soledad.
     """
 
     IV_SEPARATOR = ":"
@@ -288,7 +296,7 @@ class SoledadSecrets(object):
             self._secrets[self._secret_id] += new_piece
             enlarged = True
         # store and save in shared db if needed
-        if mac is False or enlarged is True:
+        if not mac or enlarged:
             self._store_secrets()
             self._put_secrets_in_shared_db()
 
@@ -443,13 +451,17 @@ class SoledadSecrets(object):
                 raise WrongMac('Could not authenticate recovery document\'s '
                                'contents.')
         # include secrets in the secret pool.
-        secrets = 0
+        secret_count = 0
         for secret_id, encrypted_secret in data[self.STORAGE_SECRETS_KEY].items():
             if secret_id not in self._secrets:
-                secrets += 1
-                self._secrets[secret_id] = \
-                    self._decrypt_storage_secret(encrypted_secret)
-        return secrets, mac
+                try:
+                    self._secrets[secret_id] = \
+                        self._decrypt_storage_secret(encrypted_secret)
+                    secret_count += 1
+                except SecretsException as e:
+                    logger.error("Failed to decrypt storage secret: %s"
+                                 % str(e))
+        return secret_count, mac
 
     def _get_secrets_from_shared_db(self):
         """
@@ -512,10 +524,13 @@ class SoledadSecrets(object):
 
         :return: The decrypted storage secret.
         :rtype: str
+
+        :raise SecretsException: Raised in case the decryption of the storage
+                                 secret fails for some reason.
         """
         # calculate the encryption key
         if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT:
-            raise Exception("Unknown KDF in stored secret.")
+            raise SecretsException("Unknown KDF in stored secret.")
         key = scrypt.hash(
             self._passphrase_as_string(),
             # the salt is stored base64 encoded
@@ -524,16 +539,16 @@ class SoledadSecrets(object):
             buflen=32,  # we need a key with 256 bits (32 bytes).
         )
         if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key):
-            raise Exception("Wrong length of decryption key.")
+            raise SecretsException("Wrong length of decryption key.")
         if encrypted_secret_dict[self.CIPHER_KEY] != self.CIPHER_AES256:
-            raise Exception("Unknown cipher in stored secret.")
+            raise SecretsException("Unknown cipher in stored secret.")
         # recover the initial value and ciphertext
         iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split(
             self.IV_SEPARATOR, 1)
         ciphertext = binascii.a2b_base64(ciphertext)
         decrypted_secret = self._crypto.decrypt_sym(ciphertext, key, iv=iv)
         if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret):
-            raise Exception("Wrong length of decrypted secret.")
+            raise SecretsException("Wrong length of decrypted secret.")
         return decrypted_secret
 
     def _encrypt_storage_secret(self, decrypted_secret):
@@ -729,8 +744,8 @@ class SoledadSecrets(object):
         :rtype: str
         """
         return scrypt.hash(
-            self._get_local_storage_secret(),  # the password
-            self._get_local_storage_salt(),    # the salt
+            password=self._get_local_storage_secret(),
+            salt=self._get_local_storage_salt(),
             buflen=32,  # we need a key with 256 bits (32 bytes)
         )
 
@@ -755,7 +770,7 @@ class SoledadSecrets(object):
         :rtype: str
         """
         return scrypt.hash(
-            self._get_local_storage_secret(),  # the password
-            self._get_sync_db_salt(),    # the salt
+            password=self._get_local_storage_secret(),
+            salt=self._get_sync_db_salt(),
             buflen=32,  # we need a key with 256 bits (32 bytes)
         )
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index a7ddab24..b7de2fba 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -63,6 +63,7 @@ from leap.soledad.client.target import SoledadSyncTarget
 from leap.soledad.client.target import PendingReceivedDocsSyncError
 from leap.soledad.client.sync import SoledadSynchronizer
 from leap.soledad.client.mp_safe_db import MPSafeSQLiteDB
+from leap.soledad.common import soledad_assert
 from leap.soledad.common.document import SoledadDocument
 
 
@@ -262,13 +263,16 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             self._crypto = crypto
 
         # define sync-db attrs
+        self._sqlcipher_file = sqlcipher_file
+        self._sync_db_key = sync_db_key
         self._sync_db = None
         self._sync_db_write_lock = None
         self._sync_enc_pool = None
-        self._init_sync_db(sqlcipher_file, sync_db_key)
+        self.sync_queue = None
 
         if self.defer_encryption:
             # initialize sync db
+            self._init_sync_db()
             # initialize syncing queue encryption pool
             self._sync_enc_pool = SyncEncrypterPool(
                 self._crypto, self._sync_db, self._sync_db_write_lock)
@@ -471,6 +475,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         res = None
         # the following context manager blocks until the syncing lock can be
         # acquired.
+        if defer_decryption:
+            self._init_sync_db()
         with self.syncer(url, creds=creds) as syncer:
             # XXX could mark the critical section here...
             try:
@@ -564,28 +570,27 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             'ALTER TABLE document '
             'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
 
-    def _init_sync_db(self, sqlcipher_file, sync_db_password):
+    def _init_sync_db(self):
         """
         Initialize the Symmetrically-Encrypted document to be synced database,
         and the queue to communicate with subprocess workers.
-
-        :param sqlcipher_file: The path for the SQLCipher file.
-        :type sqlcipher_file: str
         """
-        sync_db_path = None
-        if sqlcipher_file != ":memory:":
-            sync_db_path = "%s-sync" % sqlcipher_file
-        else:
-            sync_db_path = ":memory:"
-        self._sync_db = MPSafeSQLiteDB(sync_db_path)
-        # protect the sync db with a password
-        if sync_db_password is not None:
-            self._set_crypto_pragmas(
-                self._sync_db, sync_db_password, True,
-                'aes-256-cbc', 4000, 1024)
-        self._sync_db_write_lock = threading.Lock()
-        self._create_sync_db_tables()
-        self.sync_queue = multiprocessing.Queue()
+        if self._sync_db is None:
+            soledad_assert(self._sync_db_key is not None)
+            sync_db_path = None
+            if self._sqlcipher_file != ":memory:":
+                sync_db_path = "%s-sync" % self._sqlcipher_file
+            else:
+                sync_db_path = ":memory:"
+            self._sync_db = MPSafeSQLiteDB(sync_db_path)
+            # protect the sync db with a password
+            if self._sync_db_key is not None:
+                self._set_crypto_pragmas(
+                    self._sync_db, self._sync_db_key, False,
+                    'aes-256-cbc', 4000, 1024)
+            self._sync_db_write_lock = threading.Lock()
+            self._create_sync_db_tables()
+            self.sync_queue = multiprocessing.Queue()
 
     def _create_sync_db_tables(self):
         """
@@ -1106,24 +1111,30 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         """
         Close db_handle and close syncer.
         """
-        logger.debug("Sqlcipher backend: closing")
+        if logger is not None:  # logger might be none if called from __del__
+            logger.debug("Sqlcipher backend: closing")
         # stop the sync watcher for deferred encryption
         if self._sync_watcher is not None:
             self._sync_watcher.stop()
             self._sync_watcher.shutdown()
+            self._sync_watcher = None
         # close all open syncers
         for url in self._syncers:
             _, syncer = self._syncers[url]
             syncer.close()
+        self._syncers = []
         # stop the encryption pool
         if self._sync_enc_pool is not None:
             self._sync_enc_pool.close()
+            self._sync_enc_pool = None
         # close the actual database
         if self._db_handle is not None:
             self._db_handle.close()
+            self._db_handle = None
         # close the sync database
         if self._sync_db is not None:
             self._sync_db.close()
+            self._sync_db = None
         # close the sync queue
         if self.sync_queue is not None:
             self.sync_queue.close()
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 1cb02856..ae2010a6 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -807,12 +807,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
             self._sync_db = sync_db
             self._sync_db_write_lock = sync_db_write_lock
 
-    def _setup_sync_decr_pool(self, last_known_generation):
+    def _setup_sync_decr_pool(self):
         """
         Set up the SyncDecrypterPool for deferred decryption.
-
-        :param last_known_generation: Target's last known generation.
-        :type last_known_generation: int
         """
         if self._sync_decr_pool is None:
             # initialize syncing queue decryption pool
@@ -1133,7 +1130,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
 
         if defer_decryption and self._sync_db is not None:
             self._sync_exchange_lock.acquire()
-            self._setup_sync_decr_pool(last_known_generation)
+            self._setup_sync_decr_pool()
             self._setup_sync_watcher()
             self._defer_decryption = True
         else:
-- 
cgit v1.2.3