[bug] avoid double decryption of documents

Because of how the incoming document queue is implemented, it could be the
case that a document was sent to async decryption queue more than once. This
commit creates a list of documents to be decrypted, so we avoid sending the
same document to the queue more than once.

3 files changed, 10 insertions, 11 deletions

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index c437645f..def2841a 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -304,6 +304,8 @@ class Soledad(object):
         # close the sync database
         self._sync_db.close()
         self._sync_db = None
+        if self._defer_encryption:
+            self._sync_enc_pool.close()
 
     #
     # ILocalStorage
diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 0b3e7eb8..010d4f78 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -162,6 +162,8 @@ class SyncEncrypterPool(SyncEncryptDecryptPool):
         logger.debug("Starting the encryption loop...")
         reactor.callWhenRunning(self._maybe_encrypt_and_recurse)
 
+        # TODO delete already synced files from database
+
     def enqueue_doc_for_encryption(self, doc):
         """
         Enqueue a document for encryption.
@@ -256,7 +258,6 @@ class SyncEncrypterPool(SyncEncryptDecryptPool):
                  db.
         :rtype: twisted.internet.defer.Deferred
         """
-        logger.debug("Trying to get encrypted doc from sync db: %s" % doc_id)
         query = "SELECT content FROM %s WHERE doc_id=? and rev=?" \
                 % self.TABLE_NAME
         result = yield self._runQuery(query, (doc_id, doc_rev))
@@ -350,7 +351,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         4. When we have processed as many documents as we should, the loop
            finishes.
     """
-    # TODO implement throttling to reduce cpu usage??
     TABLE_NAME = "docs_received"
     FIELD_NAMES = "doc_id PRIMARY KEY, rev, content, gen, " \
                   "trans_id, encrypted, idx"
@@ -382,6 +382,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         self._docs_to_process = None
         self._processed_docs = 0
         self._last_inserted_idx = 0
+        self._decrypting_docs = []
 
         self._async_results = []
 
@@ -551,6 +552,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         doc_id, rev, content, gen, trans_id, idx = result
         logger.debug("Sync decrypter pool: decrypted doc %s: %s %s %s"
                      % (doc_id, rev, gen, trans_id))
+        self._decrypting_docs.remove((doc_id, rev))
         return self.insert_received_doc(
             doc_id, rev, content, gen, trans_id, idx)
 
@@ -619,8 +621,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         """
         docs = yield self._get_docs(encrypted=True)
         for doc_id, rev, content, gen, trans_id, _, idx in docs:
-            self._async_decrypt_doc(
-                doc_id, rev, content, gen, trans_id, idx)
+            if (doc_id, rev) not in self._decrypting_docs:
+                self._decrypting_docs.append((doc_id, rev))
+                self._async_decrypt_doc(
+                    doc_id, rev, content, gen, trans_id, idx)
 
     @defer.inlineCallbacks
     def _process_decrypted_docs(self):
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index de752267..34fd90b4 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -62,12 +62,10 @@ from twisted.internet.threads import deferToThreadPool
 from twisted.python.threadpool import ThreadPool
 from twisted.enterprise import adbapi
 
-from leap.soledad.client import encdecpool
 from leap.soledad.client.http_target import SoledadHTTPSyncTarget
 from leap.soledad.client.sync import SoledadSynchronizer
 
 from leap.soledad.client import pragmas
-from leap.soledad.common import soledad_assert
 from leap.soledad.common.document import SoledadDocument
 
 
@@ -637,11 +635,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
         for url in self._syncers.keys():
             del self._syncers[url]
 
-        # stop the encryption pool
-        if self._sync_enc_pool is not None:
-            self._sync_enc_pool.close()
-            self._sync_enc_pool = None
-
 
 class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase):
     """