[bug] move sync db and encpool creation to api

Deferred encryption was disabled because the soledad u1db wrapper for adbapi
did not correctly udated the parameter that controls it. Also, it did not
contain the encrypter pool. This commit moves the sync db and encrypt pool to
the main api, so they can be passed to the wrapper and deferred encryption
can work.

4 files changed, 91 insertions, 74 deletions

diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py
index bc0ab7a5..9675c048 100644
--- a/client/src/leap/soledad/client/adbapi.py
+++ b/client/src/leap/soledad/client/adbapi.py
@@ -54,7 +54,8 @@ How many times a SQLCipher query should be retried in case of timeout.
 SQLCIPHER_MAX_RETRIES = 10
 
 
-def getConnectionPool(opts, openfun=None, driver="pysqlcipher"):
+def getConnectionPool(opts, openfun=None, driver="pysqlcipher",
+        sync_enc_pool=None):
     """
     Return a connection pool.
 
@@ -75,8 +76,8 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"):
     if openfun is None and driver == "pysqlcipher":
         openfun = partial(set_init_pragmas, opts=opts)
     return U1DBConnectionPool(
-        "%s.dbapi2" % driver, database=opts.path,
-        check_same_thread=False, cp_openfun=openfun,
+        "%s.dbapi2" % driver, opts=opts, sync_enc_pool=sync_enc_pool,
+        database=opts.path, check_same_thread=False, cp_openfun=openfun,
         timeout=SQLCIPHER_CONNECTION_TIMEOUT)
 
 
@@ -90,7 +91,7 @@ class U1DBConnection(adbapi.Connection):
     The U1DB wrapper to use.
     """
 
-    def __init__(self, pool, init_u1db=False):
+    def __init__(self, pool, sync_enc_pool, init_u1db=False):
         """
         :param pool: The pool of connections to that owns this connection.
         :type pool: adbapi.ConnectionPool
@@ -98,6 +99,7 @@ class U1DBConnection(adbapi.Connection):
         :type init_u1db: bool
         """
         self.init_u1db = init_u1db
+        self._sync_enc_pool = sync_enc_pool
         adbapi.Connection.__init__(self, pool)
 
     def reconnect(self):
@@ -109,7 +111,10 @@ class U1DBConnection(adbapi.Connection):
         self._connection = self._pool.connect()
 
         if self.init_u1db:
-            self._u1db = self.u1db_wrapper(self._connection)
+            self._u1db = self.u1db_wrapper(
+                self._connection,
+                self._pool.opts,
+                self._sync_enc_pool)
 
     def __getattr__(self, name):
         """
@@ -158,14 +163,20 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
         """
         Initialize the connection pool.
         """
+        # extract soledad-specific objects from keyword arguments
+        self.opts = kwargs.pop("opts")
+        self._sync_enc_pool = kwargs.pop("sync_enc_pool")
+
         adbapi.ConnectionPool.__init__(self, *args, **kwargs)
+
         # all u1db connections, hashed by thread-id
         self._u1dbconnections = {}
 
         # The replica uid, primed by the connections on init.
         self.replica_uid = ProxyBase(None)
 
-        conn = self.connectionFactory(self, init_u1db=True)
+        conn = self.connectionFactory(
+            self, self._sync_enc_pool, init_u1db=True)
         replica_uid = conn._u1db._real_replica_uid
         setProxiedObject(self.replica_uid, replica_uid)
 
@@ -235,7 +246,8 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
         """
         tid = self.threadID()
         u1db = self._u1dbconnections.get(tid)
-        conn = self.connectionFactory(self, init_u1db=not bool(u1db))
+        conn = self.connectionFactory(
+            self, self._sync_enc_pool, init_u1db=not bool(u1db))
 
         if self.replica_uid is None:
             replica_uid = conn._u1db._real_replica_uid
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 6c2b3673..2bc524c3 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -43,7 +43,6 @@ from itertools import chain
 from StringIO import StringIO
 from u1db.remote import http_client
 from u1db.remote.ssl_match_hostname import match_hostname
-from twisted.plugin import getPlugins
 from zope.interface import implements
 
 from leap.common.config import get_path_prefix
@@ -59,7 +58,8 @@ from leap.soledad.client import interfaces as soledad_interfaces
 from leap.soledad.client.crypto import SoledadCrypto
 from leap.soledad.client.secrets import SoledadSecrets
 from leap.soledad.client.shared_db import SoledadSharedDatabase
-from leap.soledad.client.sqlcipher import SQLCipherOptions, SQLCipherU1DBSync
+from leap.soledad.client import sqlcipher
+from leap.soledad.client import encdecpool
 
 logger = logging.getLogger(name=__name__)
 
@@ -175,6 +175,7 @@ class Soledad(object):
         self._server_url = server_url
         self._defer_encryption = defer_encryption
         self._secrets_path = None
+        self._sync_enc_pool = None
 
         self.shared_db = shared_db
 
@@ -259,24 +260,32 @@ class Soledad(object):
         key = tohex(self._secrets.get_local_storage_key())
         sync_db_key = tohex(self._secrets.get_sync_db_key())
 
-        opts = SQLCipherOptions(
+        opts = sqlcipher.SQLCipherOptions(
             self._local_db_path, key,
             is_raw_key=True, create=True,
             defer_encryption=self._defer_encryption,
             sync_db_key=sync_db_key,
         )
         self._sqlcipher_opts = opts
-        self._dbpool = adbapi.getConnectionPool(opts)
+
+        # the sync_db is used both for deferred encryption and decryption, so
+        # we want to initialize it anyway to allow for all combinations of
+        # deferred encryption and decryption configurations.
+        self._initialize_sync_db(opts)
+        self._dbpool = adbapi.getConnectionPool(
+           opts, sync_enc_pool=self._sync_enc_pool)
 
     def _init_u1db_syncer(self):
         """
         Initialize the U1DB synchronizer.
         """
         replica_uid = self._dbpool.replica_uid
-        self._dbsyncer = SQLCipherU1DBSync(
+        self._dbsyncer = sqlcipher.SQLCipherU1DBSync(
             self._sqlcipher_opts, self._crypto, replica_uid,
             SOLEDAD_CERT,
-            defer_encryption=self._defer_encryption)
+            defer_encryption=self._defer_encryption,
+            sync_db=self._sync_db,
+            sync_enc_pool=self._sync_enc_pool)
 
     #
     # Closing methods
@@ -290,6 +299,9 @@ class Soledad(object):
         self._dbpool.close()
         if getattr(self, '_dbsyncer', None):
             self._dbsyncer.close()
+        # close the sync database
+        self._sync_db.close()
+        self._sync_db = None
 
     #
     # ILocalStorage
@@ -729,6 +741,52 @@ class Soledad(object):
 
     token = property(_get_token, _set_token, doc='The authentication Token.')
 
+    def _initialize_sync_db(self, opts):
+        """
+        Initialize the Symmetrically-Encrypted document to be synced database,
+        and the queue to communicate with subprocess workers.
+
+        :param opts:
+        :type opts: SQLCipherOptions
+        """
+        soledad_assert(opts.sync_db_key is not None)
+        sync_db_path = None
+        if opts.path != ":memory:":
+            sync_db_path = "%s-sync" % opts.path
+        else:
+            sync_db_path = ":memory:"
+
+        # we copy incoming options because the opts object might be used
+        # somewhere else
+        sync_opts = sqlcipher.SQLCipherOptions.copy(
+            opts, path=sync_db_path, create=True)
+        self._sync_db = sqlcipher.getConnectionPool(
+            sync_opts, extra_queries=self._sync_db_extra_init)
+        if self._defer_encryption:
+            # initialize syncing queue encryption pool
+            self._sync_enc_pool = encdecpool.SyncEncrypterPool(
+                self._crypto, self._sync_db)
+
+
+    @property
+    def _sync_db_extra_init(self):
+        """
+        Queries for creating tables for the local sync documents db if needed.
+        They are passed as extra initialization to initialize_sqlciphjer_db
+
+        :rtype: tuple of strings
+        """
+        maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)"
+        encr = encdecpool.SyncEncrypterPool
+        decr = encdecpool.SyncDecrypterPool
+        sql_encr_table_query = (maybe_create % (
+            encr.TABLE_NAME, encr.FIELD_NAMES))
+        sql_decr_table_query = (maybe_create % (
+            decr.TABLE_NAME, decr.FIELD_NAMES))
+        return (sql_encr_table_query, sql_decr_table_query)
+
+
+
     #
     # ISecretsStorage
     #
diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 7923bf70..0b3e7eb8 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -261,8 +261,10 @@ class SyncEncrypterPool(SyncEncryptDecryptPool):
                 % self.TABLE_NAME
         result = yield self._runQuery(query, (doc_id, doc_rev))
         if result:
+            logger.debug("Found doc on sync db: %s" % doc_id)
             val = result.pop()
             defer.returnValue(val[0])
+        logger.debug("Did not find doc on sync db: %s" % doc_id)
         defer.returnValue(None)
 
     def delete_encrypted_doc(self, doc_id, doc_rev):
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index 75d786a6..68802dd6 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -414,8 +414,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
     Soledad syncer implementation.
     """
 
-    _sync_enc_pool = None
-
     """
     The name of the local symmetrically encrypted documents to
     sync database file.
@@ -435,17 +433,16 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
     syncing_lock = defaultdict(threading.Lock)
 
     def __init__(self, opts, soledad_crypto, replica_uid, cert_file,
-                 defer_encryption=False):
+                 defer_encryption=False, sync_db=None, sync_enc_pool=None):
 
         self._opts = opts
         self._path = opts.path
         self._crypto = soledad_crypto
         self.__replica_uid = replica_uid
         self._cert_file = cert_file
+        self._sync_enc_pool = sync_enc_pool
 
-        self._sync_db_key = opts.sync_db_key
-        self._sync_db = None
-        self._sync_enc_pool = None
+        self._sync_db = sync_db
 
         # we store syncers in a dictionary indexed by the target URL. We also
         # store a hash of the auth info in case auth info expires and we need
@@ -469,16 +466,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
         self._db_handle = None
         self._initialize_main_db()
 
-        # the sync_db is used both for deferred encryption and decryption, so
-        # we want to initialize it anyway to allow for all combinations of
-        # deferred encryption and decryption configurations.
-        self._initialize_sync_db(opts)
-
-        if defer_encryption:
-            # initialize syncing queue encryption pool
-            self._sync_enc_pool = encdecpool.SyncEncrypterPool(
-                self._crypto, self._sync_db)
-
         self.shutdownID = None
 
     @property
@@ -521,45 +508,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
         #     before the database has been initialized.
         self._sync_threadpool = ThreadPool(0, 1)
 
-    def _initialize_sync_db(self, opts):
-        """
-        Initialize the Symmetrically-Encrypted document to be synced database,
-        and the queue to communicate with subprocess workers.
-
-        :param opts:
-        :type opts: SQLCipherOptions
-        """
-        soledad_assert(opts.sync_db_key is not None)
-        sync_db_path = None
-        if opts.path != ":memory:":
-            sync_db_path = "%s-sync" % opts.path
-        else:
-            sync_db_path = ":memory:"
-
-        # we copy incoming options because the opts object might be used
-        # somewhere else
-        sync_opts = SQLCipherOptions.copy(
-            opts, path=sync_db_path, create=True)
-        self._sync_db = getConnectionPool(
-            sync_opts, extra_queries=self._sync_db_extra_init)
-
-    @property
-    def _sync_db_extra_init(self):
-        """
-        Queries for creating tables for the local sync documents db if needed.
-        They are passed as extra initialization to initialize_sqlciphjer_db
-
-        :rtype: tuple of strings
-        """
-        maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)"
-        encr = encdecpool.SyncEncrypterPool
-        decr = encdecpool.SyncDecrypterPool
-        sql_encr_table_query = (maybe_create % (
-            encr.TABLE_NAME, encr.FIELD_NAMES))
-        sql_decr_table_query = (maybe_create % (
-            decr.TABLE_NAME, decr.FIELD_NAMES))
-        return (sql_encr_table_query, sql_decr_table_query)
-
     def sync(self, url, creds=None, defer_decryption=True):
         """
         Synchronize documents with remote replica exposed at url.
@@ -695,11 +643,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
             self._sync_enc_pool.close()
             self._sync_enc_pool = None
 
-        # close the sync database
-        if self._sync_db is not None:
-            self._sync_db.close()
-            self._sync_db = None
-
 
 class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase):
     """
@@ -729,12 +672,14 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase):
     It can be used from adbapi to initialize a soledad database after
     getting a regular connection to a sqlcipher database.
     """
-    def __init__(self, conn):
+    def __init__(self, conn, opts, sync_enc_pool):
         self._db_handle = conn
         self._real_replica_uid = None
         self._ensure_schema()
         self.set_document_factory(soledad_doc_factory)
         self._prime_replica_uid()
+        self.defer_encryption = opts.defer_encryption
+        self._sync_enc_pool = sync_enc_pool
 
 
 def _assert_db_is_encrypted(opts):