From e35341d57ac5fa87be5de64d62e219a105389c4c Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 22 Jul 2015 17:52:15 -0300 Subject: [bug] move sync db and encpool creation to api Deferred encryption was disabled because the soledad u1db wrapper for adbapi did not correctly udated the parameter that controls it. Also, it did not contain the encrypter pool. This commit moves the sync db and encrypt pool to the main api, so they can be passed to the wrapper and deferred encryption can work. --- client/src/leap/soledad/client/adbapi.py | 26 ++++++++--- client/src/leap/soledad/client/api.py | 70 +++++++++++++++++++++++++--- client/src/leap/soledad/client/encdecpool.py | 2 + client/src/leap/soledad/client/sqlcipher.py | 67 +++----------------------- 4 files changed, 91 insertions(+), 74 deletions(-) (limited to 'client/src/leap') diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index bc0ab7a5..9675c048 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -54,7 +54,8 @@ How many times a SQLCipher query should be retried in case of timeout. SQLCIPHER_MAX_RETRIES = 10 -def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): +def getConnectionPool(opts, openfun=None, driver="pysqlcipher", + sync_enc_pool=None): """ Return a connection pool. @@ -75,8 +76,8 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): if openfun is None and driver == "pysqlcipher": openfun = partial(set_init_pragmas, opts=opts) return U1DBConnectionPool( - "%s.dbapi2" % driver, database=opts.path, - check_same_thread=False, cp_openfun=openfun, + "%s.dbapi2" % driver, opts=opts, sync_enc_pool=sync_enc_pool, + database=opts.path, check_same_thread=False, cp_openfun=openfun, timeout=SQLCIPHER_CONNECTION_TIMEOUT) @@ -90,7 +91,7 @@ class U1DBConnection(adbapi.Connection): The U1DB wrapper to use. """ - def __init__(self, pool, init_u1db=False): + def __init__(self, pool, sync_enc_pool, init_u1db=False): """ :param pool: The pool of connections to that owns this connection. :type pool: adbapi.ConnectionPool @@ -98,6 +99,7 @@ class U1DBConnection(adbapi.Connection): :type init_u1db: bool """ self.init_u1db = init_u1db + self._sync_enc_pool = sync_enc_pool adbapi.Connection.__init__(self, pool) def reconnect(self): @@ -109,7 +111,10 @@ class U1DBConnection(adbapi.Connection): self._connection = self._pool.connect() if self.init_u1db: - self._u1db = self.u1db_wrapper(self._connection) + self._u1db = self.u1db_wrapper( + self._connection, + self._pool.opts, + self._sync_enc_pool) def __getattr__(self, name): """ @@ -158,14 +163,20 @@ class U1DBConnectionPool(adbapi.ConnectionPool): """ Initialize the connection pool. """ + # extract soledad-specific objects from keyword arguments + self.opts = kwargs.pop("opts") + self._sync_enc_pool = kwargs.pop("sync_enc_pool") + adbapi.ConnectionPool.__init__(self, *args, **kwargs) + # all u1db connections, hashed by thread-id self._u1dbconnections = {} # The replica uid, primed by the connections on init. self.replica_uid = ProxyBase(None) - conn = self.connectionFactory(self, init_u1db=True) + conn = self.connectionFactory( + self, self._sync_enc_pool, init_u1db=True) replica_uid = conn._u1db._real_replica_uid setProxiedObject(self.replica_uid, replica_uid) @@ -235,7 +246,8 @@ class U1DBConnectionPool(adbapi.ConnectionPool): """ tid = self.threadID() u1db = self._u1dbconnections.get(tid) - conn = self.connectionFactory(self, init_u1db=not bool(u1db)) + conn = self.connectionFactory( + self, self._sync_enc_pool, init_u1db=not bool(u1db)) if self.replica_uid is None: replica_uid = conn._u1db._real_replica_uid diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 6c2b3673..2bc524c3 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -43,7 +43,6 @@ from itertools import chain from StringIO import StringIO from u1db.remote import http_client from u1db.remote.ssl_match_hostname import match_hostname -from twisted.plugin import getPlugins from zope.interface import implements from leap.common.config import get_path_prefix @@ -59,7 +58,8 @@ from leap.soledad.client import interfaces as soledad_interfaces from leap.soledad.client.crypto import SoledadCrypto from leap.soledad.client.secrets import SoledadSecrets from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions, SQLCipherU1DBSync +from leap.soledad.client import sqlcipher +from leap.soledad.client import encdecpool logger = logging.getLogger(name=__name__) @@ -175,6 +175,7 @@ class Soledad(object): self._server_url = server_url self._defer_encryption = defer_encryption self._secrets_path = None + self._sync_enc_pool = None self.shared_db = shared_db @@ -259,24 +260,32 @@ class Soledad(object): key = tohex(self._secrets.get_local_storage_key()) sync_db_key = tohex(self._secrets.get_sync_db_key()) - opts = SQLCipherOptions( + opts = sqlcipher.SQLCipherOptions( self._local_db_path, key, is_raw_key=True, create=True, defer_encryption=self._defer_encryption, sync_db_key=sync_db_key, ) self._sqlcipher_opts = opts - self._dbpool = adbapi.getConnectionPool(opts) + + # the sync_db is used both for deferred encryption and decryption, so + # we want to initialize it anyway to allow for all combinations of + # deferred encryption and decryption configurations. + self._initialize_sync_db(opts) + self._dbpool = adbapi.getConnectionPool( + opts, sync_enc_pool=self._sync_enc_pool) def _init_u1db_syncer(self): """ Initialize the U1DB synchronizer. """ replica_uid = self._dbpool.replica_uid - self._dbsyncer = SQLCipherU1DBSync( + self._dbsyncer = sqlcipher.SQLCipherU1DBSync( self._sqlcipher_opts, self._crypto, replica_uid, SOLEDAD_CERT, - defer_encryption=self._defer_encryption) + defer_encryption=self._defer_encryption, + sync_db=self._sync_db, + sync_enc_pool=self._sync_enc_pool) # # Closing methods @@ -290,6 +299,9 @@ class Soledad(object): self._dbpool.close() if getattr(self, '_dbsyncer', None): self._dbsyncer.close() + # close the sync database + self._sync_db.close() + self._sync_db = None # # ILocalStorage @@ -729,6 +741,52 @@ class Soledad(object): token = property(_get_token, _set_token, doc='The authentication Token.') + def _initialize_sync_db(self, opts): + """ + Initialize the Symmetrically-Encrypted document to be synced database, + and the queue to communicate with subprocess workers. + + :param opts: + :type opts: SQLCipherOptions + """ + soledad_assert(opts.sync_db_key is not None) + sync_db_path = None + if opts.path != ":memory:": + sync_db_path = "%s-sync" % opts.path + else: + sync_db_path = ":memory:" + + # we copy incoming options because the opts object might be used + # somewhere else + sync_opts = sqlcipher.SQLCipherOptions.copy( + opts, path=sync_db_path, create=True) + self._sync_db = sqlcipher.getConnectionPool( + sync_opts, extra_queries=self._sync_db_extra_init) + if self._defer_encryption: + # initialize syncing queue encryption pool + self._sync_enc_pool = encdecpool.SyncEncrypterPool( + self._crypto, self._sync_db) + + + @property + def _sync_db_extra_init(self): + """ + Queries for creating tables for the local sync documents db if needed. + They are passed as extra initialization to initialize_sqlciphjer_db + + :rtype: tuple of strings + """ + maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)" + encr = encdecpool.SyncEncrypterPool + decr = encdecpool.SyncDecrypterPool + sql_encr_table_query = (maybe_create % ( + encr.TABLE_NAME, encr.FIELD_NAMES)) + sql_decr_table_query = (maybe_create % ( + decr.TABLE_NAME, decr.FIELD_NAMES)) + return (sql_encr_table_query, sql_decr_table_query) + + + # # ISecretsStorage # diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 7923bf70..0b3e7eb8 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -261,8 +261,10 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): % self.TABLE_NAME result = yield self._runQuery(query, (doc_id, doc_rev)) if result: + logger.debug("Found doc on sync db: %s" % doc_id) val = result.pop() defer.returnValue(val[0]) + logger.debug("Did not find doc on sync db: %s" % doc_id) defer.returnValue(None) def delete_encrypted_doc(self, doc_id, doc_rev): diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 75d786a6..68802dd6 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -414,8 +414,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): Soledad syncer implementation. """ - _sync_enc_pool = None - """ The name of the local symmetrically encrypted documents to sync database file. @@ -435,17 +433,16 @@ class SQLCipherU1DBSync(SQLCipherDatabase): syncing_lock = defaultdict(threading.Lock) def __init__(self, opts, soledad_crypto, replica_uid, cert_file, - defer_encryption=False): + defer_encryption=False, sync_db=None, sync_enc_pool=None): self._opts = opts self._path = opts.path self._crypto = soledad_crypto self.__replica_uid = replica_uid self._cert_file = cert_file + self._sync_enc_pool = sync_enc_pool - self._sync_db_key = opts.sync_db_key - self._sync_db = None - self._sync_enc_pool = None + self._sync_db = sync_db # we store syncers in a dictionary indexed by the target URL. We also # store a hash of the auth info in case auth info expires and we need @@ -469,16 +466,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._db_handle = None self._initialize_main_db() - # the sync_db is used both for deferred encryption and decryption, so - # we want to initialize it anyway to allow for all combinations of - # deferred encryption and decryption configurations. - self._initialize_sync_db(opts) - - if defer_encryption: - # initialize syncing queue encryption pool - self._sync_enc_pool = encdecpool.SyncEncrypterPool( - self._crypto, self._sync_db) - self.shutdownID = None @property @@ -521,45 +508,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # before the database has been initialized. self._sync_threadpool = ThreadPool(0, 1) - def _initialize_sync_db(self, opts): - """ - Initialize the Symmetrically-Encrypted document to be synced database, - and the queue to communicate with subprocess workers. - - :param opts: - :type opts: SQLCipherOptions - """ - soledad_assert(opts.sync_db_key is not None) - sync_db_path = None - if opts.path != ":memory:": - sync_db_path = "%s-sync" % opts.path - else: - sync_db_path = ":memory:" - - # we copy incoming options because the opts object might be used - # somewhere else - sync_opts = SQLCipherOptions.copy( - opts, path=sync_db_path, create=True) - self._sync_db = getConnectionPool( - sync_opts, extra_queries=self._sync_db_extra_init) - - @property - def _sync_db_extra_init(self): - """ - Queries for creating tables for the local sync documents db if needed. - They are passed as extra initialization to initialize_sqlciphjer_db - - :rtype: tuple of strings - """ - maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)" - encr = encdecpool.SyncEncrypterPool - decr = encdecpool.SyncDecrypterPool - sql_encr_table_query = (maybe_create % ( - encr.TABLE_NAME, encr.FIELD_NAMES)) - sql_decr_table_query = (maybe_create % ( - decr.TABLE_NAME, decr.FIELD_NAMES)) - return (sql_encr_table_query, sql_decr_table_query) - def sync(self, url, creds=None, defer_decryption=True): """ Synchronize documents with remote replica exposed at url. @@ -695,11 +643,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._sync_enc_pool.close() self._sync_enc_pool = None - # close the sync database - if self._sync_db is not None: - self._sync_db.close() - self._sync_db = None - class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): """ @@ -729,12 +672,14 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase): It can be used from adbapi to initialize a soledad database after getting a regular connection to a sqlcipher database. """ - def __init__(self, conn): + def __init__(self, conn, opts, sync_enc_pool): self._db_handle = conn self._real_replica_uid = None self._ensure_schema() self.set_document_factory(soledad_doc_factory) self._prime_replica_uid() + self.defer_encryption = opts.defer_encryption + self._sync_enc_pool = sync_enc_pool def _assert_db_is_encrypted(opts): -- cgit v1.2.3