[bug] properly trap db errors and close resources

SQLCipher database access errors can raise Soledad exceptions. Database access
and multithreading resources are allocated in different places, so we have to
be careful to close all multithreading mechanismis in case of database access
errors. If we don't, zombie threads may haunt the reactor.

This commit adds SQLCipher exception trapping and Soledad exception raising
for database access errors, while properly shutting down multithreading
resources.

4 files changed, 49 insertions, 28 deletions

diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py
index 328b4762..234be6b6 100644
--- a/client/src/leap/soledad/client/adbapi.py
+++ b/client/src/leap/soledad/client/adbapi.py
@@ -29,8 +29,7 @@ from twisted.enterprise import adbapi
 from twisted.internet.defer import DeferredSemaphore
 from twisted.python import log
 from zope.proxy import ProxyBase, setProxiedObject
-from pysqlcipher.dbapi2 import OperationalError
-from pysqlcipher.dbapi2 import DatabaseError
+from pysqlcipher import dbapi2
 
 from leap.soledad.common.errors import DatabaseAccessError
 
@@ -105,8 +104,10 @@ class U1DBConnection(adbapi.Connection):
         self._sync_enc_pool = sync_enc_pool
         try:
             adbapi.Connection.__init__(self, pool)
-        except DatabaseError:
-            raise DatabaseAccessError('Could not open sqlcipher database')
+        except dbapi2.DatabaseError as e:
+            raise DatabaseAccessError(
+                'Error initializing connection to sqlcipher database: %s'
+                % str(e))
 
     def reconnect(self):
         """
@@ -174,8 +175,9 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
         self._sync_enc_pool = kwargs.pop("sync_enc_pool")
         try:
             adbapi.ConnectionPool.__init__(self, *args, **kwargs)
-        except DatabaseError:
-            raise DatabaseAccessError('Could not open sqlcipher database')
+        except dbapi2.DatabaseError as e:
+            raise DatabaseAccessError(
+                'Error initializing u1db connection pool: %s' % str(e))
 
         # all u1db connections, hashed by thread-id
         self._u1dbconnections = {}
@@ -183,10 +185,15 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
         # The replica uid, primed by the connections on init.
         self.replica_uid = ProxyBase(None)
 
-        conn = self.connectionFactory(
-            self, self._sync_enc_pool, init_u1db=True)
-        replica_uid = conn._u1db._real_replica_uid
-        setProxiedObject(self.replica_uid, replica_uid)
+        try:
+            conn = self.connectionFactory(
+                self, self._sync_enc_pool, init_u1db=True)
+            replica_uid = conn._u1db._real_replica_uid
+            setProxiedObject(self.replica_uid, replica_uid)
+        except DatabaseAccessError as e:
+            self.threadpool.stop()
+            raise DatabaseAccessError(
+                "Error initializing connection factory: %s" % str(e))
 
     def runU1DBQuery(self, meth, *args, **kw):
         """
@@ -211,7 +218,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
                 self._runU1DBQuery, meth, *args, **kw)
 
         def _errback(failure):
-            failure.trap(OperationalError)
+            failure.trap(dbapi2.OperationalError)
             if failure.getErrorMessage() == "database is locked":
                 should_retry = semaphore.acquire()
                 if should_retry:
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 8c25243b..1bfbed8a 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -51,6 +51,7 @@ from leap.soledad.common import soledad_assert
 from leap.soledad.common import soledad_assert_type
 from leap.soledad.common.l2db.remote import http_client
 from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname
+from leap.soledad.common.errors import DatabaseAccessError
 
 from leap.soledad.client import adbapi
 from leap.soledad.client import events as soledad_events
@@ -213,10 +214,22 @@ class Soledad(object):
         self._init_secrets()
 
         self._crypto = SoledadCrypto(self._secrets.remote_storage_secret)
-        self._init_u1db_sqlcipher_backend()
 
-        if syncable:
-            self._init_u1db_syncer()
+        try:
+            # initialize database access, trap any problems so we can shutdown
+            # smoothly.
+            self._init_u1db_sqlcipher_backend()
+            if syncable:
+                self._init_u1db_syncer()
+        except DatabaseAccessError:
+            # oops! something went wrong with backend initialization. We
+            # have to close any thread-related stuff we have already opened
+            # here, otherwise there might be zombie threads that may clog the
+            # reactor.
+            self._sync_db.close()
+            if hasattr(self, '_dbpool'):
+                self._dbpool.close()
+            raise
 
     #
     # initialization/destruction methods
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index f36c0b6a..166c0783 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -58,6 +58,7 @@ from leap.soledad.common.document import SoledadDocument
 from leap.soledad.common import l2db
 from leap.soledad.common.l2db import errors as u1db_errors
 from leap.soledad.common.l2db.backends import sqlite_backend
+from leap.soledad.common.errors import DatabaseAccessError
 
 from leap.soledad.client.http_target import SoledadHTTPSyncTarget
 from leap.soledad.client.sync import SoledadSynchronizer
@@ -442,22 +443,19 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
         # format is the following:
         #
         #  self._syncers = {'<url>': ('<auth_hash>', syncer), ...}
-
         self._syncers = {}
-
-        # Storage for the documents received during a sync
+        # storage for the documents received during a sync
         self.received_docs = []
 
         self.running = False
+        self.shutdownID = None
+        self._db_handle = None
 
+        # initialize the main db before scheduling a start
+        self._initialize_main_db()
         self._reactor = reactor
         self._reactor.callWhenRunning(self._start)
 
-        self._db_handle = None
-        self._initialize_main_db()
-
-        self.shutdownID = None
-
         if DO_STATS:
             self.sync_phase = None
 
@@ -472,11 +470,14 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
             self.running = True
 
     def _initialize_main_db(self):
-        self._db_handle = initialize_sqlcipher_db(
-            self._opts, check_same_thread=False)
-        self._real_replica_uid = None
-        self._ensure_schema()
-        self.set_document_factory(soledad_doc_factory)
+        try:
+            self._db_handle = initialize_sqlcipher_db(
+                self._opts, check_same_thread=False)
+            self._real_replica_uid = None
+            self._ensure_schema()
+            self.set_document_factory(soledad_doc_factory)
+        except sqlcipher_dbapi2.DatabaseError as e:
+            raise DatabaseAccessError(str(e))
 
     @defer.inlineCallbacks
     def sync(self, url, creds=None, defer_decryption=True):
diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py
index f81001b9..033a55df 100644
--- a/testing/test_soledad/util.py
+++ b/testing/test_soledad/util.py
@@ -313,7 +313,7 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest):
                 self.tempdir, prefix, secrets_path),
             local_db_path=os.path.join(
                 self.tempdir, prefix, local_db_path),
-            server_url=server_url,  # Soledad will fail if not given an url.
+            server_url=server_url,  # Soledad will fail if not given an url
             cert_file=cert_file,
             defer_encryption=self.defer_sync_encryption,
             shared_db=MockSharedDB(),