Workarounds for sqlcipher concurrent init on tests

6 files changed, 76 insertions, 40 deletions

diff --git a/soledad/setup.py b/soledad/setup.py
index 6da976a9..67ef3cc8 100644
--- a/soledad/setup.py
+++ b/soledad/setup.py
@@ -74,7 +74,7 @@ setup(
         "LEAP client, an API for data storage and sync."
     ),
     namespace_packages=["leap"],
-    packages=find_packages('src', exclude=['leap.soledad.tests']),
+    packages=find_packages('src'),  # exclude=['leap.soledad.tests']),
     package_dir={'': 'src'},
     test_suite='leap.soledad.tests',
     install_requires=install_requirements,
diff --git a/soledad/src/leap/soledad/__init__.py b/soledad/src/leap/soledad/__init__.py
index 3fe9629f..94425d66 100644
--- a/soledad/src/leap/soledad/__init__.py
+++ b/soledad/src/leap/soledad/__init__.py
@@ -128,8 +128,8 @@ from leap.soledad.crypto import SoledadCrypto
 from leap.soledad.dbwrapper import SQLCipherWrapper
 from leap.soledad.document import SoledadDocument
 from leap.soledad.shared_db import SoledadSharedDatabase
-from leap.soledad.sqlcipher import open as sqlcipher_open
-from leap.soledad.sqlcipher import SQLCipherDatabase
+#from leap.soledad.sqlcipher import open as sqlcipher_open
+#from leap.soledad.sqlcipher import SQLCipherDatabase
 from leap.soledad.target import SoledadSyncTarget
 
 
@@ -650,6 +650,7 @@ class Soledad(object):
             self.SECRET_KEY: '%s%s%s' % (
                 str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
         }
+
         self._store_secrets()
         self._passphrase = new_passphrase
 
diff --git a/soledad/src/leap/soledad/dbwrapper.py b/soledad/src/leap/soledad/dbwrapper.py
index c16c4925..2a81086c 100644
--- a/soledad/src/leap/soledad/dbwrapper.py
+++ b/soledad/src/leap/soledad/dbwrapper.py
@@ -69,7 +69,12 @@ class SQLCipherWrapper(threading.Thread):
         """
         # instantiate u1db
         args, kwargs = self._wrargs
-        self._db = sqlcipher.open(*args, **kwargs)
+        try:
+            self._db = sqlcipher.open(*args, **kwargs)
+        except Exception as exc:
+            logger.debug("Error in init_db: %r" % (exc,))
+            self._stopped.set()
+            raise exc
 
     def run(self):
         """
@@ -79,13 +84,17 @@ class SQLCipherWrapper(threading.Thread):
         logger.debug("Initializing sqlcipher")
         end_mths = ("__end_thread", "_SQLCipherWrapper__end_thread")
 
-        self._init_db()
+        failed = False
+        try:
+            self._init_db()
+        except:
+            failed = True
         self._lock = threading.Lock()
 
         ct = 0
         started = False
 
-        while True:
+        while not failed:
             if self._db is None:
                 if started:
                     break
diff --git a/soledad/src/leap/soledad/sqlcipher.py b/soledad/src/leap/soledad/sqlcipher.py
index acbeabe6..ca61b4bf 100644
--- a/soledad/src/leap/soledad/sqlcipher.py
+++ b/soledad/src/leap/soledad/sqlcipher.py
@@ -43,7 +43,7 @@ So, as the statements above were introduced for backwards compatibility with
 SLCipher 1.1 databases, we do not implement them as all SQLCipher databases
 handled by Soledad should be created by SQLCipher >= 2.0.
 """
-
+import logging
 import os
 import time
 import string
@@ -51,11 +51,11 @@ import string
 
 from u1db.backends import sqlite_backend
 from pysqlcipher import dbapi2
-from u1db import (
-    errors,
-)
+from u1db import errors as u1db_errors
 from leap.soledad.document import SoledadDocument
 
+logger = logging.getLogger(__name__)
+
 
 # Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2
 sqlite_backend.dbapi2 = dbapi2
@@ -170,8 +170,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         def factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
                     syncable=True):
             return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
-                                has_conflicts=has_conflicts,
-                                syncable=syncable)
+                                   has_conflicts=has_conflicts,
+                                   syncable=syncable)
         self.set_document_factory(factory)
 
     @classmethod
@@ -205,20 +205,30 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         @rtype: SQLCipherDatabase
         """
         if not os.path.isfile(sqlcipher_file):
-            raise errors.DatabaseDoesNotExist()
-        tries = 2
+            raise u1db_errors.DatabaseDoesNotExist()
+
+        tries = 30
         while True:
             # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6)
             #       where without re-opening the database on Windows, it
             #       doesn't see the transaction that was just committed
+
             db_handle = dbapi2.connect(sqlcipher_file)
-            # set cryptographic params
-            cls._set_crypto_pragmas(
-                db_handle, password, raw_key, cipher, kdf_iter,
-                cipher_page_size)
-            c = db_handle.cursor()
-            v, err = cls._which_index_storage(c)
-            db_handle.close()
+
+            try:
+                # set cryptographic params
+                cls._set_crypto_pragmas(
+                    db_handle, password, raw_key, cipher, kdf_iter,
+                    cipher_page_size)
+                c = db_handle.cursor()
+                # XXX if we use it here, it should be public
+                v, err = cls._which_index_storage(c)
+            except Exception as exc:
+                logger.warning("ERROR OPENING DATABASE!")
+                logger.debug("error was: %r" % exc)
+                v, err = None, exc
+            finally:
+                db_handle.close()
             if v is not None:
                 break
             # possibly another process is initializing it, wait for it to be
@@ -273,7 +283,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
                 sqlcipher_file, password, document_factory=document_factory,
                 crypto=crypto, raw_key=raw_key, cipher=cipher,
                 kdf_iter=kdf_iter, cipher_page_size=cipher_page_size)
-        except errors.DatabaseDoesNotExist:
+        except u1db_errors.DatabaseDoesNotExist:
             if not create:
                 raise
             # TODO: remove backend class from here.
@@ -305,8 +315,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         return Synchronizer(
             self,
             SoledadSyncTarget(url,
-                           creds=creds,
-                           crypto=self._crypto)).sync(autocreate=autocreate)
+                              creds=creds,
+                              crypto=self._crypto)).sync(autocreate=autocreate)
 
     def _extra_schema_init(self, c):
         """
diff --git a/soledad/src/leap/soledad/tests/__init__.py b/soledad/src/leap/soledad/tests/__init__.py
index b33f866c..84882e27 100644
--- a/soledad/src/leap/soledad/tests/__init__.py
+++ b/soledad/src/leap/soledad/tests/__init__.py
@@ -3,17 +3,16 @@ Tests to make sure Soledad provides U1DB functionality and more.
 """
 
 import os
+import random
+import string
 import u1db
 from mock import Mock
 
 
 from leap.soledad import Soledad
 from leap.soledad.document import SoledadDocument
-from leap.soledad.crypto import SoledadCrypto
-from leap.soledad.target import (
-    decrypt_doc,
-    ENC_SCHEME_KEY,
-)
+from leap.soledad.target import decrypt_doc
+from leap.soledad.target import ENC_SCHEME_KEY
 from leap.common.testing.basetest import BaseLeapTest
 
 
@@ -40,16 +39,23 @@ class BaseSoledadTest(BaseLeapTest):
                               document_factory=SoledadDocument)
         self._db2 = u1db.open(self.db2_file, create=True,
                               document_factory=SoledadDocument)
+        # get a random prefix for each test, so we do not mess with
+        # concurrency during initialization and shutting down of
+        # each local db.
+        self.rand_prefix = ''.join(
+            map(lambda x: random.choice(string.ascii_letters), range(6)))
         # initialize soledad by hand so we can control keys
-        self._soledad = self._soledad_instance(user=self.email)
+        self._soledad = self._soledad_instance(
+            prefix=self.rand_prefix, user=self.email)
 
     def tearDown(self):
         self._db1.close()
         self._db2.close()
+        self._soledad.close()
+        # XXX should not access "private" attrs
         for f in [self._soledad._local_db_path, self._soledad._secrets_path]:
             if os.path.isfile(f):
                 os.unlink(f)
-        self._soledad.close()
 
     def _soledad_instance(self, user=ADDRESS, passphrase='123',
                           prefix='',
@@ -72,7 +78,8 @@ class BaseSoledadTest(BaseLeapTest):
         return Soledad(
             user,
             passphrase,
-            secrets_path=os.path.join(self.tempdir, prefix, secrets_path),
+            secrets_path=os.path.join(
+                self.tempdir, prefix, secrets_path),
             local_db_path=os.path.join(
                 self.tempdir, prefix, local_db_path),
             server_url=server_url,  # Soledad will fail if not given an url.
diff --git a/soledad/src/leap/soledad/tests/test_soledad.py b/soledad/src/leap/soledad/tests/test_soledad.py
index 63ab5551..bdda00bf 100644
--- a/soledad/src/leap/soledad/tests/test_soledad.py
+++ b/soledad/src/leap/soledad/tests/test_soledad.py
@@ -22,14 +22,10 @@ Tests for general Soledad functionality.
 
 
 import os
-import re
-import tempfile
-import simplejson as json
 from mock import Mock
 
 
 from pysqlcipher.dbapi2 import DatabaseError
-from leap.common.testing.basetest import BaseLeapTest
 from leap.common.events import events_pb2 as proto
 from leap.soledad.tests import (
     BaseSoledadTest,
@@ -62,8 +58,8 @@ class AuxMethodsTestCase(BaseSoledadTest):
             sol._gen_secret()
         sol._load_secrets()
         sol._init_db()
-        from leap.soledad.sqlcipher import SQLCipherDatabase
-        self.assertIsInstance(sol._db, SQLCipherDatabase)
+        from leap.soledad.dbwrapper import SQLCipherWrapper
+        self.assertIsInstance(sol._db, SQLCipherWrapper)
 
     def test__init_config_defaults(self):
         """
@@ -111,17 +107,30 @@ class AuxMethodsTestCase(BaseSoledadTest):
         """
         Test if passphrase can be changed.
         """
+        self._soledad.close()
         sol = self._soledad_instance(
             'leap@leap.se',
-            passphrase='123')
+            passphrase='123',
+            prefix=self.rand_prefix,
+        )
         doc = sol.create_doc({'simple': 'doc'})
         doc_id = doc.doc_id
+
         # change the passphrase
         sol.change_passphrase('654321')
+        sol.close()
+
         # assert we can not use the old passphrase anymore
+        # XXX this is failing with the new dbwrapper, and
+        # I suspect it has to do with the DatabaseError being
+        # raised in the db thread on db init and not properly
+        # propagated.
         self.assertRaises(
             DatabaseError,
-            self._soledad_instance, 'leap@leap.se', '123')
+            self._soledad_instance, 'leap@leap.se',
+            passphrase='123',
+            prefix=self.rand_prefix)
+
         # use new passphrase and retrieve doc
         sol2 = self._soledad_instance('leap@leap.se', '654321')
         doc2 = sol2.get_doc(doc_id)