Several fixes in soledad api.

  * Allow passing shared_db to Soledad constructor.
  * Close syncers on Soledad close.
  * Fix docstrings.

1 files changed, 340 insertions, 33 deletions

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 00884a12..59cbc4ca 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -46,6 +46,7 @@ from zope.interface import implements
 from twisted.python import log
 
 from leap.common.config import get_path_prefix
+
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common import soledad_assert
 from leap.soledad.common import soledad_assert_type
@@ -112,7 +113,7 @@ class Soledad(object):
     default_prefix = os.path.join(get_path_prefix(), 'leap', 'soledad')
 
     def __init__(self, uuid, passphrase, secrets_path, local_db_path,
-                 server_url, cert_file,
+                 server_url, cert_file, shared_db=None,
                  auth_token=None, defer_encryption=False, syncable=True):
         """
         Initialize configuration, cryptographic keys and dbs.
@@ -142,6 +143,10 @@ class Soledad(object):
             certificate used by the remote soledad server.
         :type cert_file: str
 
+        :param shared_db:
+            The shared database.
+        :type shared_db: HTTPDatabase
+
         :param auth_token:
             Authorization token for accessing remote databases.
         :type auth_token: str
@@ -157,8 +162,9 @@ class Soledad(object):
         :type syncable: bool
 
         :raise BootstrapSequenceError:
-            Raised when the secret generation and storage on server sequence
-            has failed for some reason.
+            Raised when the secret initialization sequence (i.e. retrieval
+            from server or generation and storage on server) has failed for
+            some reason.
         """
         # store config params
         self._uuid = uuid
@@ -168,7 +174,7 @@ class Soledad(object):
         self._defer_encryption = defer_encryption
         self._secrets_path = None
 
-        self.shared_db = None
+        self.shared_db = shared_db
 
         # configure SSL certificate
         global SOLEDAD_CERT
@@ -225,6 +231,9 @@ class Soledad(object):
             create_path_if_not_exists(path)
 
     def _init_secrets(self):
+        """
+        Initialize Soledad secrets.
+        """
         self._secrets = SoledadSecrets(
             self.uuid, self._passphrase, self._secrets_path,
             self.shared_db, self._crypto)
@@ -232,8 +241,9 @@ class Soledad(object):
 
     def _init_u1db_sqlcipher_backend(self):
         """
-        Initialize the U1DB SQLCipher database for local storage, by
-        instantiating a modified twisted adbapi that will maintain a threadpool
+        Initialize the U1DB SQLCipher database for local storage.
+
+        Instantiates a modified twisted adbapi that will maintain a threadpool
         with a u1db-sqclipher connection for each thread, and will return
         deferreds for each u1db query.
 
@@ -253,14 +263,16 @@ class Soledad(object):
             defer_encryption=self._defer_encryption,
             sync_db_key=sync_db_key,
         )
-        self._soledad_opts = opts
+        self._sqlcipher_opts = opts
         self._dbpool = adbapi.getConnectionPool(opts)
 
     def _init_u1db_syncer(self):
+        """
+        Initialize the U1DB synchronizer.
+        """
         replica_uid = self._dbpool.replica_uid
-        print "replica UID (syncer init)", replica_uid
         self._dbsyncer = SQLCipherU1DBSync(
-            self._soledad_opts, self._crypto, replica_uid,
+            self._sqlcipher_opts, self._crypto, replica_uid,
             self._defer_encryption)
 
     #
@@ -273,99 +285,351 @@ class Soledad(object):
         """
         logger.debug("Closing soledad")
         self._dbpool.close()
-
-        # TODO close syncers >>>>>>
+        self._dbsyncer.close()
 
     #
     # ILocalStorage
     #
 
     def _defer(self, meth, *args, **kw):
+        """
+        Defer a method to be run on a U1DB connection pool.
+
+        :param meth: A method to defer to the U1DB connection pool.
+        :type meth: callable
+        :return: A deferred.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._dbpool.runU1DBQuery(meth, *args, **kw)
 
     def put_doc(self, doc):
         """
+        Update a document.
+
+        If the document currently has conflicts, put will fail.
+        If the database specifies a maximum document size and the document
+        exceeds it, put will fail and raise a DocumentTooBig exception.
+
         ============================== WARNING ==============================
         This method converts the document's contents to unicode in-place. This
         means that after calling `put_doc(doc)`, the contents of the
         document, i.e. `doc.content`, might be different from before the
         call.
         ============================== WARNING ==============================
+
+        :param doc: A document with new content.
+        :type doc: leap.soledad.common.document.SoledadDocument
+        :return: A deferred whose callback will be invoked with the new
+            revision identifier for the document. The document object will
+            also be updated.
+        :rtype: twisted.internet.defer.Deferred
         """
         doc.content = _convert_to_unicode(doc.content)
         return self._defer("put_doc", doc)
 
     def delete_doc(self, doc):
-        # XXX what does this do when fired???
+        """
+        Mark a document as deleted.
+
+        Will abort if the current revision doesn't match doc.rev.
+        This will also set doc.content to None.
+
+        :param doc: A document to be deleted.
+        :type doc: leap.soledad.common.document.SoledadDocument
+        :return: A deferred.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("delete_doc", doc)
 
     def get_doc(self, doc_id, include_deleted=False):
+        """
+        Get the JSON string for the given document.
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise asking for a deleted
+            document will return None.
+        :type include_deleted: bool
+        :return: A deferred whose callback will be invoked with a document
+            object.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer(
             "get_doc", doc_id, include_deleted=include_deleted)
 
     def get_docs(
             self, doc_ids, check_for_conflicts=True, include_deleted=False):
+        """
+        Get the JSON content for many documents.
+
+        :param doc_ids: A list of document identifiers.
+        :type doc_ids: list
+        :param check_for_conflicts: If set to False, then the conflict check
+            will be skipped, and 'None' will be returned instead of True/False.
+        :type check_for_conflicts: bool
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise deleted documents will not
+            be included in the results.
+        :type include_deleted: bool
+        :return: A deferred whose callback will be invoked with an iterable
+            giving the document object for each document id in matching
+            doc_ids order.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer(
             "get_docs", doc_ids, check_for_conflicts=check_for_conflicts,
             include_deleted=include_deleted)
 
     def get_all_docs(self, include_deleted=False):
+        """
+        Get the JSON content for all documents in the database.
+
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise deleted documents will not
+            be included in the results.
+        :type include_deleted: bool
+
+        :return: A deferred which, when fired, will pass the a tuple
+            containing (generation, [Document]) to the callback, with the
+            current generation of the database, followed by a list of all the
+            documents in the database.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("get_all_docs", include_deleted)
 
     def create_doc(self, content, doc_id=None):
+        """
+        Create a new document.
+
+        You can optionally specify the document identifier, but the document
+        must not already exist. See 'put_doc' if you want to override an
+        existing document.
+        If the database specifies a maximum document size and the document
+        exceeds it, create will fail and raise a DocumentTooBig exception.
+
+        :param content: A Python dictionary.
+        :type content: dict
+        :param doc_id: An optional identifier specifying the document id.
+        :type doc_id: str
+        :return: A deferred whose callback will be invoked with a document.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer(
             "create_doc", _convert_to_unicode(content), doc_id=doc_id)
 
     def create_doc_from_json(self, json, doc_id=None):
+        """
+        Create a new document.
+
+        You can optionally specify the document identifier, but the document
+        must not already exist. See 'put_doc' if you want to override an
+        existing document.
+        If the database specifies a maximum document size and the document
+        exceeds it, create will fail and raise a DocumentTooBig exception.
+
+        :param json: The JSON document string
+        :type json: dict
+        :param doc_id: An optional identifier specifying the document id.
+        :type doc_id: str
+        :return: A deferred whose callback will be invoked with a document.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("create_doc_from_json", json, doc_id=doc_id)
 
     def create_index(self, index_name, *index_expressions):
+        """
+        Create a named index, which can then be queried for future lookups.
+
+        Creating an index which already exists is not an error, and is cheap.
+        Creating an index which does not match the index_expressions of the
+        existing index is an error.
+        Creating an index will block until the expressions have been evaluated
+        and the index generated.
+
+        :param index_name: A unique name which can be used as a key prefix
+        :type index_name: str
+        :param index_expressions: index expressions defining the index
+            information.
+
+            Examples:
+
+            "fieldname", or "fieldname.subfieldname" to index alphabetically
+            sorted on the contents of a field.
+
+            "number(fieldname, width)", "lower(fieldname)"
+        :type index_expresions: list of str
+        :return: A deferred.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("create_index", index_name, *index_expressions)
 
     def delete_index(self, index_name):
+        """
+        Remove a named index.
+
+        :param index_name: The name of the index we are removing
+        :type index_name: str
+        :return: A deferred.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("delete_index", index_name)
 
     def list_indexes(self):
+        """
+        List the definitions of all known indexes.
+
+        :return: A deferred whose callback will be invoked with a list of
+            [('index-name', ['field', 'field2'])] definitions.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("list_indexes")
 
     def get_from_index(self, index_name, *key_values):
+        """
+        Return documents that match the keys supplied.
+
+        You must supply exactly the same number of values as have been defined
+        in the index. It is possible to do a prefix match by using '*' to
+        indicate a wildcard match. You can only supply '*' to trailing entries,
+        (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.)
+        It is also possible to append a '*' to the last supplied value (eg
+        'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
+
+        :param index_name: The index to query
+        :type index_name: str
+        :param key_values: values to match. eg, if you have
+            an index with 3 fields then you would have:
+            get_from_index(index_name, val1, val2, val3)
+        :type key_values: list
+        :return: A deferred whose callback will be invoked with a list of
+            [Document].
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("get_from_index", index_name, *key_values)
 
     def get_count_from_index(self, index_name, *key_values):
+        """
+        Return the count for a given combination of index_name
+        and key values.
+
+        Extension method made from similar methods in u1db version 13.09
+
+        :param index_name: The index to query
+        :type index_name: str
+        :param key_values: values to match. eg, if you have
+                           an index with 3 fields then you would have:
+                           get_from_index(index_name, val1, val2, val3)
+        :type key_values: tuple
+        :return: A deferred whose callback will be invoked with the count.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("get_count_from_index", index_name, *key_values)
 
     def get_range_from_index(self, index_name, start_value, end_value):
+        """
+        Return documents that fall within the specified range.
+
+        Both ends of the range are inclusive. For both start_value and
+        end_value, one must supply exactly the same number of values as have
+        been defined in the index, or pass None. In case of a single column
+        index, a string is accepted as an alternative for a tuple with a single
+        value. It is possible to do a prefix match by using '*' to indicate
+        a wildcard match. You can only supply '*' to trailing entries, (eg
+        'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also
+        possible to append a '*' to the last supplied value (eg 'val*', '*',
+        '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
+
+        :param index_name: The index to query
+        :type index_name: str
+        :param start_values: tuples of values that define the lower bound of
+            the range. eg, if you have an index with 3 fields then you would
+            have: (val1, val2, val3)
+        :type start_values: tuple
+        :param end_values: tuples of values that define the upper bound of the
+            range. eg, if you have an index with 3 fields then you would have:
+            (val1, val2, val3)
+        :type end_values: tuple
+        :return: A deferred whose callback will be invoked with a list of
+            [Document].
+        :rtype: twisted.internet.defer.Deferred
+        """
+
         return self._defer(
             "get_range_from_index", index_name, start_value, end_value)
 
     def get_index_keys(self, index_name):
+        """
+        Return all keys under which documents are indexed in this index.
+
+        :param index_name: The index to query
+        :type index_name: str
+        :return: A deferred whose callback will be invoked with a list of
+            tuples of indexed keys.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("get_index_keys", index_name)
 
     def get_doc_conflicts(self, doc_id):
+        """
+        Get the list of conflicts for the given document.
+
+        The order of the conflicts is such that the first entry is the value
+        that would be returned by "get_doc".
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        :return: A deferred whose callback will be invoked with a list of the
+            Document entries that are conflicted.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("get_doc_conflicts", doc_id)
 
     def resolve_doc(self, doc, conflicted_doc_revs):
+        """
+        Mark a document as no longer conflicted.
+
+        We take the list of revisions that the client knows about that it is
+        superseding. This may be a different list from the actual current
+        conflicts, in which case only those are removed as conflicted.  This
+        may fail if the conflict list is significantly different from the
+        supplied information. (sync could have happened in the background from
+        the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
+
+        :param doc: A Document with the new content to be inserted.
+        :type doc: SoledadDocument
+        :param conflicted_doc_revs: A list of revisions that the new content
+            supersedes.
+        :type conflicted_doc_revs: list(str)
+        :return: A deferred.
+        :rtype: twisted.internet.defer.Deferred
+        """
         return self._defer("resolve_doc", doc, conflicted_doc_revs)
 
-    def _get_local_db_path(self):
+    @property
+    def local_db_path(self):
         return self._local_db_path
 
-    # XXX Do we really need all this private / property dance?
-
-    local_db_path = property(
-        _get_local_db_path,
-        doc='The path for the local database replica.')
-
-    def _get_uuid(self):
+    @property
+    def uuid(self):
         return self._uuid
 
-    uuid = property(_get_uuid, doc='The user uuid.')
-
     #
     # ISyncableStorage
     #
 
     def sync(self, defer_decryption=True):
+        """
+        Synchronize documents with the server replica.
+
+        :param defer_decryption:
+            Whether to defer decryption of documents, or do it inline while
+            syncing.
+        :type defer_decryption: bool
+        :return: A deferred whose callback will be invoked with the local
+            generation before the synchronization was performed.
+        :rtype: twisted.internet.defer.Deferred
+        """
 
         # -----------------------------------------------------------------
         # TODO this needs work.
@@ -377,7 +641,6 @@ class Soledad(object):
         #     thread)
         # (4) Check that the deferred is called with the local gen.
 
-        # TODO document that this returns a deferred
         # -----------------------------------------------------------------
 
         def on_sync_done(local_gen):
@@ -404,6 +667,12 @@ class Soledad(object):
 
     @property
     def syncing(self):
+        """
+        Return wether Soledad is currently synchronizing with the server.
+
+        :return: Wether Soledad is currently synchronizing with the server.
+        :rtype: bool
+        """
         return self._dbsyncer.syncing
 
     def _set_token(self, token):
@@ -413,10 +682,11 @@ class Soledad(object):
         Internally, this builds the credentials dictionary with the following
         format:
 
-            self._{
+            {
                 'token': {
                     'uuid': '<uuid>'
                     'token': '<token>'
+                }
             }
 
         :param token: The authentication token.
@@ -442,18 +712,38 @@ class Soledad(object):
     #
 
     def init_shared_db(self, server_url, uuid, creds, syncable=True):
-        shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME)
-        self.shared_db = SoledadSharedDatabase.open_database(
-            shared_db_url,
-            uuid,
-            creds=creds,
-            create=False,  # db should exist at this point.
-            syncable=syncable)
+        """
+        Initialize the shared database.
+
+        :param server_url: URL of the remote database.
+        :type server_url: str
+        :param uuid: The user's unique id.
+        :type uuid: str
+        :param creds: A tuple containing the authentication method and
+            credentials.
+        :type creds: tuple
+        :param syncable:
+            If syncable is False, the database will not attempt to sync against
+            a remote replica.
+        :type syncable: bool
+        """
+        # only case this is False is for testing purposes
+        if self.shared_db is None:
+            shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME)
+            self.shared_db = SoledadSharedDatabase.open_database(
+                shared_db_url,
+                uuid,
+                creds=creds,
+                create=False,  # db should exist at this point.
+                syncable=syncable)
 
     @property
     def storage_secret(self):
         """
-        Return the secret used for symmetric encryption.
+        Return the secret used for local storage encryption.
+
+        :return: The secret used for local storage encryption.
+        :rtype: str
         """
         return self._secrets.storage_secret
 
@@ -461,20 +751,37 @@ class Soledad(object):
     def remote_storage_secret(self):
         """
         Return the secret used for encryption of remotely stored data.
+
+        :return: The secret used for remote storage  encryption.
+        :rtype: str
         """
         return self._secrets.remote_storage_secret
 
     @property
     def secrets(self):
+        """
+        Return the secrets object.
+
+        :return: The secrets object.
+        :rtype: SoledadSecrets
+        """
         return self._secrets
 
     def change_passphrase(self, new_passphrase):
+        """
+        Change the passphrase that encrypts the storage secret.
+
+        :param new_passphrase: The new passphrase.
+        :type new_passphrase: unicode
+
+        :raise NoStorageSecret: Raised if there's no storage secret available.
+        """
         self._secrets.change_passphrase(new_passphrase)
 
 
 def _convert_to_unicode(content):
     """
-    Convert content to unicode (or all the strings in content)
+    Convert content to unicode (or all the strings in content).
 
     NOTE: Even though this method supports any type, it will
     currently ignore contents of lists, tuple or any other