diff options
| -rw-r--r-- | client/src/leap/soledad/client/api.py | 373 |
1 files changed, 340 insertions, 33 deletions
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 00884a12..59cbc4ca 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -46,6 +46,7 @@ from zope.interface import implements from twisted.python import log from leap.common.config import get_path_prefix + from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type @@ -112,7 +113,7 @@ class Soledad(object): default_prefix = os.path.join(get_path_prefix(), 'leap', 'soledad') def __init__(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file, + server_url, cert_file, shared_db=None, auth_token=None, defer_encryption=False, syncable=True): """ Initialize configuration, cryptographic keys and dbs. @@ -142,6 +143,10 @@ class Soledad(object): certificate used by the remote soledad server. :type cert_file: str + :param shared_db: + The shared database. + :type shared_db: HTTPDatabase + :param auth_token: Authorization token for accessing remote databases. :type auth_token: str @@ -157,8 +162,9 @@ class Soledad(object): :type syncable: bool :raise BootstrapSequenceError: - Raised when the secret generation and storage on server sequence - has failed for some reason. + Raised when the secret initialization sequence (i.e. retrieval + from server or generation and storage on server) has failed for + some reason. """ # store config params self._uuid = uuid @@ -168,7 +174,7 @@ class Soledad(object): self._defer_encryption = defer_encryption self._secrets_path = None - self.shared_db = None + self.shared_db = shared_db # configure SSL certificate global SOLEDAD_CERT @@ -225,6 +231,9 @@ class Soledad(object): create_path_if_not_exists(path) def _init_secrets(self): + """ + Initialize Soledad secrets. + """ self._secrets = SoledadSecrets( self.uuid, self._passphrase, self._secrets_path, self.shared_db, self._crypto) @@ -232,8 +241,9 @@ class Soledad(object): def _init_u1db_sqlcipher_backend(self): """ - Initialize the U1DB SQLCipher database for local storage, by - instantiating a modified twisted adbapi that will maintain a threadpool + Initialize the U1DB SQLCipher database for local storage. + + Instantiates a modified twisted adbapi that will maintain a threadpool with a u1db-sqclipher connection for each thread, and will return deferreds for each u1db query. @@ -253,14 +263,16 @@ class Soledad(object): defer_encryption=self._defer_encryption, sync_db_key=sync_db_key, ) - self._soledad_opts = opts + self._sqlcipher_opts = opts self._dbpool = adbapi.getConnectionPool(opts) def _init_u1db_syncer(self): + """ + Initialize the U1DB synchronizer. + """ replica_uid = self._dbpool.replica_uid - print "replica UID (syncer init)", replica_uid self._dbsyncer = SQLCipherU1DBSync( - self._soledad_opts, self._crypto, replica_uid, + self._sqlcipher_opts, self._crypto, replica_uid, self._defer_encryption) # @@ -273,99 +285,351 @@ class Soledad(object): """ logger.debug("Closing soledad") self._dbpool.close() - - # TODO close syncers >>>>>> + self._dbsyncer.close() # # ILocalStorage # def _defer(self, meth, *args, **kw): + """ + Defer a method to be run on a U1DB connection pool. + + :param meth: A method to defer to the U1DB connection pool. + :type meth: callable + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._dbpool.runU1DBQuery(meth, *args, **kw) def put_doc(self, doc): """ + Update a document. + + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + ============================== WARNING ============================== This method converts the document's contents to unicode in-place. This means that after calling `put_doc(doc)`, the contents of the document, i.e. `doc.content`, might be different from before the call. ============================== WARNING ============================== + + :param doc: A document with new content. + :type doc: leap.soledad.common.document.SoledadDocument + :return: A deferred whose callback will be invoked with the new + revision identifier for the document. The document object will + also be updated. + :rtype: twisted.internet.defer.Deferred """ doc.content = _convert_to_unicode(doc.content) return self._defer("put_doc", doc) def delete_doc(self, doc): - # XXX what does this do when fired??? + """ + Mark a document as deleted. + + Will abort if the current revision doesn't match doc.rev. + This will also set doc.content to None. + + :param doc: A document to be deleted. + :type doc: leap.soledad.common.document.SoledadDocument + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("delete_doc", doc) def get_doc(self, doc_id, include_deleted=False): + """ + Get the JSON string for the given document. + + :param doc_id: The unique document identifier + :type doc_id: str + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + :type include_deleted: bool + :return: A deferred whose callback will be invoked with a document + object. + :rtype: twisted.internet.defer.Deferred + """ return self._defer( "get_doc", doc_id, include_deleted=include_deleted) def get_docs( self, doc_ids, check_for_conflicts=True, include_deleted=False): + """ + Get the JSON content for many documents. + + :param doc_ids: A list of document identifiers. + :type doc_ids: list + :param check_for_conflicts: If set to False, then the conflict check + will be skipped, and 'None' will be returned instead of True/False. + :type check_for_conflicts: bool + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :type include_deleted: bool + :return: A deferred whose callback will be invoked with an iterable + giving the document object for each document id in matching + doc_ids order. + :rtype: twisted.internet.defer.Deferred + """ return self._defer( "get_docs", doc_ids, check_for_conflicts=check_for_conflicts, include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): + """ + Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :type include_deleted: bool + + :return: A deferred which, when fired, will pass the a tuple + containing (generation, [Document]) to the callback, with the + current generation of the database, followed by a list of all the + documents in the database. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_all_docs", include_deleted) def create_doc(self, content, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param content: A Python dictionary. + :type content: dict + :param doc_id: An optional identifier specifying the document id. + :type doc_id: str + :return: A deferred whose callback will be invoked with a document. + :rtype: twisted.internet.defer.Deferred + """ return self._defer( "create_doc", _convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :type json: dict + :param doc_id: An optional identifier specifying the document id. + :type doc_id: str + :return: A deferred whose callback will be invoked with a document. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("create_doc_from_json", json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): + """ + Create a named index, which can then be queried for future lookups. + + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: index expressions defining the index + information. + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + :type index_expresions: list of str + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("create_index", index_name, *index_expressions) def delete_index(self, index_name): + """ + Remove a named index. + + :param index_name: The name of the index we are removing + :type index_name: str + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("delete_index", index_name) def list_indexes(self): + """ + List the definitions of all known indexes. + + :return: A deferred whose callback will be invoked with a list of + [('index-name', ['field', 'field2'])] definitions. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("list_indexes") def get_from_index(self, index_name, *key_values): + """ + Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: list + :return: A deferred whose callback will be invoked with a list of + [Document]. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_from_index", index_name, *key_values) def get_count_from_index(self, index_name, *key_values): + """ + Return the count for a given combination of index_name + and key values. + + Extension method made from similar methods in u1db version 13.09 + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: A deferred whose callback will be invoked with the count. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_count_from_index", index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): + """ + Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :type end_values: tuple + :return: A deferred whose callback will be invoked with a list of + [Document]. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer( "get_range_from_index", index_name, start_value, end_value) def get_index_keys(self, index_name): + """ + Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :type index_name: str + :return: A deferred whose callback will be invoked with a list of + tuples of indexed keys. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_index_keys", index_name) def get_doc_conflicts(self, doc_id): + """ + Get the list of conflicts for the given document. + + The order of the conflicts is such that the first entry is the value + that would be returned by "get_doc". + + :param doc_id: The unique document identifier + :type doc_id: str + :return: A deferred whose callback will be invoked with a list of the + Document entries that are conflicted. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_doc_conflicts", doc_id) def resolve_doc(self, doc, conflicted_doc_revs): + """ + Mark a document as no longer conflicted. + + We take the list of revisions that the client knows about that it is + superseding. This may be a different list from the actual current + conflicts, in which case only those are removed as conflicted. This + may fail if the conflict list is significantly different from the + supplied information. (sync could have happened in the background from + the time you GET_DOC_CONFLICTS until the point where you RESOLVE) + + :param doc: A Document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: A list of revisions that the new content + supersedes. + :type conflicted_doc_revs: list(str) + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("resolve_doc", doc, conflicted_doc_revs) - def _get_local_db_path(self): + @property + def local_db_path(self): return self._local_db_path - # XXX Do we really need all this private / property dance? - - local_db_path = property( - _get_local_db_path, - doc='The path for the local database replica.') - - def _get_uuid(self): + @property + def uuid(self): return self._uuid - uuid = property(_get_uuid, doc='The user uuid.') - # # ISyncableStorage # def sync(self, defer_decryption=True): + """ + Synchronize documents with the server replica. + + :param defer_decryption: + Whether to defer decryption of documents, or do it inline while + syncing. + :type defer_decryption: bool + :return: A deferred whose callback will be invoked with the local + generation before the synchronization was performed. + :rtype: twisted.internet.defer.Deferred + """ # ----------------------------------------------------------------- # TODO this needs work. @@ -377,7 +641,6 @@ class Soledad(object): # thread) # (4) Check that the deferred is called with the local gen. - # TODO document that this returns a deferred # ----------------------------------------------------------------- def on_sync_done(local_gen): @@ -404,6 +667,12 @@ class Soledad(object): @property def syncing(self): + """ + Return wether Soledad is currently synchronizing with the server. + + :return: Wether Soledad is currently synchronizing with the server. + :rtype: bool + """ return self._dbsyncer.syncing def _set_token(self, token): @@ -413,10 +682,11 @@ class Soledad(object): Internally, this builds the credentials dictionary with the following format: - self._{ + { 'token': { 'uuid': '<uuid>' 'token': '<token>' + } } :param token: The authentication token. @@ -442,18 +712,38 @@ class Soledad(object): # def init_shared_db(self, server_url, uuid, creds, syncable=True): - shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) - self.shared_db = SoledadSharedDatabase.open_database( - shared_db_url, - uuid, - creds=creds, - create=False, # db should exist at this point. - syncable=syncable) + """ + Initialize the shared database. + + :param server_url: URL of the remote database. + :type server_url: str + :param uuid: The user's unique id. + :type uuid: str + :param creds: A tuple containing the authentication method and + credentials. + :type creds: tuple + :param syncable: + If syncable is False, the database will not attempt to sync against + a remote replica. + :type syncable: bool + """ + # only case this is False is for testing purposes + if self.shared_db is None: + shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) + self.shared_db = SoledadSharedDatabase.open_database( + shared_db_url, + uuid, + creds=creds, + create=False, # db should exist at this point. + syncable=syncable) @property def storage_secret(self): """ - Return the secret used for symmetric encryption. + Return the secret used for local storage encryption. + + :return: The secret used for local storage encryption. + :rtype: str """ return self._secrets.storage_secret @@ -461,20 +751,37 @@ class Soledad(object): def remote_storage_secret(self): """ Return the secret used for encryption of remotely stored data. + + :return: The secret used for remote storage encryption. + :rtype: str """ return self._secrets.remote_storage_secret @property def secrets(self): + """ + Return the secrets object. + + :return: The secrets object. + :rtype: SoledadSecrets + """ return self._secrets def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ self._secrets.change_passphrase(new_passphrase) def _convert_to_unicode(content): """ - Convert content to unicode (or all the strings in content) + Convert content to unicode (or all the strings in content). NOTE: Even though this method supports any type, it will currently ignore contents of lists, tuple or any other |