diff options
Diffstat (limited to 'common')
| -rw-r--r-- | common/changes/bug_3647-improve-u1db-data-storage-in-couch | 1 | ||||
| -rw-r--r-- | common/changes/feature_2167-make-couchdb-dependency-optional | 1 | ||||
| -rw-r--r-- | common/changes/feature_3501-add-verification-for-couch-permissions | 1 | ||||
| -rw-r--r-- | common/setup.py | 3 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/couch.py | 362 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/objectstore.py | 189 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_couch.py | 53 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_crypto.py | 23 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_server.py | 3 |
9 files changed, 405 insertions, 231 deletions
diff --git a/common/changes/bug_3647-improve-u1db-data-storage-in-couch b/common/changes/bug_3647-improve-u1db-data-storage-in-couch new file mode 100644 index 00000000..d46475cd --- /dev/null +++ b/common/changes/bug_3647-improve-u1db-data-storage-in-couch @@ -0,0 +1 @@ + o Improve u1db data storage in couch. Closes #3647. diff --git a/common/changes/feature_2167-make-couchdb-dependency-optional b/common/changes/feature_2167-make-couchdb-dependency-optional new file mode 100644 index 00000000..b5148940 --- /dev/null +++ b/common/changes/feature_2167-make-couchdb-dependency-optional @@ -0,0 +1 @@ + o Turn couchdb dependency for common into optional. Closes #2167. diff --git a/common/changes/feature_3501-add-verification-for-couch-permissions b/common/changes/feature_3501-add-verification-for-couch-permissions new file mode 100644 index 00000000..3c2687d5 --- /dev/null +++ b/common/changes/feature_3501-add-verification-for-couch-permissions @@ -0,0 +1 @@ + o Add verification for couch permissions. Closes #3501. diff --git a/common/setup.py b/common/setup.py index 0a2f138f..9af61be7 100644 --- a/common/setup.py +++ b/common/setup.py @@ -67,4 +67,7 @@ setup( install_requires=utils.parse_requirements(), tests_require=utils.parse_requirements( reqfiles=['pkg/requirements-testing.pip']), + extras_require={ + 'couchdb': ['couchdb'], + }, ) diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 9642e8f3..187d3035 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -18,19 +18,19 @@ """A U1DB backend that uses CouchDB as its persistence layer.""" -import uuid import re import simplejson as json +import socket +import logging -from base64 import b64encode, b64decode from u1db import errors from u1db.sync import Synchronizer from u1db.backends.inmemory import InMemoryIndex from u1db.remote.server_state import ServerState from u1db.errors import DatabaseDoesNotExist from couchdb.client import Server, Document as CouchDocument -from couchdb.http import ResourceNotFound +from couchdb.http import ResourceNotFound, Unauthorized from leap.soledad.common.objectstore import ( @@ -39,28 +39,155 @@ from leap.soledad.common.objectstore import ( ) +logger = logging.getLogger(__name__) + + class InvalidURLError(Exception): """ Exception raised when Soledad encounters a malformed URL. """ +def persistent_class(cls): + """ + Decorator that modifies a class to ensure u1db metadata persists on + underlying storage. + + @param cls: The class that will be modified. + @type cls: type + """ + + def _create_persistent_method(old_method_name, key, load_method_name, + dump_method_name, store): + """ + Create a persistent method to replace C{old_method_name}. + + The new method will load C{key} using C{load_method_name} and stores + it using C{dump_method_name} depending on the value of C{store}. + """ + # get methods + old_method = getattr(cls, old_method_name) + load_method = getattr(cls, load_method_name) \ + if load_method_name is not None \ + else lambda self, data: setattr(self, key, data) + dump_method = getattr(cls, dump_method_name) \ + if dump_method_name is not None \ + else lambda self: getattr(self, key) + + def _new_method(self, *args, **kwargs): + # get u1db data from couch db + doc = self._get_doc('%s%s' % + (self.U1DB_DATA_DOC_ID_PREFIX, key)) + load_method(self, doc.content['content']) + # run old method + retval = old_method(self, *args, **kwargs) + # store u1db data on couch + if store: + doc.content = {'content': dump_method(self)} + self._put_doc(doc) + return retval + + return _new_method + + # ensure the class has a persistency map + if not hasattr(cls, 'PERSISTENCY_MAP'): + logger.error('Class %s has no PERSISTENCY_MAP attribute, skipping ' + 'persistent methods substitution.' % cls) + return cls + # replace old methods with new persistent ones + for key, ((load_method_name, dump_method_name), + persistent_methods) in cls.PERSISTENCY_MAP.iteritems(): + for (method_name, store) in persistent_methods: + setattr(cls, method_name, + _create_persistent_method( + method_name, + key, + load_method_name, + dump_method_name, + store)) + return cls + + +@persistent_class class CouchDatabase(ObjectStoreDatabase): """ A U1DB backend that uses Couch as its persistence layer. """ - U1DB_TRANSACTION_LOG_KEY = 'transaction_log' - U1DB_CONFLICTS_KEY = 'conflicts' - U1DB_OTHER_GENERATIONS_KEY = 'other_generations' - U1DB_INDEXES_KEY = 'indexes' - U1DB_REPLICA_UID_KEY = 'replica_uid' + U1DB_TRANSACTION_LOG_KEY = '_transaction_log' + U1DB_CONFLICTS_KEY = '_conflicts' + U1DB_OTHER_GENERATIONS_KEY = '_other_generations' + U1DB_INDEXES_KEY = '_indexes' + U1DB_REPLICA_UID_KEY = '_replica_uid' + + U1DB_DATA_KEYS = [ + U1DB_TRANSACTION_LOG_KEY, + U1DB_CONFLICTS_KEY, + U1DB_OTHER_GENERATIONS_KEY, + U1DB_INDEXES_KEY, + U1DB_REPLICA_UID_KEY, + ] COUCH_ID_KEY = '_id' COUCH_REV_KEY = '_rev' COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json' COUCH_U1DB_REV_KEY = 'u1db_rev' + # the following map describes information about methods usage of + # properties that have to persist on the underlying database. The format + # of the map is assumed to be: + # + # { + # 'property_name': [ + # ('property_load_method_name', 'property_dump_method_name'), + # [('method_1_name', bool), + # ... + # ('method_N_name', bool)]], + # ... + # } + # + # where the booleans indicate if the property should be stored after + # each method execution (i.e. if the method alters the property). Property + # load/dump methods will be run after/before properties are read/written + # to the underlying db. + PERSISTENCY_MAP = { + U1DB_TRANSACTION_LOG_KEY: [ + ('_load_transaction_log_from_json', None), + [('_get_transaction_log', False), + ('_get_generation', False), + ('_get_generation_info', False), + ('_get_trans_id_for_gen', False), + ('whats_changed', False), + ('_put_and_update_indexes', True)]], + U1DB_CONFLICTS_KEY: [ + (None, None), + [('_has_conflicts', False), + ('get_doc_conflicts', False), + ('_prune_conflicts', False), + ('resolve_doc', False), + ('_replace_conflicts', True), + ('_force_doc_sync_conflict', True)]], + U1DB_OTHER_GENERATIONS_KEY: [ + ('_load_other_generations_from_json', None), + [('_get_replica_gen_and_trans_id', False), + ('_do_set_replica_gen_and_trans_id', True)]], + U1DB_INDEXES_KEY: [ + ('_load_indexes_from_json', '_dump_indexes_as_json'), + [('list_indexes', False), + ('get_from_index', False), + ('get_range_from_index', False), + ('get_index_keys', False), + ('_put_and_update_indexes', True), + ('create_index', True), + ('delete_index', True)]], + U1DB_REPLICA_UID_KEY: [ + (None, None), + [('_allocate_doc_rev', False), + ('_put_doc_if_newer', False), + ('_ensure_maximal_rev', False), + ('_prune_conflicts', False), + ('_set_replica_uid', True)]]} + @classmethod def open_database(cls, url, create): """ @@ -104,9 +231,11 @@ class CouchDatabase(ObjectStoreDatabase): @param session: an http.Session instance or None for a default session @type session: http.Session """ + # save params self._url = url self._full_commit = full_commit self._session = session + # configure couch self._server = Server(url=self._url, full_commit=self._full_commit, session=self._session) @@ -174,7 +303,7 @@ class CouchDatabase(ObjectStoreDatabase): generation = self._get_generation() results = [] for doc_id in self._database: - if doc_id == self.U1DB_DATA_DOC_ID: + if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX): continue doc = self._get_doc(doc_id, check_for_conflicts=True) if doc.content is None and not include_deleted: @@ -241,14 +370,12 @@ class CouchDatabase(ObjectStoreDatabase): raise errors.IndexNameTakenError index = InMemoryIndex(index_name, list(index_expressions)) for doc_id in self._database: - if doc_id == self.U1DB_DATA_DOC_ID: # skip special file - continue + if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX): + continue # skip special files doc = self._get_doc(doc_id) if doc.content is not None: index.add_json(doc_id, doc.get_json()) self._indexes[index_name] = index - # save data in object store - self._store_u1db_data() def close(self): """ @@ -290,76 +417,25 @@ class CouchDatabase(ObjectStoreDatabase): def _init_u1db_data(self): """ - Initialize U1DB info data structure in the couch db. + Initialize u1db configuration data on backend storage. A U1DB database needs to keep track of all database transactions, document conflicts, the generation of other replicas it has seen, indexes created by users and so on. - In this implementation, all this information is stored in a special - document stored in the couch db with id equals to - CouchDatabse.U1DB_DATA_DOC_ID. - - This method initializes the document that will hold such information. - """ - if self._replica_uid is None: - self._replica_uid = uuid.uuid4().hex - # TODO: prevent user from overwriting a document with the same doc_id - # as this one. - doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) - doc.content = { - self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])), - self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})), - self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})), - self.U1DB_INDEXES_KEY: b64encode(json.dumps({})), - self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid), - } - self._put_doc(doc) - - def _fetch_u1db_data(self): - """ - Fetch U1DB info from the couch db. - - See C{_init_u1db_data} documentation. - """ - # retrieve u1db data from couch db - cdoc = self._database.get(self.U1DB_DATA_DOC_ID) - jsonstr = self._database.get_attachment( - cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read() - content = json.loads(jsonstr) - # set u1db database info - self._transaction_log = json.loads( - b64decode(content[self.U1DB_TRANSACTION_LOG_KEY])) - self._conflicts = json.loads( - b64decode(content[self.U1DB_CONFLICTS_KEY])) - self._other_generations = json.loads( - b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY])) - self._indexes = self._load_indexes_from_json( - b64decode(content[self.U1DB_INDEXES_KEY])) - self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY]) - # save couch _rev - self._couch_rev = cdoc[self.COUCH_REV_KEY] - - def _store_u1db_data(self): - """ - Store U1DB info in the couch db. - - See C{_init_u1db_data} documentation. - """ - doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) - doc.content = { - # Here, the b64 encode ensures that document content - # does not cause strange behaviour in couchdb because - # of encoding. - self.U1DB_TRANSACTION_LOG_KEY: - b64encode(json.dumps(self._transaction_log)), - self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)), - self.U1DB_OTHER_GENERATIONS_KEY: - b64encode(json.dumps(self._other_generations)), - self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()), - self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid), - self.COUCH_REV_KEY: self._couch_rev} - self._put_doc(doc) + In this implementation, all this information is stored in special + documents stored in the underlying with doc_id prefix equal to + U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved: put_doc(), + get_doc() and delete_doc() will not allow documents with a doc_id with + that prefix to be accessed or modified. + """ + for key in self.U1DB_DATA_KEYS: + doc_id = '%s%s' % (self.U1DB_DATA_DOC_ID_PREFIX, key) + doc = self._get_doc(doc_id) + if doc is None: + doc = self._factory(doc_id) + doc.content = {'content': getattr(self, key)} + self._put_doc(doc) #------------------------------------------------------------------------- # Couch specific methods @@ -377,7 +453,7 @@ class CouchDatabase(ObjectStoreDatabase): def _dump_indexes_as_json(self): """ - Dump index definitions as JSON string. + Dump index definitions as JSON. """ indexes = {} for name, idx in self._indexes.iteritems(): @@ -385,31 +461,60 @@ class CouchDatabase(ObjectStoreDatabase): for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY, self.INDEX_VALUES_KEY]: indexes[name][attr] = getattr(idx, '_' + attr) - return json.dumps(indexes) + return indexes def _load_indexes_from_json(self, indexes): """ - Load index definitions from JSON string. + Load index definitions from stored JSON. - @param indexes: A JSON serialization of a list of [('index-name', - ['field', 'field2'])]. + @param indexes: A JSON representation of indexes as + [('index-name', ['field', 'field2', ...]), ...]. @type indexes: str - - @return: A dictionary with the index definitions. - @rtype: dict """ - dict = {} - for name, idx_dict in json.loads(indexes).iteritems(): + self._indexes = {} + for name, idx_dict in indexes.iteritems(): idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY]) idx._values = idx_dict[self.INDEX_VALUES_KEY] - dict[name] = idx - return dict + self._indexes[name] = idx + + def _load_transaction_log_from_json(self, transaction_log): + """ + Load transaction log from stored JSON. + + @param transaction_log: A JSON representation of transaction_log as + [('generation', 'transaction_id'), ...]. + @type transaction_log: list + """ + self._transaction_log = [] + for gen, trans_id in transaction_log: + self._transaction_log.append((gen, trans_id)) + + def _load_other_generations_from_json(self, other_generations): + """ + Load other generations from stored JSON. + + @param other_generations: A JSON representation of other_generations + as {'replica_uid': ('generation', 'transaction_id'), ...}. + @type other_generations: dict + """ + self._other_generations = {} + for replica_uid, [gen, trans_id] in other_generations.iteritems(): + self._other_generations[replica_uid] = (gen, trans_id) class CouchSyncTarget(ObjectStoreSyncTarget): """ Functionality for using a CouchDatabase as a synchronization target. """ + pass + + +class NotEnoughCouchPermissions(Exception): + """ + Raised when failing to assert for enough permissions on underlying Couch + Database. + """ + pass class CouchServerState(ServerState): @@ -417,8 +522,83 @@ class CouchServerState(ServerState): Inteface of the WSGI server with the CouchDB backend. """ - def __init__(self, couch_url): + def __init__(self, couch_url, shared_db_name, tokens_db_name, + user_db_prefix): + """ + Initialize the couch server state. + + @param couch_url: The URL for the couch database. + @type couch_url: str + @param shared_db_name: The name of the shared database. + @type shared_db_name: str + @param tokens_db_name: The name of the tokens database. + @type tokens_db_name: str + @param user_db_prefix: The prefix for user database names. + @type user_db_prefix: str + """ self._couch_url = couch_url + self._shared_db_name = shared_db_name + self._tokens_db_name = tokens_db_name + self._user_db_prefix = user_db_prefix + try: + self._check_couch_permissions() + except NotEnoughCouchPermissions: + logger.error("Not enough permissions on underlying couch " + "database (%s)." % self._couch_url) + except (socket.error, socket.gaierror, socket.herror, + socket.timeout), e: + logger.error("Socket problem while trying to reach underlying " + "couch database: (%s, %s)." % + (self._couch_url, e)) + + def _check_couch_permissions(self): + """ + Assert that Soledad Server has enough permissions on the underlying couch + database. + + Soledad Server has to be able to do the following in the couch server: + + * Create, read and write from/to 'shared' db. + * Create, read and write from/to 'user-<anything>' dbs. + * Read from 'tokens' db. + + This function tries to perform the actions above using the "low level" + couch library to ensure that Soledad Server can do everything it needs on + the underlying couch database. + + @param couch_url: The URL of the couch database. + @type couch_url: str + + @raise NotEnoughCouchPermissions: Raised in case there are not enough + permissions to read/write/create the needed couch databases. + @rtype: bool + """ + + def _open_couch_db(dbname): + server = Server(url=self._couch_url) + try: + server[dbname] + except ResourceNotFound: + server.create(dbname) + return server[dbname] + + def _create_delete_test_doc(db): + doc_id, _ = db.save({'test': 'document'}) + doc = db[doc_id] + db.delete(doc) + + try: + # test read/write auth for shared db + _create_delete_test_doc( + _open_couch_db(self._shared_db_name)) + # test read/write auth for user-<something> db + _create_delete_test_doc( + _open_couch_db('%stest-db' % self._user_db_prefix)) + # test read auth for tokens db + tokensdb = _open_couch_db(self._tokens_db_name) + tokensdb.info() + except Unauthorized: + raise NotEnoughCouchPermissions(self._couch_url) def open_database(self, dbname): """ diff --git a/common/src/leap/soledad/common/objectstore.py b/common/src/leap/soledad/common/objectstore.py index 921cf075..7aff3e32 100644 --- a/common/src/leap/soledad/common/objectstore.py +++ b/common/src/leap/soledad/common/objectstore.py @@ -20,6 +20,10 @@ Abstract U1DB backend to handle storage using object stores (like CouchDB, for example). +This backend uses special documents to store all U1DB data (replica uid, +indexes, transaction logs and info about other dbs). The id of these documents +are reserved and have prefix equal to ObjectStore.U1DB_DATA_DOC_ID_PREFIX. + Right now, this is only used by CouchDatabase backend, but can also be extended to implement OpenStack or Amazon S3 storage, for example. @@ -27,6 +31,13 @@ See U1DB documentation for more information on how to use databases. """ +from base64 import b64encode, b64decode + + +import uuid +import simplejson as json + + from u1db import errors from u1db.backends.inmemory import ( InMemoryDatabase, @@ -39,6 +50,8 @@ class ObjectStoreDatabase(InMemoryDatabase): A backend for storing u1db data in an object store. """ + U1DB_DATA_DOC_ID_PREFIX = 'u1db/' + @classmethod def open_database(cls, url, create, document_factory=None): """ @@ -71,24 +84,52 @@ class ObjectStoreDatabase(InMemoryDatabase): self, replica_uid, document_factory=document_factory) - # sync data in memory with data in object store - if not self._get_doc(self.U1DB_DATA_DOC_ID): - self._init_u1db_data() - self._fetch_u1db_data() + if self._replica_uid is None: + self._replica_uid = uuid.uuid4().hex + self._init_u1db_data() + + def _init_u1db_data(self): + """ + Initialize u1db configuration data on backend storage. + + A U1DB database needs to keep track of all database transactions, + document conflicts, the generation of other replicas it has seen, + indexes created by users and so on. + + In this implementation, all this information is stored in special + documents stored in the couch db with id prefix equal to + U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved: + put_doc(), get_doc() and delete_doc() will not allow documents with + a doc_id with that prefix to be accessed or modified. + """ + raise NotImplementedError(self._init_u1db_data) #------------------------------------------------------------------------- # methods from Database #------------------------------------------------------------------------- - def _set_replica_uid(self, replica_uid): + def put_doc(self, doc): """ - Force the replica_uid to be set. + Update a document. - @param replica_uid: The uid of the replica. - @type replica_uid: str + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + + This method prevents from updating the document with doc_id equals to + self.U1DB_DATA_DOC_ID, which contains U1DB data. + + @param doc: A Document with new content. + @type doc: Document + + @return: new_doc_rev - The new revision identifier for the document. + The Document object will also be updated. + @rtype: str """ - InMemoryDatabase._set_replica_uid(self, replica_uid) - self._store_u1db_data() + if doc.doc_id is not None and \ + doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX): + raise errors.InvalidDocId() + return InMemoryDatabase.put_doc(self, doc) def _put_doc(self, doc): """ @@ -106,6 +147,27 @@ class ObjectStoreDatabase(InMemoryDatabase): """ raise NotImplementedError(self._put_doc) + def get_doc(self, doc_id, include_deleted=False): + """ + Get the JSON string for the given document. + + This method prevents from getting the document with doc_id equals to + self.U1DB_DATA_DOC_ID, which contains U1DB data. + + @param doc_id: The unique document identifier + @type doc_id: str + @param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + @type include_deleted: bool + + @return: a Document object. + @rtype: Document + """ + if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX): + raise errors.InvalidDocId() + return InMemoryDatabase.get_doc(self, doc_id, include_deleted) + def _get_doc(self, doc_id): """ Get just the document content, without fancy handling. @@ -136,18 +198,32 @@ class ObjectStoreDatabase(InMemoryDatabase): the documents in the database. @rtype: tuple """ - raise NotImplementedError(self.get_all_docs) + generation = self._get_generation() + results = [] + for doc_id in self._database: + if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX): + continue + doc = self._get_doc(doc_id, check_for_conflicts=True) + if doc.content is None and not include_deleted: + continue + results.append(doc) + return (generation, results) def delete_doc(self, doc): """ Mark a document as deleted. + This method prevents from deleting the document with doc_id equals to + self.U1DB_DATA_DOC_ID, which contains U1DB data. + @param doc: The document to mark as deleted. @type doc: u1db.Document @return: The new revision id of the document. @type: str """ + if doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX): + raise errors.InvalidDocId() old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) if old_doc is None: raise errors.DocumentDoesNotExist @@ -177,58 +253,6 @@ class ObjectStoreDatabase(InMemoryDatabase): """ raise NotImplementedError(self.create_index) - def delete_index(self, index_name): - """ - Remove a named index. - - Here we just guarantee that the new info will be stored in the backend - db after update. - - @param index_name: The name of the index we are removing. - @type index_name: str - """ - InMemoryDatabase.delete_index(self, index_name) - self._store_u1db_data() - - def _replace_conflicts(self, doc, conflicts): - """ - Set new conflicts for a document. - - Here we just guarantee that the new info will be stored in the backend - db after update. - - @param doc: The document with a new set of conflicts. - @param conflicts: The new set of conflicts. - @type conflicts: list - """ - InMemoryDatabase._replace_conflicts(self, doc, conflicts) - self._store_u1db_data() - - def _do_set_replica_gen_and_trans_id(self, other_replica_uid, - other_generation, - other_transaction_id): - """ - Set the last-known generation and transaction id for the other - database replica. - - Here we just guarantee that the new info will be stored in the backend - db after update. - - @param other_replica_uid: The U1DB identifier for the other replica. - @type other_replica_uid: str - @param other_generation: The generation number for the other replica. - @type other_generation: int - @param other_transaction_id: The transaction id associated with the - generation. - @type other_transaction_id: str - """ - InMemoryDatabase._do_set_replica_gen_and_trans_id( - self, - other_replica_uid, - other_generation, - other_transaction_id) - self._store_u1db_data() - #------------------------------------------------------------------------- # implemented methods from CommonBackend #------------------------------------------------------------------------- @@ -250,45 +274,6 @@ class ObjectStoreDatabase(InMemoryDatabase): trans_id = self._allocate_transaction_id() self._put_doc(doc) self._transaction_log.append((doc.doc_id, trans_id)) - self._store_u1db_data() - - #------------------------------------------------------------------------- - # methods specific for object stores - #------------------------------------------------------------------------- - - U1DB_DATA_DOC_ID = 'u1db_data' - - def _fetch_u1db_data(self): - """ - Fetch u1db configuration data from backend storage. - - See C{_init_u1db_data} documentation. - """ - NotImplementedError(self._fetch_u1db_data) - - def _store_u1db_data(self): - """ - Store u1db configuration data on backend storage. - - See C{_init_u1db_data} documentation. - """ - NotImplementedError(self._store_u1db_data) - - def _init_u1db_data(self): - """ - Initialize u1db configuration data on backend storage. - - A U1DB database needs to keep track of all database transactions, - document conflicts, the generation of other replicas it has seen, - indexes created by users and so on. - - In this implementation, all this information is stored in a special - document stored in the couch db with id equals to - CouchDatabse.U1DB_DATA_DOC_ID. - - This method initializes the document that will hold such information. - """ - NotImplementedError(self._init_u1db_data) class ObjectStoreSyncTarget(InMemorySyncTarget): diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 2fb799b7..42edf9fe 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -178,7 +178,12 @@ def copy_couch_database_for_test(test, db): new_db._conflicts = copy.deepcopy(db._conflicts) new_db._other_generations = copy.deepcopy(db._other_generations) new_db._indexes = copy.deepcopy(db._indexes) - new_db._store_u1db_data() + # save u1db data on couch + for key in new_db.U1DB_DATA_KEYS: + doc_id = '%s%s' % (new_db.U1DB_DATA_DOC_ID_PREFIX, key) + doc = new_db._get_doc(doc_id) + doc.content = {'content': getattr(new_db, key)} + new_db._put_doc(doc) return new_db @@ -271,6 +276,14 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests, scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) + def setUp(self): + # we implement parents' setUp methods here to prevent from launching + # more couch instances then needed. + tests.TestCase.setUp(self) + self.server = self.server_thread = None + self.db, self.st = self.create_db_and_target(self) + self.other_changes = [] + def tearDown(self): self.db.delete_database() test_sync.DatabaseSyncTargetTests.tearDown(self) @@ -345,20 +358,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase): return [self._listify(i) for i in l] return l - def _fetch_u1db_data(self, db): - cdoc = db._database.get(db.U1DB_DATA_DOC_ID) - jsonstr = db._database.get_attachment(cdoc, 'u1db_json').getvalue() - return json.loads(jsonstr) + def _fetch_u1db_data(self, db, key): + doc = db._get_doc("%s%s" % (db.U1DB_DATA_DOC_ID_PREFIX, key)) + return doc.content['content'] def test_transaction_log_storage_after_put(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), 'u1db_tests') db.create_doc({'simple': 'doc'}) - content = self._fetch_u1db_data(db) + content = self._fetch_u1db_data(db, db.U1DB_TRANSACTION_LOG_KEY) self.assertEqual( self._listify(db._transaction_log), - self._listify( - json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY])))) + self._listify(content)) def test_conflict_log_storage_after_put_if_newer(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), @@ -367,29 +378,27 @@ class CouchDatabaseStorageTests(CouchDBTestCase): doc.set_json(nested_doc) doc.rev = db._replica_uid + ':2' db._force_doc_sync_conflict(doc) - content = self._fetch_u1db_data(db) + content = self._fetch_u1db_data(db, db.U1DB_CONFLICTS_KEY) self.assertEqual( self._listify(db._conflicts), - self._listify( - json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY])))) + self._listify(content)) def test_other_gens_storage_after_set(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), 'u1db_tests') doc = db.create_doc({'simple': 'doc'}) db._set_replica_gen_and_trans_id('a', 'b', 'c') - content = self._fetch_u1db_data(db) + content = self._fetch_u1db_data(db, db.U1DB_OTHER_GENERATIONS_KEY) self.assertEqual( self._listify(db._other_generations), - self._listify( - json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY])))) + self._listify(content)) def test_index_storage_after_create(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), 'u1db_tests') doc = db.create_doc({'name': 'john'}) db.create_index('myindex', 'name') - content = self._fetch_u1db_data(db) + content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY) myind = db._indexes['myindex'] index = { 'myindex': { @@ -400,8 +409,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase): } self.assertEqual( self._listify(index), - self._listify( - json.loads(b64decode(content[db.U1DB_INDEXES_KEY])))) + self._listify(content)) def test_index_storage_after_delete(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), @@ -410,7 +418,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase): db.create_index('myindex', 'name') db.create_index('myindex2', 'name') db.delete_index('myindex') - content = self._fetch_u1db_data(db) + content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY) myind = db._indexes['myindex2'] index = { 'myindex2': { @@ -421,16 +429,13 @@ class CouchDatabaseStorageTests(CouchDBTestCase): } self.assertEqual( self._listify(index), - self._listify( - json.loads(b64decode(content[db.U1DB_INDEXES_KEY])))) + self._listify(content)) def test_replica_uid_storage_after_db_creation(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), 'u1db_tests') - content = self._fetch_u1db_data(db) - self.assertEqual( - db._replica_uid, - b64decode(content[db.U1DB_REPLICA_UID_KEY])) + content = self._fetch_u1db_data(db, db.U1DB_REPLICA_UID_KEY) + self.assertEqual(db._replica_uid, content) load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index 01b43299..db217bb3 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -29,9 +29,6 @@ import binascii from leap.common.testing.basetest import BaseLeapTest -from Crypto import Random - - from leap.soledad.client import ( Soledad, crypto, @@ -193,7 +190,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): # generate 256-bit key - key = Random.new().read(32) + key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, method=crypto.EncryptionMethods.AES_256_CTR) @@ -206,7 +203,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): - key = Random.new().read(32) + key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, method=crypto.EncryptionMethods.AES_256_CTR) @@ -224,17 +221,17 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): - key = Random.new().read(32) + key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, method=crypto.EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') - wrongkey = Random.new().read(32) # 256-bits key + wrongkey = os.urandom(32) # 256-bits key # ensure keys are different in case we are extremely lucky while wrongkey == key: - wrongkey = Random.new().read(32) + wrongkey = os.urandom(32) plaintext = self._soledad._crypto.decrypt_sym( cyphertext, wrongkey, iv=iv, method=crypto.EncryptionMethods.AES_256_CTR) @@ -245,7 +242,7 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): # generate 256-bit key - key = Random.new().read(32) + key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, method=crypto.EncryptionMethods.XSALSA20) @@ -258,7 +255,7 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): - key = Random.new().read(32) + key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, method=crypto.EncryptionMethods.XSALSA20) @@ -276,17 +273,17 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): - key = Random.new().read(32) + key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, method=crypto.EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') - wrongkey = Random.new().read(32) # 256-bits key + wrongkey = os.urandom(32) # 256-bits key # ensure keys are different in case we are extremely lucky while wrongkey == key: - wrongkey = Random.new().read(32) + wrongkey = os.urandom(32) plaintext = self._soledad._crypto.decrypt_sym( cyphertext, wrongkey, iv=iv, method=crypto.EncryptionMethods.XSALSA20) diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index beb7e04d..1ea4d615 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -310,7 +310,8 @@ class EncryptedSyncTestCase( secret_id=secret_id) def make_app(self): - self.request_state = CouchServerState(self._couch_url) + self.request_state = CouchServerState( + self._couch_url, 'shared', 'tokens', 'user-') return self.make_app_with_state(self.request_state) def setUp(self): |