summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--common/changes/bug_3647-improve-u1db-data-storage-in-couch1
-rw-r--r--common/changes/feature_2167-make-couchdb-dependency-optional1
-rw-r--r--common/changes/feature_3501-add-verification-for-couch-permissions1
-rw-r--r--common/setup.py3
-rw-r--r--common/src/leap/soledad/common/couch.py362
-rw-r--r--common/src/leap/soledad/common/objectstore.py189
-rw-r--r--common/src/leap/soledad/common/tests/test_couch.py53
-rw-r--r--common/src/leap/soledad/common/tests/test_crypto.py23
-rw-r--r--common/src/leap/soledad/common/tests/test_server.py3
-rw-r--r--server/changes/feature_3501-add-verification-for-couch-permissions1
-rw-r--r--server/src/leap/soledad/server/__init__.py28
11 files changed, 426 insertions, 239 deletions
diff --git a/common/changes/bug_3647-improve-u1db-data-storage-in-couch b/common/changes/bug_3647-improve-u1db-data-storage-in-couch
new file mode 100644
index 00000000..d46475cd
--- /dev/null
+++ b/common/changes/bug_3647-improve-u1db-data-storage-in-couch
@@ -0,0 +1 @@
+ o Improve u1db data storage in couch. Closes #3647.
diff --git a/common/changes/feature_2167-make-couchdb-dependency-optional b/common/changes/feature_2167-make-couchdb-dependency-optional
new file mode 100644
index 00000000..b5148940
--- /dev/null
+++ b/common/changes/feature_2167-make-couchdb-dependency-optional
@@ -0,0 +1 @@
+ o Turn couchdb dependency for common into optional. Closes #2167.
diff --git a/common/changes/feature_3501-add-verification-for-couch-permissions b/common/changes/feature_3501-add-verification-for-couch-permissions
new file mode 100644
index 00000000..3c2687d5
--- /dev/null
+++ b/common/changes/feature_3501-add-verification-for-couch-permissions
@@ -0,0 +1 @@
+ o Add verification for couch permissions. Closes #3501.
diff --git a/common/setup.py b/common/setup.py
index 0a2f138f..9af61be7 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -67,4 +67,7 @@ setup(
install_requires=utils.parse_requirements(),
tests_require=utils.parse_requirements(
reqfiles=['pkg/requirements-testing.pip']),
+ extras_require={
+ 'couchdb': ['couchdb'],
+ },
)
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 9642e8f3..187d3035 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -18,19 +18,19 @@
"""A U1DB backend that uses CouchDB as its persistence layer."""
-import uuid
import re
import simplejson as json
+import socket
+import logging
-from base64 import b64encode, b64decode
from u1db import errors
from u1db.sync import Synchronizer
from u1db.backends.inmemory import InMemoryIndex
from u1db.remote.server_state import ServerState
from u1db.errors import DatabaseDoesNotExist
from couchdb.client import Server, Document as CouchDocument
-from couchdb.http import ResourceNotFound
+from couchdb.http import ResourceNotFound, Unauthorized
from leap.soledad.common.objectstore import (
@@ -39,28 +39,155 @@ from leap.soledad.common.objectstore import (
)
+logger = logging.getLogger(__name__)
+
+
class InvalidURLError(Exception):
"""
Exception raised when Soledad encounters a malformed URL.
"""
+def persistent_class(cls):
+ """
+ Decorator that modifies a class to ensure u1db metadata persists on
+ underlying storage.
+
+ @param cls: The class that will be modified.
+ @type cls: type
+ """
+
+ def _create_persistent_method(old_method_name, key, load_method_name,
+ dump_method_name, store):
+ """
+ Create a persistent method to replace C{old_method_name}.
+
+ The new method will load C{key} using C{load_method_name} and stores
+ it using C{dump_method_name} depending on the value of C{store}.
+ """
+ # get methods
+ old_method = getattr(cls, old_method_name)
+ load_method = getattr(cls, load_method_name) \
+ if load_method_name is not None \
+ else lambda self, data: setattr(self, key, data)
+ dump_method = getattr(cls, dump_method_name) \
+ if dump_method_name is not None \
+ else lambda self: getattr(self, key)
+
+ def _new_method(self, *args, **kwargs):
+ # get u1db data from couch db
+ doc = self._get_doc('%s%s' %
+ (self.U1DB_DATA_DOC_ID_PREFIX, key))
+ load_method(self, doc.content['content'])
+ # run old method
+ retval = old_method(self, *args, **kwargs)
+ # store u1db data on couch
+ if store:
+ doc.content = {'content': dump_method(self)}
+ self._put_doc(doc)
+ return retval
+
+ return _new_method
+
+ # ensure the class has a persistency map
+ if not hasattr(cls, 'PERSISTENCY_MAP'):
+ logger.error('Class %s has no PERSISTENCY_MAP attribute, skipping '
+ 'persistent methods substitution.' % cls)
+ return cls
+ # replace old methods with new persistent ones
+ for key, ((load_method_name, dump_method_name),
+ persistent_methods) in cls.PERSISTENCY_MAP.iteritems():
+ for (method_name, store) in persistent_methods:
+ setattr(cls, method_name,
+ _create_persistent_method(
+ method_name,
+ key,
+ load_method_name,
+ dump_method_name,
+ store))
+ return cls
+
+
+@persistent_class
class CouchDatabase(ObjectStoreDatabase):
"""
A U1DB backend that uses Couch as its persistence layer.
"""
- U1DB_TRANSACTION_LOG_KEY = 'transaction_log'
- U1DB_CONFLICTS_KEY = 'conflicts'
- U1DB_OTHER_GENERATIONS_KEY = 'other_generations'
- U1DB_INDEXES_KEY = 'indexes'
- U1DB_REPLICA_UID_KEY = 'replica_uid'
+ U1DB_TRANSACTION_LOG_KEY = '_transaction_log'
+ U1DB_CONFLICTS_KEY = '_conflicts'
+ U1DB_OTHER_GENERATIONS_KEY = '_other_generations'
+ U1DB_INDEXES_KEY = '_indexes'
+ U1DB_REPLICA_UID_KEY = '_replica_uid'
+
+ U1DB_DATA_KEYS = [
+ U1DB_TRANSACTION_LOG_KEY,
+ U1DB_CONFLICTS_KEY,
+ U1DB_OTHER_GENERATIONS_KEY,
+ U1DB_INDEXES_KEY,
+ U1DB_REPLICA_UID_KEY,
+ ]
COUCH_ID_KEY = '_id'
COUCH_REV_KEY = '_rev'
COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json'
COUCH_U1DB_REV_KEY = 'u1db_rev'
+ # the following map describes information about methods usage of
+ # properties that have to persist on the underlying database. The format
+ # of the map is assumed to be:
+ #
+ # {
+ # 'property_name': [
+ # ('property_load_method_name', 'property_dump_method_name'),
+ # [('method_1_name', bool),
+ # ...
+ # ('method_N_name', bool)]],
+ # ...
+ # }
+ #
+ # where the booleans indicate if the property should be stored after
+ # each method execution (i.e. if the method alters the property). Property
+ # load/dump methods will be run after/before properties are read/written
+ # to the underlying db.
+ PERSISTENCY_MAP = {
+ U1DB_TRANSACTION_LOG_KEY: [
+ ('_load_transaction_log_from_json', None),
+ [('_get_transaction_log', False),
+ ('_get_generation', False),
+ ('_get_generation_info', False),
+ ('_get_trans_id_for_gen', False),
+ ('whats_changed', False),
+ ('_put_and_update_indexes', True)]],
+ U1DB_CONFLICTS_KEY: [
+ (None, None),
+ [('_has_conflicts', False),
+ ('get_doc_conflicts', False),
+ ('_prune_conflicts', False),
+ ('resolve_doc', False),
+ ('_replace_conflicts', True),
+ ('_force_doc_sync_conflict', True)]],
+ U1DB_OTHER_GENERATIONS_KEY: [
+ ('_load_other_generations_from_json', None),
+ [('_get_replica_gen_and_trans_id', False),
+ ('_do_set_replica_gen_and_trans_id', True)]],
+ U1DB_INDEXES_KEY: [
+ ('_load_indexes_from_json', '_dump_indexes_as_json'),
+ [('list_indexes', False),
+ ('get_from_index', False),
+ ('get_range_from_index', False),
+ ('get_index_keys', False),
+ ('_put_and_update_indexes', True),
+ ('create_index', True),
+ ('delete_index', True)]],
+ U1DB_REPLICA_UID_KEY: [
+ (None, None),
+ [('_allocate_doc_rev', False),
+ ('_put_doc_if_newer', False),
+ ('_ensure_maximal_rev', False),
+ ('_prune_conflicts', False),
+ ('_set_replica_uid', True)]]}
+
@classmethod
def open_database(cls, url, create):
"""
@@ -104,9 +231,11 @@ class CouchDatabase(ObjectStoreDatabase):
@param session: an http.Session instance or None for a default session
@type session: http.Session
"""
+ # save params
self._url = url
self._full_commit = full_commit
self._session = session
+ # configure couch
self._server = Server(url=self._url,
full_commit=self._full_commit,
session=self._session)
@@ -174,7 +303,7 @@ class CouchDatabase(ObjectStoreDatabase):
generation = self._get_generation()
results = []
for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID:
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
continue
doc = self._get_doc(doc_id, check_for_conflicts=True)
if doc.content is None and not include_deleted:
@@ -241,14 +370,12 @@ class CouchDatabase(ObjectStoreDatabase):
raise errors.IndexNameTakenError
index = InMemoryIndex(index_name, list(index_expressions))
for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID: # skip special file
- continue
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ continue # skip special files
doc = self._get_doc(doc_id)
if doc.content is not None:
index.add_json(doc_id, doc.get_json())
self._indexes[index_name] = index
- # save data in object store
- self._store_u1db_data()
def close(self):
"""
@@ -290,76 +417,25 @@ class CouchDatabase(ObjectStoreDatabase):
def _init_u1db_data(self):
"""
- Initialize U1DB info data structure in the couch db.
+ Initialize u1db configuration data on backend storage.
A U1DB database needs to keep track of all database transactions,
document conflicts, the generation of other replicas it has seen,
indexes created by users and so on.
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- if self._replica_uid is None:
- self._replica_uid = uuid.uuid4().hex
- # TODO: prevent user from overwriting a document with the same doc_id
- # as this one.
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
- self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
- self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- }
- self._put_doc(doc)
-
- def _fetch_u1db_data(self):
- """
- Fetch U1DB info from the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- # retrieve u1db data from couch db
- cdoc = self._database.get(self.U1DB_DATA_DOC_ID)
- jsonstr = self._database.get_attachment(
- cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
- content = json.loads(jsonstr)
- # set u1db database info
- self._transaction_log = json.loads(
- b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
- self._conflicts = json.loads(
- b64decode(content[self.U1DB_CONFLICTS_KEY]))
- self._other_generations = json.loads(
- b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
- self._indexes = self._load_indexes_from_json(
- b64decode(content[self.U1DB_INDEXES_KEY]))
- self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
- # save couch _rev
- self._couch_rev = cdoc[self.COUCH_REV_KEY]
-
- def _store_u1db_data(self):
- """
- Store U1DB info in the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- # Here, the b64 encode ensures that document content
- # does not cause strange behaviour in couchdb because
- # of encoding.
- self.U1DB_TRANSACTION_LOG_KEY:
- b64encode(json.dumps(self._transaction_log)),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
- self.U1DB_OTHER_GENERATIONS_KEY:
- b64encode(json.dumps(self._other_generations)),
- self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- self.COUCH_REV_KEY: self._couch_rev}
- self._put_doc(doc)
+ In this implementation, all this information is stored in special
+ documents stored in the underlying with doc_id prefix equal to
+ U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved: put_doc(),
+ get_doc() and delete_doc() will not allow documents with a doc_id with
+ that prefix to be accessed or modified.
+ """
+ for key in self.U1DB_DATA_KEYS:
+ doc_id = '%s%s' % (self.U1DB_DATA_DOC_ID_PREFIX, key)
+ doc = self._get_doc(doc_id)
+ if doc is None:
+ doc = self._factory(doc_id)
+ doc.content = {'content': getattr(self, key)}
+ self._put_doc(doc)
#-------------------------------------------------------------------------
# Couch specific methods
@@ -377,7 +453,7 @@ class CouchDatabase(ObjectStoreDatabase):
def _dump_indexes_as_json(self):
"""
- Dump index definitions as JSON string.
+ Dump index definitions as JSON.
"""
indexes = {}
for name, idx in self._indexes.iteritems():
@@ -385,31 +461,60 @@ class CouchDatabase(ObjectStoreDatabase):
for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY,
self.INDEX_VALUES_KEY]:
indexes[name][attr] = getattr(idx, '_' + attr)
- return json.dumps(indexes)
+ return indexes
def _load_indexes_from_json(self, indexes):
"""
- Load index definitions from JSON string.
+ Load index definitions from stored JSON.
- @param indexes: A JSON serialization of a list of [('index-name',
- ['field', 'field2'])].
+ @param indexes: A JSON representation of indexes as
+ [('index-name', ['field', 'field2', ...]), ...].
@type indexes: str
-
- @return: A dictionary with the index definitions.
- @rtype: dict
"""
- dict = {}
- for name, idx_dict in json.loads(indexes).iteritems():
+ self._indexes = {}
+ for name, idx_dict in indexes.iteritems():
idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY])
idx._values = idx_dict[self.INDEX_VALUES_KEY]
- dict[name] = idx
- return dict
+ self._indexes[name] = idx
+
+ def _load_transaction_log_from_json(self, transaction_log):
+ """
+ Load transaction log from stored JSON.
+
+ @param transaction_log: A JSON representation of transaction_log as
+ [('generation', 'transaction_id'), ...].
+ @type transaction_log: list
+ """
+ self._transaction_log = []
+ for gen, trans_id in transaction_log:
+ self._transaction_log.append((gen, trans_id))
+
+ def _load_other_generations_from_json(self, other_generations):
+ """
+ Load other generations from stored JSON.
+
+ @param other_generations: A JSON representation of other_generations
+ as {'replica_uid': ('generation', 'transaction_id'), ...}.
+ @type other_generations: dict
+ """
+ self._other_generations = {}
+ for replica_uid, [gen, trans_id] in other_generations.iteritems():
+ self._other_generations[replica_uid] = (gen, trans_id)
class CouchSyncTarget(ObjectStoreSyncTarget):
"""
Functionality for using a CouchDatabase as a synchronization target.
"""
+ pass
+
+
+class NotEnoughCouchPermissions(Exception):
+ """
+ Raised when failing to assert for enough permissions on underlying Couch
+ Database.
+ """
+ pass
class CouchServerState(ServerState):
@@ -417,8 +522,83 @@ class CouchServerState(ServerState):
Inteface of the WSGI server with the CouchDB backend.
"""
- def __init__(self, couch_url):
+ def __init__(self, couch_url, shared_db_name, tokens_db_name,
+ user_db_prefix):
+ """
+ Initialize the couch server state.
+
+ @param couch_url: The URL for the couch database.
+ @type couch_url: str
+ @param shared_db_name: The name of the shared database.
+ @type shared_db_name: str
+ @param tokens_db_name: The name of the tokens database.
+ @type tokens_db_name: str
+ @param user_db_prefix: The prefix for user database names.
+ @type user_db_prefix: str
+ """
self._couch_url = couch_url
+ self._shared_db_name = shared_db_name
+ self._tokens_db_name = tokens_db_name
+ self._user_db_prefix = user_db_prefix
+ try:
+ self._check_couch_permissions()
+ except NotEnoughCouchPermissions:
+ logger.error("Not enough permissions on underlying couch "
+ "database (%s)." % self._couch_url)
+ except (socket.error, socket.gaierror, socket.herror,
+ socket.timeout), e:
+ logger.error("Socket problem while trying to reach underlying "
+ "couch database: (%s, %s)." %
+ (self._couch_url, e))
+
+ def _check_couch_permissions(self):
+ """
+ Assert that Soledad Server has enough permissions on the underlying couch
+ database.
+
+ Soledad Server has to be able to do the following in the couch server:
+
+ * Create, read and write from/to 'shared' db.
+ * Create, read and write from/to 'user-<anything>' dbs.
+ * Read from 'tokens' db.
+
+ This function tries to perform the actions above using the "low level"
+ couch library to ensure that Soledad Server can do everything it needs on
+ the underlying couch database.
+
+ @param couch_url: The URL of the couch database.
+ @type couch_url: str
+
+ @raise NotEnoughCouchPermissions: Raised in case there are not enough
+ permissions to read/write/create the needed couch databases.
+ @rtype: bool
+ """
+
+ def _open_couch_db(dbname):
+ server = Server(url=self._couch_url)
+ try:
+ server[dbname]
+ except ResourceNotFound:
+ server.create(dbname)
+ return server[dbname]
+
+ def _create_delete_test_doc(db):
+ doc_id, _ = db.save({'test': 'document'})
+ doc = db[doc_id]
+ db.delete(doc)
+
+ try:
+ # test read/write auth for shared db
+ _create_delete_test_doc(
+ _open_couch_db(self._shared_db_name))
+ # test read/write auth for user-<something> db
+ _create_delete_test_doc(
+ _open_couch_db('%stest-db' % self._user_db_prefix))
+ # test read auth for tokens db
+ tokensdb = _open_couch_db(self._tokens_db_name)
+ tokensdb.info()
+ except Unauthorized:
+ raise NotEnoughCouchPermissions(self._couch_url)
def open_database(self, dbname):
"""
diff --git a/common/src/leap/soledad/common/objectstore.py b/common/src/leap/soledad/common/objectstore.py
index 921cf075..7aff3e32 100644
--- a/common/src/leap/soledad/common/objectstore.py
+++ b/common/src/leap/soledad/common/objectstore.py
@@ -20,6 +20,10 @@
Abstract U1DB backend to handle storage using object stores (like CouchDB, for
example).
+This backend uses special documents to store all U1DB data (replica uid,
+indexes, transaction logs and info about other dbs). The id of these documents
+are reserved and have prefix equal to ObjectStore.U1DB_DATA_DOC_ID_PREFIX.
+
Right now, this is only used by CouchDatabase backend, but can also be
extended to implement OpenStack or Amazon S3 storage, for example.
@@ -27,6 +31,13 @@ See U1DB documentation for more information on how to use databases.
"""
+from base64 import b64encode, b64decode
+
+
+import uuid
+import simplejson as json
+
+
from u1db import errors
from u1db.backends.inmemory import (
InMemoryDatabase,
@@ -39,6 +50,8 @@ class ObjectStoreDatabase(InMemoryDatabase):
A backend for storing u1db data in an object store.
"""
+ U1DB_DATA_DOC_ID_PREFIX = 'u1db/'
+
@classmethod
def open_database(cls, url, create, document_factory=None):
"""
@@ -71,24 +84,52 @@ class ObjectStoreDatabase(InMemoryDatabase):
self,
replica_uid,
document_factory=document_factory)
- # sync data in memory with data in object store
- if not self._get_doc(self.U1DB_DATA_DOC_ID):
- self._init_u1db_data()
- self._fetch_u1db_data()
+ if self._replica_uid is None:
+ self._replica_uid = uuid.uuid4().hex
+ self._init_u1db_data()
+
+ def _init_u1db_data(self):
+ """
+ Initialize u1db configuration data on backend storage.
+
+ A U1DB database needs to keep track of all database transactions,
+ document conflicts, the generation of other replicas it has seen,
+ indexes created by users and so on.
+
+ In this implementation, all this information is stored in special
+ documents stored in the couch db with id prefix equal to
+ U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved:
+ put_doc(), get_doc() and delete_doc() will not allow documents with
+ a doc_id with that prefix to be accessed or modified.
+ """
+ raise NotImplementedError(self._init_u1db_data)
#-------------------------------------------------------------------------
# methods from Database
#-------------------------------------------------------------------------
- def _set_replica_uid(self, replica_uid):
+ def put_doc(self, doc):
"""
- Force the replica_uid to be set.
+ Update a document.
- @param replica_uid: The uid of the replica.
- @type replica_uid: str
+ If the document currently has conflicts, put will fail.
+ If the database specifies a maximum document size and the document
+ exceeds it, put will fail and raise a DocumentTooBig exception.
+
+ This method prevents from updating the document with doc_id equals to
+ self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
+ @param doc: A Document with new content.
+ @type doc: Document
+
+ @return: new_doc_rev - The new revision identifier for the document.
+ The Document object will also be updated.
+ @rtype: str
"""
- InMemoryDatabase._set_replica_uid(self, replica_uid)
- self._store_u1db_data()
+ if doc.doc_id is not None and \
+ doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ raise errors.InvalidDocId()
+ return InMemoryDatabase.put_doc(self, doc)
def _put_doc(self, doc):
"""
@@ -106,6 +147,27 @@ class ObjectStoreDatabase(InMemoryDatabase):
"""
raise NotImplementedError(self._put_doc)
+ def get_doc(self, doc_id, include_deleted=False):
+ """
+ Get the JSON string for the given document.
+
+ This method prevents from getting the document with doc_id equals to
+ self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
+ @param doc_id: The unique document identifier
+ @type doc_id: str
+ @param include_deleted: If set to True, deleted documents will be
+ returned with empty content. Otherwise asking for a deleted
+ document will return None.
+ @type include_deleted: bool
+
+ @return: a Document object.
+ @rtype: Document
+ """
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ raise errors.InvalidDocId()
+ return InMemoryDatabase.get_doc(self, doc_id, include_deleted)
+
def _get_doc(self, doc_id):
"""
Get just the document content, without fancy handling.
@@ -136,18 +198,32 @@ class ObjectStoreDatabase(InMemoryDatabase):
the documents in the database.
@rtype: tuple
"""
- raise NotImplementedError(self.get_all_docs)
+ generation = self._get_generation()
+ results = []
+ for doc_id in self._database:
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ continue
+ doc = self._get_doc(doc_id, check_for_conflicts=True)
+ if doc.content is None and not include_deleted:
+ continue
+ results.append(doc)
+ return (generation, results)
def delete_doc(self, doc):
"""
Mark a document as deleted.
+ This method prevents from deleting the document with doc_id equals to
+ self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
@param doc: The document to mark as deleted.
@type doc: u1db.Document
@return: The new revision id of the document.
@type: str
"""
+ if doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ raise errors.InvalidDocId()
old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
if old_doc is None:
raise errors.DocumentDoesNotExist
@@ -177,58 +253,6 @@ class ObjectStoreDatabase(InMemoryDatabase):
"""
raise NotImplementedError(self.create_index)
- def delete_index(self, index_name):
- """
- Remove a named index.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param index_name: The name of the index we are removing.
- @type index_name: str
- """
- InMemoryDatabase.delete_index(self, index_name)
- self._store_u1db_data()
-
- def _replace_conflicts(self, doc, conflicts):
- """
- Set new conflicts for a document.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param doc: The document with a new set of conflicts.
- @param conflicts: The new set of conflicts.
- @type conflicts: list
- """
- InMemoryDatabase._replace_conflicts(self, doc, conflicts)
- self._store_u1db_data()
-
- def _do_set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation,
- other_transaction_id):
- """
- Set the last-known generation and transaction id for the other
- database replica.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param other_replica_uid: The U1DB identifier for the other replica.
- @type other_replica_uid: str
- @param other_generation: The generation number for the other replica.
- @type other_generation: int
- @param other_transaction_id: The transaction id associated with the
- generation.
- @type other_transaction_id: str
- """
- InMemoryDatabase._do_set_replica_gen_and_trans_id(
- self,
- other_replica_uid,
- other_generation,
- other_transaction_id)
- self._store_u1db_data()
-
#-------------------------------------------------------------------------
# implemented methods from CommonBackend
#-------------------------------------------------------------------------
@@ -250,45 +274,6 @@ class ObjectStoreDatabase(InMemoryDatabase):
trans_id = self._allocate_transaction_id()
self._put_doc(doc)
self._transaction_log.append((doc.doc_id, trans_id))
- self._store_u1db_data()
-
- #-------------------------------------------------------------------------
- # methods specific for object stores
- #-------------------------------------------------------------------------
-
- U1DB_DATA_DOC_ID = 'u1db_data'
-
- def _fetch_u1db_data(self):
- """
- Fetch u1db configuration data from backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._fetch_u1db_data)
-
- def _store_u1db_data(self):
- """
- Store u1db configuration data on backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._store_u1db_data)
-
- def _init_u1db_data(self):
- """
- Initialize u1db configuration data on backend storage.
-
- A U1DB database needs to keep track of all database transactions,
- document conflicts, the generation of other replicas it has seen,
- indexes created by users and so on.
-
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- NotImplementedError(self._init_u1db_data)
class ObjectStoreSyncTarget(InMemorySyncTarget):
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 2fb799b7..42edf9fe 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -178,7 +178,12 @@ def copy_couch_database_for_test(test, db):
new_db._conflicts = copy.deepcopy(db._conflicts)
new_db._other_generations = copy.deepcopy(db._other_generations)
new_db._indexes = copy.deepcopy(db._indexes)
- new_db._store_u1db_data()
+ # save u1db data on couch
+ for key in new_db.U1DB_DATA_KEYS:
+ doc_id = '%s%s' % (new_db.U1DB_DATA_DOC_ID_PREFIX, key)
+ doc = new_db._get_doc(doc_id)
+ doc.content = {'content': getattr(new_db, key)}
+ new_db._put_doc(doc)
return new_db
@@ -271,6 +276,14 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests,
scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios))
+ def setUp(self):
+ # we implement parents' setUp methods here to prevent from launching
+ # more couch instances then needed.
+ tests.TestCase.setUp(self)
+ self.server = self.server_thread = None
+ self.db, self.st = self.create_db_and_target(self)
+ self.other_changes = []
+
def tearDown(self):
self.db.delete_database()
test_sync.DatabaseSyncTargetTests.tearDown(self)
@@ -345,20 +358,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
return [self._listify(i) for i in l]
return l
- def _fetch_u1db_data(self, db):
- cdoc = db._database.get(db.U1DB_DATA_DOC_ID)
- jsonstr = db._database.get_attachment(cdoc, 'u1db_json').getvalue()
- return json.loads(jsonstr)
+ def _fetch_u1db_data(self, db, key):
+ doc = db._get_doc("%s%s" % (db.U1DB_DATA_DOC_ID_PREFIX, key))
+ return doc.content['content']
def test_transaction_log_storage_after_put(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
db.create_doc({'simple': 'doc'})
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_TRANSACTION_LOG_KEY)
self.assertEqual(
self._listify(db._transaction_log),
- self._listify(
- json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY]))))
+ self._listify(content))
def test_conflict_log_storage_after_put_if_newer(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -367,29 +378,27 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
doc.set_json(nested_doc)
doc.rev = db._replica_uid + ':2'
db._force_doc_sync_conflict(doc)
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_CONFLICTS_KEY)
self.assertEqual(
self._listify(db._conflicts),
- self._listify(
- json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY]))))
+ self._listify(content))
def test_other_gens_storage_after_set(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
doc = db.create_doc({'simple': 'doc'})
db._set_replica_gen_and_trans_id('a', 'b', 'c')
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_OTHER_GENERATIONS_KEY)
self.assertEqual(
self._listify(db._other_generations),
- self._listify(
- json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY]))))
+ self._listify(content))
def test_index_storage_after_create(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
doc = db.create_doc({'name': 'john'})
db.create_index('myindex', 'name')
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
myind = db._indexes['myindex']
index = {
'myindex': {
@@ -400,8 +409,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
}
self.assertEqual(
self._listify(index),
- self._listify(
- json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
+ self._listify(content))
def test_index_storage_after_delete(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -410,7 +418,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
db.create_index('myindex', 'name')
db.create_index('myindex2', 'name')
db.delete_index('myindex')
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
myind = db._indexes['myindex2']
index = {
'myindex2': {
@@ -421,16 +429,13 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
}
self.assertEqual(
self._listify(index),
- self._listify(
- json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
+ self._listify(content))
def test_replica_uid_storage_after_db_creation(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
- content = self._fetch_u1db_data(db)
- self.assertEqual(
- db._replica_uid,
- b64decode(content[db.U1DB_REPLICA_UID_KEY]))
+ content = self._fetch_u1db_data(db, db.U1DB_REPLICA_UID_KEY)
+ self.assertEqual(db._replica_uid, content)
load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py
index 01b43299..db217bb3 100644
--- a/common/src/leap/soledad/common/tests/test_crypto.py
+++ b/common/src/leap/soledad/common/tests/test_crypto.py
@@ -29,9 +29,6 @@ import binascii
from leap.common.testing.basetest import BaseLeapTest
-from Crypto import Random
-
-
from leap.soledad.client import (
Soledad,
crypto,
@@ -193,7 +190,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
def test_encrypt_decrypt_sym(self):
# generate 256-bit key
- key = Random.new().read(32)
+ key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
method=crypto.EncryptionMethods.AES_256_CTR)
@@ -206,7 +203,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
self.assertEqual('data', plaintext)
def test_decrypt_with_wrong_iv_fails(self):
- key = Random.new().read(32)
+ key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
method=crypto.EncryptionMethods.AES_256_CTR)
@@ -224,17 +221,17 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
self.assertNotEqual('data', plaintext)
def test_decrypt_with_wrong_key_fails(self):
- key = Random.new().read(32)
+ key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
method=crypto.EncryptionMethods.AES_256_CTR)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
- wrongkey = Random.new().read(32) # 256-bits key
+ wrongkey = os.urandom(32) # 256-bits key
# ensure keys are different in case we are extremely lucky
while wrongkey == key:
- wrongkey = Random.new().read(32)
+ wrongkey = os.urandom(32)
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, wrongkey, iv=iv,
method=crypto.EncryptionMethods.AES_256_CTR)
@@ -245,7 +242,7 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
def test_encrypt_decrypt_sym(self):
# generate 256-bit key
- key = Random.new().read(32)
+ key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
method=crypto.EncryptionMethods.XSALSA20)
@@ -258,7 +255,7 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
self.assertEqual('data', plaintext)
def test_decrypt_with_wrong_iv_fails(self):
- key = Random.new().read(32)
+ key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
method=crypto.EncryptionMethods.XSALSA20)
@@ -276,17 +273,17 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
self.assertNotEqual('data', plaintext)
def test_decrypt_with_wrong_key_fails(self):
- key = Random.new().read(32)
+ key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
method=crypto.EncryptionMethods.XSALSA20)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
- wrongkey = Random.new().read(32) # 256-bits key
+ wrongkey = os.urandom(32) # 256-bits key
# ensure keys are different in case we are extremely lucky
while wrongkey == key:
- wrongkey = Random.new().read(32)
+ wrongkey = os.urandom(32)
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, wrongkey, iv=iv,
method=crypto.EncryptionMethods.XSALSA20)
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index beb7e04d..1ea4d615 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -310,7 +310,8 @@ class EncryptedSyncTestCase(
secret_id=secret_id)
def make_app(self):
- self.request_state = CouchServerState(self._couch_url)
+ self.request_state = CouchServerState(
+ self._couch_url, 'shared', 'tokens', 'user-')
return self.make_app_with_state(self.request_state)
def setUp(self):
diff --git a/server/changes/feature_3501-add-verification-for-couch-permissions b/server/changes/feature_3501-add-verification-for-couch-permissions
new file mode 100644
index 00000000..9206c708
--- /dev/null
+++ b/server/changes/feature_3501-add-verification-for-couch-permissions
@@ -0,0 +1 @@
+ o Verify for couch permissions when starting server. Closes #3501.
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 67b0611d..b4b715e2 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -19,12 +19,19 @@
"""
A U1DB server that stores data using CouchDB as its persistence layer.
-This should be run with:
- twistd -n web --wsgi=leap.soledad.server.application --port=2424
+This is written as a Twisted application and intended to be run using the
+twistd command. To start the soledad server, run:
+
+ twistd -n web --wsgi=leap.soledad.server.application --port=X
+
+An initscript is included and will be installed system wide to make it
+feasible to start and stop the Soledad server service using a standard
+interface.
"""
import configparser
+
from u1db.remote import http_app
@@ -116,13 +123,18 @@ def load_configuration(file_path):
# Run as Twisted WSGI Resource
#-----------------------------------------------------------------------------
-conf = load_configuration('/etc/leap/soledad-server.conf')
-state = CouchServerState(conf['couch_url'])
-
-# WSGI application that may be used by `twistd -web`
-application = SoledadTokenAuthMiddleware(SoledadApp(state))
+def application(environ, start_response):
+ conf = load_configuration('/etc/leap/soledad-server.conf')
+ state = CouchServerState(
+ conf['couch_url'],
+ SoledadApp.SHARED_DB_NAME,
+ SoledadTokenAuthMiddleware.TOKENS_DB,
+ SoledadApp.USER_DB_PREFIX)
+ # WSGI application that may be used by `twistd -web`
+ application = SoledadTokenAuthMiddleware(SoledadApp(state))
+ resource = WSGIResource(reactor, reactor.getThreadPool(), application)
+ return application(environ, start_response)
-resource = WSGIResource(reactor, reactor.getThreadPool(), application)
from ._version import get_versions
__version__ = get_versions()['version']
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-06 16:03:16 +0000