Merge tag '0.4.0' into debian-0.4.0

Tag leap.soledad version 0.4.0

10 files changed, 446 insertions, 249 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 6d797d89..d400fc52 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,13 @@
+0.4.0 Sep 20:
+Client:
+  o Remove redundant logging when creating data dirs.
+Server:
+  o Verify for couch permissions when starting server. Closes #3501.
+Common:
+  o Improve u1db data storage in couch. Closes #3647.
+  o Turn couchdb dependency for common into optional. Closes #2167.
+  o Add verification for couch permissions. Closes #3501.
+
 0.3.2 Sep 6:
 Client:
   o Use dirspec instead of plain xdg. Closes #3574.
diff --git a/README.rst b/README.rst
index 8fa966c2..61a15ee4 100644
--- a/README.rst
+++ b/README.rst
@@ -12,20 +12,24 @@ There are currently three python packages maintained under this
 repository:
 
 **leap.soledad.common** common pieces.
+
 .. image:: https://pypip.in/v/leap.soledad.common/badge.png
         :target: https://crate.io/packages/leap.soledad.common
 
 **leap.soledad.client** where the soledad client lives.
+
 .. image:: https://pypip.in/v/leap.soledad.client/badge.png
         :target: https://crate.io/packages/leap.soledad.client
 
 **leap.soledad.server** oh surprise! bits needed for the soledad server.
+
 .. image:: https://pypip.in/v/leap.soledad.server/badge.png
         :target: https://crate.io/packages/leap.soledad.server
 
 
 Library dependencies
 --------------------
+
 * ``libsqlite3-dev``
 
 Tests
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index c6fbeac4..4c6a41fc 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -14,8 +14,6 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
 """
 Soledad - Synchronization Of Locally Encrypted Data Among Devices.
 
@@ -25,22 +23,23 @@ implements (1) a SQLCipher backend for local storage in the client, (2) a
 SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for
 remote storage in the server side.
 """
-
-import os
 import binascii
-import logging
-import urlparse
-import simplejson as json
-import scrypt
+import errno
 import httplib
+import logging
+import os
 import socket
 import ssl
-import errno
+import urlparse
 
 from hashlib import sha256
+
 from u1db.remote import http_client
 from u1db.remote.ssl_match_hostname import match_hostname
 
+import scrypt
+import simplejson as json
+
 from leap.common.config import get_path_prefix
 
 #
@@ -346,8 +345,9 @@ class Soledad(object):
             lambda x: os.path.dirname(x),
             [self._local_db_path, self._secrets_path])
         for path in paths:
-            logger.info('Creating directory: %s.' % path)
             try:
+                if not os.path.isdir(path):
+                    logger.info('Creating directory: %s.' % path)
                 os.makedirs(path)
             except OSError as exc:
                 if exc.errno == errno.EEXIST and os.path.isdir(path):
diff --git a/common/setup.py b/common/setup.py
index 0a2f138f..9af61be7 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -67,4 +67,7 @@ setup(
     install_requires=utils.parse_requirements(),
     tests_require=utils.parse_requirements(
         reqfiles=['pkg/requirements-testing.pip']),
+    extras_require={
+        'couchdb': ['couchdb'],
+    },
 )
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 9642e8f3..187d3035 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -18,19 +18,19 @@
 
 """A U1DB backend that uses CouchDB as its persistence layer."""
 
-import uuid
 import re
 import simplejson as json
+import socket
+import logging
 
 
-from base64 import b64encode, b64decode
 from u1db import errors
 from u1db.sync import Synchronizer
 from u1db.backends.inmemory import InMemoryIndex
 from u1db.remote.server_state import ServerState
 from u1db.errors import DatabaseDoesNotExist
 from couchdb.client import Server, Document as CouchDocument
-from couchdb.http import ResourceNotFound
+from couchdb.http import ResourceNotFound, Unauthorized
 
 
 from leap.soledad.common.objectstore import (
@@ -39,28 +39,155 @@ from leap.soledad.common.objectstore import (
 )
 
 
+logger = logging.getLogger(__name__)
+
+
 class InvalidURLError(Exception):
     """
     Exception raised when Soledad encounters a malformed URL.
     """
 
 
+def persistent_class(cls):
+    """
+    Decorator that modifies a class to ensure u1db metadata persists on
+    underlying storage.
+
+    @param cls: The class that will be modified.
+    @type cls: type
+    """
+
+    def _create_persistent_method(old_method_name, key, load_method_name,
+                                  dump_method_name, store):
+        """
+        Create a persistent method to replace C{old_method_name}.
+        
+        The new method will load C{key} using C{load_method_name} and stores
+        it using C{dump_method_name} depending on the value of C{store}.
+        """
+        # get methods
+        old_method = getattr(cls, old_method_name)
+        load_method = getattr(cls, load_method_name) \
+            if load_method_name is not None \
+            else lambda self, data: setattr(self, key, data)
+        dump_method = getattr(cls, dump_method_name) \
+            if dump_method_name is not None \
+            else lambda self: getattr(self, key)
+
+        def _new_method(self, *args, **kwargs):
+            # get u1db data from couch db
+            doc = self._get_doc('%s%s' %
+                                (self.U1DB_DATA_DOC_ID_PREFIX, key))
+            load_method(self, doc.content['content'])
+            # run old method
+            retval = old_method(self, *args, **kwargs)
+            # store u1db data on couch
+            if store:
+                doc.content = {'content': dump_method(self)}
+                self._put_doc(doc)
+            return retval
+
+        return _new_method
+
+    # ensure the class has a persistency map
+    if not hasattr(cls, 'PERSISTENCY_MAP'):
+        logger.error('Class %s has no PERSISTENCY_MAP attribute, skipping '
+                     'persistent methods substitution.' % cls)
+        return cls
+    # replace old methods with new persistent ones
+    for key, ((load_method_name, dump_method_name),
+              persistent_methods) in cls.PERSISTENCY_MAP.iteritems():
+        for (method_name, store) in persistent_methods:
+            setattr(cls, method_name,
+                    _create_persistent_method(
+                        method_name,
+                        key,
+                        load_method_name,
+                        dump_method_name,
+                        store))
+    return cls
+
+
+@persistent_class
 class CouchDatabase(ObjectStoreDatabase):
     """
     A U1DB backend that uses Couch as its persistence layer.
     """
 
-    U1DB_TRANSACTION_LOG_KEY = 'transaction_log'
-    U1DB_CONFLICTS_KEY = 'conflicts'
-    U1DB_OTHER_GENERATIONS_KEY = 'other_generations'
-    U1DB_INDEXES_KEY = 'indexes'
-    U1DB_REPLICA_UID_KEY = 'replica_uid'
+    U1DB_TRANSACTION_LOG_KEY = '_transaction_log'
+    U1DB_CONFLICTS_KEY = '_conflicts'
+    U1DB_OTHER_GENERATIONS_KEY = '_other_generations'
+    U1DB_INDEXES_KEY = '_indexes'
+    U1DB_REPLICA_UID_KEY = '_replica_uid'
+
+    U1DB_DATA_KEYS = [
+        U1DB_TRANSACTION_LOG_KEY,
+        U1DB_CONFLICTS_KEY,
+        U1DB_OTHER_GENERATIONS_KEY,
+        U1DB_INDEXES_KEY,
+        U1DB_REPLICA_UID_KEY,
+    ]
 
     COUCH_ID_KEY = '_id'
     COUCH_REV_KEY = '_rev'
     COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json'
     COUCH_U1DB_REV_KEY = 'u1db_rev'
 
+    # the following map describes information about methods usage of
+    # properties that have to persist on the underlying database. The format
+    # of the map is assumed to be:
+    #
+    #     {
+    #         'property_name': [
+    #             ('property_load_method_name', 'property_dump_method_name'),
+    #             [('method_1_name', bool),
+    #              ...
+    #              ('method_N_name', bool)]],
+    #         ...
+    #     }
+    #
+    # where the booleans indicate if the property should be stored after
+    # each method execution (i.e. if the method alters the property). Property
+    # load/dump methods will be run after/before properties are read/written
+    # to the underlying db.
+    PERSISTENCY_MAP = {
+        U1DB_TRANSACTION_LOG_KEY: [
+            ('_load_transaction_log_from_json', None),
+            [('_get_transaction_log', False),
+             ('_get_generation', False),
+             ('_get_generation_info', False),
+             ('_get_trans_id_for_gen', False),
+             ('whats_changed', False),
+             ('_put_and_update_indexes', True)]],
+        U1DB_CONFLICTS_KEY: [
+            (None, None),
+            [('_has_conflicts', False),
+             ('get_doc_conflicts', False),
+             ('_prune_conflicts', False),
+             ('resolve_doc', False),
+             ('_replace_conflicts', True),
+             ('_force_doc_sync_conflict', True)]],
+        U1DB_OTHER_GENERATIONS_KEY: [
+            ('_load_other_generations_from_json', None),
+            [('_get_replica_gen_and_trans_id', False),
+             ('_do_set_replica_gen_and_trans_id', True)]],
+        U1DB_INDEXES_KEY: [
+            ('_load_indexes_from_json', '_dump_indexes_as_json'),
+            [('list_indexes', False),
+             ('get_from_index', False),
+             ('get_range_from_index', False),
+             ('get_index_keys', False),
+             ('_put_and_update_indexes', True),
+             ('create_index', True),
+             ('delete_index', True)]],
+        U1DB_REPLICA_UID_KEY: [
+            (None, None),
+            [('_allocate_doc_rev', False),
+             ('_put_doc_if_newer', False),
+             ('_ensure_maximal_rev', False),
+             ('_prune_conflicts', False),
+             ('_set_replica_uid', True)]]}
+
     @classmethod
     def open_database(cls, url, create):
         """
@@ -104,9 +231,11 @@ class CouchDatabase(ObjectStoreDatabase):
         @param session: an http.Session instance or None for a default session
         @type session: http.Session
         """
+        # save params
         self._url = url
         self._full_commit = full_commit
         self._session = session
+        # configure couch
         self._server = Server(url=self._url,
                               full_commit=self._full_commit,
                               session=self._session)
@@ -174,7 +303,7 @@ class CouchDatabase(ObjectStoreDatabase):
         generation = self._get_generation()
         results = []
         for doc_id in self._database:
-            if doc_id == self.U1DB_DATA_DOC_ID:
+            if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
                 continue
             doc = self._get_doc(doc_id, check_for_conflicts=True)
             if doc.content is None and not include_deleted:
@@ -241,14 +370,12 @@ class CouchDatabase(ObjectStoreDatabase):
             raise errors.IndexNameTakenError
         index = InMemoryIndex(index_name, list(index_expressions))
         for doc_id in self._database:
-            if doc_id == self.U1DB_DATA_DOC_ID:  # skip special file
-                continue
+            if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+                continue  # skip special files
             doc = self._get_doc(doc_id)
             if doc.content is not None:
                 index.add_json(doc_id, doc.get_json())
         self._indexes[index_name] = index
-        # save data in object store
-        self._store_u1db_data()
 
     def close(self):
         """
@@ -290,76 +417,25 @@ class CouchDatabase(ObjectStoreDatabase):
 
     def _init_u1db_data(self):
         """
-        Initialize U1DB info data structure in the couch db.
+        Initialize u1db configuration data on backend storage.
 
         A U1DB database needs to keep track of all database transactions,
         document conflicts, the generation of other replicas it has seen,
         indexes created by users and so on.
 
-        In this implementation, all this information is stored in a special
-        document stored in the couch db with id equals to
-        CouchDatabse.U1DB_DATA_DOC_ID.
-
-        This method initializes the document that will hold such information.
-        """
-        if self._replica_uid is None:
-            self._replica_uid = uuid.uuid4().hex
-        # TODO: prevent user from overwriting a document with the same doc_id
-        # as this one.
-        doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
-        doc.content = {
-            self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
-            self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
-            self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
-            self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
-            self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
-        }
-        self._put_doc(doc)
-
-    def _fetch_u1db_data(self):
-        """
-        Fetch U1DB info from the couch db.
-
-        See C{_init_u1db_data} documentation.
-        """
-        # retrieve u1db data from couch db
-        cdoc = self._database.get(self.U1DB_DATA_DOC_ID)
-        jsonstr = self._database.get_attachment(
-            cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
-        content = json.loads(jsonstr)
-        # set u1db database info
-        self._transaction_log = json.loads(
-            b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
-        self._conflicts = json.loads(
-            b64decode(content[self.U1DB_CONFLICTS_KEY]))
-        self._other_generations = json.loads(
-            b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
-        self._indexes = self._load_indexes_from_json(
-            b64decode(content[self.U1DB_INDEXES_KEY]))
-        self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
-        # save couch _rev
-        self._couch_rev = cdoc[self.COUCH_REV_KEY]
-
-    def _store_u1db_data(self):
-        """
-        Store U1DB info in the couch db.
-
-        See C{_init_u1db_data} documentation.
-        """
-        doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
-        doc.content = {
-            # Here, the b64 encode ensures that document content
-            # does not cause strange behaviour in couchdb because
-            # of encoding.
-            self.U1DB_TRANSACTION_LOG_KEY:
-                b64encode(json.dumps(self._transaction_log)),
-            self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
-            self.U1DB_OTHER_GENERATIONS_KEY:
-                b64encode(json.dumps(self._other_generations)),
-            self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
-            self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
-            self.COUCH_REV_KEY: self._couch_rev}
-        self._put_doc(doc)
+        In this implementation, all this information is stored in special
+        documents stored in the underlying with doc_id prefix equal to
+        U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved: put_doc(),
+        get_doc() and delete_doc() will not allow documents with a doc_id with
+        that prefix to be accessed or modified.
+        """
+        for key in self.U1DB_DATA_KEYS:
+            doc_id = '%s%s' % (self.U1DB_DATA_DOC_ID_PREFIX, key)
+            doc = self._get_doc(doc_id)
+            if doc is None:
+                doc = self._factory(doc_id)
+                doc.content = {'content': getattr(self, key)}
+                self._put_doc(doc)
 
     #-------------------------------------------------------------------------
     # Couch specific methods
@@ -377,7 +453,7 @@ class CouchDatabase(ObjectStoreDatabase):
 
     def _dump_indexes_as_json(self):
         """
-        Dump index definitions as JSON string.
+        Dump index definitions as JSON.
         """
         indexes = {}
         for name, idx in self._indexes.iteritems():
@@ -385,31 +461,60 @@ class CouchDatabase(ObjectStoreDatabase):
             for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY,
                          self.INDEX_VALUES_KEY]:
                 indexes[name][attr] = getattr(idx, '_' + attr)
-        return json.dumps(indexes)
+        return indexes
 
     def _load_indexes_from_json(self, indexes):
         """
-        Load index definitions from JSON string.
+        Load index definitions from stored JSON.
 
-        @param indexes: A JSON serialization of a list of [('index-name',
-            ['field', 'field2'])].
+        @param indexes: A JSON representation of indexes as
+            [('index-name', ['field', 'field2', ...]), ...].
         @type indexes: str
-
-        @return: A dictionary with the index definitions.
-        @rtype: dict
         """
-        dict = {}
-        for name, idx_dict in json.loads(indexes).iteritems():
+        self._indexes = {}
+        for name, idx_dict in indexes.iteritems():
             idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY])
             idx._values = idx_dict[self.INDEX_VALUES_KEY]
-            dict[name] = idx
-        return dict
+            self._indexes[name] = idx
+
+    def _load_transaction_log_from_json(self, transaction_log):
+        """
+        Load transaction log from stored JSON.
+
+        @param transaction_log: A JSON representation of transaction_log as
+            [('generation', 'transaction_id'), ...].
+        @type transaction_log: list
+        """
+        self._transaction_log = []
+        for gen, trans_id in transaction_log:
+            self._transaction_log.append((gen, trans_id))
+
+    def _load_other_generations_from_json(self, other_generations):
+        """
+        Load other generations from stored JSON.
+
+        @param other_generations: A JSON representation of other_generations
+            as {'replica_uid': ('generation', 'transaction_id'), ...}.
+        @type other_generations: dict
+        """
+        self._other_generations = {}
+        for replica_uid, [gen, trans_id] in other_generations.iteritems():
+            self._other_generations[replica_uid] = (gen, trans_id)
 
 
 class CouchSyncTarget(ObjectStoreSyncTarget):
     """
     Functionality for using a CouchDatabase as a synchronization target.
     """
+    pass
+
+
+class NotEnoughCouchPermissions(Exception):
+    """
+    Raised when failing to assert for enough permissions on underlying Couch
+    Database.
+    """
+    pass
 
 
 class CouchServerState(ServerState):
@@ -417,8 +522,83 @@ class CouchServerState(ServerState):
     Inteface of the WSGI server with the CouchDB backend.
     """
 
-    def __init__(self, couch_url):
+    def __init__(self, couch_url, shared_db_name, tokens_db_name,
+                 user_db_prefix):
+        """
+        Initialize the couch server state.
+
+        @param couch_url: The URL for the couch database.
+        @type couch_url: str
+        @param shared_db_name: The name of the shared database.
+        @type shared_db_name: str
+        @param tokens_db_name: The name of the tokens database.
+        @type tokens_db_name: str
+        @param user_db_prefix: The prefix for user database names.
+        @type user_db_prefix: str
+        """
         self._couch_url = couch_url
+        self._shared_db_name = shared_db_name
+        self._tokens_db_name = tokens_db_name
+        self._user_db_prefix = user_db_prefix
+        try:
+            self._check_couch_permissions()
+        except NotEnoughCouchPermissions:
+            logger.error("Not enough permissions on underlying couch "
+                         "database (%s)." % self._couch_url)
+        except (socket.error, socket.gaierror, socket.herror,
+                socket.timeout), e:
+            logger.error("Socket problem while trying to reach underlying "
+                         "couch database: (%s, %s)." %
+                         (self._couch_url, e))
+
+    def _check_couch_permissions(self):
+        """
+        Assert that Soledad Server has enough permissions on the underlying couch
+        database.
+
+        Soledad Server has to be able to do the following in the couch server:
+
+            * Create, read and write from/to 'shared' db.
+            * Create, read and write from/to 'user-<anything>' dbs.
+            * Read from 'tokens' db.
+
+        This function tries to perform the actions above using the "low level"
+        couch library to ensure that Soledad Server can do everything it needs on
+        the underlying couch database.
+
+        @param couch_url: The URL of the couch database.
+        @type couch_url: str
+
+        @raise NotEnoughCouchPermissions: Raised in case there are not enough
+            permissions to read/write/create the needed couch databases.
+        @rtype: bool
+        """
+
+        def _open_couch_db(dbname):
+            server = Server(url=self._couch_url)
+            try:
+                server[dbname]
+            except ResourceNotFound:
+                server.create(dbname)
+            return server[dbname]
+
+        def _create_delete_test_doc(db):
+            doc_id, _ = db.save({'test': 'document'})
+            doc = db[doc_id]
+            db.delete(doc)
+
+        try:
+            # test read/write auth for shared db
+            _create_delete_test_doc(
+                _open_couch_db(self._shared_db_name))
+            # test read/write auth for user-<something> db
+            _create_delete_test_doc(
+                _open_couch_db('%stest-db' % self._user_db_prefix))
+            # test read auth for tokens db
+            tokensdb = _open_couch_db(self._tokens_db_name)
+            tokensdb.info()
+        except Unauthorized:
+            raise NotEnoughCouchPermissions(self._couch_url)
 
     def open_database(self, dbname):
         """
diff --git a/common/src/leap/soledad/common/objectstore.py b/common/src/leap/soledad/common/objectstore.py
index 921cf075..7aff3e32 100644
--- a/common/src/leap/soledad/common/objectstore.py
+++ b/common/src/leap/soledad/common/objectstore.py
@@ -20,6 +20,10 @@
 Abstract U1DB backend to handle storage using object stores (like CouchDB, for
 example).
 
+This backend uses special documents to store all U1DB data (replica uid,
+indexes, transaction logs and info about other dbs). The id of these documents
+are reserved and have prefix equal to ObjectStore.U1DB_DATA_DOC_ID_PREFIX.
+
 Right now, this is only used by CouchDatabase backend, but can also be
 extended to implement OpenStack or Amazon S3 storage, for example.
 
@@ -27,6 +31,13 @@ See U1DB documentation for more information on how to use databases.
 """
 
 
+from base64 import b64encode, b64decode
+
+
+import uuid
+import simplejson as json
+
+
 from u1db import errors
 from u1db.backends.inmemory import (
     InMemoryDatabase,
@@ -39,6 +50,8 @@ class ObjectStoreDatabase(InMemoryDatabase):
     A backend for storing u1db data in an object store.
     """
 
+    U1DB_DATA_DOC_ID_PREFIX = 'u1db/'
+
     @classmethod
     def open_database(cls, url, create, document_factory=None):
         """
@@ -71,24 +84,52 @@ class ObjectStoreDatabase(InMemoryDatabase):
             self,
             replica_uid,
             document_factory=document_factory)
-        # sync data in memory with data in object store
-        if not self._get_doc(self.U1DB_DATA_DOC_ID):
-            self._init_u1db_data()
-        self._fetch_u1db_data()
+        if self._replica_uid is None:
+            self._replica_uid = uuid.uuid4().hex
+        self._init_u1db_data()
+
+    def _init_u1db_data(self):
+        """
+        Initialize u1db configuration data on backend storage.
+
+        A U1DB database needs to keep track of all database transactions,
+        document conflicts, the generation of other replicas it has seen,
+        indexes created by users and so on.
+
+        In this implementation, all this information is stored in special
+        documents stored in the couch db with id prefix equal to
+        U1DB_DATA_DOC_ID_PREFIX.  Those documents ids are reserved:
+        put_doc(), get_doc() and delete_doc() will not allow documents with
+        a doc_id with that prefix to be accessed or modified.
+        """
+        raise NotImplementedError(self._init_u1db_data)
 
     #-------------------------------------------------------------------------
     # methods from Database
     #-------------------------------------------------------------------------
 
-    def _set_replica_uid(self, replica_uid):
+    def put_doc(self, doc):
         """
-        Force the replica_uid to be set.
+        Update a document.
 
-        @param replica_uid: The uid of the replica.
-        @type replica_uid: str
+        If the document currently has conflicts, put will fail.
+        If the database specifies a maximum document size and the document
+        exceeds it, put will fail and raise a DocumentTooBig exception.
+
+        This method prevents from updating the document with doc_id equals to
+        self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
+        @param doc: A Document with new content.
+        @type doc: Document
+
+        @return: new_doc_rev - The new revision identifier for the document.
+            The Document object will also be updated.
+        @rtype: str
         """
-        InMemoryDatabase._set_replica_uid(self, replica_uid)
-        self._store_u1db_data()
+        if doc.doc_id is not None and \
+                doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+            raise errors.InvalidDocId()
+        return InMemoryDatabase.put_doc(self, doc)
 
     def _put_doc(self, doc):
         """
@@ -106,6 +147,27 @@ class ObjectStoreDatabase(InMemoryDatabase):
         """
         raise NotImplementedError(self._put_doc)
 
+    def get_doc(self, doc_id, include_deleted=False):
+        """
+        Get the JSON string for the given document.
+
+        This method prevents from getting the document with doc_id equals to
+        self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
+        @param doc_id: The unique document identifier
+        @type doc_id: str
+        @param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise asking for a deleted
+            document will return None.
+        @type include_deleted: bool
+
+        @return: a Document object.
+        @rtype: Document
+        """
+        if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+            raise errors.InvalidDocId()
+        return InMemoryDatabase.get_doc(self, doc_id, include_deleted)
+
     def _get_doc(self, doc_id):
         """
         Get just the document content, without fancy handling.
@@ -136,18 +198,32 @@ class ObjectStoreDatabase(InMemoryDatabase):
             the documents in the database.
         @rtype: tuple
         """
-        raise NotImplementedError(self.get_all_docs)
+        generation = self._get_generation()
+        results = []
+        for doc_id in self._database:
+            if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+                continue
+            doc = self._get_doc(doc_id, check_for_conflicts=True)
+            if doc.content is None and not include_deleted:
+                continue
+            results.append(doc)
+        return (generation, results)
 
     def delete_doc(self, doc):
         """
         Mark a document as deleted.
 
+        This method prevents from deleting the document with doc_id equals to
+        self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
         @param doc: The document to mark as deleted.
         @type doc: u1db.Document
 
         @return: The new revision id of the document.
         @type: str
         """
+        if doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+            raise errors.InvalidDocId()
         old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
         if old_doc is None:
             raise errors.DocumentDoesNotExist
@@ -177,58 +253,6 @@ class ObjectStoreDatabase(InMemoryDatabase):
         """
         raise NotImplementedError(self.create_index)
 
-    def delete_index(self, index_name):
-        """
-        Remove a named index.
-
-        Here we just guarantee that the new info will be stored in the backend
-        db after update.
-
-        @param index_name: The name of the index we are removing.
-        @type index_name: str
-        """
-        InMemoryDatabase.delete_index(self, index_name)
-        self._store_u1db_data()
-
-    def _replace_conflicts(self, doc, conflicts):
-        """
-        Set new conflicts for a document.
-
-        Here we just guarantee that the new info will be stored in the backend
-        db after update.
-
-        @param doc: The document with a new set of conflicts.
-        @param conflicts: The new set of conflicts.
-        @type conflicts: list
-        """
-        InMemoryDatabase._replace_conflicts(self, doc, conflicts)
-        self._store_u1db_data()
-
-    def _do_set_replica_gen_and_trans_id(self, other_replica_uid,
-                                         other_generation,
-                                         other_transaction_id):
-        """
-        Set the last-known generation and transaction id for the other
-        database replica.
-
-        Here we just guarantee that the new info will be stored in the backend
-        db after update.
-
-        @param other_replica_uid: The U1DB identifier for the other replica.
-        @type other_replica_uid: str
-        @param other_generation: The generation number for the other replica.
-        @type other_generation: int
-        @param other_transaction_id: The transaction id associated with the
-            generation.
-        @type other_transaction_id: str
-        """
-        InMemoryDatabase._do_set_replica_gen_and_trans_id(
-            self,
-            other_replica_uid,
-            other_generation,
-            other_transaction_id)
-        self._store_u1db_data()
-
     #-------------------------------------------------------------------------
     # implemented methods from CommonBackend
     #-------------------------------------------------------------------------
@@ -250,45 +274,6 @@ class ObjectStoreDatabase(InMemoryDatabase):
         trans_id = self._allocate_transaction_id()
         self._put_doc(doc)
         self._transaction_log.append((doc.doc_id, trans_id))
-        self._store_u1db_data()
-
-    #-------------------------------------------------------------------------
-    # methods specific for object stores
-    #-------------------------------------------------------------------------
-
-    U1DB_DATA_DOC_ID = 'u1db_data'
-
-    def _fetch_u1db_data(self):
-        """
-        Fetch u1db configuration data from backend storage.
-
-        See C{_init_u1db_data} documentation.
-        """
-        NotImplementedError(self._fetch_u1db_data)
-
-    def _store_u1db_data(self):
-        """
-        Store u1db configuration data on backend storage.
-
-        See C{_init_u1db_data} documentation.
-        """
-        NotImplementedError(self._store_u1db_data)
-
-    def _init_u1db_data(self):
-        """
-        Initialize u1db configuration data on backend storage.
-
-        A U1DB database needs to keep track of all database transactions,
-        document conflicts, the generation of other replicas it has seen,
-        indexes created by users and so on.
-
-        In this implementation, all this information is stored in a special
-        document stored in the couch db with id equals to
-        CouchDatabse.U1DB_DATA_DOC_ID.
-
-        This method initializes the document that will hold such information.
-        """
-        NotImplementedError(self._init_u1db_data)
 
 
 class ObjectStoreSyncTarget(InMemorySyncTarget):
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 2fb799b7..42edf9fe 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -178,7 +178,12 @@ def copy_couch_database_for_test(test, db):
     new_db._conflicts = copy.deepcopy(db._conflicts)
     new_db._other_generations = copy.deepcopy(db._other_generations)
     new_db._indexes = copy.deepcopy(db._indexes)
-    new_db._store_u1db_data()
+    # save u1db data on couch
+    for key in new_db.U1DB_DATA_KEYS:
+        doc_id = '%s%s' % (new_db.U1DB_DATA_DOC_ID_PREFIX, key)
+        doc = new_db._get_doc(doc_id)
+        doc.content = {'content': getattr(new_db, key)}
+        new_db._put_doc(doc)
     return new_db
 
 
@@ -271,6 +276,14 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests,
 
     scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios))
 
+    def setUp(self):
+        # we implement parents' setUp methods here to prevent from launching
+        # more couch instances then needed.
+        tests.TestCase.setUp(self)
+        self.server = self.server_thread = None
+        self.db, self.st = self.create_db_and_target(self)
+        self.other_changes = []
+
     def tearDown(self):
         self.db.delete_database()
         test_sync.DatabaseSyncTargetTests.tearDown(self)
@@ -345,20 +358,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
             return [self._listify(i) for i in l]
         return l
 
-    def _fetch_u1db_data(self, db):
-        cdoc = db._database.get(db.U1DB_DATA_DOC_ID)
-        jsonstr = db._database.get_attachment(cdoc, 'u1db_json').getvalue()
-        return json.loads(jsonstr)
+    def _fetch_u1db_data(self, db, key):
+        doc = db._get_doc("%s%s" % (db.U1DB_DATA_DOC_ID_PREFIX, key))
+        return doc.content['content']
 
     def test_transaction_log_storage_after_put(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
                                  'u1db_tests')
         db.create_doc({'simple': 'doc'})
-        content = self._fetch_u1db_data(db)
+        content = self._fetch_u1db_data(db, db.U1DB_TRANSACTION_LOG_KEY)
         self.assertEqual(
             self._listify(db._transaction_log),
-            self._listify(
-                json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY]))))
+            self._listify(content))
 
     def test_conflict_log_storage_after_put_if_newer(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -367,29 +378,27 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         doc.set_json(nested_doc)
         doc.rev = db._replica_uid + ':2'
         db._force_doc_sync_conflict(doc)
-        content = self._fetch_u1db_data(db)
+        content = self._fetch_u1db_data(db, db.U1DB_CONFLICTS_KEY)
         self.assertEqual(
             self._listify(db._conflicts),
-            self._listify(
-                json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY]))))
+            self._listify(content))
 
     def test_other_gens_storage_after_set(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
                                  'u1db_tests')
         doc = db.create_doc({'simple': 'doc'})
         db._set_replica_gen_and_trans_id('a', 'b', 'c')
-        content = self._fetch_u1db_data(db)
+        content = self._fetch_u1db_data(db, db.U1DB_OTHER_GENERATIONS_KEY)
         self.assertEqual(
             self._listify(db._other_generations),
-            self._listify(
-                json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY]))))
+            self._listify(content))
 
     def test_index_storage_after_create(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
                                  'u1db_tests')
         doc = db.create_doc({'name': 'john'})
         db.create_index('myindex', 'name')
-        content = self._fetch_u1db_data(db)
+        content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
         myind = db._indexes['myindex']
         index = {
             'myindex': {
@@ -400,8 +409,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         }
         self.assertEqual(
             self._listify(index),
-            self._listify(
-                json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
+            self._listify(content))
 
     def test_index_storage_after_delete(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -410,7 +418,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         db.create_index('myindex', 'name')
         db.create_index('myindex2', 'name')
         db.delete_index('myindex')
-        content = self._fetch_u1db_data(db)
+        content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
         myind = db._indexes['myindex2']
         index = {
             'myindex2': {
@@ -421,16 +429,13 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         }
         self.assertEqual(
             self._listify(index),
-            self._listify(
-                json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
+            self._listify(content))
 
     def test_replica_uid_storage_after_db_creation(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
                                  'u1db_tests')
-        content = self._fetch_u1db_data(db)
-        self.assertEqual(
-            db._replica_uid,
-            b64decode(content[db.U1DB_REPLICA_UID_KEY]))
+        content = self._fetch_u1db_data(db, db.U1DB_REPLICA_UID_KEY)
+        self.assertEqual(db._replica_uid, content)
 
 
 load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py
index 01b43299..db217bb3 100644
--- a/common/src/leap/soledad/common/tests/test_crypto.py
+++ b/common/src/leap/soledad/common/tests/test_crypto.py
@@ -29,9 +29,6 @@ import binascii
 
 
 from leap.common.testing.basetest import BaseLeapTest
-from Crypto import Random
-
-
 from leap.soledad.client import (
     Soledad,
     crypto,
@@ -193,7 +190,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
 
     def test_encrypt_decrypt_sym(self):
         # generate 256-bit key
-        key = Random.new().read(32)
+        key = os.urandom(32)
         iv, cyphertext = self._soledad._crypto.encrypt_sym(
             'data', key,
             method=crypto.EncryptionMethods.AES_256_CTR)
@@ -206,7 +203,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
         self.assertEqual('data', plaintext)
 
     def test_decrypt_with_wrong_iv_fails(self):
-        key = Random.new().read(32)
+        key = os.urandom(32)
         iv, cyphertext = self._soledad._crypto.encrypt_sym(
             'data', key,
             method=crypto.EncryptionMethods.AES_256_CTR)
@@ -224,17 +221,17 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
         self.assertNotEqual('data', plaintext)
 
     def test_decrypt_with_wrong_key_fails(self):
-        key = Random.new().read(32)
+        key = os.urandom(32)
         iv, cyphertext = self._soledad._crypto.encrypt_sym(
             'data', key,
             method=crypto.EncryptionMethods.AES_256_CTR)
         self.assertTrue(cyphertext is not None)
         self.assertTrue(cyphertext != '')
         self.assertTrue(cyphertext != 'data')
-        wrongkey = Random.new().read(32)  # 256-bits key
+        wrongkey = os.urandom(32)  # 256-bits key
         # ensure keys are different in case we are extremely lucky
         while wrongkey == key:
-            wrongkey = Random.new().read(32)
+            wrongkey = os.urandom(32)
         plaintext = self._soledad._crypto.decrypt_sym(
             cyphertext, wrongkey, iv=iv,
             method=crypto.EncryptionMethods.AES_256_CTR)
@@ -245,7 +242,7 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
 
     def test_encrypt_decrypt_sym(self):
         # generate 256-bit key
-        key = Random.new().read(32)
+        key = os.urandom(32)
         iv, cyphertext = self._soledad._crypto.encrypt_sym(
             'data', key,
             method=crypto.EncryptionMethods.XSALSA20)
@@ -258,7 +255,7 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
         self.assertEqual('data', plaintext)
 
     def test_decrypt_with_wrong_iv_fails(self):
-        key = Random.new().read(32)
+        key = os.urandom(32)
         iv, cyphertext = self._soledad._crypto.encrypt_sym(
             'data', key,
             method=crypto.EncryptionMethods.XSALSA20)
@@ -276,17 +273,17 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
         self.assertNotEqual('data', plaintext)
 
     def test_decrypt_with_wrong_key_fails(self):
-        key = Random.new().read(32)
+        key = os.urandom(32)
         iv, cyphertext = self._soledad._crypto.encrypt_sym(
             'data', key,
             method=crypto.EncryptionMethods.XSALSA20)
         self.assertTrue(cyphertext is not None)
         self.assertTrue(cyphertext != '')
         self.assertTrue(cyphertext != 'data')
-        wrongkey = Random.new().read(32)  # 256-bits key
+        wrongkey = os.urandom(32)  # 256-bits key
         # ensure keys are different in case we are extremely lucky
         while wrongkey == key:
-            wrongkey = Random.new().read(32)
+            wrongkey = os.urandom(32)
         plaintext = self._soledad._crypto.decrypt_sym(
             cyphertext, wrongkey, iv=iv,
             method=crypto.EncryptionMethods.XSALSA20)
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index beb7e04d..1ea4d615 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -310,7 +310,8 @@ class EncryptedSyncTestCase(
             secret_id=secret_id)
 
     def make_app(self):
-        self.request_state = CouchServerState(self._couch_url)
+        self.request_state = CouchServerState(
+            self._couch_url, 'shared', 'tokens', 'user-')
         return self.make_app_with_state(self.request_state)
 
     def setUp(self):
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 67b0611d..b4b715e2 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -19,12 +19,19 @@
 """
 A U1DB server that stores data using CouchDB as its persistence layer.
 
-This should be run with:
-    twistd -n web --wsgi=leap.soledad.server.application --port=2424
+This is written as a Twisted application and intended to be run using the
+twistd command. To start the soledad server, run:
+
+    twistd -n web --wsgi=leap.soledad.server.application --port=X
+
+An initscript is included and will be installed system wide to make it
+feasible to start and stop the Soledad server service using a standard
+interface.
 """
 
 import configparser
 
+
 from u1db.remote import http_app
 
 
@@ -116,13 +123,18 @@ def load_configuration(file_path):
 # Run as Twisted WSGI Resource
 #-----------------------------------------------------------------------------
 
-conf = load_configuration('/etc/leap/soledad-server.conf')
-state = CouchServerState(conf['couch_url'])
-
-# WSGI application that may be used by `twistd -web`
-application = SoledadTokenAuthMiddleware(SoledadApp(state))
+def application(environ, start_response):
+    conf = load_configuration('/etc/leap/soledad-server.conf')
+    state = CouchServerState(
+        conf['couch_url'],
+        SoledadApp.SHARED_DB_NAME,
+        SoledadTokenAuthMiddleware.TOKENS_DB,
+        SoledadApp.USER_DB_PREFIX)
+    # WSGI application that may be used by `twistd -web`
+    application = SoledadTokenAuthMiddleware(SoledadApp(state))
+    resource = WSGIResource(reactor, reactor.getThreadPool(), application)
+    return application(environ, start_response)
 
-resource = WSGIResource(reactor, reactor.getThreadPool(), application)
 
 from ._version import get_versions
 __version__ = get_versions()['version']