Add shared db locking.

  * Improve bootstrap sequence:
    - stages are more organized.
    - there are less useless requests to server.
  * Improve shared db access:
    - instantiate the shared db only once.
    - also results in less requests to server.
  * Handle unicode passphrases.
  * Move some common functions and global variables to common.
  * Improve security of recovery document:
    - access to the recovery document now depends on the user password.
  * Improve documentation.

3 files changed, 286 insertions, 46 deletions

diff --git a/server/changes/feature_4097-allow-for-locking-creation-of-secrets-in-server b/server/changes/feature_4097-allow-for-locking-creation-of-secrets-in-server
new file mode 100644
index 00000000..48b367d7
--- /dev/null
+++ b/server/changes/feature_4097-allow-for-locking-creation-of-secrets-in-server
@@ -0,0 +1 @@
+  o Allow for locking the shared database. Closes #4097.
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index b4b715e2..c80b4c68 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -19,6 +19,9 @@
 """
 A U1DB server that stores data using CouchDB as its persistence layer.
 
+General information
+===================
+
 This is written as a Twisted application and intended to be run using the
 twistd command. To start the soledad server, run:
 
@@ -27,14 +30,69 @@ twistd command. To start the soledad server, run:
 An initscript is included and will be installed system wide to make it
 feasible to start and stop the Soledad server service using a standard
 interface.
+
+Server database organization
+============================
+
+Soledad Server works with one database per user and one shared database in
+which user's encrypted secrets might be stored.
+
+User database
+-------------
+
+Users' databases in the server are named 'user-<uuid>' and Soledad Client
+may perform synchronization between its local replicas and the user's
+database in the server. Authorization for creating, updating, deleting and
+retrieving information about the user database as well as performing
+synchronization is handled by the `leap.soledad.server.auth` module.
+
+Shared database
+---------------
+
+Each user may store password-encrypted recovery data in the shared database,
+as well as obtain a lock on the shared database in order to prevent creation
+of multiple secrets in parallel.
+
+Recovery documents are stored in the database without any information that
+may identify the user. In order to achieve this, the doc_id of recovery
+documents are obtained as a hash of the user's uid and the user's password.
+User's must have a valid token to interact with recovery documents, but the
+server does not perform further authentication because it has no way to know
+which recovery document belongs to each user.
+
+This has some implications:
+
+  * The security of the recovery document doc_id, and thus of access to the
+    recovery document (encrypted) content, as well as tampering with the
+    stored data, all rely on the difficulty of obtaining the user's password
+    (supposing the user's uid is somewhat public) and the security of the hash
+    function used to calculate the doc_id.
+
+  * The security of the content of a recovery document relies on the
+    difficulty of obtaining the user's password.
+
+  * If the user looses his/her password, he/she will not be able to obtain the
+    recovery document.
+
+  * Because of the above, it is recommended that recovery documents expire
+    (not implemented yet) to prevent excess storage.
+
+Lock documents, on the other hand, may be more thoroughly protected by the
+server. Their doc_id's are calculated from the SHARED_DB_LOCK_DOC_ID_PREFIX
+and the user's uid.
+
+The authorization for creating, updating, deleting and retrieving recovery
+and lock documents on the shared database is handled by
+`leap.soledad.server.auth` module.
 """
 
 import configparser
-
+import time
+import hashlib
+import os
 
 from u1db.remote import http_app
 
-
 # Keep OpenSSL's tsafe before importing Twisted submodules so we can put
 # it back if Twisted==12.0.0 messes with it.
 from OpenSSL import tsafe
@@ -42,6 +100,8 @@ old_tsafe = tsafe
 
 from twisted.web.wsgi import WSGIResource
 from twisted.internet import reactor
+from twisted.internet.error import TimeoutError
+from twisted.python.lockfile import FilesystemLock
 from twisted import version
 if version.base() == "12.0.0":
     # Put OpenSSL's tsafe back into place. This can probably be removed if we
@@ -49,9 +109,17 @@ if version.base() == "12.0.0":
     import sys
     sys.modules['OpenSSL.tsafe'] = old_tsafe
 
-
 from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+from leap.soledad.common import (
+    SHARED_DB_NAME,
+    SHARED_DB_LOCK_DOC_ID_PREFIX,
+)
 from leap.soledad.common.couch import CouchServerState
+from leap.soledad.common.errors import (
+    InvalidTokenError,
+    NotLockedError,
+    AlreadyLockedError,
+)
 
 
 #-----------------------------------------------------------------------------
@@ -63,16 +131,11 @@ class SoledadApp(http_app.HTTPApp):
     Soledad WSGI application
     """
 
-    SHARED_DB_NAME = 'shared'
+    SHARED_DB_NAME = SHARED_DB_NAME
     """
     The name of the shared database that holds user's encrypted secrets.
     """
 
-    USER_DB_PREFIX = 'user-'
-    """
-    The string prefix of users' databases.
-    """
-
     def __call__(self, environ, start_response):
         """
         Handle a WSGI call to the Soledad application.
@@ -91,6 +154,192 @@ class SoledadApp(http_app.HTTPApp):
         return http_app.HTTPApp.__call__(self, environ, start_response)
 
 
+#
+# LockResource: a lock based on a document in the shared database.
+#
+
+@http_app.url_to_resource.register
+class LockResource(object):
+    """
+    Handle requests for locking documents.
+
+    This class uses Twisted's Filesystem lock to manage a lock in the shared
+    database.
+    """
+
+    url_pattern = '/%s/lock/{uuid}' % SoledadApp.SHARED_DB_NAME
+    """
+    """
+
+    TIMEOUT = 300  # XXX is 5 minutes reasonable?
+    """
+    The timeout after which the lock expires.
+    """
+
+    # used for lock doc storage
+    TIMESTAMP_KEY = '_timestamp'
+    LOCK_TOKEN_KEY = '_token'
+
+    FILESYSTEM_LOCK_TRIES = 5
+    FILESYSTEM_LOCK_SLEEP_SECONDS = 1
+
+
+    def __init__(self, uuid, state, responder):
+        """
+        Initialize the lock resource. Parameters to this constructor are
+        automatically passed by u1db.
+
+        :param uuid: The user unique id.
+        :type uuid: str
+        :param state: The backend database state.
+        :type state: u1db.remote.ServerState
+        :param responder: The infrastructure to send responses to client.
+        :type responder: u1db.remote.HTTPResponder
+        """
+        self._shared_db = state.open_database(SoledadApp.SHARED_DB_NAME)
+        self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid)
+        self._lock = FilesystemLock(
+            hashlib.sha512(self._lock_doc_id).hexdigest())
+        self._state = state
+        self._responder = responder
+
+    @http_app.http_method(content=str)
+    def put(self, content=None):
+        """
+        Handle a PUT request to the lock document.
+
+        A lock is a document in the shared db with doc_id equal to
+        'lock-<uuid>' and the timestamp of its creation as content. This
+        method obtains a threaded-lock and creates a lock document if it does
+        not exist or if it has expired.
+
+        It returns '201 Created' and a pair containing a token to unlock and
+        the lock timeout, or '403 AlreadyLockedError' and the remaining amount
+        of seconds the lock will still be valid.
+
+        :param content: The content of the PUT request. It is only here
+                        because PUT requests with empty content are considered
+                        invalid requests by u1db.
+        :type content: str
+        """
+        # obtain filesystem lock
+        if not self._try_obtain_filesystem_lock():
+            self._responder.send_response_json(408)  # error: request timeout
+            return
+
+        created_lock = False
+        now = time.time()
+        token = hashlib.sha256(os.urandom(10)).hexdigest()  # for releasing
+        lock_doc = self._shared_db.get_doc(self._lock_doc_id)
+        remaining = self._remaining(lock_doc, now)
+
+        # if there's no lock, create one
+        if lock_doc is None:
+            lock_doc = self._shared_db.create_doc(
+                {
+                    self.TIMESTAMP_KEY: now,
+                    self.LOCK_TOKEN_KEY: token,
+                },
+                doc_id=self._lock_doc_id)
+            created_lock = True
+        else:
+            if remaining == 0:
+                # lock expired, create new one
+                lock_doc.content = {
+                    self.TIMESTAMP_KEY: now,
+                    self.LOCK_TOKEN_KEY: token,
+                }
+                self._shared_db.put_doc(lock_doc)
+                created_lock = True
+
+        self._try_release_filesystem_lock()
+
+        # send response to client
+        if created_lock is True:
+            self._responder.send_response_json(
+                201, timeout=self.TIMEOUT, token=token)  # success: created
+        else:
+            wire_descr = AlreadyLockedError.wire_description
+            self._responder.send_response_json(
+                AlreadyLockedError.status,  # error: forbidden
+                error=AlreadyLockedError.wire_description, remaining=remaining)
+
+    @http_app.http_method(token=str)
+    def delete(self, token=None):
+        """
+        Delete the lock if the C{token} is valid.
+
+        Delete the lock document in case C{token} is equal to the token stored
+        in the lock document.
+
+        :param token: The token returned when locking.
+        :type token: str
+
+        :raise NotLockedError: Raised in case the lock is not locked.
+        :raise InvalidTokenError: Raised in case the token is invalid for
+                                  unlocking.
+        """
+        lock_doc = self._shared_db.get_doc(self._lock_doc_id)
+        if lock_doc is None or self._remaining(lock_doc, time.time()) == 0:
+            self._responder.send_response_json(
+                NotLockedError.status,  # error: not found
+                error=NotLockedError.wire_description)
+        elif token != lock_doc.content[self.LOCK_TOKEN_KEY]:
+            self._responder.send_response_json(
+                InvalidTokenError.status,  # error: unauthorized
+                error=InvalidTokenError.wire_description)
+        else:
+            self._shared_db.delete_doc(lock_doc)
+            self._responder.send_response_json(200)  # success: should use 204
+                                                     # but u1db does not
+                                                     # support it.
+
+    def _remaining(self, lock_doc, now):
+        """
+        Return the number of seconds the lock contained in C{lock_doc} is
+        still valid, when compared to C{now}.
+
+        :param lock_doc: The document containing the lock.
+        :type lock_doc: u1db.Document
+        :param now: The time to which to compare the lock timestamp.
+        :type now: float
+
+        :return: The amount of seconds the lock is still valid.
+        :rtype: float
+        """
+        if lock_doc is not None:
+            lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY]
+            remaining = lock_timestamp + self.TIMEOUT - now
+            return remaining if remaining > 0 else 0.0
+        return 0.0
+
+    def _try_obtain_filesystem_lock(self):
+        """
+        Try to obtain the file system lock.
+
+        @return: Whether the lock was succesfully obtained.
+        @rtype: bool
+        """
+        tries = self.FILESYSTEM_LOCK_TRIES
+        while tries > 0:
+            try:
+                return self._lock.lock()
+            except Exception as e:
+                tries -= 1
+                time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS)
+        return False
+
+    def _try_release_filesystem_lock(self):
+        """
+        Release the filesystem lock.
+        """
+        try:
+            self._lock.unlock()
+            return True
+        except Exception:
+            return False
+
+
 #-----------------------------------------------------------------------------
 # Auxiliary functions
 #-----------------------------------------------------------------------------
@@ -128,8 +377,7 @@ def application(environ, start_response):
     state = CouchServerState(
         conf['couch_url'],
         SoledadApp.SHARED_DB_NAME,
-        SoledadTokenAuthMiddleware.TOKENS_DB,
-        SoledadApp.USER_DB_PREFIX)
+        SoledadTokenAuthMiddleware.TOKENS_DB)
     # WSGI application that may be used by `twistd -web`
     application = SoledadTokenAuthMiddleware(SoledadApp(state))
     resource = WSGIResource(reactor, reactor.getThreadPool(), application)
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index 3bcfcf04..0ae49576 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -32,6 +32,13 @@ from couchdb.client import Server
 from twisted.python import log
 
 
+from leap.soledad.common import (
+    SHARED_DB_NAME,
+    SHARED_DB_LOCK_DOC_ID_PREFIX,
+    USER_DB_PREFIX,
+)
+
+
 #-----------------------------------------------------------------------------
 # Authentication
 #-----------------------------------------------------------------------------
@@ -52,7 +59,7 @@ class URLToAuthorization(object):
     HTTP_METHOD_DELETE = 'DELETE'
     HTTP_METHOD_POST = 'POST'
 
-    def __init__(self, uuid, shared_db_name, user_db_prefix):
+    def __init__(self, uuid):
         """
         Initialize the mapper.
 
@@ -61,16 +68,13 @@ class URLToAuthorization(object):
 
         @param uuid: The user uuid.
         @type uuid: str
-        @param shared_db_name: The name of the shared database that holds
-            user's encrypted secrets.
-        @type shared_db_name: str
         @param user_db_prefix: The string prefix of users' databases.
         @type user_db_prefix: str
         """
         self._map = Mapper(controller_scan=None)
-        self._user_db_prefix = user_db_prefix
-        self._shared_db_name = shared_db_name
-        self._register_auth_info(self._uuid_dbname(uuid))
+        self._user_db_name = "%s%s" % (USER_DB_PREFIX, uuid)
+        self._uuid = uuid
+        self._register_auth_info()
 
     def is_authorized(self, environ):
         """
@@ -99,22 +103,10 @@ class URLToAuthorization(object):
             conditions=dict(method=http_methods),
             requirements={'dbname': DBNAME_CONSTRAINTS})
 
-    def _uuid_dbname(self, uuid):
-        """
-        Return the database name corresponding to C{uuid}.
-
-        @param uuid: The user uid.
-        @type uuid: str
-
-        @return: The database name corresponding to C{uuid}.
-        @rtype: str
-        """
-        return '%s%s' % (self._user_db_prefix, uuid)
-
-    def _register_auth_info(self, dbname):
+    def _register_auth_info(self):
         """
-        Register the authorization info in the mapper using C{dbname} as the
-        user's database name.
+        Register the authorization info in the mapper using C{SHARED_DB_NAME}
+        as the user's database name.
 
         This method sets up the following authorization rules:
 
@@ -123,35 +115,37 @@ class URLToAuthorization(object):
             /                             | GET
             /shared-db                    | GET
             /shared-db/docs               | -
-            /shared-db/doc/{id}           | GET, PUT, DELETE
+            /shared-db/doc/{any_id}       | GET, PUT, DELETE
             /shared-db/sync-from/{source} | -
+            /shared-db/lock/{uuid}        | PUT, DELETE
             /user-db                      | GET, PUT, DELETE
             /user-db/docs                 | -
             /user-db/doc/{id}             | -
             /user-db/sync-from/{source}   | GET, PUT, POST
-
-        @param dbname: The name of the user's database.
-        @type dbname: str
         """
         # auth info for global resource
         self._register('/', [self.HTTP_METHOD_GET])
         # auth info for shared-db database resource
         self._register(
-            '/%s' % self._shared_db_name,
+            '/%s' % SHARED_DB_NAME,
             [self.HTTP_METHOD_GET])
         # auth info for shared-db doc resource
         self._register(
-            '/%s/doc/{id:.*}' % self._shared_db_name,
+            '/%s/doc/{id:.*}' % SHARED_DB_NAME,
             [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
              self.HTTP_METHOD_DELETE])
+        # auth info for shared-db lock resource
+        self._register(
+            '/%s/lock/%s' % (SHARED_DB_NAME, self._uuid),
+            [self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE])
         # auth info for user-db database resource
         self._register(
-            '/%s' % dbname,
+            '/%s' % self._user_db_name,
             [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
              self.HTTP_METHOD_DELETE])
         # auth info for user-db sync resource
         self._register(
-            '/%s/sync-from/{source_replica_uid}' % dbname,
+            '/%s/sync-from/{source_replica_uid}' % self._user_db_name,
             [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
              self.HTTP_METHOD_POST])
         # generate the regular expressions
@@ -165,7 +159,7 @@ class SoledadAuthMiddleware(object):
     This class must be extended to implement specific authentication methods
     (see SoledadTokenAuthMiddleware below).
 
-    It expects an HTTP_AUTHORIZATION header containing the the concatenation of
+    It expects an HTTP_AUTHORIZATION header containing the concatenation of
     the following strings:
 
         1. The authentication scheme. It will be verified by the
@@ -342,10 +336,7 @@ class SoledadAuthMiddleware(object):
             over the requested db.
         @rtype: bool
         """
-        return URLToAuthorization(
-            uuid, self._app.SHARED_DB_NAME,
-            self._app.USER_DB_PREFIX
-        ).is_authorized(environ)
+        return URLToAuthorization(uuid).is_authorized(environ)
 
     @abstractmethod
     def _get_auth_error_string(self):