Merge tag '0.8.1'

0.8.1

11 files changed, 73 insertions, 297 deletions

diff --git a/server/changes/next-changelog.rst b/server/changes/next-changelog.rst
index bdc9f893..fc4cbc30 100644
--- a/server/changes/next-changelog.rst
+++ b/server/changes/next-changelog.rst
@@ -1,4 +1,4 @@
-0.8.1 - ...
+0.8.2 - ...
 ++++++++++++++++++++
 
 Please add lines to this file, they will be moved to the CHANGELOG.rst during
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index 54856643..5e48d4de 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -25,6 +25,9 @@ from leap.soledad.common.couch import list_users_dbs
 from leap.soledad.server import load_configuration
 
 
+BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False)
+
+
 description = """
 Creates a user database.
 This is meant to be used by Soledad Server.
@@ -40,16 +43,23 @@ NETRC_PATH = CONF['soledad-server']['admin_netrc']
 
 
 def url_for_db(dbname):
-    if not os.path.exists(NETRC_PATH):
-        print ('netrc not found in %s' % NETRC_PATH)
-        sys.exit(1)
-    parsed_netrc = netrc.netrc(NETRC_PATH)
-    host, (login, _, password) = parsed_netrc.hosts.items()[0]
-    url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % {
-           'login': login,
-           'password': password,
-           'host': host,
-           'dbname': dbname})
+    if BYPASS_AUTH:
+        login = ''
+        password = ''
+        host = 'localhost'
+        url = 'http://localhost:5984/%(dbname)s' % {
+            'dbname': dbname}
+    else:
+        if not os.path.exists(NETRC_PATH):
+            print ('netrc not found in %s' % NETRC_PATH)
+            sys.exit(1)
+        parsed_netrc = netrc.netrc(NETRC_PATH)
+        host, (login, _, password) = parsed_netrc.hosts.items()[0]
+        url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % {
+               'login': login,
+               'password': password,
+               'host': host,
+               'dbname': dbname})
     return url
 
 
diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh
index e29c327e..a13e2c7a 100755
--- a/server/pkg/generate_wheels.sh
+++ b/server/pkg/generate_wheels.sh
@@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then
 fi
 
 pip wheel --wheel-dir $WHEELHOUSE pip
-pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip
+pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip
 if [ -f pkg/requirements-testing.pip ]; then
    pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip
 fi
diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh
index d0479365..f4b5f67a 100755
--- a/server/pkg/pip_install_requirements.sh
+++ b/server/pkg/pip_install_requirements.sh
@@ -4,7 +4,7 @@
 # Use at your own risk.
 # See $usage for help
 
-insecure_packages="u1db dirspec"
+insecure_packages=""
 leap_wheelhouse=https://lizard.leap.se/wheels
 
 show_help() {
@@ -80,5 +80,5 @@ insecure_flags=`return_insecure_flags`
 packages=`return_packages`
 
 pip install -U wheel
-pip install $install_options pip
+pip install -U pip
 pip install $install_options $insecure_flags $packages
diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip
index a629aa57..46a7ccba 100644
--- a/server/pkg/requirements-latest.pip
+++ b/server/pkg/requirements-latest.pip
@@ -1,8 +1,5 @@
 --index-url https://pypi.python.org/simple/
 
---allow-external u1db  --allow-unverified u1db
---allow-external dirspec  --allow-unverified dirspec
-
 -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common'
 -e '../common'
 -e .
diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index f9cce08e..2d845f24 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -1,13 +1,6 @@
 configparser
-u1db
-routes
 PyOpenSSL
 twisted>=12.3.0
 #pinned for wheezy compatibility
 Beaker==1.6.3 #wheezy
 couchdb==0.8 #wheezy
-
-# XXX -- fix me!
-# oauth is not strictly needed by us, but we need it until u1db adds it to its
-# release as a dep.
-oauth
diff --git a/server/setup.py b/server/setup.py
index 8a7fbe45..b3b26010 100644
--- a/server/setup.py
+++ b/server/setup.py
@@ -77,14 +77,20 @@ class freeze_debianver(Command):
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
 
-version_version = '{version}'
-full_revisionid = '{full_revisionid}'
-"""
-    templatefun = r"""
-
-def get_versions(default={}, verbose=False):
-        return {'version': version_version,
-                'full-revisionid': full_revisionid}
+import json
+import sys
+
+version_json = '''
+{
+ "dirty": false,
+ "error": null,
+ "full-revisionid": "FULL_REVISIONID",
+ "version": "VERSION_STRING"
+}
+'''  # END VERSION_JSON
+
+def get_versions():
+    return json.loads(version_json)
 """
 
     def initialize_options(self):
@@ -99,9 +105,9 @@ def get_versions(default={}, verbose=False):
         if proceed != "y":
             print("He. You scared. Aborting.")
             return
-        subst_template = self.template.format(
-            version=VERSION_SHORT,
-            full_revisionid=VERSION_REVISION) + self.templatefun
+        subst_template = self.template.replace(
+            'VERSION_STRING', VERSION_SHORT).replace(
+            'FULL_REVISIONID', VERSION_REVISION)
         versioneer_cfg = versioneer.get_config_from_root('.')
         with open(versioneer_cfg.versionfile_source, 'w') as f:
             f.write(subst_template)
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 22894dac..34570b52 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -49,9 +49,7 @@ synchronization is handled by the `leap.soledad.server.auth` module.
 Shared database
 ---------------
 
-Each user may store password-encrypted recovery data in the shared database,
-as well as obtain a lock on the shared database in order to prevent creation
-of multiple secrets in parallel.
+Each user may store password-encrypted recovery data in the shared database.
 
 Recovery documents are stored in the database without any information that
 may identify the user. In order to achieve this, the doc_id of recovery
@@ -77,26 +75,19 @@ This has some implications:
   * Because of the above, it is recommended that recovery documents expire
     (not implemented yet) to prevent excess storage.
 
-Lock documents, on the other hand, may be more thoroughly protected by the
-server. Their doc_id's are calculated from the SHARED_DB_LOCK_DOC_ID_PREFIX
-and the user's uid.
-
 The authorization for creating, updating, deleting and retrieving recovery
-and lock documents on the shared database is handled by
-`leap.soledad.server.auth` module.
+documents on the shared database is handled by `leap.soledad.server.auth`
+module.
 """
 
 import configparser
 import urlparse
 import sys
 
-from u1db.remote import http_app, utils
-
-from ._version import get_versions
+from leap.soledad.common.l2db.remote import http_app, utils
 
 from leap.soledad.server.auth import SoledadTokenAuthMiddleware
 from leap.soledad.server.gzip_middleware import GzipMiddleware
-from leap.soledad.server.lock_resource import LockResource
 from leap.soledad.server.sync import (
     SyncResource,
     MAX_REQUEST_SIZE,
@@ -107,6 +98,8 @@ from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common.backend import SoledadBackend
 from leap.soledad.common.couch.state import CouchServerState
 
+from ._version import get_versions
+
 # ----------------------------------------------------------------------------
 # Soledad WSGI application
 # ----------------------------------------------------------------------------
@@ -155,7 +148,6 @@ http_app.url_to_resource.register(http_app.DocsResource)
 http_app.url_to_resource.register(http_app.DocResource)
 
 # register Soledad's new or modified resources
-http_app.url_to_resource.register(LockResource)
 http_app.url_to_resource.register(SyncResource)
 
 
@@ -312,16 +304,34 @@ def load_configuration(file_path):
 # Run as Twisted WSGI Resource
 # ----------------------------------------------------------------------------
 
-def application(environ, start_response):
+
+def _load_config():
     conf = load_configuration('/etc/soledad/soledad-server.conf')
-    conf = conf['soledad-server']
+    return conf['soledad-server']
+
+
+def _get_couch_state():
+    conf = _load_config()
     state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
-    SoledadBackend.BATCH_SUPPORT = conf['batching']
-    # WSGI application that may be used by `twistd -web`
+    SoledadBackend.BATCH_SUPPORT = conf.get('batching', False)
+    return state
+
+
+def application(environ, start_response):
+    """return WSGI application that may be used by `twistd -web`"""
+    state = _get_couch_state()
     application = GzipMiddleware(
         SoledadTokenAuthMiddleware(SoledadApp(state)))
+    return application(environ, start_response)
+
 
+def debug_local_application_do_not_use(environ, start_response):
+    """in where we bypass token auth middleware for ease of mind while
+    debugging in your local environment"""
+    state = _get_couch_state()
+    application = SoledadApp(state)
     return application(environ, start_response)
 
+
 __version__ = get_versions()['version']
 del get_versions
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index ccbd6fbd..ecee2d5d 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -14,21 +14,17 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
 """
 Authentication facilities for Soledad Server.
 """
-
-
 import httplib
 import json
 
-from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors
 from abc import ABCMeta, abstractmethod
 from routes.mapper import Mapper
 from twisted.python import log
 
+from leap.soledad.common.l2db import DBNAME_CONSTRAINTS, errors as u1db_errors
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common import USER_DB_PREFIX
 
@@ -101,7 +97,6 @@ class URLToAuthorization(object):
             /shared-db/docs               | -
             /shared-db/doc/{any_id}       | GET, PUT, DELETE
             /shared-db/sync-from/{source} | -
-            /shared-db/lock/{uuid}        | PUT, DELETE
             /user-db                      | GET, PUT, DELETE
             /user-db/docs                 | -
             /user-db/doc/{id}             | -
@@ -118,10 +113,6 @@ class URLToAuthorization(object):
             '/%s/doc/{id:.*}' % SHARED_DB_NAME,
             [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
              self.HTTP_METHOD_DELETE])
-        # auth info for shared-db lock resource
-        self._register(
-            '/%s/lock/%s' % (SHARED_DB_NAME, self._uuid),
-            [self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE])
         # auth info for user-db database resource
         self._register(
             '/%s' % self._user_db_name,
diff --git a/server/src/leap/soledad/server/lock_resource.py b/server/src/leap/soledad/server/lock_resource.py
deleted file mode 100644
index 0a602e26..00000000
--- a/server/src/leap/soledad/server/lock_resource.py
+++ /dev/null
@@ -1,231 +0,0 @@
-# -*- coding: utf-8 -*-
-# lock_resource.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-LockResource: a lock based on a document in the shared database.
-"""
-
-
-import hashlib
-import time
-import os
-import tempfile
-import errno
-
-
-from u1db.remote import http_app
-from twisted.python.lockfile import FilesystemLock
-
-
-from leap.soledad.common import (
-    SHARED_DB_NAME,
-    SHARED_DB_LOCK_DOC_ID_PREFIX,
-)
-from leap.soledad.common.errors import (
-    InvalidTokenError,
-    NotLockedError,
-    AlreadyLockedError,
-    LockTimedOutError,
-    CouldNotObtainLockError,
-)
-
-
-class LockResource(object):
-    """
-    Handle requests for locking documents.
-
-    This class uses Twisted's Filesystem lock to manage a lock in the shared
-    database.
-    """
-
-    url_pattern = '/%s/lock/{uuid}' % SHARED_DB_NAME
-    """
-    """
-
-    TIMEOUT = 300  # XXX is 5 minutes reasonable?
-    """
-    The timeout after which the lock expires.
-    """
-
-    # used for lock doc storage
-    TIMESTAMP_KEY = '_timestamp'
-    LOCK_TOKEN_KEY = '_token'
-
-    FILESYSTEM_LOCK_TRIES = 5
-    FILESYSTEM_LOCK_SLEEP_SECONDS = 1
-
-    def __init__(self, uuid, state, responder):
-        """
-        Initialize the lock resource. Parameters to this constructor are
-        automatically passed by u1db.
-
-        :param uuid: The user unique id.
-        :type uuid: str
-        :param state: The backend database state.
-        :type state: u1db.remote.ServerState
-        :param responder: The infrastructure to send responses to client.
-        :type responder: u1db.remote.HTTPResponder
-        """
-        self._shared_db = state.open_database(SHARED_DB_NAME)
-        self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid)
-        self._lock = FilesystemLock(
-            os.path.join(
-                tempfile.gettempdir(),
-                hashlib.sha512(self._lock_doc_id).hexdigest()))
-        self._state = state
-        self._responder = responder
-
-    @http_app.http_method(content=str)
-    def put(self, content=None):
-        """
-        Handle a PUT request to the lock document.
-
-        A lock is a document in the shared db with doc_id equal to
-        'lock-<uuid>' and the timestamp of its creation as content. This
-        method obtains a threaded-lock and creates a lock document if it does
-        not exist or if it has expired.
-
-        It returns '201 Created' and a pair containing a token to unlock and
-        the lock timeout, or '403 AlreadyLockedError' and the remaining amount
-        of seconds the lock will still be valid.
-
-        :param content: The content of the PUT request. It is only here
-                        because PUT requests with empty content are considered
-                        invalid requests by u1db.
-        :type content: str
-        """
-        # obtain filesystem lock
-        if not self._try_obtain_filesystem_lock():
-            self._responder.send_response_json(
-                LockTimedOutError.status,  # error: request timeout
-                error=LockTimedOutError.wire_description)
-            return
-
-        created_lock = False
-        now = time.time()
-        token = hashlib.sha256(os.urandom(10)).hexdigest()  # for releasing
-        lock_doc = self._shared_db.get_doc(self._lock_doc_id)
-        remaining = self._remaining(lock_doc, now)
-
-        # if there's no lock, create one
-        if lock_doc is None:
-            lock_doc = self._shared_db.create_doc(
-                {
-                    self.TIMESTAMP_KEY: now,
-                    self.LOCK_TOKEN_KEY: token,
-                },
-                doc_id=self._lock_doc_id)
-            created_lock = True
-        else:
-            if remaining == 0:
-                # lock expired, create new one
-                lock_doc.content = {
-                    self.TIMESTAMP_KEY: now,
-                    self.LOCK_TOKEN_KEY: token,
-                }
-                self._shared_db.put_doc(lock_doc)
-                created_lock = True
-
-        self._try_release_filesystem_lock()
-
-        # send response to client
-        if created_lock is True:
-            self._responder.send_response_json(
-                201, timeout=self.TIMEOUT, token=token)  # success: created
-        else:
-            self._responder.send_response_json(
-                AlreadyLockedError.status,  # error: forbidden
-                error=AlreadyLockedError.wire_description, remaining=remaining)
-
-    @http_app.http_method(token=str)
-    def delete(self, token=None):
-        """
-        Delete the lock if the C{token} is valid.
-
-        Delete the lock document in case C{token} is equal to the token stored
-        in the lock document.
-
-        :param token: The token returned when locking.
-        :type token: str
-
-        :raise NotLockedError: Raised in case the lock is not locked.
-        :raise InvalidTokenError: Raised in case the token is invalid for
-                                  unlocking.
-        """
-        lock_doc = self._shared_db.get_doc(self._lock_doc_id)
-        if lock_doc is None or self._remaining(lock_doc, time.time()) == 0:
-            self._responder.send_response_json(
-                NotLockedError.status,  # error: not found
-                error=NotLockedError.wire_description)
-        elif token != lock_doc.content[self.LOCK_TOKEN_KEY]:
-            self._responder.send_response_json(
-                InvalidTokenError.status,  # error: unauthorized
-                error=InvalidTokenError.wire_description)
-        else:
-            self._shared_db.delete_doc(lock_doc)
-            # respond success: should use 204 but u1db does not support it.
-            self._responder.send_response_json(200)
-
-    def _remaining(self, lock_doc, now):
-        """
-        Return the number of seconds the lock contained in C{lock_doc} is
-        still valid, when compared to C{now}.
-
-        :param lock_doc: The document containing the lock.
-        :type lock_doc: u1db.Document
-        :param now: The time to which to compare the lock timestamp.
-        :type now: float
-
-        :return: The amount of seconds the lock is still valid.
-        :rtype: float
-        """
-        if lock_doc is not None:
-            lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY]
-            remaining = lock_timestamp + self.TIMEOUT - now
-            return remaining if remaining > 0 else 0.0
-        return 0.0
-
-    def _try_obtain_filesystem_lock(self):
-        """
-        Try to obtain the file system lock.
-
-        @return: Whether the lock was succesfully obtained.
-        @rtype: bool
-        """
-        tries = self.FILESYSTEM_LOCK_TRIES
-        while tries > 0:
-            try:
-                return self._lock.lock()
-            except OSError as e:
-                tries -= 1
-                if tries == 0:
-                    raise CouldNotObtainLockError(e.message)
-                time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS)
-        return False
-
-    def _try_release_filesystem_lock(self):
-        """
-        Release the filesystem lock.
-        """
-        try:
-            self._lock.unlock()
-            return True
-        except OSError as e:
-            if e.errno == errno.ENOENT:
-                return True
-            return False
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 96f65912..3f5c4aba 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -17,10 +17,10 @@
 """
 Server side synchronization infrastructure.
 """
-from u1db import sync, Document
-from u1db.remote import http_app
-from leap.soledad.server.state import ServerSyncState
+from leap.soledad.common.l2db import sync, Document
+from leap.soledad.common.l2db.remote import http_app
 from leap.soledad.server.caching import get_cache_for
+from leap.soledad.server.state import ServerSyncState
 
 
 MAX_REQUEST_SIZE = 200  # in Mb