Merge tag '0.7.2' into debian/experimental

Tag leap.soledad version 0.7.2

13 files changed, 201 insertions, 60 deletions

diff --git a/server/AUTHORS b/server/AUTHORS
new file mode 100644
index 00000000..934cf129
--- /dev/null
+++ b/server/AUTHORS
@@ -0,0 +1,10 @@
+drebs <drebs@leap.se>
+Tomás Touceda <chiiph@leap.se>
+Kali Kaneko <kali@leap.se>
+Ivan Alejandro <ivanalejandro0@gmail.com>
+Micah Anderson <micah@riseup.net>
+Victor Shyba <victor.shyba@gmail.com>
+Bruno Wagner <bwgpro@gmail.com>
+Ruben Pollan <meskio@sindominio.net>
+Duda Dornelles <ddornell@thoughtworks.com>
+antialias <antialias@leap.se>
diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh
new file mode 100755
index 00000000..e29c327e
--- /dev/null
+++ b/server/pkg/generate_wheels.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Generate wheels for dependencies
+# Use at your own risk.
+
+if [ "$WHEELHOUSE" = "" ]; then
+    WHEELHOUSE=$HOME/wheelhouse
+fi
+
+pip wheel --wheel-dir $WHEELHOUSE pip
+pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip
+if [ -f pkg/requirements-testing.pip ]; then
+   pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip
+fi
diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh
new file mode 100755
index 00000000..d0479365
--- /dev/null
+++ b/server/pkg/pip_install_requirements.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+# Update pip and install LEAP base/testing requirements.
+# For convenience, $insecure_packages are allowed with insecure flags enabled.
+# Use at your own risk.
+# See $usage for help
+
+insecure_packages="u1db dirspec"
+leap_wheelhouse=https://lizard.leap.se/wheels
+
+show_help() {
+    usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n
+\t\t\tOtherwise, it will install requirements.pip\n
+--use-leap-wheels\tUse wheels from leap.se"
+    echo -e $usage
+
+    exit 1
+}
+
+process_arguments() {
+    testing=false
+    while [ "$#" -gt 0 ]; do
+	# From http://stackoverflow.com/a/31443098
+	case "$1" in
+	    --help) show_help;;
+	    --testing) testing=true; shift 1;;
+	    --use-leap-wheels) use_leap_wheels=true; shift 1;;
+
+	    -h) show_help;;
+	    -*) echo "unknown option: $1" >&2; exit 1;;
+	esac
+    done
+}
+
+return_wheelhouse() {
+    if $use_leap_wheels ; then
+	WHEELHOUSE=$leap_wheelhouse
+    elif [ "$WHEELHOUSE" = "" ]; then
+	WHEELHOUSE=$HOME/wheelhouse
+    fi
+
+    # Tested with bash and zsh
+    if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then
+	    mkdir $WHEELHOUSE
+    fi
+
+    echo "$WHEELHOUSE"
+}
+
+return_install_options() {
+    wheelhouse=`return_wheelhouse`
+    install_options="-U --find-links=$wheelhouse"
+    if $use_leap_wheels ; then
+	install_options="$install_options --trusted-host lizard.leap.se"
+    fi
+
+    echo $install_options
+}
+
+return_insecure_flags() {
+    for insecure_package in $insecure_packages; do
+	flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package"
+    done
+
+    echo $flags
+}
+
+return_packages() {
+    if $testing ; then
+	packages="-r pkg/requirements-testing.pip"
+    else
+	packages="-r pkg/requirements.pip"
+    fi
+
+    echo $packages
+}
+
+process_arguments $@
+install_options=`return_install_options`
+insecure_flags=`return_insecure_flags`
+packages=`return_packages`
+
+pip install -U wheel
+pip install $install_options pip
+pip install $install_options $insecure_flags $packages
diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip
new file mode 100644
index 00000000..a629aa57
--- /dev/null
+++ b/server/pkg/requirements-latest.pip
@@ -0,0 +1,8 @@
+--index-url https://pypi.python.org/simple/
+
+--allow-external u1db  --allow-unverified u1db
+--allow-external dirspec  --allow-unverified dirspec
+
+-e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common'
+-e '../common'
+-e .
diff --git a/server/pkg/requirements-leap.pip b/server/pkg/requirements-leap.pip
new file mode 100644
index 00000000..aaad340c
--- /dev/null
+++ b/server/pkg/requirements-leap.pip
@@ -0,0 +1 @@
+leap.soledad.common>=0.6.5
diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index 28717664..ca8ba42a 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -1,22 +1,10 @@
 configparser
 couchdb
-simplejson
 u1db
 routes
-PyOpenSSL<0.14
-
-# TODO: maybe we just want twisted-web?
-twisted>=12.0.0
-
-# leap deps -- bump me!
-leap.soledad.common>=0.6.0
-
-#
-# Things yet to fix:
-#
-
-# oauth is not strictly needed by us, but we need it
-# until u1db adds it to its release as a dep.
-
+PyOpenSSL
+twisted
+# XXX -- fix me!
+# oauth is not strictly needed by us, but we need it until u1db adds it to its
+# release as a dep.
 oauth
-
diff --git a/server/pkg/utils.py b/server/pkg/utils.py
index deace14b..d1680102 100644
--- a/server/pkg/utils.py
+++ b/server/pkg/utils.py
@@ -14,20 +14,34 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
 Utils to help in the setup process
 """
-
 import os
 import re
 import sys
 
 
+def is_develop_mode():
+    """
+    Returns True if we're calling the setup script using the argument for
+    setuptools development mode.
+
+    This avoids messing up with dependency pinning and order, the
+    responsibility of installing the leap dependencies is left to the
+    developer.
+    """
+    args = sys.argv
+    devflags = "setup.py", "develop"
+    if (args[0], args[1]) == devflags:
+        return True
+    return False
+
+
 def get_reqs_from_files(reqfiles):
     """
     Returns the contents of the top requirement file listed as a
-    string list with the lines
+    string list with the lines.
 
     @param reqfiles: requirement files to parse
     @type reqfiles: list of str
@@ -43,6 +57,9 @@ def parse_requirements(reqfiles=['requirements.txt',
     """
     Parses the requirement files provided.
 
+    The passed reqfiles list is a list of possible locations to try, the
+    function will return the contents of the first path found.
+
     Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should
     return PySide as a dep or not. Don't set, or set to 0 if you want
     to install it through pip.
@@ -58,9 +75,9 @@ def parse_requirements(reqfiles=['requirements.txt',
         if re.match(r'\s*-e\s+', line):
             pass
             # do not try to do anything with externals on vcs
-            #requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
-                                #line))
-        # http://foo.bar/baz/foobar/zipball/master#egg=foobar
+            # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
+            # line))
+            # http://foo.bar/baz/foobar/zipball/master#egg=foobar
         elif re.match(r'\s*https?:', line):
             requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1',
                                 line))
diff --git a/server/setup.cfg b/server/setup.cfg
new file mode 100644
index 00000000..6b530888
--- /dev/null
+++ b/server/setup.cfg
@@ -0,0 +1,7 @@
+[pep8]
+exclude = versioneer.py,_version.py,ddocs.py,*.egg,build
+ignore = E731
+
+[flake8]
+exclude = versioneer.py,_version.py,ddocs.py,*.egg,build
+ignore = E731
diff --git a/server/setup.py b/server/setup.py
index 124ddd32..b3942551 100644
--- a/server/setup.py
+++ b/server/setup.py
@@ -21,6 +21,9 @@ import os
 import re
 from setuptools import setup
 from setuptools import find_packages
+from setuptools import Command
+
+from pkg import utils
 
 import versioneer
 versioneer.versionfile_source = 'src/leap/soledad/server/_version.py'
@@ -28,8 +31,6 @@ versioneer.versionfile_build = 'leap/soledad/server/_version.py'
 versioneer.tag_prefix = ''  # tags are like 1.2.0
 versioneer.parentdir_prefix = 'leap.soledad.server-'
 
-from pkg import utils
-
 isset = lambda var: os.environ.get(var, None)
 if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'):
     data_files = None
@@ -68,9 +69,6 @@ if len(_version_short) > 0:
 cmdclass = versioneer.get_cmdclass()
 
 
-from setuptools import Command
-
-
 class freeze_debianver(Command):
     """
     Freezes the version in a debian branch.
@@ -116,6 +114,22 @@ cmdclass["freeze_debianver"] = freeze_debianver
 
 # XXX add ref to docs
 
+requirements = utils.parse_requirements()
+
+if utils.is_develop_mode():
+    print
+    print ("[WARNING] Skipping leap-specific dependencies "
+           "because development mode is detected.")
+    print ("[WARNING] You can install "
+           "the latest published versions with "
+           "'pip install -r pkg/requirements-leap.pip'")
+    print ("[WARNING] Or you can instead do 'python setup.py develop' "
+           "from the parent folder of each one of them.")
+    print
+else:
+    requirements += utils.parse_requirements(
+        reqfiles=["pkg/requirements-leap.pip"])
+
 setup(
     name='leap.soledad.server',
     version=VERSION,
@@ -138,6 +152,6 @@ setup(
     namespace_packages=["leap", "leap.soledad"],
     packages=find_packages('src'),
     package_dir={'': 'src'},
-    install_requires=utils.parse_requirements(),
+    install_requires=requirements,
     data_files=data_files
 )
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index adb5b561..7a03f6fb 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -92,16 +92,13 @@ import sys
 
 from u1db.remote import http_app, utils
 
+from ._version import get_versions
+
 # Keep OpenSSL's tsafe before importing Twisted submodules so we can put
 # it back if Twisted==12.0.0 messes with it.
 from OpenSSL import tsafe
-old_tsafe = tsafe
 
 from twisted import version
-if version.base() == "12.0.0":
-    # Put OpenSSL's tsafe back into place. This can probably be removed if we
-    # come to use Twisted>=12.3.0.
-    sys.modules['OpenSSL.tsafe'] = old_tsafe
 
 from leap.soledad.server.auth import SoledadTokenAuthMiddleware
 from leap.soledad.server.gzip_middleware import GzipMiddleware
@@ -115,11 +112,18 @@ from leap.soledad.server.sync import (
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common.couch import CouchServerState
 
+old_tsafe = tsafe
+
+if version.base() == "12.0.0":
+    # Put OpenSSL's tsafe back into place. This can probably be removed if we
+    # come to use Twisted>=12.3.0.
+    sys.modules['OpenSSL.tsafe'] = old_tsafe
 
 # ----------------------------------------------------------------------------
 # Soledad WSGI application
 # ----------------------------------------------------------------------------
 
+
 class SoledadApp(http_app.HTTPApp):
     """
     Soledad WSGI application
@@ -303,7 +307,5 @@ def application(environ, start_response):
 
     return application(environ, start_response)
 
-
-from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
diff --git a/server/src/leap/soledad/server/_version.py b/server/src/leap/soledad/server/_version.py
index 5b2e62d4..d9fb8322 100644
--- a/server/src/leap/soledad/server/_version.py
+++ b/server/src/leap/soledad/server/_version.py
@@ -1,4 +1,3 @@
-
 # This file was generated by the `freeze_debianver` command in setup.py
 # Using 'versioneer.py' (0.7+) from
 # revision-control system data, or from the parent directory name of an
@@ -8,6 +7,3 @@
 version_version = '0.6.5'
 version_full = '963717dc05824bcb4c69b912d948647ece3cf3aa'
 
-
-def get_versions(default={}, verbose=False):
-        return {'version': version_version, 'full': version_full}
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index 57f600a1..02b54cca 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -21,9 +21,9 @@ Authentication facilities for Soledad Server.
 """
 
 
+import time
 import httplib
-import simplejson as json
-
+import json
 
 from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors
 from abc import ABCMeta, abstractmethod
@@ -32,12 +32,8 @@ from couchdb.client import Server
 from twisted.python import log
 from hashlib import sha512
 
-
-from leap.soledad.common import (
-    SHARED_DB_NAME,
-    SHARED_DB_LOCK_DOC_ID_PREFIX,
-    USER_DB_PREFIX,
-)
+from leap.soledad.common import SHARED_DB_NAME
+from leap.soledad.common import USER_DB_PREFIX
 from leap.soledad.common.errors import InvalidAuthTokenError
 
 
@@ -268,7 +264,8 @@ class SoledadAuthMiddleware(object):
         scheme, encoded = auth.split(None, 1)
         uuid, auth_data = encoded.decode('base64').split(':', 1)
         if not self._verify_authentication_scheme(scheme):
-            return self._unauthorized_error("Wrong authentication scheme")
+            return self._unauthorized_error(
+                start_response, "Wrong authentication scheme")
 
         # verify if user is athenticated
         try:
@@ -354,7 +351,8 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
     Token based authentication.
     """
 
-    TOKENS_DB = "tokens"
+    TOKENS_DB_PREFIX = "tokens_"
+    TOKENS_DB_EXPIRE = 30 * 24 * 3600  # 30 days in seconds
     TOKENS_TYPE_KEY = "type"
     TOKENS_TYPE_DEF = "Token"
     TOKENS_USER_ID_KEY = "user_id"
@@ -414,7 +412,14 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
                                       invalid.
         """
         server = Server(url=self._app.state.couch_url)
-        dbname = self.TOKENS_DB
+        # the tokens db rotates every 30 days, and the current db name is
+        # "tokens_NNN", where NNN is the number of seconds since epoch divided
+        # by the rotate period in seconds. When rotating, old and new tokens
+        # db coexist during a certain window of time and valid tokens are
+        # replicated from the old db to the new one. See:
+        # https://leap.se/code/issues/6785
+        dbname = self.TOKENS_DB_PREFIX + \
+            str(int(time.time() / self.TOKENS_DB_EXPIRE))
         db = server[dbname]
         # lookup key is a hash of the token to prevent timing attacks.
         token = db.get(sha512(token).hexdigest())
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 6dc99b5a..18c4ee40 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -14,17 +14,12 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
 """
 Server side synchronization infrastructure.
 """
-
 import json
 
-
 from leap.soledad.common.couch import CouchDatabase
-from itertools import izip
 from u1db import sync, Document
 from u1db.remote import http_app
 
@@ -224,7 +219,6 @@ class SyncExchange(sync.SyncExchange):
         self._sync_state = ServerSyncState(
             self._db, self.source_replica_uid, sync_id)
 
-
     def find_changes_to_return(self, received):
         """
         Find changes to return.
@@ -286,7 +280,8 @@ class SyncExchange(sync.SyncExchange):
             doc = self._db.get_doc(changed_doc_id, include_deleted=True)
             return_doc_cb(doc, gen, trans_id)
 
-    def insert_doc_from_source(self, doc, source_gen, trans_id,
+    def insert_doc_from_source(
+            self, doc, source_gen, trans_id,
             number_of_docs=None, doc_idx=None, sync_id=None):
         """Try to insert synced document from source.
 
@@ -371,8 +366,9 @@ class SyncResource(http_app.SyncResource):
         self._sync_id = sync_id
 
     @http_app.http_method(content_as_args=True)
-    def post_put(self, id, rev, content, gen, trans_id, number_of_docs,
-            doc_idx):
+    def post_put(
+            self, id, rev, content, gen,
+            trans_id, number_of_docs, doc_idx):
         """
         Put one incoming document into the server replica.