diff options
Diffstat (limited to 'server')
| -rw-r--r-- | server/AUTHORS | 10 | ||||
| -rwxr-xr-x | server/pkg/generate_wheels.sh | 13 | ||||
| -rwxr-xr-x | server/pkg/pip_install_requirements.sh | 84 | ||||
| -rw-r--r-- | server/pkg/requirements-latest.pip | 8 | ||||
| -rw-r--r-- | server/pkg/requirements-leap.pip | 1 | ||||
| -rw-r--r-- | server/pkg/requirements.pip | 22 | ||||
| -rw-r--r-- | server/pkg/utils.py | 29 | ||||
| -rw-r--r-- | server/setup.cfg | 7 | ||||
| -rw-r--r-- | server/setup.py | 26 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/__init__.py | 16 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/_version.py | 4 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/auth.py | 27 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/sync.py | 14 |
13 files changed, 201 insertions, 60 deletions
diff --git a/server/AUTHORS b/server/AUTHORS new file mode 100644 index 00000000..934cf129 --- /dev/null +++ b/server/AUTHORS @@ -0,0 +1,10 @@ +drebs <drebs@leap.se> +Tomás Touceda <chiiph@leap.se> +Kali Kaneko <kali@leap.se> +Ivan Alejandro <ivanalejandro0@gmail.com> +Micah Anderson <micah@riseup.net> +Victor Shyba <victor.shyba@gmail.com> +Bruno Wagner <bwgpro@gmail.com> +Ruben Pollan <meskio@sindominio.net> +Duda Dornelles <ddornell@thoughtworks.com> +antialias <antialias@leap.se> diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh new file mode 100755 index 00000000..e29c327e --- /dev/null +++ b/server/pkg/generate_wheels.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Generate wheels for dependencies +# Use at your own risk. + +if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse +fi + +pip wheel --wheel-dir $WHEELHOUSE pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +if [ -f pkg/requirements-testing.pip ]; then + pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip +fi diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh new file mode 100755 index 00000000..d0479365 --- /dev/null +++ b/server/pkg/pip_install_requirements.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Update pip and install LEAP base/testing requirements. +# For convenience, $insecure_packages are allowed with insecure flags enabled. +# Use at your own risk. +# See $usage for help + +insecure_packages="u1db dirspec" +leap_wheelhouse=https://lizard.leap.se/wheels + +show_help() { + usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n +\t\t\tOtherwise, it will install requirements.pip\n +--use-leap-wheels\tUse wheels from leap.se" + echo -e $usage + + exit 1 +} + +process_arguments() { + testing=false + while [ "$#" -gt 0 ]; do + # From http://stackoverflow.com/a/31443098 + case "$1" in + --help) show_help;; + --testing) testing=true; shift 1;; + --use-leap-wheels) use_leap_wheels=true; shift 1;; + + -h) show_help;; + -*) echo "unknown option: $1" >&2; exit 1;; + esac + done +} + +return_wheelhouse() { + if $use_leap_wheels ; then + WHEELHOUSE=$leap_wheelhouse + elif [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + # Tested with bash and zsh + if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +return_install_options() { + wheelhouse=`return_wheelhouse` + install_options="-U --find-links=$wheelhouse" + if $use_leap_wheels ; then + install_options="$install_options --trusted-host lizard.leap.se" + fi + + echo $install_options +} + +return_insecure_flags() { + for insecure_package in $insecure_packages; do + flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" + done + + echo $flags +} + +return_packages() { + if $testing ; then + packages="-r pkg/requirements-testing.pip" + else + packages="-r pkg/requirements.pip" + fi + + echo $packages +} + +process_arguments $@ +install_options=`return_install_options` +insecure_flags=`return_insecure_flags` +packages=`return_packages` + +pip install -U wheel +pip install $install_options pip +pip install $install_options $insecure_flags $packages diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip new file mode 100644 index 00000000..a629aa57 --- /dev/null +++ b/server/pkg/requirements-latest.pip @@ -0,0 +1,8 @@ +--index-url https://pypi.python.org/simple/ + +--allow-external u1db --allow-unverified u1db +--allow-external dirspec --allow-unverified dirspec + +-e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' +-e '../common' +-e . diff --git a/server/pkg/requirements-leap.pip b/server/pkg/requirements-leap.pip new file mode 100644 index 00000000..aaad340c --- /dev/null +++ b/server/pkg/requirements-leap.pip @@ -0,0 +1 @@ +leap.soledad.common>=0.6.5 diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index 28717664..ca8ba42a 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -1,22 +1,10 @@ configparser couchdb -simplejson u1db routes -PyOpenSSL<0.14 - -# TODO: maybe we just want twisted-web? -twisted>=12.0.0 - -# leap deps -- bump me! -leap.soledad.common>=0.6.0 - -# -# Things yet to fix: -# - -# oauth is not strictly needed by us, but we need it -# until u1db adds it to its release as a dep. - +PyOpenSSL +twisted +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. oauth - diff --git a/server/pkg/utils.py b/server/pkg/utils.py index deace14b..d1680102 100644 --- a/server/pkg/utils.py +++ b/server/pkg/utils.py @@ -14,20 +14,34 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - """ Utils to help in the setup process """ - import os import re import sys +def is_develop_mode(): + """ + Returns True if we're calling the setup script using the argument for + setuptools development mode. + + This avoids messing up with dependency pinning and order, the + responsibility of installing the leap dependencies is left to the + developer. + """ + args = sys.argv + devflags = "setup.py", "develop" + if (args[0], args[1]) == devflags: + return True + return False + + def get_reqs_from_files(reqfiles): """ Returns the contents of the top requirement file listed as a - string list with the lines + string list with the lines. @param reqfiles: requirement files to parse @type reqfiles: list of str @@ -43,6 +57,9 @@ def parse_requirements(reqfiles=['requirements.txt', """ Parses the requirement files provided. + The passed reqfiles list is a list of possible locations to try, the + function will return the contents of the first path found. + Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should return PySide as a dep or not. Don't set, or set to 0 if you want to install it through pip. @@ -58,9 +75,9 @@ def parse_requirements(reqfiles=['requirements.txt', if re.match(r'\s*-e\s+', line): pass # do not try to do anything with externals on vcs - #requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - #line)) - # http://foo.bar/baz/foobar/zipball/master#egg=foobar + # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + # line)) + # http://foo.bar/baz/foobar/zipball/master#egg=foobar elif re.match(r'\s*https?:', line): requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', line)) diff --git a/server/setup.cfg b/server/setup.cfg new file mode 100644 index 00000000..6b530888 --- /dev/null +++ b/server/setup.cfg @@ -0,0 +1,7 @@ +[pep8] +exclude = versioneer.py,_version.py,ddocs.py,*.egg,build +ignore = E731 + +[flake8] +exclude = versioneer.py,_version.py,ddocs.py,*.egg,build +ignore = E731 diff --git a/server/setup.py b/server/setup.py index 124ddd32..b3942551 100644 --- a/server/setup.py +++ b/server/setup.py @@ -21,6 +21,9 @@ import os import re from setuptools import setup from setuptools import find_packages +from setuptools import Command + +from pkg import utils import versioneer versioneer.versionfile_source = 'src/leap/soledad/server/_version.py' @@ -28,8 +31,6 @@ versioneer.versionfile_build = 'leap/soledad/server/_version.py' versioneer.tag_prefix = '' # tags are like 1.2.0 versioneer.parentdir_prefix = 'leap.soledad.server-' -from pkg import utils - isset = lambda var: os.environ.get(var, None) if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'): data_files = None @@ -68,9 +69,6 @@ if len(_version_short) > 0: cmdclass = versioneer.get_cmdclass() -from setuptools import Command - - class freeze_debianver(Command): """ Freezes the version in a debian branch. @@ -116,6 +114,22 @@ cmdclass["freeze_debianver"] = freeze_debianver # XXX add ref to docs +requirements = utils.parse_requirements() + +if utils.is_develop_mode(): + print + print ("[WARNING] Skipping leap-specific dependencies " + "because development mode is detected.") + print ("[WARNING] You can install " + "the latest published versions with " + "'pip install -r pkg/requirements-leap.pip'") + print ("[WARNING] Or you can instead do 'python setup.py develop' " + "from the parent folder of each one of them.") + print +else: + requirements += utils.parse_requirements( + reqfiles=["pkg/requirements-leap.pip"]) + setup( name='leap.soledad.server', version=VERSION, @@ -138,6 +152,6 @@ setup( namespace_packages=["leap", "leap.soledad"], packages=find_packages('src'), package_dir={'': 'src'}, - install_requires=utils.parse_requirements(), + install_requires=requirements, data_files=data_files ) diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index adb5b561..7a03f6fb 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -92,16 +92,13 @@ import sys from u1db.remote import http_app, utils +from ._version import get_versions + # Keep OpenSSL's tsafe before importing Twisted submodules so we can put # it back if Twisted==12.0.0 messes with it. from OpenSSL import tsafe -old_tsafe = tsafe from twisted import version -if version.base() == "12.0.0": - # Put OpenSSL's tsafe back into place. This can probably be removed if we - # come to use Twisted>=12.3.0. - sys.modules['OpenSSL.tsafe'] = old_tsafe from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware @@ -115,11 +112,18 @@ from leap.soledad.server.sync import ( from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import CouchServerState +old_tsafe = tsafe + +if version.base() == "12.0.0": + # Put OpenSSL's tsafe back into place. This can probably be removed if we + # come to use Twisted>=12.3.0. + sys.modules['OpenSSL.tsafe'] = old_tsafe # ---------------------------------------------------------------------------- # Soledad WSGI application # ---------------------------------------------------------------------------- + class SoledadApp(http_app.HTTPApp): """ Soledad WSGI application @@ -303,7 +307,5 @@ def application(environ, start_response): return application(environ, start_response) - -from ._version import get_versions __version__ = get_versions()['version'] del get_versions diff --git a/server/src/leap/soledad/server/_version.py b/server/src/leap/soledad/server/_version.py index 5b2e62d4..d9fb8322 100644 --- a/server/src/leap/soledad/server/_version.py +++ b/server/src/leap/soledad/server/_version.py @@ -1,4 +1,3 @@ - # This file was generated by the `freeze_debianver` command in setup.py # Using 'versioneer.py' (0.7+) from # revision-control system data, or from the parent directory name of an @@ -8,6 +7,3 @@ version_version = '0.6.5' version_full = '963717dc05824bcb4c69b912d948647ece3cf3aa' - -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 57f600a1..02b54cca 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -21,9 +21,9 @@ Authentication facilities for Soledad Server. """ +import time import httplib -import simplejson as json - +import json from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod @@ -32,12 +32,8 @@ from couchdb.client import Server from twisted.python import log from hashlib import sha512 - -from leap.soledad.common import ( - SHARED_DB_NAME, - SHARED_DB_LOCK_DOC_ID_PREFIX, - USER_DB_PREFIX, -) +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common import USER_DB_PREFIX from leap.soledad.common.errors import InvalidAuthTokenError @@ -268,7 +264,8 @@ class SoledadAuthMiddleware(object): scheme, encoded = auth.split(None, 1) uuid, auth_data = encoded.decode('base64').split(':', 1) if not self._verify_authentication_scheme(scheme): - return self._unauthorized_error("Wrong authentication scheme") + return self._unauthorized_error( + start_response, "Wrong authentication scheme") # verify if user is athenticated try: @@ -354,7 +351,8 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): Token based authentication. """ - TOKENS_DB = "tokens" + TOKENS_DB_PREFIX = "tokens_" + TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds TOKENS_TYPE_KEY = "type" TOKENS_TYPE_DEF = "Token" TOKENS_USER_ID_KEY = "user_id" @@ -414,7 +412,14 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): invalid. """ server = Server(url=self._app.state.couch_url) - dbname = self.TOKENS_DB + # the tokens db rotates every 30 days, and the current db name is + # "tokens_NNN", where NNN is the number of seconds since epoch divided + # by the rotate period in seconds. When rotating, old and new tokens + # db coexist during a certain window of time and valid tokens are + # replicated from the old db to the new one. See: + # https://leap.se/code/issues/6785 + dbname = self.TOKENS_DB_PREFIX + \ + str(int(time.time() / self.TOKENS_DB_EXPIRE)) db = server[dbname] # lookup key is a hash of the token to prevent timing attacks. token = db.get(sha512(token).hexdigest()) diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 6dc99b5a..18c4ee40 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -14,17 +14,12 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - - """ Server side synchronization infrastructure. """ - import json - from leap.soledad.common.couch import CouchDatabase -from itertools import izip from u1db import sync, Document from u1db.remote import http_app @@ -224,7 +219,6 @@ class SyncExchange(sync.SyncExchange): self._sync_state = ServerSyncState( self._db, self.source_replica_uid, sync_id) - def find_changes_to_return(self, received): """ Find changes to return. @@ -286,7 +280,8 @@ class SyncExchange(sync.SyncExchange): doc = self._db.get_doc(changed_doc_id, include_deleted=True) return_doc_cb(doc, gen, trans_id) - def insert_doc_from_source(self, doc, source_gen, trans_id, + def insert_doc_from_source( + self, doc, source_gen, trans_id, number_of_docs=None, doc_idx=None, sync_id=None): """Try to insert synced document from source. @@ -371,8 +366,9 @@ class SyncResource(http_app.SyncResource): self._sync_id = sync_id @http_app.http_method(content_as_args=True) - def post_put(self, id, rev, content, gen, trans_id, number_of_docs, - doc_idx): + def post_put( + self, id, rev, content, gen, + trans_id, number_of_docs, doc_idx): """ Put one incoming document into the server replica. |