diff options
Diffstat (limited to 'server')
| -rw-r--r-- | server/changes/next-changelog.rst | 2 | ||||
| -rwxr-xr-x | server/pkg/create-user-db | 30 | ||||
| -rwxr-xr-x | server/pkg/generate_wheels.sh | 2 | ||||
| -rwxr-xr-x | server/pkg/pip_install_requirements.sh | 4 | ||||
| -rw-r--r-- | server/pkg/requirements-latest.pip | 3 | ||||
| -rw-r--r-- | server/pkg/requirements.pip | 7 | ||||
| -rw-r--r-- | server/setup.py | 28 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/__init__.py | 46 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/auth.py | 11 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/lock_resource.py | 231 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/sync.py | 6 |
11 files changed, 73 insertions, 297 deletions
diff --git a/server/changes/next-changelog.rst b/server/changes/next-changelog.rst index bdc9f893..fc4cbc30 100644 --- a/server/changes/next-changelog.rst +++ b/server/changes/next-changelog.rst @@ -1,4 +1,4 @@ -0.8.1 - ... +0.8.2 - ... ++++++++++++++++++++ Please add lines to this file, they will be moved to the CHANGELOG.rst during diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 54856643..5e48d4de 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -25,6 +25,9 @@ from leap.soledad.common.couch import list_users_dbs from leap.soledad.server import load_configuration +BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False) + + description = """ Creates a user database. This is meant to be used by Soledad Server. @@ -40,16 +43,23 @@ NETRC_PATH = CONF['soledad-server']['admin_netrc'] def url_for_db(dbname): - if not os.path.exists(NETRC_PATH): - print ('netrc not found in %s' % NETRC_PATH) - sys.exit(1) - parsed_netrc = netrc.netrc(NETRC_PATH) - host, (login, _, password) = parsed_netrc.hosts.items()[0] - url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % { - 'login': login, - 'password': password, - 'host': host, - 'dbname': dbname}) + if BYPASS_AUTH: + login = '' + password = '' + host = 'localhost' + url = 'http://localhost:5984/%(dbname)s' % { + 'dbname': dbname} + else: + if not os.path.exists(NETRC_PATH): + print ('netrc not found in %s' % NETRC_PATH) + sys.exit(1) + parsed_netrc = netrc.netrc(NETRC_PATH) + host, (login, _, password) = parsed_netrc.hosts.items()[0] + url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % { + 'login': login, + 'password': password, + 'host': host, + 'dbname': dbname}) return url diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh index e29c327e..a13e2c7a 100755 --- a/server/pkg/generate_wheels.sh +++ b/server/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh index d0479365..f4b5f67a 100755 --- a/server/pkg/pip_install_requirements.sh +++ b/server/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { @@ -80,5 +80,5 @@ insecure_flags=`return_insecure_flags` packages=`return_packages` pip install -U wheel -pip install $install_options pip +pip install -U pip pip install $install_options $insecure_flags $packages diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip index a629aa57..46a7ccba 100644 --- a/server/pkg/requirements-latest.pip +++ b/server/pkg/requirements-latest.pip @@ -1,8 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db ---allow-external dirspec --allow-unverified dirspec - -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -e '../common' -e . diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index f9cce08e..2d845f24 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -1,13 +1,6 @@ configparser -u1db -routes PyOpenSSL twisted>=12.3.0 #pinned for wheezy compatibility Beaker==1.6.3 #wheezy couchdb==0.8 #wheezy - -# XXX -- fix me! -# oauth is not strictly needed by us, but we need it until u1db adds it to its -# release as a dep. -oauth diff --git a/server/setup.py b/server/setup.py index 8a7fbe45..b3b26010 100644 --- a/server/setup.py +++ b/server/setup.py @@ -77,14 +77,20 @@ class freeze_debianver(Command): # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '{version}' -full_revisionid = '{full_revisionid}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, - 'full-revisionid': full_revisionid} +import json +import sys + +version_json = ''' +{ + "dirty": false, + "error": null, + "full-revisionid": "FULL_REVISIONID", + "version": "VERSION_STRING" +} +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ def initialize_options(self): @@ -99,9 +105,9 @@ def get_versions(default={}, verbose=False): if proceed != "y": print("He. You scared. Aborting.") return - subst_template = self.template.format( - version=VERSION_SHORT, - full_revisionid=VERSION_REVISION) + self.templatefun + subst_template = self.template.replace( + 'VERSION_STRING', VERSION_SHORT).replace( + 'FULL_REVISIONID', VERSION_REVISION) versioneer_cfg = versioneer.get_config_from_root('.') with open(versioneer_cfg.versionfile_source, 'w') as f: f.write(subst_template) diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 22894dac..34570b52 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -49,9 +49,7 @@ synchronization is handled by the `leap.soledad.server.auth` module. Shared database --------------- -Each user may store password-encrypted recovery data in the shared database, -as well as obtain a lock on the shared database in order to prevent creation -of multiple secrets in parallel. +Each user may store password-encrypted recovery data in the shared database. Recovery documents are stored in the database without any information that may identify the user. In order to achieve this, the doc_id of recovery @@ -77,26 +75,19 @@ This has some implications: * Because of the above, it is recommended that recovery documents expire (not implemented yet) to prevent excess storage. -Lock documents, on the other hand, may be more thoroughly protected by the -server. Their doc_id's are calculated from the SHARED_DB_LOCK_DOC_ID_PREFIX -and the user's uid. - The authorization for creating, updating, deleting and retrieving recovery -and lock documents on the shared database is handled by -`leap.soledad.server.auth` module. +documents on the shared database is handled by `leap.soledad.server.auth` +module. """ import configparser import urlparse import sys -from u1db.remote import http_app, utils - -from ._version import get_versions +from leap.soledad.common.l2db.remote import http_app, utils from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware -from leap.soledad.server.lock_resource import LockResource from leap.soledad.server.sync import ( SyncResource, MAX_REQUEST_SIZE, @@ -107,6 +98,8 @@ from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState +from ._version import get_versions + # ---------------------------------------------------------------------------- # Soledad WSGI application # ---------------------------------------------------------------------------- @@ -155,7 +148,6 @@ http_app.url_to_resource.register(http_app.DocsResource) http_app.url_to_resource.register(http_app.DocResource) # register Soledad's new or modified resources -http_app.url_to_resource.register(LockResource) http_app.url_to_resource.register(SyncResource) @@ -312,16 +304,34 @@ def load_configuration(file_path): # Run as Twisted WSGI Resource # ---------------------------------------------------------------------------- -def application(environ, start_response): + +def _load_config(): conf = load_configuration('/etc/soledad/soledad-server.conf') - conf = conf['soledad-server'] + return conf['soledad-server'] + + +def _get_couch_state(): + conf = _load_config() state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) - SoledadBackend.BATCH_SUPPORT = conf['batching'] - # WSGI application that may be used by `twistd -web` + SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) + return state + + +def application(environ, start_response): + """return WSGI application that may be used by `twistd -web`""" + state = _get_couch_state() application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) + return application(environ, start_response) + +def debug_local_application_do_not_use(environ, start_response): + """in where we bypass token auth middleware for ease of mind while + debugging in your local environment""" + state = _get_couch_state() + application = SoledadApp(state) return application(environ, start_response) + __version__ = get_versions()['version'] del get_versions diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index ccbd6fbd..ecee2d5d 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -14,21 +14,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - - """ Authentication facilities for Soledad Server. """ - - import httplib import json -from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper from twisted.python import log +from leap.soledad.common.l2db import DBNAME_CONSTRAINTS, errors as u1db_errors from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import USER_DB_PREFIX @@ -101,7 +97,6 @@ class URLToAuthorization(object): /shared-db/docs | - /shared-db/doc/{any_id} | GET, PUT, DELETE /shared-db/sync-from/{source} | - - /shared-db/lock/{uuid} | PUT, DELETE /user-db | GET, PUT, DELETE /user-db/docs | - /user-db/doc/{id} | - @@ -118,10 +113,6 @@ class URLToAuthorization(object): '/%s/doc/{id:.*}' % SHARED_DB_NAME, [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) - # auth info for shared-db lock resource - self._register( - '/%s/lock/%s' % (SHARED_DB_NAME, self._uuid), - [self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) # auth info for user-db database resource self._register( '/%s' % self._user_db_name, diff --git a/server/src/leap/soledad/server/lock_resource.py b/server/src/leap/soledad/server/lock_resource.py deleted file mode 100644 index 0a602e26..00000000 --- a/server/src/leap/soledad/server/lock_resource.py +++ /dev/null @@ -1,231 +0,0 @@ -# -*- coding: utf-8 -*- -# lock_resource.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -LockResource: a lock based on a document in the shared database. -""" - - -import hashlib -import time -import os -import tempfile -import errno - - -from u1db.remote import http_app -from twisted.python.lockfile import FilesystemLock - - -from leap.soledad.common import ( - SHARED_DB_NAME, - SHARED_DB_LOCK_DOC_ID_PREFIX, -) -from leap.soledad.common.errors import ( - InvalidTokenError, - NotLockedError, - AlreadyLockedError, - LockTimedOutError, - CouldNotObtainLockError, -) - - -class LockResource(object): - """ - Handle requests for locking documents. - - This class uses Twisted's Filesystem lock to manage a lock in the shared - database. - """ - - url_pattern = '/%s/lock/{uuid}' % SHARED_DB_NAME - """ - """ - - TIMEOUT = 300 # XXX is 5 minutes reasonable? - """ - The timeout after which the lock expires. - """ - - # used for lock doc storage - TIMESTAMP_KEY = '_timestamp' - LOCK_TOKEN_KEY = '_token' - - FILESYSTEM_LOCK_TRIES = 5 - FILESYSTEM_LOCK_SLEEP_SECONDS = 1 - - def __init__(self, uuid, state, responder): - """ - Initialize the lock resource. Parameters to this constructor are - automatically passed by u1db. - - :param uuid: The user unique id. - :type uuid: str - :param state: The backend database state. - :type state: u1db.remote.ServerState - :param responder: The infrastructure to send responses to client. - :type responder: u1db.remote.HTTPResponder - """ - self._shared_db = state.open_database(SHARED_DB_NAME) - self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) - self._lock = FilesystemLock( - os.path.join( - tempfile.gettempdir(), - hashlib.sha512(self._lock_doc_id).hexdigest())) - self._state = state - self._responder = responder - - @http_app.http_method(content=str) - def put(self, content=None): - """ - Handle a PUT request to the lock document. - - A lock is a document in the shared db with doc_id equal to - 'lock-<uuid>' and the timestamp of its creation as content. This - method obtains a threaded-lock and creates a lock document if it does - not exist or if it has expired. - - It returns '201 Created' and a pair containing a token to unlock and - the lock timeout, or '403 AlreadyLockedError' and the remaining amount - of seconds the lock will still be valid. - - :param content: The content of the PUT request. It is only here - because PUT requests with empty content are considered - invalid requests by u1db. - :type content: str - """ - # obtain filesystem lock - if not self._try_obtain_filesystem_lock(): - self._responder.send_response_json( - LockTimedOutError.status, # error: request timeout - error=LockTimedOutError.wire_description) - return - - created_lock = False - now = time.time() - token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing - lock_doc = self._shared_db.get_doc(self._lock_doc_id) - remaining = self._remaining(lock_doc, now) - - # if there's no lock, create one - if lock_doc is None: - lock_doc = self._shared_db.create_doc( - { - self.TIMESTAMP_KEY: now, - self.LOCK_TOKEN_KEY: token, - }, - doc_id=self._lock_doc_id) - created_lock = True - else: - if remaining == 0: - # lock expired, create new one - lock_doc.content = { - self.TIMESTAMP_KEY: now, - self.LOCK_TOKEN_KEY: token, - } - self._shared_db.put_doc(lock_doc) - created_lock = True - - self._try_release_filesystem_lock() - - # send response to client - if created_lock is True: - self._responder.send_response_json( - 201, timeout=self.TIMEOUT, token=token) # success: created - else: - self._responder.send_response_json( - AlreadyLockedError.status, # error: forbidden - error=AlreadyLockedError.wire_description, remaining=remaining) - - @http_app.http_method(token=str) - def delete(self, token=None): - """ - Delete the lock if the C{token} is valid. - - Delete the lock document in case C{token} is equal to the token stored - in the lock document. - - :param token: The token returned when locking. - :type token: str - - :raise NotLockedError: Raised in case the lock is not locked. - :raise InvalidTokenError: Raised in case the token is invalid for - unlocking. - """ - lock_doc = self._shared_db.get_doc(self._lock_doc_id) - if lock_doc is None or self._remaining(lock_doc, time.time()) == 0: - self._responder.send_response_json( - NotLockedError.status, # error: not found - error=NotLockedError.wire_description) - elif token != lock_doc.content[self.LOCK_TOKEN_KEY]: - self._responder.send_response_json( - InvalidTokenError.status, # error: unauthorized - error=InvalidTokenError.wire_description) - else: - self._shared_db.delete_doc(lock_doc) - # respond success: should use 204 but u1db does not support it. - self._responder.send_response_json(200) - - def _remaining(self, lock_doc, now): - """ - Return the number of seconds the lock contained in C{lock_doc} is - still valid, when compared to C{now}. - - :param lock_doc: The document containing the lock. - :type lock_doc: u1db.Document - :param now: The time to which to compare the lock timestamp. - :type now: float - - :return: The amount of seconds the lock is still valid. - :rtype: float - """ - if lock_doc is not None: - lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY] - remaining = lock_timestamp + self.TIMEOUT - now - return remaining if remaining > 0 else 0.0 - return 0.0 - - def _try_obtain_filesystem_lock(self): - """ - Try to obtain the file system lock. - - @return: Whether the lock was succesfully obtained. - @rtype: bool - """ - tries = self.FILESYSTEM_LOCK_TRIES - while tries > 0: - try: - return self._lock.lock() - except OSError as e: - tries -= 1 - if tries == 0: - raise CouldNotObtainLockError(e.message) - time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS) - return False - - def _try_release_filesystem_lock(self): - """ - Release the filesystem lock. - """ - try: - self._lock.unlock() - return True - except OSError as e: - if e.errno == errno.ENOENT: - return True - return False diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 96f65912..3f5c4aba 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -17,10 +17,10 @@ """ Server side synchronization infrastructure. """ -from u1db import sync, Document -from u1db.remote import http_app -from leap.soledad.server.state import ServerSyncState +from leap.soledad.common.l2db import sync, Document +from leap.soledad.common.l2db.remote import http_app from leap.soledad.server.caching import get_cache_for +from leap.soledad.server.state import ServerSyncState MAX_REQUEST_SIZE = 200 # in Mb |