diff options
| -rw-r--r-- | CHANGELOG | 14 | ||||
| -rw-r--r-- | client/setup.py | 70 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/__init__.py | 189 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/shared_db.py | 40 | ||||
| -rw-r--r-- | common/setup.py | 70 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/__init__.py | 16 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/couch.py | 19 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/errors.py | 70 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/__init__.py | 10 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_server.py | 189 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_soledad.py | 94 | ||||
| -rw-r--r-- | server/setup.py | 71 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/__init__.py | 270 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/auth.py | 61 |
14 files changed, 1017 insertions, 166 deletions
@@ -1,3 +1,17 @@ +0.4.2 Nov 1: +Client: + o Support non-ascii passwords. Closes #4001. + o Change error severity for missing secrets path. + o Use chardet as fallback if cchardet not found. + o Improve bootstrap sequence and allow for locking the shared + database while creating/uploading the encryption secret. Closes + #4097. +Common: + o Move some common functions and global variables to + leap.soledad.common. +Server: + o Allow for locking the shared database. Closes #4097. + 0.4.1 Oct 4: Client: o Save only UTF8 strings. Related to #3660. diff --git a/client/setup.py b/client/setup.py index 4f62809f..c3e4936f 100644 --- a/client/setup.py +++ b/client/setup.py @@ -17,6 +17,7 @@ """ setup file for leap.soledad.client """ +import re from setuptools import setup from setuptools import find_packages @@ -43,18 +44,83 @@ trove_classifiers = ( "Topic :: Software Development :: Libraries :: Python Modules" ) +DOWNLOAD_BASE = ('https://github.com/leapcode/soledad/' + 'archive/%s.tar.gz') +_versions = versioneer.get_versions() +VERSION = _versions['version'] +VERSION_FULL = _versions['full'] +DOWNLOAD_URL = "" + +# get the short version for the download url +_version_short = re.findall('\d+\.\d+\.\d+', VERSION) +if len(_version_short) > 0: + VERSION_SHORT = _version_short[0] + DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT + +cmdclass = versioneer.get_cmdclass() + + +from setuptools import Command + + +class freeze_debianver(Command): + """ + Freezes the version in a debian branch. + To be used after merging the development branch onto the debian one. + """ + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +version_full = '{version_full}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} +""" + subst_template = template.format( + version=VERSION_SHORT, + version_full=VERSION_FULL) + templatefun + with open(versioneer.versionfile_source, 'w') as f: + f.write(subst_template) + + +cmdclass["freeze_debianver"] = freeze_debianver + # XXX add ref to docs setup( name='leap.soledad.client', - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=VERSION, + cmdclass=cmdclass, url='https://leap.se/', + download_url=DOWNLOAD_URL, license='GPLv3+', description='Synchronization of locally encrypted data among devices ' '(client components).', author='The LEAP Encryption Access Project', author_email='info@leap.se', + maintainer='Kali Kaneko', + maintainer_email='kali@leap.se', long_description=( "Soledad is the part of LEAP that allows application data to be " "securely shared among devices. It provides, to other parts of the " diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 13a3b68f..534040ef 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -32,10 +32,13 @@ import socket import ssl import urlparse -import cchardet - from hashlib import sha256 +try: + import cchardet as chardet +except ImportError: + import chardet + from u1db.remote import http_client from u1db.remote.ssl_match_hostname import match_hostname @@ -43,6 +46,12 @@ import scrypt import simplejson as json from leap.common.config import get_path_prefix +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.errors import ( + InvalidTokenError, + NotLockedError, + AlreadyLockedError, +) # # Signaling function @@ -103,8 +112,6 @@ Path to the certificate file used to certify the SSL connection between Soledad client and server. """ -SECRETS_DOC_ID_HASH_PREFIX = 'uuid-' - # # Soledad: local encrypted storage and remote encrypted sync. @@ -124,6 +131,13 @@ class PassphraseTooShort(Exception): """ +class BootstrapSequenceError(Exception): + """ + Raised when an attempt to generate a secret and store it in a recovery + documents on server failed. + """ + + class Soledad(object): """ Soledad provides encrypted data storage and sync. @@ -232,7 +246,7 @@ class Soledad(object): :type uuid: str :param passphrase: The passphrase for locking and unlocking encryption secrets for local and remote storage. - :type passphrase: str + :type passphrase: unicode :param secrets_path: Path for storing encrypted key used for symmetric encryption. :type secrets_path: str @@ -248,9 +262,13 @@ class Soledad(object): :type cert_file: str :param auth_token: Authorization token for accessing remote databases. :type auth_token: str + + :raise BootstrapSequenceError: Raised when the secret generation and + storage on server sequence has failed for some reason. """ # get config params self._uuid = uuid + soledad_assert_type(passphrase, unicode) self._passphrase = passphrase # init crypto variables self._secrets = {} @@ -258,11 +276,12 @@ class Soledad(object): # init config (possibly with default values) self._init_config(secrets_path, local_db_path, server_url) self._set_token(auth_token) + self._shared_db_instance = None # configure SSL certificate global SOLEDAD_CERT SOLEDAD_CERT = cert_file # initiate bootstrap sequence - self._bootstrap() + self._bootstrap() # might raise BootstrapSequenceError() def _init_config(self, secrets_path, local_db_path, server_url): """ @@ -297,45 +316,93 @@ class Soledad(object): * stage 0 - local environment setup. - directory initialization. - crypto submodule initialization - * stage 1 - secret generation/loading: + * stage 1 - local secret loading: - if secrets exist locally, load them. + * stage 2 - remote secret loading: - else, if secrets exist in server, download them. - - else, generate a new secret. - * stage 2 - store secrets in server. - * stage 3 - database initialization. + * stage 3 - secret generation: + - else, generate a new secret and store in server. + * stage 4 - database initialization. This method decides which bootstrap stages have already been performed and performs the missing ones in order. + + :raise BootstrapSequenceError: Raised when the secret generation and + storage on server sequence has failed for some reason. """ - # TODO: make sure key storage always happens (even if this method is - # interrupted). - # TODO: write tests for bootstrap stages. - # TODO: log each bootstrap step. - # stage 0 - socal environment setup + # STAGE 0 - local environment setup self._init_dirs() self._crypto = SoledadCrypto(self) - # stage 1 - secret generation/loading + + # STAGE 1 - verify if secrets exist locally if not self._has_secret(): # try to load from local storage. + + # STAGE 2 - there are no secrets in local storage, so try to fetch + # encrypted secrets from server. logger.info( 'Trying to fetch cryptographic secrets from shared recovery ' 'database...') - # there are no secrets in local storage, so try to fetch encrypted - # secrets from server. + + # --- start of atomic operation in shared db --- + + # obtain lock on shared db + token = timeout = None + try: + token, timeout = self._shared_db.lock() + except AlreadyLockedError: + raise BootstrapSequenceError('Database is already locked.') + doc = self._get_secrets_from_shared_db() if doc: - # found secrets in server, so import them. logger.info( 'Found cryptographic secrets in shared recovery ' 'database.') self.import_recovery_document(doc.content) else: - # there are no secrets in server also, so generate a secret. + # STAGE 3 - there are no secrets in server also, so + # generate a secret and store it in remote db. logger.info( - 'No cryptographic secrets found, creating new secrets...') + 'No cryptographic secrets found, creating new ' + ' secrets...') self._set_secret_id(self._gen_secret()) - # Stage 2 - storage of encrypted secrets in the server. - self._put_secrets_in_shared_db() - # Stage 3 - Local database initialization + try: + self._put_secrets_in_shared_db() + except Exception: + # storing generated secret in shared db failed for + # some reason, so we erase the generated secret and + # raise. + try: + os.unlink(self._secrets_path) + except OSError as e: + if errno == 2: # no such file or directory + pass + raise BootstrapSequenceError( + 'Could not store generated secret in the shared ' + 'database, bailing out...') + + # release the lock on shared db + try: + self._shared_db.unlock(token) + except NotLockedError: + # for some reason the lock expired. Despite that, secret + # loading or generation/storage must have been executed + # successfully, so we pass. + pass + except InvalidTokenError: + # here, our lock has not only expired but also some other + # client application has obtained a new lock and is currently + # doing its thing in the shared database. Using the same + # reasoning as above, we assume everything went smooth and + # pass. + pass + except Exception as e: + logger.error("Unhandled exception when unlocking shared " + "database.") + logger.exception(e) + + # --- end of atomic operation in shared db --- + + # STAGE 4 - local database initialization self._init_db() def _init_dirs(self): @@ -426,7 +493,7 @@ class Soledad(object): """ # calculate the encryption key key = scrypt.hash( - self._passphrase, + self._passphrase_as_string(), # the salt is stored base64 encoded binascii.a2b_base64( self._secrets[self._secret_id][self.KDF_SALT_KEY]), @@ -488,11 +555,11 @@ class Soledad(object): try: self._load_secrets() # try to load from disk except IOError, e: - logger.error('IOError: %s' % str(e)) + logger.warning('IOError: %s' % str(e)) try: self._get_storage_secret() return True - except: + except Exception: return False def _gen_secret(self): @@ -528,7 +595,7 @@ class Soledad(object): # generate random salt salt = os.urandom(self.SALT_LENGTH) # get a 256-bit key - key = scrypt.hash(self._passphrase, salt, buflen=32) + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) iv, ciphertext = self._crypto.encrypt_sym(secret, key) self._secrets[secret_id] = { # leap.soledad.crypto submodule uses AES256 for symmetric @@ -575,13 +642,13 @@ class Soledad(object): Change the passphrase that encrypts the storage secret. :param new_passphrase: The new passphrase. - :type new_passphrase: str + :type new_passphrase: unicode :raise NoStorageSecret: Raised if there's no storage secret available. """ # maybe we want to add more checks to guarantee passphrase is # reasonable? - soledad_assert_type(new_passphrase, str) + soledad_assert_type(new_passphrase, unicode) if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: raise PassphraseTooShort( 'Passphrase must be at least %d characters long!' % @@ -593,7 +660,7 @@ class Soledad(object): # generate random salt new_salt = os.urandom(self.SALT_LENGTH) # get a 256-bit key - key = scrypt.hash(new_passphrase, new_salt, buflen=32) + key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) iv, ciphertext = self._crypto.encrypt_sym(secret, key) self._secrets[self._secret_id] = { # leap.soledad.crypto submodule uses AES256 for symmetric @@ -614,28 +681,32 @@ class Soledad(object): # General crypto utility methods. # - def _uuid_hash(self): + @property + def _shared_db(self): """ - Calculate a hash for storing/retrieving key material on shared - database, based on user's uuid. + Return an instance of the shared recovery database object. - :return: the hash - :rtype: str + :return: The shared database. + :rtype: SoledadSharedDatabase """ - return sha256( - '%s%s' % ( - SECRETS_DOC_ID_HASH_PREFIX, - self._uuid)).hexdigest() + if self._shared_db_instance is None: + self._shared_db_instance = SoledadSharedDatabase.open_database( + urlparse.urljoin(self.server_url, SHARED_DB_NAME), + self._uuid, + False, # db should exist at this point. + creds=self._creds) + return self._shared_db_instance - def _shared_db(self): + def _shared_db_doc_id(self): """ - Return an instance of the shared recovery database object. + Calculate the doc_id of the document in the shared db that stores key + material. + + :return: the hash + :rtype: str """ - if self.server_url: - return SoledadSharedDatabase.open_database( - urlparse.urljoin(self.server_url, 'shared'), - False, # TODO: eliminate need to create db here. - creds=self._creds) + return sha256('%s%s' % + (self._passphrase_as_string(), self.uuid)).hexdigest() def _get_secrets_from_shared_db(self): """ @@ -646,11 +717,11 @@ class Soledad(object): :rtype: SoledadDocument """ signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid) - db = self._shared_db() + db = self._shared_db if not db: logger.warning('No shared db found') return - doc = db.get_doc(self._uuid_hash()) + doc = db.get_doc(self._shared_db_doc_id()) signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) return doc @@ -661,7 +732,6 @@ class Soledad(object): Try to fetch keys from shared recovery database. If they already exist in the remote db, assert that that data is the same as local data. Otherwise, upload keys to shared recovery database. - """ soledad_assert( self._has_secret(), @@ -670,12 +740,13 @@ class Soledad(object): # try to get secrets doc from server, otherwise create it doc = self._get_secrets_from_shared_db() if doc is None: - doc = SoledadDocument(doc_id=self._uuid_hash()) + doc = SoledadDocument( + doc_id=self._shared_db_doc_id()) # fill doc with encrypted secrets doc.content = self.export_recovery_document(include_uuid=False) # upload secrets to server signal(SOLEDAD_UPLOADING_KEYS, self._uuid) - db = self._shared_db() + db = self._shared_db if not db: logger.warning('No shared db found') return @@ -757,7 +828,7 @@ class Soledad(object): """ return self._db.get_all_docs(include_deleted) - def _convert_to_utf8(self, content): + def _convert_to_unicode(self, content): """ Converts content to utf8 (or all the strings in content) @@ -771,12 +842,11 @@ class Soledad(object): :rtype: object """ - if isinstance(content, unicode): return content elif isinstance(content, str): try: - result = cchardet.detect(content) + result = chardet.detect(content) content = content.decode(result["encoding"]).encode("utf-8")\ .decode("utf-8") except UnicodeError: @@ -785,7 +855,7 @@ class Soledad(object): else: if isinstance(content, dict): for key in content.keys(): - content[key] = self._convert_to_utf8(content[key]) + content[key] = self._convert_to_unicode(content[key]) return content def create_doc(self, content, doc_id=None): @@ -800,7 +870,8 @@ class Soledad(object): :return: the new document :rtype: SoledadDocument """ - return self._db.create_doc(self._convert_to_utf8(content), doc_id=doc_id) + return self._db.create_doc( + self._convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -1027,6 +1098,7 @@ class Soledad(object): # # Recovery document export and import methods # + def export_recovery_document(self, include_uuid=True): """ Export the storage secrets and (optionally) the uuid. @@ -1125,6 +1197,9 @@ class Soledad(object): doc='The passphrase for locking and unlocking encryption secrets for ' 'local and remote storage.') + def _passphrase_as_string(self): + return self._passphrase.encode('utf-8') + #----------------------------------------------------------------------------- # Monkey patching u1db to be able to provide a custom SSL cert diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index adcde4e2..0753cbb5 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -26,6 +26,7 @@ import simplejson as json from u1db.remote import http_database +from leap.soledad.common import SHARED_DB_LOCK_DOC_ID_PREFIX from leap.soledad.client.auth import TokenBasedAuth @@ -89,7 +90,7 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, create, creds=None): + def open_database(url, uuid, create, creds=None): # TODO: users should not be able to create the shared database, so we # have to remove this from here in the future. """ @@ -97,8 +98,10 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :param url: URL of the remote database. :type url: str + :param uuid: The user's unique id. + :type uuid: str :param create: Should the database be created if it does not already - exist? + exist? :type create: bool :param token: An authentication token for accessing the shared db. :type token: str @@ -106,7 +109,7 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ - db = SoledadSharedDatabase(url, creds=creds) + db = SoledadSharedDatabase(url, uuid, creds=creds) db.open(create) return db @@ -122,12 +125,14 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): """ raise Unauthorized("Can't delete shared database.") - def __init__(self, url, document_factory=None, creds=None): + def __init__(self, url, uuid, document_factory=None, creds=None): """ Initialize database with auth token and encryption powers. :param url: URL of the remote database. :type url: str + :param uuid: The user's unique id. + :type uuid: str :param document_factory: A factory for U1BD documents. :type document_factory: u1db.Document :param creds: A tuple containing the authentication method and @@ -136,3 +141,30 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): """ http_database.HTTPDatabase.__init__(self, url, document_factory, creds) + self._uuid = uuid + + def lock(self): + """ + Obtain a lock on document with id C{doc_id}. + + :return: A tuple containing the token to unlock and the timeout until + lock expiration. + :rtype: (str, float) + + :raise HTTPError: Raised if any HTTP error occurs. + """ + res, headers = self._request_json('PUT', ['lock', self._uuid], + body={}) + return res['token'], res['timeout'] + + def unlock(self, token): + """ + Release the lock on shared database. + + :param token: The token returned by a previous call to lock(). + :type token: str + + :raise HTTPError: + """ + res, headers = self._request_json('DELETE', ['lock', self._uuid], + params={'token': token}) diff --git a/common/setup.py b/common/setup.py index 9af61be7..bcc2b4b3 100644 --- a/common/setup.py +++ b/common/setup.py @@ -17,6 +17,7 @@ """ setup file for leap.soledad.common """ +import re from setuptools import setup from setuptools import find_packages @@ -42,18 +43,83 @@ trove_classifiers = ( "Topic :: Software Development :: Libraries :: Python Modules" ) +DOWNLOAD_BASE = ('https://github.com/leapcode/soledad/' + 'archive/%s.tar.gz') +_versions = versioneer.get_versions() +VERSION = _versions['version'] +VERSION_FULL = _versions['full'] +DOWNLOAD_URL = "" + +# get the short version for the download url +_version_short = re.findall('\d+\.\d+\.\d+', VERSION) +if len(_version_short) > 0: + VERSION_SHORT = _version_short[0] + DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT + +cmdclass = versioneer.get_cmdclass() + + +from setuptools import Command + + +class freeze_debianver(Command): + """ + Freezes the version in a debian branch. + To be used after merging the development branch onto the debian one. + """ + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +version_full = '{version_full}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} +""" + subst_template = template.format( + version=VERSION_SHORT, + version_full=VERSION_FULL) + templatefun + with open(versioneer.versionfile_source, 'w') as f: + f.write(subst_template) + + +cmdclass["freeze_debianver"] = freeze_debianver + # XXX add ref to docs setup( name='leap.soledad.common', - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=VERSION, + cmdclass=cmdclass, url='https://leap.se/', + download_url=DOWNLOAD_URL, license='GPLv3+', description='Synchronization of locally encrypted data among devices ' '(common files).', author='The LEAP Encryption Access Project', author_email='info@leap.se', + maintainer='Kali Kaneko', + maintainer_email='kali@leap.se', long_description=( "Soledad is the part of LEAP that allows application data to be " "securely shared among devices. It provides, to other parts of the " diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index 26467740..23d28e76 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -21,8 +21,21 @@ Soledad routines common to client and server. """ +from hashlib import sha256 + + # -# Assert functions +# Global constants +# + + +SHARED_DB_NAME = 'shared' +SHARED_DB_LOCK_DOC_ID_PREFIX = 'lock-' +USER_DB_PREFIX = 'user-' + + +# +# Global functions # # we want to use leap.common.check.leap_assert in case it is available, @@ -63,6 +76,7 @@ except ImportError: "Expected type %r instead of %r" % (expectedType, type(var))) + from ._version import get_versions __version__ = get_versions()['version'] del get_versions diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 187d3035..1396f4d7 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -33,6 +33,7 @@ from couchdb.client import Server, Document as CouchDocument from couchdb.http import ResourceNotFound, Unauthorized +from leap.soledad.common import USER_DB_PREFIX from leap.soledad.common.objectstore import ( ObjectStoreDatabase, ObjectStoreSyncTarget, @@ -61,7 +62,7 @@ def persistent_class(cls): dump_method_name, store): """ Create a persistent method to replace C{old_method_name}. - + The new method will load C{key} using C{load_method_name} and stores it using C{dump_method_name} depending on the value of C{store}. """ @@ -522,8 +523,7 @@ class CouchServerState(ServerState): Inteface of the WSGI server with the CouchDB backend. """ - def __init__(self, couch_url, shared_db_name, tokens_db_name, - user_db_prefix): + def __init__(self, couch_url, shared_db_name, tokens_db_name): """ Initialize the couch server state. @@ -533,13 +533,10 @@ class CouchServerState(ServerState): @type shared_db_name: str @param tokens_db_name: The name of the tokens database. @type tokens_db_name: str - @param user_db_prefix: The prefix for user database names. - @type user_db_prefix: str """ self._couch_url = couch_url self._shared_db_name = shared_db_name self._tokens_db_name = tokens_db_name - self._user_db_prefix = user_db_prefix try: self._check_couch_permissions() except NotEnoughCouchPermissions: @@ -553,8 +550,8 @@ class CouchServerState(ServerState): def _check_couch_permissions(self): """ - Assert that Soledad Server has enough permissions on the underlying couch - database. + Assert that Soledad Server has enough permissions on the underlying + couch database. Soledad Server has to be able to do the following in the couch server: @@ -563,8 +560,8 @@ class CouchServerState(ServerState): * Read from 'tokens' db. This function tries to perform the actions above using the "low level" - couch library to ensure that Soledad Server can do everything it needs on - the underlying couch database. + couch library to ensure that Soledad Server can do everything it needs + on the underlying couch database. @param couch_url: The URL of the couch database. @type couch_url: str @@ -593,7 +590,7 @@ class CouchServerState(ServerState): _open_couch_db(self._shared_db_name)) # test read/write auth for user-<something> db _create_delete_test_doc( - _open_couch_db('%stest-db' % self._user_db_prefix)) + _open_couch_db('%stest-db' % USER_DB_PREFIX)) # test read auth for tokens db tokensdb = _open_couch_db(self._tokens_db_name) tokensdb.info() diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py new file mode 100644 index 00000000..7c2d7296 --- /dev/null +++ b/common/src/leap/soledad/common/errors.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# errors.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Soledad errors. +""" + + +from u1db import errors +from u1db.remote import http_errors + + +# +# LockResource: a lock based on a document in the shared database. +# + +class InvalidTokenError(errors.U1DBError): + """ + Exception raised when trying to unlock shared database with invalid token. + """ + + wire_description = "unlock unauthorized" + status = 401 + + +class NotLockedError(errors.U1DBError): + """ + Exception raised when trying to unlock shared database when it is not + locked. + """ + + wire_description = "lock not found" + status = 404 + + +class AlreadyLockedError(errors.U1DBError): + """ + Exception raised when trying to lock shared database but it is already + locked. + """ + + wire_description = "lock is locked" + status = 403 + +# update u1db "wire description to status" and "wire description to exception" +# maps. +for e in [InvalidTokenError, NotLockedError, AlreadyLockedError]: + http_errors.wire_description_to_status.update({ + e.wire_description: e.status}) + errors.wire_description_to_exc.update({ + e.wire_description: e}) + +# u1db error statuses also have to be updated +http_errors.ERROR_STATUSES = set( + http_errors.wire_description_to_status.values()) diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index 9f47d74a..88f98272 100644 --- a/common/src/leap/soledad/common/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -60,11 +60,12 @@ class BaseSoledadTest(BaseLeapTest): if os.path.isfile(f): os.unlink(f) - def _soledad_instance(self, user=ADDRESS, passphrase='123', + def _soledad_instance(self, user=ADDRESS, passphrase=u'123', prefix='', secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, local_db_path='soledad.u1db', server_url='', - cert_file=None, secret_id=None): + cert_file=None, secret_id=None, + shared_db_class=None): def _put_doc_side_effect(doc): self._doc_put = doc @@ -73,10 +74,15 @@ class BaseSoledadTest(BaseLeapTest): get_doc = Mock(return_value=None) put_doc = Mock(side_effect=_put_doc_side_effect) + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) def __call__(self): return self + if shared_db_class is not None: + MockSharedDB = shared_db_class + Soledad._shared_db = MockSharedDB() return Soledad( user, diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 1ea4d615..83df192b 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -24,6 +24,7 @@ import os import tempfile import simplejson as json import mock +import time from leap.common.testing.basetest import BaseLeapTest @@ -45,7 +46,7 @@ from leap.soledad.client import ( Soledad, target, ) -from leap.soledad.server import SoledadApp +from leap.soledad.server import SoledadApp, LockResource from leap.soledad.server.auth import URLToAuthorization @@ -86,9 +87,8 @@ class ServerAuthorizationTestCase(BaseLeapTest): /user-db/sync-from/{source} | GET, PUT, POST """ uuid = 'myuuid' - authmap = URLToAuthorization( - uuid, SoledadApp.SHARED_DB_NAME, SoledadApp.USER_DB_PREFIX) - dbname = authmap._uuid_dbname(uuid) + authmap = URLToAuthorization(uuid,) + dbname = authmap._user_db_name # test global auth self.assertTrue( authmap.is_authorized(self._make_environ('/', 'GET'))) @@ -202,8 +202,7 @@ class ServerAuthorizationTestCase(BaseLeapTest): Test if authorization fails for a wrong dbname. """ uuid = 'myuuid' - authmap = URLToAuthorization( - uuid, SoledadApp.SHARED_DB_NAME, SoledadApp.USER_DB_PREFIX) + authmap = URLToAuthorization(uuid) dbname = 'somedb' # test wrong-db database resource auth self.assertFalse( @@ -273,7 +272,7 @@ class EncryptedSyncTestCase( sync_target = token_leap_sync_target - def _soledad_instance(self, user='user-uuid', passphrase='123', + def _soledad_instance(self, user='user-uuid', passphrase=u'123', prefix='', secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, local_db_path='soledad.u1db', server_url='', @@ -293,6 +292,8 @@ class EncryptedSyncTestCase( get_doc = mock.Mock(return_value=None) put_doc = mock.Mock(side_effect=_put_doc_side_effect) + lock = mock.Mock(return_value=('atoken', 300)) + unlock = mock.Mock() def __call__(self): return self @@ -310,8 +311,8 @@ class EncryptedSyncTestCase( secret_id=secret_id) def make_app(self): - self.request_state = CouchServerState( - self._couch_url, 'shared', 'tokens', 'user-') + self.request_state = CouchServerState(self._couch_url, 'shared', + 'tokens') return self.make_app_with_state(self.request_state) def setUp(self): @@ -375,3 +376,173 @@ class EncryptedSyncTestCase( doc2 = doclist[0] # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) + + def test_encrypted_sym_sync_with_unicode_passphrase(self): + """ + Test the complete syncing chain between two soledad dbs using a + Soledad server backed by a couch database, using an unicode + passphrase. + """ + self.startServer() + # instantiate soledad and create a document + sol1 = self._soledad_instance( + # token is verified in test_target.make_token_soledad_app + auth_token='auth-token', + passphrase=u'ãáàäéàëíìïóòöõúùüñç', + ) + _, doclist = sol1.get_all_docs() + self.assertEqual([], doclist) + doc1 = sol1.create_doc(json.loads(simple_doc)) + # sync with server + sol1._server_url = self.getURL() + sol1.sync() + # assert doc was sent to couch db + db = CouchDatabase( + self._couch_url, + # the name of the user database is "user-<uuid>". + 'user-user-uuid', + ) + _, doclist = db.get_all_docs() + self.assertEqual(1, len(doclist)) + couchdoc = doclist[0] + # assert document structure in couch server + self.assertEqual(doc1.doc_id, couchdoc.doc_id) + self.assertEqual(doc1.rev, couchdoc.rev) + self.assertEqual(6, len(couchdoc.content)) + self.assertTrue(target.ENC_JSON_KEY in couchdoc.content) + self.assertTrue(target.ENC_SCHEME_KEY in couchdoc.content) + self.assertTrue(target.ENC_METHOD_KEY in couchdoc.content) + self.assertTrue(target.ENC_IV_KEY in couchdoc.content) + self.assertTrue(target.MAC_KEY in couchdoc.content) + self.assertTrue(target.MAC_METHOD_KEY in couchdoc.content) + # instantiate soledad with empty db, but with same secrets path + sol2 = self._soledad_instance( + prefix='x', + auth_token='auth-token', + passphrase=u'ãáàäéàëíìïóòöõúùüñç', + ) + _, doclist = sol2.get_all_docs() + self.assertEqual([], doclist) + sol2._secrets_path = sol1.secrets_path + sol2._load_secrets() + sol2._set_secret_id(sol1._secret_id) + # sync the new instance + sol2._server_url = self.getURL() + sol2.sync() + _, doclist = sol2.get_all_docs() + self.assertEqual(1, len(doclist)) + doc2 = doclist[0] + # assert incoming doc is equal to the first sent doc + self.assertEqual(doc1, doc2) + + +class LockResourceTestCase( + CouchDBTestCase, TestCaseWithServer): + """ + Tests for use of PUT and DELETE on lock resource. + """ + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_leap_document_for_test + + sync_target = token_leap_sync_target + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self._couch_url = 'http://localhost:' + str(self.wrapper.port) + self._state = CouchServerState( + self._couch_url, 'shared', 'tokens') + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def test__try_obtain_filesystem_lock(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + self.assertFalse(lr._lock.locked) + self.assertTrue(lr._try_obtain_filesystem_lock()) + self.assertTrue(lr._lock.locked) + lr._try_release_filesystem_lock() + + def test__try_release_filesystem_lock(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + lr._try_obtain_filesystem_lock() + self.assertTrue(lr._lock.locked) + lr._try_release_filesystem_lock() + self.assertFalse(lr._lock.locked) + + def test_put(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + # lock! + lr.put({}, None) + # assert lock document was correctly written + lock_doc = lr._shared_db.get_doc('lock-uuid') + self.assertIsNotNone(lock_doc) + self.assertTrue(LockResource.TIMESTAMP_KEY in lock_doc.content) + self.assertTrue(LockResource.LOCK_TOKEN_KEY in lock_doc.content) + timestamp = lock_doc.content[LockResource.TIMESTAMP_KEY] + token = lock_doc.content[LockResource.LOCK_TOKEN_KEY] + self.assertTrue(timestamp < time.time()) + self.assertTrue(time.time() < timestamp + LockResource.TIMEOUT) + # assert response to user + responder.send_response_json.assert_called_with( + 201, token=token, + timeout=LockResource.TIMEOUT) + + def test_delete(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + # lock! + lr.put({}, None) + lock_doc = lr._shared_db.get_doc('lock-uuid') + token = lock_doc.content[LockResource.LOCK_TOKEN_KEY] + # unlock! + lr.delete({'token': token}, None) + self.assertFalse(lr._lock.locked) + self.assertIsNone(lr._shared_db.get_doc('lock-uuid')) + responder.send_response_json.assert_called_with(200) + + def test_put_while_locked_fails(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + # lock! + lr.put({}, None) + # try to lock again! + lr.put({}, None) + self.assertEqual( + len(responder.send_response_json.call_args), 2) + self.assertEqual( + responder.send_response_json.call_args[0], (403,)) + self.assertEqual( + len(responder.send_response_json.call_args[1]), 2) + self.assertTrue( + 'remaining' in responder.send_response_json.call_args[1]) + self.assertTrue( + responder.send_response_json.call_args[1]['remaining'] > 0) + + def test_unlock_unexisting_lock_fails(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + # unlock! + lr.delete({'token': 'anything'}, None) + responder.send_response_json.assert_called_with( + 404, error='lock not found') + + def test_unlock_with_wrong_token_fails(self): + responder = mock.Mock() + lr = LockResource('uuid', self._state, responder) + # lock! + lr.put({}, None) + # unlock! + lr.delete({'token': 'wrongtoken'}, None) + self.assertIsNotNone(lr._shared_db.get_doc('lock-uuid')) + responder.send_response_json.assert_called_with( + 401, error='unlock unauthorized') diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 0b753647..8970a437 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -90,7 +90,7 @@ class AuxMethodsTestCase(BaseSoledadTest): """ sol = self._soledad_instance( 'leap@leap.se', - passphrase='123', + passphrase=u'123', secrets_path='value_3', local_db_path='value_2', server_url='value_1', @@ -109,25 +109,25 @@ class AuxMethodsTestCase(BaseSoledadTest): """ sol = self._soledad_instance( 'leap@leap.se', - passphrase='123', + passphrase=u'123', prefix=self.rand_prefix, ) doc = sol.create_doc({'simple': 'doc'}) doc_id = doc.doc_id # change the passphrase - sol.change_passphrase('654321') + sol.change_passphrase(u'654321') self.assertRaises( DatabaseError, self._soledad_instance, 'leap@leap.se', - passphrase='123', + passphrase=u'123', prefix=self.rand_prefix) # use new passphrase and retrieve doc sol2 = self._soledad_instance( 'leap@leap.se', - passphrase='654321', + passphrase=u'654321', prefix=self.rand_prefix) doc2 = sol2.get_doc(doc_id) self.assertEqual(doc, doc2) @@ -139,11 +139,11 @@ class AuxMethodsTestCase(BaseSoledadTest): """ sol = self._soledad_instance( 'leap@leap.se', - passphrase='123') + passphrase=u'123') # check that soledad complains about new passphrase length self.assertRaises( PassphraseTooShort, - sol.change_passphrase, '54321') + sol.change_passphrase, u'54321') def test_get_passphrase(self): """ @@ -161,13 +161,14 @@ class SoledadSharedDBTestCase(BaseSoledadTest): def setUp(self): BaseSoledadTest.setUp(self) self._shared_db = SoledadSharedDatabase( - 'https://provider/', SoledadDocument, None) + 'https://provider/', ADDRESS, document_factory=SoledadDocument, + creds=None) def test__get_secrets_from_shared_db(self): """ Ensure the shared db is queried with the correct doc_id. """ - doc_id = self._soledad._uuid_hash() + doc_id = self._soledad._shared_db_doc_id() self._soledad._get_secrets_from_shared_db() self.assertTrue( self._soledad._shared_db().get_doc.assert_called_with( @@ -178,7 +179,7 @@ class SoledadSharedDBTestCase(BaseSoledadTest): """ Ensure recovery document is put into shared recover db. """ - doc_id = self._soledad._uuid_hash() + doc_id = self._soledad._shared_db_doc_id() self._soledad._put_secrets_in_shared_db() self.assertTrue( self._soledad._shared_db().get_doc.assert_called_with( @@ -201,9 +202,10 @@ class SoledadSignalingTestCase(BaseSoledadTest): EVENTS_SERVER_PORT = 8090 def setUp(self): - BaseSoledadTest.setUp(self) # mock signaling soledad.client.signal = Mock() + # run parent's setUp + BaseSoledadTest.setUp(self) def tearDown(self): pass @@ -213,22 +215,28 @@ class SoledadSignalingTestCase(BaseSoledadTest): mocked.mock_calls.pop() mocked.call_args = mocked.call_args_list[-1] - def test_stage2_bootstrap_signals(self): + def test_stage3_bootstrap_signals(self): """ Test that a fresh soledad emits all bootstrap signals. + + Signals are: + - downloading keys / done downloading keys. + - creating keys / done creating keys. + - downloading keys / done downloading keys. + - uploading keys / done uploading keys. """ soledad.client.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( - secrets_path='alternative.json', - local_db_path='alternative.u1db') + secrets_path='alternative_stage3.json', + local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected soledad.client.signal.mock_calls.reverse() soledad.client.signal.call_args = \ soledad.client.signal.call_args_list[0] soledad.client.signal.call_args_list.reverse() - # assert signals + # downloading keys signals soledad.client.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, @@ -238,6 +246,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) + # creating keys signals self._pop_mock_call(soledad.client.signal) soledad.client.signal.assert_called_with( proto.SOLEDAD_CREATING_KEYS, @@ -248,6 +257,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_CREATING_KEYS, ADDRESS, ) + # downloading once more (inside _put_keys_in_shared_db) self._pop_mock_call(soledad.client.signal) soledad.client.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, @@ -258,6 +268,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) + # uploading keys signals self._pop_mock_call(soledad.client.signal) soledad.client.signal.assert_called_with( proto.SOLEDAD_UPLOADING_KEYS, @@ -268,21 +279,45 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_UPLOADING_KEYS, ADDRESS, ) + # assert db was locked and unlocked + sol._shared_db.lock.assert_called_with() + sol._shared_db.unlock.assert_called_with('atoken') - def test_stage1_bootstrap_signals(self): + def test_stage2_bootstrap_signals(self): """ - Test that an existent soledad emits some of the bootstrap signals. + Test that if there are keys in server, soledad will download them and + emit corresponding signals. """ - soledad.client.signal.reset_mock() - # get an existent instance so it emits only some of bootstrap signals + # get existing instance so we have access to keys sol = self._soledad_instance() + # create a document with secrets + doc = SoledadDocument(doc_id=sol._shared_db_doc_id()) + doc.content = sol.export_recovery_document(include_uuid=False) + + class Stage2MockSharedDB(object): + + get_doc = Mock(return_value=doc) + put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock() + + def __call__(self): + return self + + # reset mock + soledad.client.signal.reset_mock() + # get a fresh instance so it emits all bootstrap signals + sol = self._soledad_instance( + secrets_path='alternative_stage2.json', + local_db_path='alternative_stage2.u1db', + shared_db_class=Stage2MockSharedDB) # reverse call order so we can verify in the order the signals were # expected soledad.client.signal.mock_calls.reverse() soledad.client.signal.call_args = \ soledad.client.signal.call_args_list[0] soledad.client.signal.call_args_list.reverse() - # assert signals + # assert download keys signals soledad.client.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, @@ -292,16 +327,15 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_UPLOADING_KEYS, - ADDRESS, - ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_UPLOADING_KEYS, - ADDRESS, - ) + + def test_stage1_bootstrap_signals(self): + """ + Test that if soledad already has a local secret, it emits no signals. + """ + soledad.client.signal.reset_mock() + # get an existent instance so it emits only some of bootstrap signals + sol = self._soledad_instance() + self.assertEqual([], soledad.client.signal.mock_calls) def test_sync_signals(self): """ diff --git a/server/setup.py b/server/setup.py index 348aa838..573622ce 100644 --- a/server/setup.py +++ b/server/setup.py @@ -18,6 +18,7 @@ setup file for leap.soledad.server """ import os +import re from setuptools import setup from setuptools import find_packages @@ -51,17 +52,83 @@ trove_classifiers = ( "Topic :: Software Development :: Libraries :: Python Modules" ) +DOWNLOAD_BASE = ('https://github.com/leapcode/soledad/' + 'archive/%s.tar.gz') +_versions = versioneer.get_versions() +VERSION = _versions['version'] +VERSION_FULL = _versions['full'] +DOWNLOAD_URL = "" + +# get the short version for the download url +_version_short = re.findall('\d+\.\d+\.\d+', VERSION) +if len(_version_short) > 0: + VERSION_SHORT = _version_short[0] + DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT + +cmdclass = versioneer.get_cmdclass() + + +from setuptools import Command + + +class freeze_debianver(Command): + """ + Freezes the version in a debian branch. + To be used after merging the development branch onto the debian one. + """ + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +version_full = '{version_full}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} +""" + subst_template = template.format( + version=VERSION_SHORT, + version_full=VERSION_FULL) + templatefun + with open(versioneer.versionfile_source, 'w') as f: + f.write(subst_template) + + +cmdclass["freeze_debianver"] = freeze_debianver + +# XXX add ref to docs setup( name='leap.soledad.server', - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=VERSION, + cmdclass=cmdclass, url='https://leap.se/', + download_url=DOWNLOAD_URL, license='GPLv3+', description='Synchronization of locally encrypted data among devices ' '(server components)', author='The LEAP Encryption Access Project', author_email='info@leap.se', + maintainer='Kali Kaneko', + maintainer_email='kali@leap.se', long_description=( "Soledad is the part of LEAP that allows application data to be " "securely shared among devices. It provides, to other parts of the " diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index b4b715e2..c80b4c68 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -19,6 +19,9 @@ """ A U1DB server that stores data using CouchDB as its persistence layer. +General information +=================== + This is written as a Twisted application and intended to be run using the twistd command. To start the soledad server, run: @@ -27,14 +30,69 @@ twistd command. To start the soledad server, run: An initscript is included and will be installed system wide to make it feasible to start and stop the Soledad server service using a standard interface. + +Server database organization +============================ + +Soledad Server works with one database per user and one shared database in +which user's encrypted secrets might be stored. + +User database +------------- + +Users' databases in the server are named 'user-<uuid>' and Soledad Client +may perform synchronization between its local replicas and the user's +database in the server. Authorization for creating, updating, deleting and +retrieving information about the user database as well as performing +synchronization is handled by the `leap.soledad.server.auth` module. + +Shared database +--------------- + +Each user may store password-encrypted recovery data in the shared database, +as well as obtain a lock on the shared database in order to prevent creation +of multiple secrets in parallel. + +Recovery documents are stored in the database without any information that +may identify the user. In order to achieve this, the doc_id of recovery +documents are obtained as a hash of the user's uid and the user's password. +User's must have a valid token to interact with recovery documents, but the +server does not perform further authentication because it has no way to know +which recovery document belongs to each user. + +This has some implications: + + * The security of the recovery document doc_id, and thus of access to the + recovery document (encrypted) content, as well as tampering with the + stored data, all rely on the difficulty of obtaining the user's password + (supposing the user's uid is somewhat public) and the security of the hash + function used to calculate the doc_id. + + * The security of the content of a recovery document relies on the + difficulty of obtaining the user's password. + + * If the user looses his/her password, he/she will not be able to obtain the + recovery document. + + * Because of the above, it is recommended that recovery documents expire + (not implemented yet) to prevent excess storage. + +Lock documents, on the other hand, may be more thoroughly protected by the +server. Their doc_id's are calculated from the SHARED_DB_LOCK_DOC_ID_PREFIX +and the user's uid. + +The authorization for creating, updating, deleting and retrieving recovery +and lock documents on the shared database is handled by +`leap.soledad.server.auth` module. """ import configparser - +import time +import hashlib +import os from u1db.remote import http_app - # Keep OpenSSL's tsafe before importing Twisted submodules so we can put # it back if Twisted==12.0.0 messes with it. from OpenSSL import tsafe @@ -42,6 +100,8 @@ old_tsafe = tsafe from twisted.web.wsgi import WSGIResource from twisted.internet import reactor +from twisted.internet.error import TimeoutError +from twisted.python.lockfile import FilesystemLock from twisted import version if version.base() == "12.0.0": # Put OpenSSL's tsafe back into place. This can probably be removed if we @@ -49,9 +109,17 @@ if version.base() == "12.0.0": import sys sys.modules['OpenSSL.tsafe'] = old_tsafe - from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from leap.soledad.common import ( + SHARED_DB_NAME, + SHARED_DB_LOCK_DOC_ID_PREFIX, +) from leap.soledad.common.couch import CouchServerState +from leap.soledad.common.errors import ( + InvalidTokenError, + NotLockedError, + AlreadyLockedError, +) #----------------------------------------------------------------------------- @@ -63,16 +131,11 @@ class SoledadApp(http_app.HTTPApp): Soledad WSGI application """ - SHARED_DB_NAME = 'shared' + SHARED_DB_NAME = SHARED_DB_NAME """ The name of the shared database that holds user's encrypted secrets. """ - USER_DB_PREFIX = 'user-' - """ - The string prefix of users' databases. - """ - def __call__(self, environ, start_response): """ Handle a WSGI call to the Soledad application. @@ -91,6 +154,192 @@ class SoledadApp(http_app.HTTPApp): return http_app.HTTPApp.__call__(self, environ, start_response) +# +# LockResource: a lock based on a document in the shared database. +# + +@http_app.url_to_resource.register +class LockResource(object): + """ + Handle requests for locking documents. + + This class uses Twisted's Filesystem lock to manage a lock in the shared + database. + """ + + url_pattern = '/%s/lock/{uuid}' % SoledadApp.SHARED_DB_NAME + """ + """ + + TIMEOUT = 300 # XXX is 5 minutes reasonable? + """ + The timeout after which the lock expires. + """ + + # used for lock doc storage + TIMESTAMP_KEY = '_timestamp' + LOCK_TOKEN_KEY = '_token' + + FILESYSTEM_LOCK_TRIES = 5 + FILESYSTEM_LOCK_SLEEP_SECONDS = 1 + + + def __init__(self, uuid, state, responder): + """ + Initialize the lock resource. Parameters to this constructor are + automatically passed by u1db. + + :param uuid: The user unique id. + :type uuid: str + :param state: The backend database state. + :type state: u1db.remote.ServerState + :param responder: The infrastructure to send responses to client. + :type responder: u1db.remote.HTTPResponder + """ + self._shared_db = state.open_database(SoledadApp.SHARED_DB_NAME) + self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) + self._lock = FilesystemLock( + hashlib.sha512(self._lock_doc_id).hexdigest()) + self._state = state + self._responder = responder + + @http_app.http_method(content=str) + def put(self, content=None): + """ + Handle a PUT request to the lock document. + + A lock is a document in the shared db with doc_id equal to + 'lock-<uuid>' and the timestamp of its creation as content. This + method obtains a threaded-lock and creates a lock document if it does + not exist or if it has expired. + + It returns '201 Created' and a pair containing a token to unlock and + the lock timeout, or '403 AlreadyLockedError' and the remaining amount + of seconds the lock will still be valid. + + :param content: The content of the PUT request. It is only here + because PUT requests with empty content are considered + invalid requests by u1db. + :type content: str + """ + # obtain filesystem lock + if not self._try_obtain_filesystem_lock(): + self._responder.send_response_json(408) # error: request timeout + return + + created_lock = False + now = time.time() + token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing + lock_doc = self._shared_db.get_doc(self._lock_doc_id) + remaining = self._remaining(lock_doc, now) + + # if there's no lock, create one + if lock_doc is None: + lock_doc = self._shared_db.create_doc( + { + self.TIMESTAMP_KEY: now, + self.LOCK_TOKEN_KEY: token, + }, + doc_id=self._lock_doc_id) + created_lock = True + else: + if remaining == 0: + # lock expired, create new one + lock_doc.content = { + self.TIMESTAMP_KEY: now, + self.LOCK_TOKEN_KEY: token, + } + self._shared_db.put_doc(lock_doc) + created_lock = True + + self._try_release_filesystem_lock() + + # send response to client + if created_lock is True: + self._responder.send_response_json( + 201, timeout=self.TIMEOUT, token=token) # success: created + else: + wire_descr = AlreadyLockedError.wire_description + self._responder.send_response_json( + AlreadyLockedError.status, # error: forbidden + error=AlreadyLockedError.wire_description, remaining=remaining) + + @http_app.http_method(token=str) + def delete(self, token=None): + """ + Delete the lock if the C{token} is valid. + + Delete the lock document in case C{token} is equal to the token stored + in the lock document. + + :param token: The token returned when locking. + :type token: str + + :raise NotLockedError: Raised in case the lock is not locked. + :raise InvalidTokenError: Raised in case the token is invalid for + unlocking. + """ + lock_doc = self._shared_db.get_doc(self._lock_doc_id) + if lock_doc is None or self._remaining(lock_doc, time.time()) == 0: + self._responder.send_response_json( + NotLockedError.status, # error: not found + error=NotLockedError.wire_description) + elif token != lock_doc.content[self.LOCK_TOKEN_KEY]: + self._responder.send_response_json( + InvalidTokenError.status, # error: unauthorized + error=InvalidTokenError.wire_description) + else: + self._shared_db.delete_doc(lock_doc) + self._responder.send_response_json(200) # success: should use 204 + # but u1db does not + # support it. + + def _remaining(self, lock_doc, now): + """ + Return the number of seconds the lock contained in C{lock_doc} is + still valid, when compared to C{now}. + + :param lock_doc: The document containing the lock. + :type lock_doc: u1db.Document + :param now: The time to which to compare the lock timestamp. + :type now: float + + :return: The amount of seconds the lock is still valid. + :rtype: float + """ + if lock_doc is not None: + lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY] + remaining = lock_timestamp + self.TIMEOUT - now + return remaining if remaining > 0 else 0.0 + return 0.0 + + def _try_obtain_filesystem_lock(self): + """ + Try to obtain the file system lock. + + @return: Whether the lock was succesfully obtained. + @rtype: bool + """ + tries = self.FILESYSTEM_LOCK_TRIES + while tries > 0: + try: + return self._lock.lock() + except Exception as e: + tries -= 1 + time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS) + return False + + def _try_release_filesystem_lock(self): + """ + Release the filesystem lock. + """ + try: + self._lock.unlock() + return True + except Exception: + return False + + #----------------------------------------------------------------------------- # Auxiliary functions #----------------------------------------------------------------------------- @@ -128,8 +377,7 @@ def application(environ, start_response): state = CouchServerState( conf['couch_url'], SoledadApp.SHARED_DB_NAME, - SoledadTokenAuthMiddleware.TOKENS_DB, - SoledadApp.USER_DB_PREFIX) + SoledadTokenAuthMiddleware.TOKENS_DB) # WSGI application that may be used by `twistd -web` application = SoledadTokenAuthMiddleware(SoledadApp(state)) resource = WSGIResource(reactor, reactor.getThreadPool(), application) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 3bcfcf04..0ae49576 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -32,6 +32,13 @@ from couchdb.client import Server from twisted.python import log +from leap.soledad.common import ( + SHARED_DB_NAME, + SHARED_DB_LOCK_DOC_ID_PREFIX, + USER_DB_PREFIX, +) + + #----------------------------------------------------------------------------- # Authentication #----------------------------------------------------------------------------- @@ -52,7 +59,7 @@ class URLToAuthorization(object): HTTP_METHOD_DELETE = 'DELETE' HTTP_METHOD_POST = 'POST' - def __init__(self, uuid, shared_db_name, user_db_prefix): + def __init__(self, uuid): """ Initialize the mapper. @@ -61,16 +68,13 @@ class URLToAuthorization(object): @param uuid: The user uuid. @type uuid: str - @param shared_db_name: The name of the shared database that holds - user's encrypted secrets. - @type shared_db_name: str @param user_db_prefix: The string prefix of users' databases. @type user_db_prefix: str """ self._map = Mapper(controller_scan=None) - self._user_db_prefix = user_db_prefix - self._shared_db_name = shared_db_name - self._register_auth_info(self._uuid_dbname(uuid)) + self._user_db_name = "%s%s" % (USER_DB_PREFIX, uuid) + self._uuid = uuid + self._register_auth_info() def is_authorized(self, environ): """ @@ -99,22 +103,10 @@ class URLToAuthorization(object): conditions=dict(method=http_methods), requirements={'dbname': DBNAME_CONSTRAINTS}) - def _uuid_dbname(self, uuid): - """ - Return the database name corresponding to C{uuid}. - - @param uuid: The user uid. - @type uuid: str - - @return: The database name corresponding to C{uuid}. - @rtype: str - """ - return '%s%s' % (self._user_db_prefix, uuid) - - def _register_auth_info(self, dbname): + def _register_auth_info(self): """ - Register the authorization info in the mapper using C{dbname} as the - user's database name. + Register the authorization info in the mapper using C{SHARED_DB_NAME} + as the user's database name. This method sets up the following authorization rules: @@ -123,35 +115,37 @@ class URLToAuthorization(object): / | GET /shared-db | GET /shared-db/docs | - - /shared-db/doc/{id} | GET, PUT, DELETE + /shared-db/doc/{any_id} | GET, PUT, DELETE /shared-db/sync-from/{source} | - + /shared-db/lock/{uuid} | PUT, DELETE /user-db | GET, PUT, DELETE /user-db/docs | - /user-db/doc/{id} | - /user-db/sync-from/{source} | GET, PUT, POST - - @param dbname: The name of the user's database. - @type dbname: str """ # auth info for global resource self._register('/', [self.HTTP_METHOD_GET]) # auth info for shared-db database resource self._register( - '/%s' % self._shared_db_name, + '/%s' % SHARED_DB_NAME, [self.HTTP_METHOD_GET]) # auth info for shared-db doc resource self._register( - '/%s/doc/{id:.*}' % self._shared_db_name, + '/%s/doc/{id:.*}' % SHARED_DB_NAME, [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) + # auth info for shared-db lock resource + self._register( + '/%s/lock/%s' % (SHARED_DB_NAME, self._uuid), + [self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) # auth info for user-db database resource self._register( - '/%s' % dbname, + '/%s' % self._user_db_name, [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) # auth info for user-db sync resource self._register( - '/%s/sync-from/{source_replica_uid}' % dbname, + '/%s/sync-from/{source_replica_uid}' % self._user_db_name, [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, self.HTTP_METHOD_POST]) # generate the regular expressions @@ -165,7 +159,7 @@ class SoledadAuthMiddleware(object): This class must be extended to implement specific authentication methods (see SoledadTokenAuthMiddleware below). - It expects an HTTP_AUTHORIZATION header containing the the concatenation of + It expects an HTTP_AUTHORIZATION header containing the concatenation of the following strings: 1. The authentication scheme. It will be verified by the @@ -342,10 +336,7 @@ class SoledadAuthMiddleware(object): over the requested db. @rtype: bool """ - return URLToAuthorization( - uuid, self._app.SHARED_DB_NAME, - self._app.USER_DB_PREFIX - ).is_authorized(environ) + return URLToAuthorization(uuid).is_authorized(environ) @abstractmethod def _get_auth_error_string(self): |