-- cgit v1.2.3 From ff85c2a41fe933d9959fb84a0df2a13a6e199cec Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 5 Dec 2016 09:10:47 -0200 Subject: [refactor] improve secrets generation and storage code --- .../src/leap/soledad/client/_secrets/__init__.py | 132 ++++ client/src/leap/soledad/client/_secrets/crypto.py | 123 ++++ client/src/leap/soledad/client/_secrets/storage.py | 124 ++++ client/src/leap/soledad/client/_secrets/util.py | 46 ++ client/src/leap/soledad/client/api.py | 100 +-- client/src/leap/soledad/client/interfaces.py | 20 - client/src/leap/soledad/client/secrets.py | 794 --------------------- client/src/leap/soledad/client/shared_db.py | 12 +- testing/tests/client/test_aux_methods.py | 27 +- testing/tests/client/test_crypto.py | 227 +++--- testing/tests/client/test_deprecated_crypto.py | 6 +- testing/tests/client/test_shared_db.py | 38 +- testing/tests/client/test_signals.py | 98 ++- testing/tests/sync/test_sync_target.py | 2 +- 14 files changed, 643 insertions(+), 1106 deletions(-) create mode 100644 client/src/leap/soledad/client/_secrets/__init__.py create mode 100644 client/src/leap/soledad/client/_secrets/crypto.py create mode 100644 client/src/leap/soledad/client/_secrets/storage.py create mode 100644 client/src/leap/soledad/client/_secrets/util.py delete mode 100644 client/src/leap/soledad/client/secrets.py diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py new file mode 100644 index 00000000..f9da8423 --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# _secrets/__init__.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from collections import namedtuple + +from leap.soledad.common.log import getLogger + +from leap.soledad.client._secrets.storage import SecretsStorage +from leap.soledad.client._secrets.crypto import SecretsCrypto +from leap.soledad.client._secrets.util import emit + + +logger = getLogger(__name__) + + +SecretLength = namedtuple('SecretLength', 'name length') + + +class Secrets(object): + + lengths = { + 'remote': 512, + 'salt': 64, + 'local': 448, + } + + def __init__(self, uuid, passphrase, url, local_path, creds, userid, + shared_db=None): + self._passphrase = passphrase + self._secrets = {} + self._user_data = {'uuid': uuid, 'userid': userid} + self.crypto = SecretsCrypto(self.get_passphrase) + self.storage = SecretsStorage( + uuid, self.get_passphrase, url, local_path, creds, userid, + shared_db=shared_db) + self._bootstrap() + + # + # bootstrap + # + + def _bootstrap(self): + force_storage = False + + # attempt to load secrets from local storage + encrypted = self.storage.load_local() + + # if not found, attempt to load secrets from remote storage + if not encrypted: + encrypted = self.storage.load_remote() + + if not encrypted: + # if not found, generate new secrets + secrets = self._generate() + encrypted = self.crypto.encrypt(secrets) + force_storage = True + else: + # decrypt secrets found either in local or remote storage + secrets = self.crypto.decrypt(encrypted) + + self._secrets = secrets + + if encrypted['version'] < self.crypto.VERSION or force_storage: + self.storage.save_local(encrypted) + self.storage.save_remote(encrypted) + + # + # generation + # + + @emit('creating') + def _generate(self): + logger.info("generating new set of secrets...") + secrets = {} + for name, length in self.lengths.iteritems(): + secret = os.urandom(length) + secrets[name] = secret + logger.info("new set of secrets successfully generated") + return secrets + + # + # crypto + # + + def _encrypt(self): + # encrypt secrets + secrets = self._secrets + encrypted = self.crypto.encrypt(secrets) + # create the recovery document + data = {'secret': encrypted, 'version': 2} + return data + + def get_passphrase(self): + return self._passphrase.encode('utf-8') + + @property + def passphrase(self): + return self.get_passphrase() + + def change_passphrase(self, new_passphrase): + self._passphrase = new_passphrase + encrypted = self.crypto.encrypt(self._secrets) + self.storage.save_local(encrypted) + self.storage.save_remote(encrypted) + + @property + def remote(self): + return self._secrets.get('remote') + + @property + def salt(self): + return self._secrets.get('salt') + + @property + def local(self): + return self._secrets.get('local') diff --git a/client/src/leap/soledad/client/_secrets/crypto.py b/client/src/leap/soledad/client/_secrets/crypto.py new file mode 100644 index 00000000..76e80222 --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/crypto.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# _secrets/crypto.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import binascii +import json +import os +import scrypt + +from leap.soledad.common import soledad_assert +from leap.soledad.common.log import getLogger + +from leap.soledad.client._crypto import encrypt_sym, decrypt_sym, ENC_METHOD +from leap.soledad.client._secrets.util import SecretsError + + +logger = getLogger(__name__) + + +class SecretsCrypto(object): + + VERSION = 2 + + def __init__(self, get_pass): + self._get_pass = get_pass + + def _get_key(self, salt): + key = scrypt.hash(self._get_pass(), salt, buflen=32) + return key + + # + # encryption + # + + def encrypt(self, secrets): + encoded = {} + for name, value in secrets.iteritems(): + encoded[name] = binascii.b2a_base64(value) + plaintext = json.dumps(encoded) + salt = os.urandom(64) # TODO: get salt length from somewhere else + key = self._get_key(salt) + iv, ciphertext = encrypt_sym(plaintext, key, + method=ENC_METHOD.aes_256_gcm) + encrypted = { + 'version': self.VERSION, + 'kdf': 'scrypt', + 'kdf_salt': binascii.b2a_base64(salt), + 'kdf_length': len(key), + 'cipher': 'aes_256_gcm', + 'length': len(plaintext), + 'iv': str(iv), + 'secrets': binascii.b2a_base64(ciphertext), + } + return encrypted + + # + # decryption + # + + def decrypt(self, data): + version = data.get('version') + method = getattr(self, '_decrypt_v%d' % version) + try: + return method(data) + except Exception as e: + logger.error('error decrypting secrets: %r' % e) + raise SecretsError(e) + + def _decrypt_v1(self, data): + secret_id = data['active_secret'] + encrypted = data['storage_secrets'][secret_id] + soledad_assert(encrypted['cipher'] == 'aes256') + + salt = binascii.a2b_base64(encrypted['kdf_salt']) + key = self._get_key(salt) + separator = ':' + iv, ciphertext = encrypted['secret'].split(separator, 1) + ciphertext = binascii.a2b_base64(ciphertext) + plaintext = self._decrypt( + key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_ctr) + secrets = { + 'remote': plaintext[0:512], + 'salt': plaintext[512:576], + 'local': plaintext[576:1024], + } + return secrets + + def _decrypt_v2(self, encrypted): + soledad_assert(encrypted['cipher'] == 'aes_256_gcm') + + salt = binascii.a2b_base64(encrypted['kdf_salt']) + key = self._get_key(salt) + iv = encrypted['iv'] + ciphertext = binascii.a2b_base64(encrypted['secrets']) + plaintext = self._decrypt( + key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_gcm) + encoded = json.loads(plaintext) + secrets = {} + for name, value in encoded.iteritems(): + secrets[name] = binascii.a2b_base64(value) + return secrets + + def _decrypt(self, key, iv, ciphertext, encrypted, method): + # assert some properties of the stored secret + soledad_assert(encrypted['kdf'] == 'scrypt') + soledad_assert(encrypted['kdf_length'] == len(key)) + # decrypt + plaintext = decrypt_sym(ciphertext, key, iv, method) + soledad_assert(encrypted['length'] == len(plaintext)) + return plaintext diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py new file mode 100644 index 00000000..da3aa9d7 --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +# _secrets/storage.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import urlparse + +from hashlib import sha256 + +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.log import getLogger + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client._secrets.util import emit + + +logger = getLogger(__name__) + + +class SecretsStorage(object): + + def __init__(self, uuid, get_pass, url, local_path, creds, userid, + shared_db=None): + self._uuid = uuid + self._get_pass = get_pass + self._local_path = local_path + self._userid = userid + + self._shared_db = shared_db or self._init_shared_db(url, creds) + self.__remote_doc = None + + # + # properties + # + + @property + def _user_data(self): + return {'uuid': self._uuid, 'userid': self._userid} + + # + # local storage + # + + def load_local(self): + logger.info("trying to load secrets from disk: %s" % self._local_path) + try: + with open(self._local_path, 'r') as f: + encrypted = json.loads(f.read()) + logger.info("secrets loaded successfully from disk") + return encrypted + except IOError: + logger.warn("secrets not found in disk") + return None + + def save_local(self, encrypted): + json_data = json.dumps(encrypted) + with open(self._local_path, 'w') as f: + f.write(json_data) + + # + # remote storage + # + + def _init_shared_db(self, url, creds): + url = urlparse.urljoin(url, SHARED_DB_NAME) + db = SoledadSharedDatabase.open_database( + url, self._uuid, creds=creds) + self._shared_db = db + + def _remote_doc_id(self): + passphrase = self._get_pass() + text = '%s%s' % (passphrase, self._uuid) + digest = sha256(text).hexdigest() + return digest + + @property + def _remote_doc(self): + if not self.__remote_doc and self._shared_db: + doc = self._get_remote_doc() + self.__remote_doc = doc + return self.__remote_doc + + @emit('downloading') + def _get_remote_doc(self): + logger.info('trying to load secrets from server...') + doc = self._shared_db.get_doc(self._remote_doc_id()) + if doc: + logger.info('secrets loaded successfully from server') + else: + logger.warn('secrets not found in server') + return doc + + def load_remote(self): + doc = self._remote_doc + if not doc: + return None + encrypted = doc.content + return encrypted + + @emit('uploading') + def save_remote(self, encrypted): + doc = self._remote_doc + if not doc: + doc = SoledadDocument(doc_id=self._remote_doc_id()) + doc.content = encrypted + db = self._shared_db + if not db: + logger.warn('no shared db found') + return + db.put_doc(doc) diff --git a/client/src/leap/soledad/client/_secrets/util.py b/client/src/leap/soledad/client/_secrets/util.py new file mode 100644 index 00000000..f75b2bb6 --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/util.py @@ -0,0 +1,46 @@ +# -*- coding:utf-8 -*- +# _secrets/util.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from leap.soledad.client import events + + +class SecretsError(Exception): + pass + + +def emit(verb): + def _decorator(method): + def _decorated(self, *args, **kwargs): + + # emit starting event + user_data = self._user_data + name = 'SOLEDAD_' + verb.upper() + '_KEYS' + event = getattr(events, name) + events.emit_async(event, user_data) + + # run the method + result = method(self, *args, **kwargs) + + # emit a finished event + name = 'SOLEDAD_DONE_' + verb.upper() + '_KEYS' + event = getattr(events, name) + events.emit_async(event, user_data) + + return result + return _decorated + return _decorator diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index da6eec66..2e1d1cd3 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -45,7 +45,6 @@ from zope.interface import implements from leap.common.config import get_path_prefix from leap.common.plugins import collect_plugins -from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type from leap.soledad.common.log import getLogger @@ -57,8 +56,7 @@ from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events from leap.soledad.client import interfaces as soledad_interfaces from leap.soledad.client import sqlcipher -from leap.soledad.client.secrets import SoledadSecrets -from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client._secrets import Secrets from leap.soledad.client._crypto import SoledadCrypto logger = getLogger(__name__) @@ -130,7 +128,7 @@ class Soledad(object): def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, - auth_token=None, syncable=True): + auth_token=None): """ Initialize configuration, cryptographic keys and dbs. @@ -185,8 +183,6 @@ class Soledad(object): self._secrets_path = None self._dbsyncer = None - self.shared_db = shared_db - # configure SSL certificate global SOLEDAD_CERT SOLEDAD_CERT = cert_file @@ -198,21 +194,15 @@ class Soledad(object): self._secrets_path = secrets_path - # Initialize shared recovery database - self.init_shared_db(server_url, uuid, self._creds, syncable=syncable) - - # The following can raise BootstrapSequenceError, that will be - # propagated upwards. - self._init_secrets() + self._init_secrets(shared_db=shared_db) - self._crypto = SoledadCrypto(self._secrets.remote_storage_secret) + self._crypto = SoledadCrypto(self._secrets.remote) try: # initialize database access, trap any problems so we can shutdown # smoothly. self._init_u1db_sqlcipher_backend() - if syncable: - self._init_u1db_syncer() + self._init_u1db_syncer() except DatabaseAccessError: # oops! something went wrong with backend initialization. We # have to close any thread-related stuff we have already opened @@ -255,14 +245,13 @@ class Soledad(object): for path in paths: create_path_if_not_exists(path) - def _init_secrets(self): + def _init_secrets(self, shared_db=None): """ Initialize Soledad secrets. """ - self._secrets = SoledadSecrets( - self.uuid, self._passphrase, self._secrets_path, - self.shared_db, userid=self.userid) - self._secrets.bootstrap() + self._secrets = Secrets( + self._uuid, self._passphrase, self._server_url, self._secrets_path, + self._creds, self.userid, shared_db=shared_db) def _init_u1db_sqlcipher_backend(self): """ @@ -279,7 +268,7 @@ class Soledad(object): """ tohex = binascii.b2a_hex # sqlcipher only accepts the hex version - key = tohex(self._secrets.get_local_storage_key()) + key = tohex(self._secrets.local) opts = sqlcipher.SQLCipherOptions( self._local_db_path, key, @@ -659,21 +648,6 @@ class Soledad(object): # ISyncableStorage # - def set_syncable(self, syncable): - """ - Toggle the syncable state for this database. - - This can be used to start a database with offline state and switch it - online afterwards. Or the opposite: stop syncs when connection is lost. - - :param syncable: new status for syncable. - :type syncable: bool - """ - # TODO should check that we've got a token! - self.shared_db.syncable = syncable - if syncable and not self._dbsyncer: - self._init_u1db_syncer() - def sync(self): """ Synchronize documents with the server replica. @@ -760,13 +734,6 @@ class Soledad(object): """ return self.sync_lock.locked - @property - def syncable(self): - if self.shared_db: - return self.shared_db.syncable - else: - return False - def _set_token(self, token): """ Set the authentication token for remote database access. @@ -803,58 +770,13 @@ class Soledad(object): # ISecretsStorage # - def init_shared_db(self, server_url, uuid, creds, syncable=True): - """ - Initialize the shared database. - - :param server_url: URL of the remote database. - :type server_url: str - :param uuid: The user's unique id. - :type uuid: str - :param creds: A tuple containing the authentication method and - credentials. - :type creds: tuple - :param syncable: - If syncable is False, the database will not attempt to sync against - a remote replica. - :type syncable: bool - """ - # only case this is False is for testing purposes - if self.shared_db is None: - shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) - self.shared_db = SoledadSharedDatabase.open_database( - shared_db_url, - uuid, - creds=creds, - syncable=syncable) - - @property - def storage_secret(self): - """ - Return the secret used for local storage encryption. - - :return: The secret used for local storage encryption. - :rtype: str - """ - return self._secrets.storage_secret - - @property - def remote_storage_secret(self): - """ - Return the secret used for encryption of remotely stored data. - - :return: The secret used for remote storage encryption. - :rtype: str - """ - return self._secrets.remote_storage_secret - @property def secrets(self): """ Return the secrets object. :return: The secrets object. - :rtype: SoledadSecrets + :rtype: Secrets """ return self._secrets diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py index 82927ff4..1be47df7 100644 --- a/client/src/leap/soledad/client/interfaces.py +++ b/client/src/leap/soledad/client/interfaces.py @@ -351,28 +351,12 @@ class ISecretsStorage(Interface): secrets_file_name = Attribute( "The name of the file where the storage secrets will be stored") - storage_secret = Attribute("") - remote_storage_secret = Attribute("") - shared_db = Attribute("The shared db object") - # XXX this used internally from secrets, so it might be good to preserve # as a public boundary with other components. # We should also probably document its interface. secrets = Attribute("A SoledadSecrets object containing access to secrets") - def init_shared_db(self, server_url, uuid, creds): - """ - Initialize the shared recovery database. - - :param server_url: - :type server_url: - :param uuid: - :type uuid: - :param creds: - :type creds: - """ - def change_passphrase(self, new_passphrase): """ Change the passphrase that encrypts the storage secret. @@ -382,7 +366,3 @@ class ISecretsStorage(Interface): :raise NoStorageSecret: Raised if there's no storage secret available. """ - - # XXX not in use. Uncomment if we ever decide to allow - # multiple secrets. - # secret_id = Attribute("The id of the storage secret to be used") diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py deleted file mode 100644 index 3fe98c64..00000000 --- a/client/src/leap/soledad/client/secrets.py +++ /dev/null @@ -1,794 +0,0 @@ -# -*- coding: utf-8 -*- -# secrets.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Soledad secrets handling. -""" - - -import os -import scrypt -import binascii -import errno -import json - -from hashlib import sha256 - -from leap.soledad.common import soledad_assert -from leap.soledad.common import soledad_assert_type -from leap.soledad.common import document -from leap.soledad.common.log import getLogger -from leap.soledad.client import events -from leap.soledad.client import _crypto - - -logger = getLogger(__name__) - - -# -# Exceptions -# - - -class SecretsException(Exception): - - """ - Generic exception type raised by this module. - """ - - -class NoStorageSecret(SecretsException): - - """ - Raised when trying to use a storage secret but none is available. - """ - pass - - -class PassphraseTooShort(SecretsException): - - """ - Raised when trying to change the passphrase but the provided passphrase is - too short. - """ - - -class BootstrapSequenceError(SecretsException): - - """ - Raised when an attempt to generate a secret and store it in a recovery - document on server failed. - """ - - -# -# Secrets handler -# - - -class SoledadSecrets(object): - - """ - Soledad secrets handler. - - The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage - secret are used for remote storage encryption. We use the next - C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage. - From these bytes, the first C{self.SALT_LENGTH} bytes are used as the - salt and the rest as the password for the scrypt hashing. - """ - - LOCAL_STORAGE_SECRET_LENGTH = 512 - """ - The length, in bytes, of the secret used to derive a passphrase for the - SQLCipher database. - """ - - REMOTE_STORAGE_SECRET_LENGTH = 512 - """ - The length, in bytes, of the secret used to derive an encryption key for - remote storage. - """ - - SALT_LENGTH = 64 - """ - The length, in bytes, of the salt used to derive the key for the storage - secret encryption. - """ - - GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \ - + REMOTE_STORAGE_SECRET_LENGTH \ - + SALT_LENGTH # for sync db - """ - The length, in bytes, of the secret to be generated. This includes local - and remote secrets, and the salt for deriving the sync db secret. - """ - - MINIMUM_PASSPHRASE_LENGTH = 6 - """ - The minimum length, in bytes, for a passphrase. The passphrase length is - only checked when the user changes her passphrase, not when she - instantiates Soledad. - """ - - SEPARATOR = ":" - """ - A separator used for storing the encryption initial value prepended to the - ciphertext. - """ - - UUID_KEY = 'uuid' - STORAGE_SECRETS_KEY = 'storage_secrets' - ACTIVE_SECRET_KEY = 'active_secret' - SECRET_KEY = 'secret' - CIPHER_KEY = 'cipher' - LENGTH_KEY = 'length' - KDF_KEY = 'kdf' - KDF_SALT_KEY = 'kdf_salt' - KDF_LENGTH_KEY = 'kdf_length' - KDF_SCRYPT = 'scrypt' - CIPHER_AES256 = 'aes256' # deprecated, AES-CTR - CIPHER_AES256_GCM = _crypto.ENC_METHOD.aes_256_gcm - RECOVERY_DOC_VERSION_KEY = 'version' - RECOVERY_DOC_VERSION = 1 - """ - Keys used to access storage secrets in recovery documents. - """ - - def __init__(self, uuid, passphrase, secrets_path, shared_db, userid=None): - """ - Initialize the secrets manager. - - :param uuid: User's unique id. - :type uuid: str - :param passphrase: The passphrase for locking and unlocking encryption - secrets for local and remote storage. - :type passphrase: unicode - :param secrets_path: Path for storing encrypted key used for - symmetric encryption. - :type secrets_path: str - :param shared_db: The shared database that stores user secrets. - :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase - """ - self._uuid = uuid - self._userid = userid - self._passphrase = passphrase - self._secrets_path = secrets_path - self._shared_db = shared_db - self._secrets = {} - self._secret_id = None - - def bootstrap(self): - """ - Bootstrap secrets. - - Soledad secrets bootstrap is the following sequence of stages: - - * stage 1 - local secret loading: - - if secrets exist locally, load them. - * stage 2 - remote secret loading: - - else, if secrets exist in server, download them. - * stage 3 - secret generation: - - else, generate a new secret and store in server. - - This method decides which bootstrap stages have already been performed - and performs the missing ones in order. - - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed for some reason. - """ - # STAGE 1 - verify if secrets exist locally - try: - logger.info("trying to load secrets from local storage...") - version = self._load_secrets_from_local_file() - # eventually migrate local and remote stored documents from old - # format version - if version < self.RECOVERY_DOC_VERSION: - self._store_secrets() - self._upload_crypto_secrets() - logger.info("found secrets in local storage") - return - - except NoStorageSecret: - logger.info("could not find secrets in local storage") - - # STAGE 2 - there are no secrets in local storage and this is the - # first time we are running soledad with the specified - # secrets_path. Try to fetch encrypted secrets from - # server. - try: - logger.info('trying to fetch secrets from remote storage...') - version = self._download_crypto_secrets() - self._store_secrets() - # eventually migrate remote stored document from old format - # version - if version < self.RECOVERY_DOC_VERSION: - self._upload_crypto_secrets() - logger.info('found secrets in remote storage.') - return - except NoStorageSecret: - logger.info("could not find secrets in remote storage.") - - # STAGE 3 - there are no secrets in server also, so we want to - # generate the secrets and store them in the remote - # db. - logger.info("generating secrets...") - self._gen_crypto_secrets() - logger.info("uploading secrets...") - self._upload_crypto_secrets() - - def _has_secret(self): - """ - Return whether there is a storage secret available for use or not. - - :return: Whether there's a storage secret for symmetric encryption. - :rtype: bool - """ - return self.storage_secret is not None - - def _maybe_set_active_secret(self, active_secret): - """ - If no secret_id is already set, choose the passed active secret, or - just choose first secret available if none. - """ - if not self._secret_id: - if not active_secret: - active_secret = self._secrets.items()[0][0] - self.set_secret_id(active_secret) - - def _load_secrets_from_local_file(self): - """ - Load storage secrets from local file. - - :return version: The version of the locally stored recovery document. - - :raise NoStorageSecret: Raised if there are no secrets available in - local storage. - """ - # check if secrets file exists and we can read it - if not os.path.isfile(self._secrets_path): - raise NoStorageSecret - - # read storage secrets from file - content = None - with open(self._secrets_path, 'r') as f: - content = json.loads(f.read()) - _, active_secret, version = self._import_recovery_document(content) - - self._maybe_set_active_secret(active_secret) - - return version - - def _download_crypto_secrets(self): - """ - Download crypto secrets. - - :return version: The version of the remotelly stored recovery document. - - :raise NoStorageSecret: Raised if there are no secrets available in - remote storage. - """ - doc = None - if self._shared_db.syncable: - doc = self._get_secrets_from_shared_db() - - if doc is None: - raise NoStorageSecret - - _, active_secret, version = self._import_recovery_document(doc.content) - self._maybe_set_active_secret(active_secret) - - return version - - def _gen_crypto_secrets(self): - """ - Generate the crypto secrets. - """ - logger.info('no cryptographic secrets found, creating new secrets...') - secret_id = self._gen_secret() - self.set_secret_id(secret_id) - - def _upload_crypto_secrets(self): - """ - Send crypto secrets to shared db. - - :raises BootstrapSequenceError: Raised when unable to store secrets in - shared database. - """ - if self._shared_db.syncable: - try: - self._put_secrets_in_shared_db() - except Exception as ex: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. - try: - os.unlink(self._secrets_path) - except OSError as e: - if e.errno != errno.ENOENT: - # no such file or directory - logger.exception(e) - logger.exception(ex) - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') - - # - # Shared DB related methods - # - - def _shared_db_doc_id(self): - """ - Calculate the doc_id of the document in the shared db that stores key - material. - - :return: the hash - :rtype: str - """ - return sha256( - '%s%s' % - (self._passphrase_as_string(), self._uuid)).hexdigest() - - def _export_recovery_document(self, cipher=None): - """ - Export the storage secrets. - - Current format of recovery document has the following structure: - - { - 'storage_secrets': { - '': { - 'cipher': 'aes256', - 'length': , - 'secret': '', - }, - }, - 'active_secret': '', - 'version': '', - } - - Note that multiple storage secrets might be stored in one recovery - document. - - :param cipher: (Optional) The ciper to use. Defaults to AES256 - :type cipher: str - - :return: The recovery document. - :rtype: dict - """ - # encrypt secrets - encrypted_secrets = {} - for secret_id in self._secrets: - encrypted_secrets[secret_id] = self._encrypt_storage_secret( - self._secrets[secret_id], doc_cipher=cipher) - # create the recovery document - data = { - self.STORAGE_SECRETS_KEY: encrypted_secrets, - self.ACTIVE_SECRET_KEY: self._secret_id, - self.RECOVERY_DOC_VERSION_KEY: self.RECOVERY_DOC_VERSION, - } - return data - - def _import_recovery_document(self, data): - """ - Import storage secrets for symmetric encryption from a recovery - document. - - Note that this method does not store the imported data on disk. For - that, use C{self._store_secrets()}. - - :param data: The recovery document. - :type data: dict - - :return: A tuple containing the number of imported secrets, the - secret_id of the last active secret, and the recovery - document format version. - :rtype: (int, str, int) - """ - soledad_assert(self.STORAGE_SECRETS_KEY in data) - version = data.get(self.RECOVERY_DOC_VERSION_KEY, 1) - meth = getattr(self, '_import_recovery_document_version_%d' % version) - secret_count, active_secret = meth(data) - return secret_count, active_secret, version - - def _import_recovery_document_version_1(self, data): - """ - Import storage secrets for symmetric encryption from a recovery - document with format version 1. - - Version 1 of recovery document has the following structure: - - { - 'storage_secrets': { - '': { - 'cipher': 'aes256', - 'length': , - 'secret': '', - }, - }, - 'active_secret': '', - 'version': '', - } - - :param data: The recovery document. - :type data: dict - - :return: A tuple containing the number of imported secrets, the - secret_id of the last active secret, and the recovery - document format version. - :rtype: (int, str, int) - """ - # include secrets in the secret pool. - secret_count = 0 - secrets = data[self.STORAGE_SECRETS_KEY].items() - active_secret = None - # XXX remove check for existence of key (included for backwards - # compatibility) - if self.ACTIVE_SECRET_KEY in data: - active_secret = data[self.ACTIVE_SECRET_KEY] - for secret_id, encrypted_secret in secrets: - if secret_id not in self._secrets: - try: - self._secrets[secret_id] = \ - self._decrypt_storage_secret_version_1( - encrypted_secret) - secret_count += 1 - except SecretsException as e: - logger.error("failed to decrypt storage secret: %s" - % str(e)) - raise e - return secret_count, active_secret - - def _get_secrets_from_shared_db(self): - """ - Retrieve the document with encrypted key material from the shared - database. - - :return: a document with encrypted key material in its contents - :rtype: document.SoledadDocument - """ - user_data = self._get_user_data() - events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, user_data) - db = self._shared_db - if not db: - logger.warn('no shared db found') - return - doc = db.get_doc(self._shared_db_doc_id()) - user_data = {'userid': self._userid, 'uuid': self._uuid} - events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) - return doc - - def _put_secrets_in_shared_db(self): - """ - Assert local keys are the same as shared db's ones. - - Try to fetch keys from shared recovery database. If they already exist - in the remote db, assert that that data is the same as local data. - Otherwise, upload keys to shared recovery database. - """ - soledad_assert( - self._has_secret(), - 'Tried to send keys to server but they don\'t exist in local ' - 'storage.') - # try to get secrets doc from server, otherwise create it - doc = self._get_secrets_from_shared_db() - if doc is None: - doc = document.SoledadDocument( - doc_id=self._shared_db_doc_id()) - # fill doc with encrypted secrets - doc.content = self._export_recovery_document() - # upload secrets to server - user_data = self._get_user_data() - events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data) - db = self._shared_db - if not db: - logger.warn('no shared db found') - return - db.put_doc(doc) - events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data) - - # - # Management of secret for symmetric encryption. - # - - def _decrypt_storage_secret_version_1(self, encrypted_secret_dict): - """ - Decrypt the storage secret. - - Storage secret is encrypted before being stored. This method decrypts - and returns the decrypted storage secret. - - Version 1 of storage secret format has the following structure: - - '': { - 'cipher': 'aes256', - 'length': , - 'secret': '', - }, - - :param encrypted_secret_dict: The encrypted storage secret. - :type encrypted_secret_dict: dict - - :return: The decrypted storage secret. - :rtype: str - - :raise SecretsException: Raised in case the decryption of the storage - secret fails for some reason. - """ - # calculate the encryption key - if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT: - raise SecretsException("Unknown KDF in stored secret.") - key = scrypt.hash( - self._passphrase_as_string(), - # the salt is stored base64 encoded - binascii.a2b_base64( - encrypted_secret_dict[self.KDF_SALT_KEY]), - buflen=32, # we need a key with 256 bits (32 bytes). - ) - if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key): - raise SecretsException("Wrong length of decryption key.") - supported_ciphers = [self.CIPHER_AES256, self.CIPHER_AES256_GCM] - doc_cipher = encrypted_secret_dict[self.CIPHER_KEY] - if doc_cipher not in supported_ciphers: - raise SecretsException("Unknown cipher in stored secret.") - # recover the initial value and ciphertext - iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split( - self.SEPARATOR, 1) - ciphertext = binascii.a2b_base64(ciphertext) - try: - decrypted_secret = _crypto.decrypt_sym( - ciphertext, key, iv, doc_cipher) - except Exception as e: - logger.error(e) - raise SecretsException("Unable to decrypt secret.") - if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret): - raise SecretsException("Wrong length of decrypted secret.") - return decrypted_secret - - def _encrypt_storage_secret(self, decrypted_secret, doc_cipher=None): - """ - Encrypt the storage secret. - - An encrypted secret has the following structure: - - { - '': { - 'kdf': 'scrypt', - 'kdf_salt': '' - 'kdf_length': - 'cipher': 'aes256', - 'length': , - 'secret': '', - } - } - - :param decrypted_secret: The decrypted storage secret. - :type decrypted_secret: str - :param cipher: (Optional) The ciper to use. Defaults to AES256 - :type cipher: str - - :return: The encrypted storage secret. - :rtype: dict - """ - # generate random salt - salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - doc_cipher = doc_cipher or self.CIPHER_AES256_GCM - iv, ciphertext = _crypto.encrypt_sym(decrypted_secret, key, doc_cipher) - ciphertext = binascii.b2a_base64(ciphertext) - encrypted_secret_dict = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: doc_cipher, - self.LENGTH_KEY: len(decrypted_secret), - self.SECRET_KEY: self.SEPARATOR.join([str(iv), ciphertext]) - } - return encrypted_secret_dict - - @property - def storage_secret(self): - """ - Return the storage secret. - - :return: The decrypted storage secret. - :rtype: str - """ - return self._secrets.get(self._secret_id) - - def set_secret_id(self, secret_id): - """ - Define the id of the storage secret to be used. - - This method will also replace the secret in the crypto object. - - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - """ - self._secret_id = secret_id - - def _gen_secret(self): - """ - Generate a secret for symmetric encryption and store in a local - encrypted file. - - This method emits the following events.signals: - - * SOLEDAD_CREATING_KEYS - * SOLEDAD_DONE_CREATING_KEYS - - :return: The id of the generated secret. - :rtype: str - """ - user_data = self._get_user_data() - events.emit_async(events.SOLEDAD_CREATING_KEYS, user_data) - # generate random secret - secret = os.urandom(self.GEN_SECRET_LENGTH) - secret_id = sha256(secret).hexdigest() - self._secrets[secret_id] = secret - self._store_secrets() - events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, user_data) - return secret_id - - def _store_secrets(self): - """ - Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. - """ - with open(self._secrets_path, 'w') as f: - f.write( - json.dumps( - self._export_recovery_document())) - - def change_passphrase(self, new_passphrase): - """ - Change the passphrase that encrypts the storage secret. - - :param new_passphrase: The new passphrase. - :type new_passphrase: unicode - - :raise NoStorageSecret: Raised if there's no storage secret available. - """ - # TODO: maybe we want to add more checks to guarantee passphrase is - # reasonable? - soledad_assert_type(new_passphrase, unicode) - if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: - raise PassphraseTooShort( - 'Passphrase must be at least %d characters long!' % - self.MINIMUM_PASSPHRASE_LENGTH) - # ensure there's a secret for which the passphrase will be changed. - if not self._has_secret(): - raise NoStorageSecret() - self._passphrase = new_passphrase - self._store_secrets() - self._put_secrets_in_shared_db() - - # - # Setters and getters - # - - @property - def secret_id(self): - return self._secret_id - - def _get_secrets_path(self): - return self._secrets_path - - def _set_secrets_path(self, secrets_path): - self._secrets_path = secrets_path - - secrets_path = property( - _get_secrets_path, - _set_secrets_path, - doc='The path for the file containing the encrypted symmetric secret.') - - @property - def passphrase(self): - """ - Return the passphrase for locking and unlocking encryption secrets for - local and remote storage. - """ - return self._passphrase - - def _passphrase_as_string(self): - return self._passphrase.encode('utf-8') - - # - # remote storage secret - # - - @property - def remote_storage_secret(self): - """ - Return the secret for remote storage. - """ - key_start = 0 - key_end = self.REMOTE_STORAGE_SECRET_LENGTH - return self.storage_secret[key_start:key_end] - - # - # local storage key - # - - def _get_local_storage_secret(self): - """ - Return the local storage secret. - - :return: The local storage secret. - :rtype: str - """ - secret_len = self.REMOTE_STORAGE_SECRET_LENGTH - lsecret_len = self.LOCAL_STORAGE_SECRET_LENGTH - pwd_start = secret_len + self.SALT_LENGTH - pwd_end = secret_len + lsecret_len - return self.storage_secret[pwd_start:pwd_end] - - def _get_local_storage_salt(self): - """ - Return the local storage salt. - - :return: The local storage salt. - :rtype: str - """ - salt_start = self.REMOTE_STORAGE_SECRET_LENGTH - salt_end = salt_start + self.SALT_LENGTH - return self.storage_secret[salt_start:salt_end] - - def get_local_storage_key(self): - """ - Return the local storage key derived from the local storage secret. - - :return: The key for protecting the local database. - :rtype: str - """ - return scrypt.hash( - password=self._get_local_storage_secret(), - salt=self._get_local_storage_salt(), - buflen=32, # we need a key with 256 bits (32 bytes) - ) - - # - # sync db key - # - - def _get_sync_db_salt(self): - """ - Return the salt for sync db. - """ - salt_start = self.LOCAL_STORAGE_SECRET_LENGTH \ - + self.REMOTE_STORAGE_SECRET_LENGTH - salt_end = salt_start + self.SALT_LENGTH - return self.storage_secret[salt_start:salt_end] - - def get_sync_db_key(self): - """ - Return the key for protecting the sync database. - - :return: The key for protecting the sync database. - :rtype: str - """ - return scrypt.hash( - password=self._get_local_storage_secret(), - salt=self._get_sync_db_salt(), - buflen=32, # we need a key with 256 bits (32 bytes) - ) - - def _get_user_data(self): - return {'uuid': self._uuid, 'userid': self._userid} diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index d43db045..52b226b9 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -17,7 +17,7 @@ """ A shared database for storing/retrieving encrypted key material. """ -from leap.soledad.common.l2db.remote import http_database +from leap.soledad.common.l2db.remote.http_database import HTTPDatabase from leap.soledad.client.auth import TokenBasedAuth @@ -47,7 +47,7 @@ class ImproperlyConfiguredError(Exception): """ -class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): +class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): """ This is a shared recovery database that enables users to store their encryption secrets in the server and retrieve them afterwards. @@ -134,20 +134,16 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): """ raise Unauthorized("Can't delete shared database.") - def __init__(self, url, uuid, document_factory=None, creds=None): + def __init__(self, url, document_factory=None, creds=None): """ Initialize database with auth token and encryption powers. :param url: URL of the remote database. :type url: str - :param uuid: The user's unique id. - :type uuid: str :param document_factory: A factory for U1BD documents. :type document_factory: u1db.Document :param creds: A tuple containing the authentication method and credentials. :type creds: tuple """ - http_database.HTTPDatabase.__init__(self, url, document_factory, - creds) - self._uuid = uuid + HTTPDatabase.__init__(self, url, document_factory, creds) diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py index 9b4a175f..a08f7d36 100644 --- a/testing/tests/client/test_aux_methods.py +++ b/testing/tests/client/test_aux_methods.py @@ -19,12 +19,11 @@ Tests for general Soledad functionality. """ import os -from twisted.internet import defer +from pytest import inlineCallbacks from leap.soledad.client import Soledad from leap.soledad.client.adbapi import U1DBConnectionPool -from leap.soledad.client.secrets import PassphraseTooShort -from leap.soledad.client.secrets import SecretsException +from leap.soledad.client._secrets.util import SecretsError from test_soledad.util import BaseSoledadTest @@ -34,7 +33,7 @@ class AuxMethodsTestCase(BaseSoledadTest): def test__init_dirs(self): sol = self._soledad_instance(prefix='_init_dirs') local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets.secrets_path) + secrets_path = os.path.dirname(sol.secrets.storage._local_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) @@ -85,14 +84,14 @@ class AuxMethodsTestCase(BaseSoledadTest): cert_file=None) self.assertEqual( os.path.join(self.tempdir, 'value_3'), - sol.secrets.secrets_path) + sol.secrets.storage._local_path) self.assertEqual( os.path.join(self.tempdir, 'value_2'), sol.local_db_path) self.assertEqual('value_1', sol._server_url) sol.close() - @defer.inlineCallbacks + @inlineCallbacks def test_change_passphrase(self): """ Test if passphrase can be changed. @@ -108,7 +107,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol.change_passphrase(u'654321') sol.close() - with self.assertRaises(SecretsException): + with self.assertRaises(SecretsError): self._soledad_instance( 'leap@leap.se', passphrase=u'123', @@ -124,20 +123,6 @@ class AuxMethodsTestCase(BaseSoledadTest): sol2.close() - def test_change_passphrase_with_short_passphrase_raises(self): - """ - Test if attempt to change passphrase passing a short passphrase - raises. - """ - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123') - # check that soledad complains about new passphrase length - self.assertRaises( - PassphraseTooShort, - sol.change_passphrase, u'54321') - sol.close() - def test_get_passphrase(self): """ Assert passphrase getter works fine. diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 49a61438..379475cd 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -19,9 +19,9 @@ Tests for cryptographic related stuff. """ import binascii import base64 -import hashlib import json import os +import scrypt from io import BytesIO @@ -34,6 +34,7 @@ from cryptography.exceptions import InvalidTag from leap.soledad.common.document import SoledadDocument from test_soledad.util import BaseSoledadTest from leap.soledad.client import _crypto +from leap.soledad.client._secrets import SecretsCrypto from twisted.trial import unittest from twisted.internet import defer @@ -186,99 +187,145 @@ class BlobTestCase(unittest.TestCase): yield crypto.decrypt_doc(doc2) -class RecoveryDocumentTestCase(BaseSoledadTest): - - def test_export_recovery_document_raw(self): - rd = self._soledad.secrets._export_recovery_document() - secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] - # assert exported secret is the same - secret = self._soledad.secrets._decrypt_storage_secret_version_1( - rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) - self.assertEqual(secret_id, self._soledad.secrets._secret_id) - self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) - # assert recovery document structure - encrypted_secret = rd[ - self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] - self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) - self.assertEquals( - _crypto.ENC_METHOD.aes_256_gcm, - encrypted_secret[self._soledad.secrets.CIPHER_KEY]) - self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) - self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) - - def test_import_recovery_document(self, cipher='aes256'): - rd = self._soledad.secrets._export_recovery_document(cipher) - s = self._soledad_instance() - s.secrets._import_recovery_document(rd) - s.secrets.set_secret_id(self._soledad.secrets._secret_id) - self.assertEqual(self._soledad.storage_secret, - s.storage_secret, - 'Failed settinng secret for symmetric encryption.') - s.close() - - def test_import_GCM_recovery_document(self): - cipher = self._soledad.secrets.CIPHER_AES256_GCM - self.test_import_recovery_document(cipher) - - def test_import_legacy_CTR_recovery_document(self): - cipher = self._soledad.secrets.CIPHER_AES256 - self.test_import_recovery_document(cipher) +class SecretsCryptoTestCase(unittest.TestCase): + + SECRETS = {'remote': 'a' * 512, 'salt': 'b' * 64, 'local': 'c' * 448} + ENCRYPTED_V2 = { + 'cipher': 'aes_256_gcm', + 'length': 1417, + 'kdf_salt': '3DCkfecls0GcX2RadA04FAC2cqkI+vpGwwCLwffdRI6vpO5SPxaw/eM0/' + 'z3GUADm3If3YCQBldKXNdqHQLsU1Q==\n', + 'iv': 'rRwCDw5Rbp5+J3QwjQ46Hw==', + 'secrets': 'lxf6yrGDcBr8XFWNDgsCoO2XPGfDJndviL9Y2GmHcSEBWnO2dm2sieuPoq' + 'PwSHRSJSrzM4Ezgdaan7X8+ErnuRLUVqbPAqPl8xx8FdCjnid4vFyFYNFI' + '/dmo8SQAf8O9vdlVEPZ5Nk2DuWIrh+oPlrSUOmR6XzI0YVdoJDmGWowygU' + 'MR0R9Bi9xFGlG135NVcNP8KGdnQDkI0V+U/3qm3tctbo4LRCxxJ60wdi0M' + 'DA6iYFI/IMshxI/ZXFHp5/YPk0k2m0i6z71kMVksgjIMMgT5Kmz7WR54na' + 'IkWbvNkbYRFR/Hbg9p6Bs7NjJlOLTjnwGJNYPbdyfJXKd1R/S8Mg7ZqsyQ' + 'VbBqXHwEN7gYlMZ66D8wu8LOK70mN7LLiSz5J8tXO3rDT1mIIf3IvNhv/j' + 'rEZHf1fTFPRp+ZVEt/hJKyPv71ua4p2lgdgNlCs2IsACk9ku/LQwXP6uZr' + 'hMJsTvniTQoCVXFYVN/jKo7Pz/+uT5wOXOXtL7smpBE/2r3uoERNM+Zw11' + 'SA8UzzMZQMxJQKVNwLmKtwvztN5dxVXhxCUyeLmeQc84VzV7NK0WMUOdfA' + '18I0HS6rHLKcdsvrPAdzvGim7tiE8TBdp8ITNQ8yMFNiGNyOVliTSTwQFf' + 'sCj6m5nYcjvprNQ8RkeitvicrtI1Ylc8CfFK50xPV77XVmlgvNsfm54msN' + 'tV0K5+XwaNgimlh/1m2bVEYj55gO0twVASwRuZj3sSY2z669iuXRk7EPyT' + 'jcE2NnfW+lqOQkJ73N7pv73t6OjiEnrKx7VmH94zYlY8ZReVVn4RTZhare' + 'D7rqCmGPhsPaCPaAfotfNBBa0w6p6L9ZlNxpIesnMObtyGob1g4Vcu8O6K' + '2Q1Ldj95+Q53tJDpx2NLP/5tfAUlbehD3whKwKOz/rGKEfhgE+Nx32RR0y' + 'YM4aJ7CYI/U3YH82xqGoa1ufIJbSBt965CVIHSVJt/mYfilhMACV/wBlvL' + 'ua08iKpHwc7suMc9DuFS4s/bAzc128L8wtfNvNiP6zhAV+UvfgUmyNKjgl' + '0be9Ke2pCNChEQmViNal3zbWNcBrXYQpFpX1lWNkx/OuQalxzSaqmZiOR5' + 'eRwqRDZ3R9EpkOFj2ZXS1NlJg1kYXL/ibS8uvjKgJFPrZQzwaKmPNsZyGc' + 'CnHupfgC2iRIu97wnvmDxWQ9Cs62NSynr0IYGkTLN5PZU6Z5gd1F7zV6uh' + 'oFiHOYidj2EoUj7xnb8GHi5U6PQzaC97nSCR4CFnmcpfv+XcRIWe8nrM8G' + 'AVdcUob8pofUlnyGV6GEGlO3mnb7ls5B6lvuZqB/x6UqZiNKwmZvxvS11X' + 'AGkhfBGTfFZeqRlLwXvXWnOUOO0KJ8h3gSlc1gFVY+4HCbTOqjUASWw0mV' + 'JP+U0anK9wu9B/icLDUZxM/NRdbTQFmcfvABjwdm2GTmwGpQek/H0wN3dO' + 'terlTiS7arMUft7A6hkhkmLb0iDfWPWdN50V+XOMpdZtaJSGqwNHokc75p' + '3zYll0/ZpxTgmWXariOkKxr6KHHjml89QNQSBE2TJW/YnQ5SrkaHLHKdcy' + 'PqQtcXDz/WxKquQfRF+fsvcwqaeqlAWOxUXHU77cBvDGPU5O3uvEIJnHr1' + 'kuabqRQbJIV5Uzo4sEW828r2IWQnUd4Om79y+9yp/aT10DusEmvOgS3oSp' + '3eYkhvlVULeCQEJoI41t4nGLhHiiK4xBG8yFknuV7nF4k2O+EbyCXsJeeD' + 'qlGok91zEhQl1MlQA8ZofRK7bDPcn97USiJMss81s5bwIv4yN8s0QL62Ha' + 'vrIYG7C26DV6c0GxULu02H1YOnoPf6JsGC/2+zA+b7a+4O0EP0BXU3FYCb' + 'iEDbDpB3dFe63ed+ml2HQjqzOLAtKVXzAQq5UNV4m2zY0/y7gV7qSrM=' + '\n', + 'version': 2, + 'kdf': 'scrypt', + 'kdf_length': 32 + } + + ENCRYPTED_V1 = { + 'version': 1, + 'active_secret': 'secret_id', + 'storage_secrets': { + 'secret_id': { + 'kdf': 'scrypt', + 'secret': 'u31ObvxNU8jB0HgMj3TVwQ==:JQwlYq6sAQmHYS3x2CJzObT9h' + 'j1iiHthvrMh887qedNCcOfJyCA3jpRkc0vjd2Qk/2HSJ+JxM2F' + 'MrPzzx5O34EHlgF2scen34guZRRIf42WpnMy+PrL4cnMlZLgCh' + 'H1Jz6wcIMEpU9LQ8OaCShk1/yJ6qcVHOV4DDt3mTF7ttiqI5cp' + 'msaVtxxYCcpxFiWSeSCEgr0h4/Ih1qHuM6vk+CQjf/zg1f/7HR' + 'imIyNYXit9Fw3YTkxBen1wG3f5L7OAODRTuqnWpkQFOmclx050' + 'k0frKRcX6UWhIOWpW2mqJXnvzDtQQVGzqIdSgGTGtUDGQ7Onnc' + 'NkUlSnuVC7PkDNNRuwit3pCB9YWBWyPAQgs0kLqoV4YcuSctz6' + 'SAf76ozdcK5/SrOzutOfyPag4V3AYKMv6rCKALJ10OnFJ61FL9' + 'kd6JZam7WOlEUXyO7Gdgvz+eKiQMTZXbtO2kAKqel513MedPXC' + 'dzajUe1U2JaGg86UdiDWoPYOiWxnAPwfNJk+1QuNy5NZ7PaMtF' + 'IKT3/Xema2U8mufS0FbvJyK2flP1VUWcCzHKTSqX6+kU7UpoWa' + 'hYa7PlO40El+putTQLBmNaEeaWFngO+XB4TReICHSiCdcAb3pw' + 'sabjtxt+OpK4vbj3yBSfpiZTpVbEjt9U/tUpVp/T2M66lMi3ZC' + 'oHLlhu45Zo0aEq3UmQ/WBXu6EkO2eLYz2br9YQwRbSJ6z5CHmu' + 'hjKBQlpvGNfZYObx5lY4o6Ab4f/N8gyukskjmAFAf7Fr8cEog/' + 'oxmbagoCtUGRYJp2paooqH8L6xXp0Y8+23g7WJaAIr1i4V4aKS' + 'r9x7iUK6prcZTtMJZEHCswkLN/+DU6/FX3YZcOjseC+Qv3P+9v' + 'zIDp/92KJzqVqITGwrsc6ZsglMW37qxs6albtw3lMWSHlkcLbj' + 'Xf/iHPeKnb2WNLdkFNQ1J5OaTJR+E1CrXN+pm1JtB6XaUbaLGV' + 'CGUo13lAPVDtXcPbo64kMrQtQu4m9m8X8t8tfuJmINfwBnrKzk' + 'O6pl+LwimFaFEArV6wcaMxmwi0lM7mt4U1u9OIQjghQ/dEmOyV' + 'dZBnvyG7T/oRuLdUyZ/QGXZMlPQ3lAZ0ONn1Mk4bmKToW8ToE8' + 'ylld3rLlWDjjoQP8mP05Izg3mguLHXUhikUL8MD5NdYyeZJ1XZ' + '0OZ5S9uncurYj2ABWJoVaq/tFCdCEo9bbjWsePei26GZjaM3Fx' + 'RkAICXe/bt6/uLgaPZtO+sdARDuU3DRKMIdgM9NBaIn0kC7Wk4' + 'bnYShZ/rbhVt2/ds5XinnDBZsxSR3s553DixJ9v6w9Db++9Stw' + '4DgePd9lLy+6WuVBlKmcNflx9zg7US0AOarX2UNiQ==', + 'kdf_length': 32, + 'kdf_salt': 'MYH68QH48nRFMWH44piFWqBnKtU8KCz6Ajh24otrvzJlqPgB' + 'v6bvFJjRvjRp/0/v1j2nt40RZ6H5hfoKmore0g==\n', + 'length': 1024, + 'cipher': 'aes256', + } + } + } + + def setUp(self): + def _get_pass(): + return '123' + self._crypto = SecretsCrypto(_get_pass) + + def test__get_pass(self): + self.assertEqual(self._crypto._get_pass(), '123') + + def test__get_key(self): + salt = 'abc' + expected = scrypt.hash('123', salt, buflen=32) + key = self._crypto._get_key(salt) + self.assertEqual(expected, key) + + def test_encrypt(self): + info = self._crypto.encrypt(self.SECRETS) + self.assertEqual(8, len(info)) + for key, value in [ + ('kdf', 'scrypt'), + ('kdf_salt', None), + ('kdf_length', None), + ('cipher', 'aes_256_gcm'), + ('length', None), + ('iv', None), + ('secrets', None), + ('version', 2)]: + self.assertTrue(key in info) + if value: + self.assertEqual(info[key], value) + + def test__decrypt_v2(self): + encrypted = self.ENCRYPTED_V2 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) + + def test__decrypt_v1(self): + encrypted = self.ENCRYPTED_V1 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) class SoledadSecretsTestCase(BaseSoledadTest): - def test_new_soledad_instance_generates_one_secret(self): - self.assertTrue( - self._soledad.storage_secret is not None, - "Expected secret to be something different than None") - number_of_secrets = len(self._soledad.secrets._secrets) - self.assertTrue( - number_of_secrets == 1, - "Expected exactly 1 secret, got %d instead." % number_of_secrets) - - def test_generated_secret_is_of_correct_type(self): - expected_type = str - self.assertIsInstance( - self._soledad.storage_secret, expected_type, - "Expected secret to be of type %s" % expected_type) - - def test_generated_secret_has_correct_lengt(self): - expected_length = self._soledad.secrets.GEN_SECRET_LENGTH - actual_length = len(self._soledad.storage_secret) - self.assertTrue( - expected_length == actual_length, - "Expected secret with length %d, got %d instead." - % (expected_length, actual_length)) - - def test_generated_secret_id_is_sha256_hash_of_secret(self): - generated = self._soledad.secrets.secret_id - expected = hashlib.sha256(self._soledad.storage_secret).hexdigest() - self.assertTrue( - generated == expected, - "Expeceted generated secret id to be sha256 hash, got something " - "else instead.") - - def test_generate_new_secret_generates_different_secret_id(self): - # generate new secret - secret_id_1 = self._soledad.secrets.secret_id - secret_id_2 = self._soledad.secrets._gen_secret() - self.assertTrue( - len(self._soledad.secrets._secrets) == 2, - "Expected exactly 2 secrets.") - self.assertTrue( - secret_id_1 != secret_id_2, - "Expected IDs of secrets to be distinct.") - self.assertTrue( - secret_id_1 in self._soledad.secrets._secrets, - "Expected to find ID of first secret in Soledad Secrets.") - self.assertTrue( - secret_id_2 in self._soledad.secrets._secrets, - "Expected to find ID of second secret in Soledad Secrets.") - - def test__has_secret(self): - self.assertTrue( - self._soledad._secrets._has_secret(), - "Should have a secret at this point") + def test_generated_secrets_have_correct_length(self): + expected = self._soledad.secrets.lengths + for name, length in expected.iteritems(): + secret = getattr(self._soledad.secrets, name) + self.assertEqual(length, len(secret)) class SoledadCryptoAESTestCase(BaseSoledadTest): diff --git a/testing/tests/client/test_deprecated_crypto.py b/testing/tests/client/test_deprecated_crypto.py index 8ee3735c..8c711c22 100644 --- a/testing/tests/client/test_deprecated_crypto.py +++ b/testing/tests/client/test_deprecated_crypto.py @@ -1,5 +1,7 @@ import json -from twisted.internet import defer + +from pytest import inlineCallbacks + from uuid import uuid4 from urlparse import urljoin @@ -39,7 +41,7 @@ class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): def make_app_with_state(state): return make_token_soledad_app(state) - @defer.inlineCallbacks + @inlineCallbacks def test_touch_updates_remote_representation(self): self.startTwistedServer() user = 'user-' + uuid4().hex diff --git a/testing/tests/client/test_shared_db.py b/testing/tests/client/test_shared_db.py index aac766c2..b045e524 100644 --- a/testing/tests/client/test_shared_db.py +++ b/testing/tests/client/test_shared_db.py @@ -2,7 +2,6 @@ from leap.soledad.common.document import SoledadDocument from leap.soledad.client.shared_db import SoledadSharedDatabase from test_soledad.util import BaseSoledadTest -from test_soledad.util import ADDRESS class SoledadSharedDBTestCase(BaseSoledadTest): @@ -14,37 +13,28 @@ class SoledadSharedDBTestCase(BaseSoledadTest): def setUp(self): BaseSoledadTest.setUp(self) self._shared_db = SoledadSharedDatabase( - 'https://provider/', ADDRESS, document_factory=SoledadDocument, + 'https://provider/', document_factory=SoledadDocument, creds=None) def tearDown(self): BaseSoledadTest.tearDown(self) - def test__get_secrets_from_shared_db(self): + def test__get_remote_doc(self): """ Ensure the shared db is queried with the correct doc_id. """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._get_secrets_from_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - - def test__put_secrets_in_shared_db(self): + doc_id = self._soledad.secrets.storage._remote_doc_id() + self._soledad.secrets.storage._get_remote_doc() + self._soledad.secrets.storage._shared_db.get_doc.assert_called_with( + doc_id) + + def test_save_remote(self): """ Ensure recovery document is put into shared recover db. """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._put_secrets_in_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - self.assertTrue( - self._soledad.shared_db.put_doc.assert_called_with( - self._doc_put) is None, - 'Wrong document when putting recovery document.') - self.assertTrue( - self._doc_put.doc_id == doc_id, - 'Wrong doc_id when putting recovery document.') + doc_id = self._soledad.secrets.storage._remote_doc_id() + storage = self._soledad.secrets.storage + storage.save_remote({'content': 'blah'}) + storage._shared_db.get_doc.assert_called_with(doc_id) + storage._shared_db.put_doc.assert_called_with(self._doc_put) + self.assertTrue(self._doc_put.doc_id == doc_id) diff --git a/testing/tests/client/test_signals.py b/testing/tests/client/test_signals.py index 4e9ebfd0..c7609a74 100644 --- a/testing/tests/client/test_signals.py +++ b/testing/tests/client/test_signals.py @@ -20,7 +20,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def setUp(self): # mock signaling soledad.client.signal = Mock() - soledad.client.secrets.events.emit_async = Mock() + soledad.client._secrets.util.events.emit_async = Mock() # run parent's setUp BaseSoledadTest.setUp(self) @@ -42,55 +42,36 @@ class SoledadSignalingTestCase(BaseSoledadTest): - downloading keys / done downloading keys. - uploading keys / done uploading keys. """ - soledad.client.secrets.events.emit_async.reset_mock() + soledad.client._secrets.util.events.emit_async.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage3.json', local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() + soledad.client._secrets.util.events.emit_async.mock_calls.reverse() + soledad.client._secrets.util.events.emit_async.call_args = \ + soledad.client._secrets.util.events.emit_async.call_args_list[0] + soledad.client._secrets.util.events.emit_async.call_args_list.reverse() user_data = {'userid': ADDRESS, 'uuid': ADDRESS} - # downloading keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # creating keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_CREATING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_CREATING_KEYS, user_data - ) - # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # uploading keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_UPLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data - ) + def _assert(*args, **kwargs): + mocked = soledad.client._secrets.util.events.emit_async + mocked.assert_called_with(*args) + pop = kwargs.get('pop') + if pop or pop is None: + self._pop_mock_call(mocked) + + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_CREATING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_CREATING_KEYS, user_data) + _assert(catalog.SOLEDAD_UPLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data, pop=False) + sol.close() def test_stage2_bootstrap_signals(self): @@ -101,11 +82,11 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get existing instance so we have access to keys sol = self._soledad_instance() # create a document with secrets - doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) - doc.content = sol.secrets._export_recovery_document() + doc = SoledadDocument(doc_id=sol.secrets.storage._remote_doc_id()) + doc.content = sol.secrets.crypto.encrypt(sol.secrets._secrets) sol.close() # reset mock - soledad.client.secrets.events.emit_async.reset_mock() + soledad.client._secrets.util.events.emit_async.reset_mock() # get a fresh instance so it emits all bootstrap signals shared_db = self.get_default_shared_mock(get_doc_return_value=doc) sol = self._soledad_instance( @@ -114,20 +95,23 @@ class SoledadSignalingTestCase(BaseSoledadTest): shared_db_class=shared_db) # reverse call order so we can verify in the order the signals were # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() + mocked = soledad.client._secrets.util.events.emit_async + mocked.mock_calls.reverse() + mocked.call_args = mocked.call_args_list[0] + mocked.call_args_list.reverse() + + def _assert(*args, **kwargs): + mocked = soledad.client._secrets.util.events.emit_async + mocked.assert_called_with(*args) + pop = kwargs.get('pop') + if pop or pop is None: + self._pop_mock_call(mocked) + # assert download keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS} - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS}, - ) + user_data = {'userid': ADDRESS, 'uuid': ADDRESS} + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data, pop=False) + sol.close() def test_stage1_bootstrap_signals(self): diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 6ce9a5c5..302a16b8 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -838,7 +838,7 @@ class TestSoledadDbSync( # already created on some setUp method. import binascii tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) + key = tohex(self._soledad.secrets.local) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( -- cgit v1.2.3 From 8d9782c689daa14aca495d7b6b2598b2743c4e7c Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 24 Dec 2016 14:05:15 -0200 Subject: [bug] use derived key for local storage --- .../src/leap/soledad/client/_secrets/__init__.py | 36 +++++++--- client/src/leap/soledad/client/_secrets/crypto.py | 6 +- client/src/leap/soledad/client/api.py | 4 +- testing/tests/client/test_crypto.py | 82 ++++++++++++---------- 4 files changed, 75 insertions(+), 53 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index f9da8423..42fe5a2d 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -16,6 +16,7 @@ # along with this program. If not, see . import os +import scrypt from collections import namedtuple @@ -34,10 +35,12 @@ SecretLength = namedtuple('SecretLength', 'name length') class Secrets(object): + # remote secret is used + lengths = { - 'remote': 512, - 'salt': 64, - 'local': 448, + 'remote_secret': 512, # remote_secret is used to encrypt remote data. + 'local_salt': 64, # local_salt is used in conjunction with + 'local_secret': 448, # local_secret to derive a local_key for storage } def __init__(self, uuid, passphrase, url, local_path, creds, userid, @@ -119,14 +122,29 @@ class Secrets(object): self.storage.save_local(encrypted) self.storage.save_remote(encrypted) + # + # secrets + # + + @property + def remote_secret(self): + return self._secrets.get('remote_secret') + @property - def remote(self): - return self._secrets.get('remote') + def local_salt(self): + return self._secrets.get('local_salt') @property - def salt(self): - return self._secrets.get('salt') + def local_secret(self): + return self._secrets.get('local_secret') @property - def local(self): - return self._secrets.get('local') + def local_key(self): + # local storage key is scrypt-derived from `local_secret` and + # `local_salt` above + secret = scrypt.hash( + password=self.local_secret, + salt=self.local_salt, + buflen=32, # we need a key with 256 bits (32 bytes) + ) + return secret diff --git a/client/src/leap/soledad/client/_secrets/crypto.py b/client/src/leap/soledad/client/_secrets/crypto.py index 76e80222..88f32507 100644 --- a/client/src/leap/soledad/client/_secrets/crypto.py +++ b/client/src/leap/soledad/client/_secrets/crypto.py @@ -92,9 +92,9 @@ class SecretsCrypto(object): plaintext = self._decrypt( key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_ctr) secrets = { - 'remote': plaintext[0:512], - 'salt': plaintext[512:576], - 'local': plaintext[576:1024], + 'remote_secret': plaintext[0:512], + 'local_salt': plaintext[512:576], + 'local_secret': plaintext[576:1024], } return secrets diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 2e1d1cd3..54cbcd9d 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -196,7 +196,7 @@ class Soledad(object): self._init_secrets(shared_db=shared_db) - self._crypto = SoledadCrypto(self._secrets.remote) + self._crypto = SoledadCrypto(self._secrets.remote_secret) try: # initialize database access, trap any problems so we can shutdown @@ -268,7 +268,7 @@ class Soledad(object): """ tohex = binascii.b2a_hex # sqlcipher only accepts the hex version - key = tohex(self._secrets.local) + key = tohex(self._secrets.local_key) opts = sqlcipher.SQLCipherOptions( self._local_db_path, key, diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 379475cd..d161052a 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -189,47 +189,51 @@ class BlobTestCase(unittest.TestCase): class SecretsCryptoTestCase(unittest.TestCase): - SECRETS = {'remote': 'a' * 512, 'salt': 'b' * 64, 'local': 'c' * 448} + SECRETS = { + 'remote_secret': 'a' * 512, + 'local_salt': 'b' * 64, + 'local_secret': 'c' * 448 + } ENCRYPTED_V2 = { 'cipher': 'aes_256_gcm', - 'length': 1417, - 'kdf_salt': '3DCkfecls0GcX2RadA04FAC2cqkI+vpGwwCLwffdRI6vpO5SPxaw/eM0/' - 'z3GUADm3If3YCQBldKXNdqHQLsU1Q==\n', - 'iv': 'rRwCDw5Rbp5+J3QwjQ46Hw==', - 'secrets': 'lxf6yrGDcBr8XFWNDgsCoO2XPGfDJndviL9Y2GmHcSEBWnO2dm2sieuPoq' - 'PwSHRSJSrzM4Ezgdaan7X8+ErnuRLUVqbPAqPl8xx8FdCjnid4vFyFYNFI' - '/dmo8SQAf8O9vdlVEPZ5Nk2DuWIrh+oPlrSUOmR6XzI0YVdoJDmGWowygU' - 'MR0R9Bi9xFGlG135NVcNP8KGdnQDkI0V+U/3qm3tctbo4LRCxxJ60wdi0M' - 'DA6iYFI/IMshxI/ZXFHp5/YPk0k2m0i6z71kMVksgjIMMgT5Kmz7WR54na' - 'IkWbvNkbYRFR/Hbg9p6Bs7NjJlOLTjnwGJNYPbdyfJXKd1R/S8Mg7ZqsyQ' - 'VbBqXHwEN7gYlMZ66D8wu8LOK70mN7LLiSz5J8tXO3rDT1mIIf3IvNhv/j' - 'rEZHf1fTFPRp+ZVEt/hJKyPv71ua4p2lgdgNlCs2IsACk9ku/LQwXP6uZr' - 'hMJsTvniTQoCVXFYVN/jKo7Pz/+uT5wOXOXtL7smpBE/2r3uoERNM+Zw11' - 'SA8UzzMZQMxJQKVNwLmKtwvztN5dxVXhxCUyeLmeQc84VzV7NK0WMUOdfA' - '18I0HS6rHLKcdsvrPAdzvGim7tiE8TBdp8ITNQ8yMFNiGNyOVliTSTwQFf' - 'sCj6m5nYcjvprNQ8RkeitvicrtI1Ylc8CfFK50xPV77XVmlgvNsfm54msN' - 'tV0K5+XwaNgimlh/1m2bVEYj55gO0twVASwRuZj3sSY2z669iuXRk7EPyT' - 'jcE2NnfW+lqOQkJ73N7pv73t6OjiEnrKx7VmH94zYlY8ZReVVn4RTZhare' - 'D7rqCmGPhsPaCPaAfotfNBBa0w6p6L9ZlNxpIesnMObtyGob1g4Vcu8O6K' - '2Q1Ldj95+Q53tJDpx2NLP/5tfAUlbehD3whKwKOz/rGKEfhgE+Nx32RR0y' - 'YM4aJ7CYI/U3YH82xqGoa1ufIJbSBt965CVIHSVJt/mYfilhMACV/wBlvL' - 'ua08iKpHwc7suMc9DuFS4s/bAzc128L8wtfNvNiP6zhAV+UvfgUmyNKjgl' - '0be9Ke2pCNChEQmViNal3zbWNcBrXYQpFpX1lWNkx/OuQalxzSaqmZiOR5' - 'eRwqRDZ3R9EpkOFj2ZXS1NlJg1kYXL/ibS8uvjKgJFPrZQzwaKmPNsZyGc' - 'CnHupfgC2iRIu97wnvmDxWQ9Cs62NSynr0IYGkTLN5PZU6Z5gd1F7zV6uh' - 'oFiHOYidj2EoUj7xnb8GHi5U6PQzaC97nSCR4CFnmcpfv+XcRIWe8nrM8G' - 'AVdcUob8pofUlnyGV6GEGlO3mnb7ls5B6lvuZqB/x6UqZiNKwmZvxvS11X' - 'AGkhfBGTfFZeqRlLwXvXWnOUOO0KJ8h3gSlc1gFVY+4HCbTOqjUASWw0mV' - 'JP+U0anK9wu9B/icLDUZxM/NRdbTQFmcfvABjwdm2GTmwGpQek/H0wN3dO' - 'terlTiS7arMUft7A6hkhkmLb0iDfWPWdN50V+XOMpdZtaJSGqwNHokc75p' - '3zYll0/ZpxTgmWXariOkKxr6KHHjml89QNQSBE2TJW/YnQ5SrkaHLHKdcy' - 'PqQtcXDz/WxKquQfRF+fsvcwqaeqlAWOxUXHU77cBvDGPU5O3uvEIJnHr1' - 'kuabqRQbJIV5Uzo4sEW828r2IWQnUd4Om79y+9yp/aT10DusEmvOgS3oSp' - '3eYkhvlVULeCQEJoI41t4nGLhHiiK4xBG8yFknuV7nF4k2O+EbyCXsJeeD' - 'qlGok91zEhQl1MlQA8ZofRK7bDPcn97USiJMss81s5bwIv4yN8s0QL62Ha' - 'vrIYG7C26DV6c0GxULu02H1YOnoPf6JsGC/2+zA+b7a+4O0EP0BXU3FYCb' - 'iEDbDpB3dFe63ed+ml2HQjqzOLAtKVXzAQq5UNV4m2zY0/y7gV7qSrM=' - '\n', + 'length': 1437, + 'kdf_salt': 'TSgNLeAGFeITeSgNzmYZHh+mzmkZPOqao7CAV/tx3KZCLwsrT0HmWtVK3' + 'TyWHWNgVdeamMZYRuvZavE2sp0DGw==\n', + 'iv': 'TKZQKIlRgdnXFhJf08qswg==', + 'secrets': 'ZNZRi72VDtwZqyuU+uf3yzZt23vCtMS3Ki2bnZyeHUOSGVweJeDadF4oqE' + 'BW87NN00j9E49BzyzLr9SNgwZjPp0wlUm7kt+s8EUfJUdH8nxaQ+9iqGXM' + 'cCHmBM8L8DRN2m3BrPGx7m+QGlN9sbrRpl7fqc46RWcYuTEpm4upjdtI7O' + 'jDd0JG3C0rUzIuKJn9w4rEpX3tLEKXVdZfLvRXS5roR0cauazsDO69E13q' + 'a01vDuY+UJ+buLQ3FluPnnk8QE7ztPVUmRJJ76yAIhjVX9owiwlp9GnUJY' + 'sETRCqdRSTwUcHIkzVR0zAvtxTX7eGTitzf4gCYEC4T9v5N/jHxEfPdx28' + 'MM4KShWN2nFxNFQLQUpMN2OrM7UyUw+DQ3ydqBeBPKPHRN5s05kIK7P/Ra' + 'aLNcrJWa7DopLbgLlei0Jd7S4sjv1ufaRY7v0qJaVkhh/VaCylTSVw1rv5' + 'YzSWcHHcLuC0R8xLadz6T+EpsVYxgPYCS7w5xoE82zwNQzw/EBxLIcyLPl' + 'ipKnr2dttrmm3KXUOT1IdbSbI5elF6yQTAusdqiXuypey+MDqHYWEYWkCn' + 'e9/uGM9FjklDLE0RtPEDxhq64tw6u2Xu7RzDzyQDI8EIoTdU+4zEMTnelZ' + 'fKEwdG58EDxTXfUk6IDcRUupz3YuToSMhIOkqgXnbWl/nrK0O9v4JMhQjI' + 'r+oPICYfFr14kvJXBsfntILTJCxzbqTQcNba3jc8rGqCZ6gM0u4PndwTG2' + 'UiCqPU2HMnWvVGQOXeLdQY+EqqXQiRDi0DrDmkVwFf+27dPXxmZ43C48W3' + 'lMhTKXl0rdBFnOD5jiMh0X6q/KYXonyEtMZMsjT7dFePcCy4wQRhuut+ac' + '/TJWyrr+/IB45E+LZbhV7xCy1dYsbdb52jTRJFpaQ83sj6Iv6SYdiqqXzL' + 'F5JGMyuovTjwAoIIQzpLv36xY2wGGAH1V8c7QmDR2qumXrHD9R68WjBoSY' + '7IFM0TFAGZNun56y/zQ4r8yOMSAId+j4kuRH0fENEi0FJ+SpmSdHfpvBhE' + 'MdGh927E9enEYWmUQMmkxXIw6E+O3cmOWt2hsMbUAikDCpQOnVP2BD55HT' + '6FfbW7ITVwPINHYexmy2Xcm8H5zzGFSp+uYIyPBYDKA+VJ+QQI8bud9K+T' + 'NBybUv9u6LbB6BsLpwLoxMPJu0WsN2HpmLYgrg2ML1huMF1OtaGRuUr2PL' + 'NBaZaL6VOztYrVtQG1+tNyRxn8XQTtx0l6n+EihGVe9Sk5XF6DJA9ZN7uO' + 'svTUFJ5qG3Erf4AmbUJWoOR/NvapBtifidM7gPZZ6NqBs6v72rU1pGy+p7' + 'o84KrmB2MNf3yJ0BvKxPvFmltF3Dc7LB5TN8ycbmFM6hgrLvvhPxiHEnG/' + '8Qcrg0nUXOipFGNgZEU7t7Mz6RJ189Z2Kx1HVGrkAzEgqwZYqijAPlsgzO' + 'bg6DwzwC7stolQWGCDQUtJVlE8FZ/Up8zFYYZKn52WzjmSN4/hHhEvdkck' + 'Nez/JVev6fMcVrgdrTZ+uCwxjN/4xPdgog2HV470ea1bvIkSNOOrhm194M' + '40GmvmBhiSSMjdRQCQtM0t9bUuSQLPDzEiCA9QaLyygtlz9uRR/dXgkEg0' + 'J4YNpZvhE0wbyp4GHytbPaAmrcd7im9+buTuMwhXpZl0stmfkJxVHJSZ8Y' + 'IHakHs3W1fdYyI3wxGpel/9eYO3ISukolwrHXESP65wVNKfBwbqVJzQmts' + 'pyDBOI6DcLKZfE1EVg0+uwQ/5PKZbn0TwlXO1YE3NL3mAply3zQR9hyBrY' + '6f1jkHVD3irIlWkSiPJsP8sW+nrK8c/Ha8F+dua6DTZmg594OIaQj8mPiY' + 'GcIusiARWocR5/MmSjupGOgFx4HtmckTJtAta3XP4elOx04teH/P9Cgr1x' + 'XYf+cEX6gp92L9rTo0FCz3Hw==\n', 'version': 2, 'kdf': 'scrypt', 'kdf_length': 32 -- cgit v1.2.3 From fffaf266ef401cc7b081085929f42f6ca8af73b6 Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 24 Dec 2016 14:17:46 -0200 Subject: [test] fix test after secrets refactor --- testing/tests/sync/test_sync_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 302a16b8..34702362 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -838,7 +838,7 @@ class TestSoledadDbSync( # already created on some setUp method. import binascii tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.local) + key = tohex(self._soledad.secrets.local_key) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( -- cgit v1.2.3 From 728697634428f6a357c491cfed8860f4896f8911 Mon Sep 17 00:00:00 2001 From: Thais Siqueira Date: Wed, 11 Jan 2017 15:26:11 -0200 Subject: [bug] Fix import for load_configuration on migration script Related with https://leap.se/code/issues/8742 --- scripts/migration/0.9.0/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migration/0.9.0/migrate.py b/scripts/migration/0.9.0/migrate.py index 6ad5bc2d..9a29d8b5 100755 --- a/scripts/migration/0.9.0/migrate.py +++ b/scripts/migration/0.9.0/migrate.py @@ -27,7 +27,7 @@ import os from argparse import ArgumentParser -from leap.soledad.server import load_configuration +from leap.soledad.server.config import load_configuration from migrate_couch_schema import migrate -- cgit v1.2.3 From 1aa4fa6861626bd9ece719513698ef494cc23610 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 20 Jan 2017 19:43:25 -0200 Subject: [bug] several fixes for secrets refactor - store ENC_METHOD value instead of string in secrets file - allow for migration of not-activated secrets - allow migration of 'aes256' and ENC_METHOD secrets cipher --- client/src/leap/soledad/client/_secrets/crypto.py | 24 ++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/crypto.py b/client/src/leap/soledad/client/_secrets/crypto.py index 88f32507..dc80cf0b 100644 --- a/client/src/leap/soledad/client/_secrets/crypto.py +++ b/client/src/leap/soledad/client/_secrets/crypto.py @@ -59,7 +59,7 @@ class SecretsCrypto(object): 'kdf': 'scrypt', 'kdf_salt': binascii.b2a_base64(salt), 'kdf_length': len(key), - 'cipher': 'aes_256_gcm', + 'cipher': ENC_METHOD.aes_256_gcm, 'length': len(plaintext), 'iv': str(iv), 'secrets': binascii.b2a_base64(ciphertext), @@ -80,17 +80,26 @@ class SecretsCrypto(object): raise SecretsError(e) def _decrypt_v1(self, data): - secret_id = data['active_secret'] + # get encrypted secret from dictionary + secret_id = data['storage_secrets'].keys().pop() encrypted = data['storage_secrets'][secret_id] - soledad_assert(encrypted['cipher'] == 'aes256') + # assert that we know how to decrypt the secret + soledad_assert('cipher' in encrypted) + cipher = encrypted['cipher'] + if cipher == 'aes256': + cipher = ENC_METHOD.aes_256_ctr + soledad_assert(cipher in ENC_METHOD) + + # decrypt salt = binascii.a2b_base64(encrypted['kdf_salt']) key = self._get_key(salt) separator = ':' iv, ciphertext = encrypted['secret'].split(separator, 1) ciphertext = binascii.a2b_base64(ciphertext) - plaintext = self._decrypt( - key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_ctr) + plaintext = self._decrypt(key, iv, ciphertext, encrypted, cipher) + + # create secrets dictionary secrets = { 'remote_secret': plaintext[0:512], 'local_salt': plaintext[512:576], @@ -99,14 +108,15 @@ class SecretsCrypto(object): return secrets def _decrypt_v2(self, encrypted): - soledad_assert(encrypted['cipher'] == 'aes_256_gcm') + cipher = encrypted['cipher'] + soledad_assert(cipher in ENC_METHOD) salt = binascii.a2b_base64(encrypted['kdf_salt']) key = self._get_key(salt) iv = encrypted['iv'] ciphertext = binascii.a2b_base64(encrypted['secrets']) plaintext = self._decrypt( - key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_gcm) + key, iv, ciphertext, encrypted, cipher) encoded = json.loads(plaintext) secrets = {} for name, value in encoded.iteritems(): -- cgit v1.2.3 From 71ab022aa12d5754d5845eae757b2f3531b4e50a Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 21 Jan 2017 20:13:27 -0200 Subject: [test] move client secrets tests to its own file --- testing/tests/client/test_crypto.py | 138 ------------------------------ testing/tests/client/test_secrets.py | 161 +++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 138 deletions(-) create mode 100644 testing/tests/client/test_secrets.py diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index d161052a..5411a4e8 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -21,7 +21,6 @@ import binascii import base64 import json import os -import scrypt from io import BytesIO @@ -34,7 +33,6 @@ from cryptography.exceptions import InvalidTag from leap.soledad.common.document import SoledadDocument from test_soledad.util import BaseSoledadTest from leap.soledad.client import _crypto -from leap.soledad.client._secrets import SecretsCrypto from twisted.trial import unittest from twisted.internet import defer @@ -187,142 +185,6 @@ class BlobTestCase(unittest.TestCase): yield crypto.decrypt_doc(doc2) -class SecretsCryptoTestCase(unittest.TestCase): - - SECRETS = { - 'remote_secret': 'a' * 512, - 'local_salt': 'b' * 64, - 'local_secret': 'c' * 448 - } - ENCRYPTED_V2 = { - 'cipher': 'aes_256_gcm', - 'length': 1437, - 'kdf_salt': 'TSgNLeAGFeITeSgNzmYZHh+mzmkZPOqao7CAV/tx3KZCLwsrT0HmWtVK3' - 'TyWHWNgVdeamMZYRuvZavE2sp0DGw==\n', - 'iv': 'TKZQKIlRgdnXFhJf08qswg==', - 'secrets': 'ZNZRi72VDtwZqyuU+uf3yzZt23vCtMS3Ki2bnZyeHUOSGVweJeDadF4oqE' - 'BW87NN00j9E49BzyzLr9SNgwZjPp0wlUm7kt+s8EUfJUdH8nxaQ+9iqGXM' - 'cCHmBM8L8DRN2m3BrPGx7m+QGlN9sbrRpl7fqc46RWcYuTEpm4upjdtI7O' - 'jDd0JG3C0rUzIuKJn9w4rEpX3tLEKXVdZfLvRXS5roR0cauazsDO69E13q' - 'a01vDuY+UJ+buLQ3FluPnnk8QE7ztPVUmRJJ76yAIhjVX9owiwlp9GnUJY' - 'sETRCqdRSTwUcHIkzVR0zAvtxTX7eGTitzf4gCYEC4T9v5N/jHxEfPdx28' - 'MM4KShWN2nFxNFQLQUpMN2OrM7UyUw+DQ3ydqBeBPKPHRN5s05kIK7P/Ra' - 'aLNcrJWa7DopLbgLlei0Jd7S4sjv1ufaRY7v0qJaVkhh/VaCylTSVw1rv5' - 'YzSWcHHcLuC0R8xLadz6T+EpsVYxgPYCS7w5xoE82zwNQzw/EBxLIcyLPl' - 'ipKnr2dttrmm3KXUOT1IdbSbI5elF6yQTAusdqiXuypey+MDqHYWEYWkCn' - 'e9/uGM9FjklDLE0RtPEDxhq64tw6u2Xu7RzDzyQDI8EIoTdU+4zEMTnelZ' - 'fKEwdG58EDxTXfUk6IDcRUupz3YuToSMhIOkqgXnbWl/nrK0O9v4JMhQjI' - 'r+oPICYfFr14kvJXBsfntILTJCxzbqTQcNba3jc8rGqCZ6gM0u4PndwTG2' - 'UiCqPU2HMnWvVGQOXeLdQY+EqqXQiRDi0DrDmkVwFf+27dPXxmZ43C48W3' - 'lMhTKXl0rdBFnOD5jiMh0X6q/KYXonyEtMZMsjT7dFePcCy4wQRhuut+ac' - '/TJWyrr+/IB45E+LZbhV7xCy1dYsbdb52jTRJFpaQ83sj6Iv6SYdiqqXzL' - 'F5JGMyuovTjwAoIIQzpLv36xY2wGGAH1V8c7QmDR2qumXrHD9R68WjBoSY' - '7IFM0TFAGZNun56y/zQ4r8yOMSAId+j4kuRH0fENEi0FJ+SpmSdHfpvBhE' - 'MdGh927E9enEYWmUQMmkxXIw6E+O3cmOWt2hsMbUAikDCpQOnVP2BD55HT' - '6FfbW7ITVwPINHYexmy2Xcm8H5zzGFSp+uYIyPBYDKA+VJ+QQI8bud9K+T' - 'NBybUv9u6LbB6BsLpwLoxMPJu0WsN2HpmLYgrg2ML1huMF1OtaGRuUr2PL' - 'NBaZaL6VOztYrVtQG1+tNyRxn8XQTtx0l6n+EihGVe9Sk5XF6DJA9ZN7uO' - 'svTUFJ5qG3Erf4AmbUJWoOR/NvapBtifidM7gPZZ6NqBs6v72rU1pGy+p7' - 'o84KrmB2MNf3yJ0BvKxPvFmltF3Dc7LB5TN8ycbmFM6hgrLvvhPxiHEnG/' - '8Qcrg0nUXOipFGNgZEU7t7Mz6RJ189Z2Kx1HVGrkAzEgqwZYqijAPlsgzO' - 'bg6DwzwC7stolQWGCDQUtJVlE8FZ/Up8zFYYZKn52WzjmSN4/hHhEvdkck' - 'Nez/JVev6fMcVrgdrTZ+uCwxjN/4xPdgog2HV470ea1bvIkSNOOrhm194M' - '40GmvmBhiSSMjdRQCQtM0t9bUuSQLPDzEiCA9QaLyygtlz9uRR/dXgkEg0' - 'J4YNpZvhE0wbyp4GHytbPaAmrcd7im9+buTuMwhXpZl0stmfkJxVHJSZ8Y' - 'IHakHs3W1fdYyI3wxGpel/9eYO3ISukolwrHXESP65wVNKfBwbqVJzQmts' - 'pyDBOI6DcLKZfE1EVg0+uwQ/5PKZbn0TwlXO1YE3NL3mAply3zQR9hyBrY' - '6f1jkHVD3irIlWkSiPJsP8sW+nrK8c/Ha8F+dua6DTZmg594OIaQj8mPiY' - 'GcIusiARWocR5/MmSjupGOgFx4HtmckTJtAta3XP4elOx04teH/P9Cgr1x' - 'XYf+cEX6gp92L9rTo0FCz3Hw==\n', - 'version': 2, - 'kdf': 'scrypt', - 'kdf_length': 32 - } - - ENCRYPTED_V1 = { - 'version': 1, - 'active_secret': 'secret_id', - 'storage_secrets': { - 'secret_id': { - 'kdf': 'scrypt', - 'secret': 'u31ObvxNU8jB0HgMj3TVwQ==:JQwlYq6sAQmHYS3x2CJzObT9h' - 'j1iiHthvrMh887qedNCcOfJyCA3jpRkc0vjd2Qk/2HSJ+JxM2F' - 'MrPzzx5O34EHlgF2scen34guZRRIf42WpnMy+PrL4cnMlZLgCh' - 'H1Jz6wcIMEpU9LQ8OaCShk1/yJ6qcVHOV4DDt3mTF7ttiqI5cp' - 'msaVtxxYCcpxFiWSeSCEgr0h4/Ih1qHuM6vk+CQjf/zg1f/7HR' - 'imIyNYXit9Fw3YTkxBen1wG3f5L7OAODRTuqnWpkQFOmclx050' - 'k0frKRcX6UWhIOWpW2mqJXnvzDtQQVGzqIdSgGTGtUDGQ7Onnc' - 'NkUlSnuVC7PkDNNRuwit3pCB9YWBWyPAQgs0kLqoV4YcuSctz6' - 'SAf76ozdcK5/SrOzutOfyPag4V3AYKMv6rCKALJ10OnFJ61FL9' - 'kd6JZam7WOlEUXyO7Gdgvz+eKiQMTZXbtO2kAKqel513MedPXC' - 'dzajUe1U2JaGg86UdiDWoPYOiWxnAPwfNJk+1QuNy5NZ7PaMtF' - 'IKT3/Xema2U8mufS0FbvJyK2flP1VUWcCzHKTSqX6+kU7UpoWa' - 'hYa7PlO40El+putTQLBmNaEeaWFngO+XB4TReICHSiCdcAb3pw' - 'sabjtxt+OpK4vbj3yBSfpiZTpVbEjt9U/tUpVp/T2M66lMi3ZC' - 'oHLlhu45Zo0aEq3UmQ/WBXu6EkO2eLYz2br9YQwRbSJ6z5CHmu' - 'hjKBQlpvGNfZYObx5lY4o6Ab4f/N8gyukskjmAFAf7Fr8cEog/' - 'oxmbagoCtUGRYJp2paooqH8L6xXp0Y8+23g7WJaAIr1i4V4aKS' - 'r9x7iUK6prcZTtMJZEHCswkLN/+DU6/FX3YZcOjseC+Qv3P+9v' - 'zIDp/92KJzqVqITGwrsc6ZsglMW37qxs6albtw3lMWSHlkcLbj' - 'Xf/iHPeKnb2WNLdkFNQ1J5OaTJR+E1CrXN+pm1JtB6XaUbaLGV' - 'CGUo13lAPVDtXcPbo64kMrQtQu4m9m8X8t8tfuJmINfwBnrKzk' - 'O6pl+LwimFaFEArV6wcaMxmwi0lM7mt4U1u9OIQjghQ/dEmOyV' - 'dZBnvyG7T/oRuLdUyZ/QGXZMlPQ3lAZ0ONn1Mk4bmKToW8ToE8' - 'ylld3rLlWDjjoQP8mP05Izg3mguLHXUhikUL8MD5NdYyeZJ1XZ' - '0OZ5S9uncurYj2ABWJoVaq/tFCdCEo9bbjWsePei26GZjaM3Fx' - 'RkAICXe/bt6/uLgaPZtO+sdARDuU3DRKMIdgM9NBaIn0kC7Wk4' - 'bnYShZ/rbhVt2/ds5XinnDBZsxSR3s553DixJ9v6w9Db++9Stw' - '4DgePd9lLy+6WuVBlKmcNflx9zg7US0AOarX2UNiQ==', - 'kdf_length': 32, - 'kdf_salt': 'MYH68QH48nRFMWH44piFWqBnKtU8KCz6Ajh24otrvzJlqPgB' - 'v6bvFJjRvjRp/0/v1j2nt40RZ6H5hfoKmore0g==\n', - 'length': 1024, - 'cipher': 'aes256', - } - } - } - - def setUp(self): - def _get_pass(): - return '123' - self._crypto = SecretsCrypto(_get_pass) - - def test__get_pass(self): - self.assertEqual(self._crypto._get_pass(), '123') - - def test__get_key(self): - salt = 'abc' - expected = scrypt.hash('123', salt, buflen=32) - key = self._crypto._get_key(salt) - self.assertEqual(expected, key) - - def test_encrypt(self): - info = self._crypto.encrypt(self.SECRETS) - self.assertEqual(8, len(info)) - for key, value in [ - ('kdf', 'scrypt'), - ('kdf_salt', None), - ('kdf_length', None), - ('cipher', 'aes_256_gcm'), - ('length', None), - ('iv', None), - ('secrets', None), - ('version', 2)]: - self.assertTrue(key in info) - if value: - self.assertEqual(info[key], value) - - def test__decrypt_v2(self): - encrypted = self.ENCRYPTED_V2 - decrypted = self._crypto.decrypt(encrypted) - self.assertEqual(decrypted, self.SECRETS) - - def test__decrypt_v1(self): - encrypted = self.ENCRYPTED_V1 - decrypted = self._crypto.decrypt(encrypted) - self.assertEqual(decrypted, self.SECRETS) - - class SoledadSecretsTestCase(BaseSoledadTest): def test_generated_secrets_have_correct_length(self): diff --git a/testing/tests/client/test_secrets.py b/testing/tests/client/test_secrets.py new file mode 100644 index 00000000..bbeb1fc2 --- /dev/null +++ b/testing/tests/client/test_secrets.py @@ -0,0 +1,161 @@ +# -*- CODing: utf-8 -*- +# test_secrets.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for secrets encryption and decryption. +""" +import scrypt + +from twisted.trial import unittest + +from leap.soledad.client._crypto import ENC_METHOD +from leap.soledad.client._secrets import SecretsCrypto + + +class SecretsCryptoTestCase(unittest.TestCase): + + SECRETS = { + 'remote_secret': 'a' * 512, + 'local_salt': 'b' * 64, + 'local_secret': 'c' * 448 + } + ENCRYPTED_V2 = { + 'cipher': 2, + 'length': 1437, + 'kdf_salt': 'TSgNLeAGFeITeSgNzmYZHh+mzmkZPOqao7CAV/tx3KZCLwsrT0HmWtVK3' + 'TyWHWNgVdeamMZYRuvZavE2sp0DGw==\n', + 'iv': 'TKZQKIlRgdnXFhJf08qswg==', + 'secrets': 'ZNZRi72VDtwZqyuU+uf3yzZt23vCtMS3Ki2bnZyeHUOSGVweJeDadF4oqE' + 'BW87NN00j9E49BzyzLr9SNgwZjPp0wlUm7kt+s8EUfJUdH8nxaQ+9iqGXM' + 'cCHmBM8L8DRN2m3BrPGx7m+QGlN9sbrRpl7fqc46RWcYuTEpm4upjdtI7O' + 'jDd0JG3C0rUzIuKJn9w4rEpX3tLEKXVdZfLvRXS5roR0cauazsDO69E13q' + 'a01vDuY+UJ+buLQ3FluPnnk8QE7ztPVUmRJJ76yAIhjVX9owiwlp9GnUJY' + 'sETRCqdRSTwUcHIkzVR0zAvtxTX7eGTitzf4gCYEC4T9v5N/jHxEfPdx28' + 'MM4KShWN2nFxNFQLQUpMN2OrM7UyUw+DQ3ydqBeBPKPHRN5s05kIK7P/Ra' + 'aLNcrJWa7DopLbgLlei0Jd7S4sjv1ufaRY7v0qJaVkhh/VaCylTSVw1rv5' + 'YzSWcHHcLuC0R8xLadz6T+EpsVYxgPYCS7w5xoE82zwNQzw/EBxLIcyLPl' + 'ipKnr2dttrmm3KXUOT1IdbSbI5elF6yQTAusdqiXuypey+MDqHYWEYWkCn' + 'e9/uGM9FjklDLE0RtPEDxhq64tw6u2Xu7RzDzyQDI8EIoTdU+4zEMTnelZ' + 'fKEwdG58EDxTXfUk6IDcRUupz3YuToSMhIOkqgXnbWl/nrK0O9v4JMhQjI' + 'r+oPICYfFr14kvJXBsfntILTJCxzbqTQcNba3jc8rGqCZ6gM0u4PndwTG2' + 'UiCqPU2HMnWvVGQOXeLdQY+EqqXQiRDi0DrDmkVwFf+27dPXxmZ43C48W3' + 'lMhTKXl0rdBFnOD5jiMh0X6q/KYXonyEtMZMsjT7dFePcCy4wQRhuut+ac' + '/TJWyrr+/IB45E+LZbhV7xCy1dYsbdb52jTRJFpaQ83sj6Iv6SYdiqqXzL' + 'F5JGMyuovTjwAoIIQzpLv36xY2wGGAH1V8c7QmDR2qumXrHD9R68WjBoSY' + '7IFM0TFAGZNun56y/zQ4r8yOMSAId+j4kuRH0fENEi0FJ+SpmSdHfpvBhE' + 'MdGh927E9enEYWmUQMmkxXIw6E+O3cmOWt2hsMbUAikDCpQOnVP2BD55HT' + '6FfbW7ITVwPINHYexmy2Xcm8H5zzGFSp+uYIyPBYDKA+VJ+QQI8bud9K+T' + 'NBybUv9u6LbB6BsLpwLoxMPJu0WsN2HpmLYgrg2ML1huMF1OtaGRuUr2PL' + 'NBaZaL6VOztYrVtQG1+tNyRxn8XQTtx0l6n+EihGVe9Sk5XF6DJA9ZN7uO' + 'svTUFJ5qG3Erf4AmbUJWoOR/NvapBtifidM7gPZZ6NqBs6v72rU1pGy+p7' + 'o84KrmB2MNf3yJ0BvKxPvFmltF3Dc7LB5TN8ycbmFM6hgrLvvhPxiHEnG/' + '8Qcrg0nUXOipFGNgZEU7t7Mz6RJ189Z2Kx1HVGrkAzEgqwZYqijAPlsgzO' + 'bg6DwzwC7stolQWGCDQUtJVlE8FZ/Up8zFYYZKn52WzjmSN4/hHhEvdkck' + 'Nez/JVev6fMcVrgdrTZ+uCwxjN/4xPdgog2HV470ea1bvIkSNOOrhm194M' + '40GmvmBhiSSMjdRQCQtM0t9bUuSQLPDzEiCA9QaLyygtlz9uRR/dXgkEg0' + 'J4YNpZvhE0wbyp4GHytbPaAmrcd7im9+buTuMwhXpZl0stmfkJxVHJSZ8Y' + 'IHakHs3W1fdYyI3wxGpel/9eYO3ISukolwrHXESP65wVNKfBwbqVJzQmts' + 'pyDBOI6DcLKZfE1EVg0+uwQ/5PKZbn0TwlXO1YE3NL3mAply3zQR9hyBrY' + '6f1jkHVD3irIlWkSiPJsP8sW+nrK8c/Ha8F+dua6DTZmg594OIaQj8mPiY' + 'GcIusiARWocR5/MmSjupGOgFx4HtmckTJtAta3XP4elOx04teH/P9Cgr1x' + 'XYf+cEX6gp92L9rTo0FCz3Hw==\n', + 'version': 2, + 'kdf': 'scrypt', + 'kdf_length': 32 + } + + ENCRYPTED_V1 = { + 'version': 1, + 'active_secret': 'secret_id', + 'storage_secrets': { + 'secret_id': { + 'kdf': 'scrypt', + 'secret': 'u31ObvxNU8jB0HgMj3TVwQ==:JQwlYq6sAQmHYS3x2CJzObT9h' + 'j1iiHthvrMh887qedNCcOfJyCA3jpRkc0vjd2Qk/2HSJ+JxM2F' + 'MrPzzx5O34EHlgF2scen34guZRRIf42WpnMy+PrL4cnMlZLgCh' + 'H1Jz6wcIMEpU9LQ8OaCShk1/yJ6qcVHOV4DDt3mTF7ttiqI5cp' + 'msaVtxxYCcpxFiWSeSCEgr0h4/Ih1qHuM6vk+CQjf/zg1f/7HR' + 'imIyNYXit9Fw3YTkxBen1wG3f5L7OAODRTuqnWpkQFOmclx050' + 'k0frKRcX6UWhIOWpW2mqJXnvzDtQQVGzqIdSgGTGtUDGQ7Onnc' + 'NkUlSnuVC7PkDNNRuwit3pCB9YWBWyPAQgs0kLqoV4YcuSctz6' + 'SAf76ozdcK5/SrOzutOfyPag4V3AYKMv6rCKALJ10OnFJ61FL9' + 'kd6JZam7WOlEUXyO7Gdgvz+eKiQMTZXbtO2kAKqel513MedPXC' + 'dzajUe1U2JaGg86UdiDWoPYOiWxnAPwfNJk+1QuNy5NZ7PaMtF' + 'IKT3/Xema2U8mufS0FbvJyK2flP1VUWcCzHKTSqX6+kU7UpoWa' + 'hYa7PlO40El+putTQLBmNaEeaWFngO+XB4TReICHSiCdcAb3pw' + 'sabjtxt+OpK4vbj3yBSfpiZTpVbEjt9U/tUpVp/T2M66lMi3ZC' + 'oHLlhu45Zo0aEq3UmQ/WBXu6EkO2eLYz2br9YQwRbSJ6z5CHmu' + 'hjKBQlpvGNfZYObx5lY4o6Ab4f/N8gyukskjmAFAf7Fr8cEog/' + 'oxmbagoCtUGRYJp2paooqH8L6xXp0Y8+23g7WJaAIr1i4V4aKS' + 'r9x7iUK6prcZTtMJZEHCswkLN/+DU6/FX3YZcOjseC+Qv3P+9v' + 'zIDp/92KJzqVqITGwrsc6ZsglMW37qxs6albtw3lMWSHlkcLbj' + 'Xf/iHPeKnb2WNLdkFNQ1J5OaTJR+E1CrXN+pm1JtB6XaUbaLGV' + 'CGUo13lAPVDtXcPbo64kMrQtQu4m9m8X8t8tfuJmINfwBnrKzk' + 'O6pl+LwimFaFEArV6wcaMxmwi0lM7mt4U1u9OIQjghQ/dEmOyV' + 'dZBnvyG7T/oRuLdUyZ/QGXZMlPQ3lAZ0ONn1Mk4bmKToW8ToE8' + 'ylld3rLlWDjjoQP8mP05Izg3mguLHXUhikUL8MD5NdYyeZJ1XZ' + '0OZ5S9uncurYj2ABWJoVaq/tFCdCEo9bbjWsePei26GZjaM3Fx' + 'RkAICXe/bt6/uLgaPZtO+sdARDuU3DRKMIdgM9NBaIn0kC7Wk4' + 'bnYShZ/rbhVt2/ds5XinnDBZsxSR3s553DixJ9v6w9Db++9Stw' + '4DgePd9lLy+6WuVBlKmcNflx9zg7US0AOarX2UNiQ==', + 'kdf_length': 32, + 'kdf_salt': 'MYH68QH48nRFMWH44piFWqBnKtU8KCz6Ajh24otrvzJlqPgB' + 'v6bvFJjRvjRp/0/v1j2nt40RZ6H5hfoKmore0g==\n', + 'length': 1024, + 'cipher': 'aes256', + } + } + } + + def setUp(self): + def _get_pass(): + return '123' + self._crypto = SecretsCrypto(_get_pass) + + def test__get_pass(self): + self.assertEqual(self._crypto._get_pass(), '123') + + def test__get_key(self): + salt = 'abc' + expected = scrypt.hash('123', salt, buflen=32) + key = self._crypto._get_key(salt) + self.assertEqual(expected, key) + + def test_encrypt(self): + info = self._crypto.encrypt(self.SECRETS) + self.assertEqual(8, len(info)) + for key, value in [ + ('kdf', 'scrypt'), + ('kdf_salt', None), + ('kdf_length', None), + ('cipher', ENC_METHOD.aes_256_gcm), + ('length', None), + ('iv', None), + ('secrets', None), + ('version', 2)]: + self.assertTrue(key in info) + if value: + self.assertEqual(info[key], value) + + def test__decrypt_v2(self): + encrypted = self.ENCRYPTED_V2 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) + + def test__decrypt_v1(self): + encrypted = self.ENCRYPTED_V1 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) -- cgit v1.2.3 From 3787a645993ea36bbecebc850296de4b0fdd3620 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Jan 2017 10:52:20 -0200 Subject: [doc] improve comment for client secrets file migration function --- client/src/leap/soledad/client/_secrets/crypto.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/_secrets/crypto.py b/client/src/leap/soledad/client/_secrets/crypto.py index dc80cf0b..02d7dc02 100644 --- a/client/src/leap/soledad/client/_secrets/crypto.py +++ b/client/src/leap/soledad/client/_secrets/crypto.py @@ -80,7 +80,11 @@ class SecretsCrypto(object): raise SecretsError(e) def _decrypt_v1(self, data): - # get encrypted secret from dictionary + # get encrypted secret from dictionary: the old format allowed for + # storage of more than one secret, but this feature was never used and + # soledad has been using only one secret so far. As there is a corner + # case where the old 'active_secret' key might not be set, we just + # ignore it and pop the only secret found in the 'storage_secrets' key. secret_id = data['storage_secrets'].keys().pop() encrypted = data['storage_secrets'][secret_id] -- cgit v1.2.3 From db7607768310c9f9993d771cf1951d396be2554b Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 20 Dec 2016 13:37:58 -0200 Subject: [pkg] improve migration script logging --- .../0.9.0/migrate_couch_schema/__init__.py | 35 ++++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/scripts/migration/0.9.0/migrate_couch_schema/__init__.py b/scripts/migration/0.9.0/migrate_couch_schema/__init__.py index f0b456e4..d1560c59 100644 --- a/scripts/migration/0.9.0/migrate_couch_schema/__init__.py +++ b/scripts/migration/0.9.0/migrate_couch_schema/__init__.py @@ -30,7 +30,7 @@ def _get_couch_server(couch_url): return Server(couch_url) -def _is_migrateable(db): +def _has_u1db_config_doc(db): config_doc = db.get('u1db_config') return bool(config_doc) @@ -63,6 +63,35 @@ def _get_user_dbs(server): # migration main functions # +def _report_missing_u1db_config_doc(dbname, db): + config_doc = db.get(CONFIG_DOC_ID) + if not config_doc: + logger.warning( + "[%s] no '%s' or '%s' documents found, possibly an empty db? I " + "don't know what to do with this db, so I am skipping it." + % (dbname, 'u1db_config', CONFIG_DOC_ID)) + else: + if SCHEMA_VERSION_KEY in config_doc: + version = config_doc[SCHEMA_VERSION_KEY] + if version == SCHEMA_VERSION: + logger.info( + "[%s] '%s' document exists, and schema versions match " + "(expected %r and found %r). This database reports to be " + "using the new schema version, so I am skipping it." + % (dbname, CONFIG_DOC_ID)) + else: + logger.error( + "[%s] '%s' document exists, but schema versions don't " + "match (expected %r, found %r instead). I don't know " + "how to migrate such a db, so I am skipping it." + % (dbname, CONFIG_DOC_ID, SCHEMA_VERSION, version)) + else: + logger.error( + "[%s] '%s' document exists, but has no schema version " + "information in it. I don't know how to migrate such a db, " + "so I am skipping it." % (dbname, CONFIG_DOC_ID)) + + def migrate(args, target_version): server = _get_couch_server(args.couch_url) logger.info('starting couch schema migration to %s' % target_version) @@ -71,8 +100,8 @@ def migrate(args, target_version): user_dbs = _get_user_dbs(server) for dbname in user_dbs: db = server[dbname] - if not _is_migrateable(db): - logger.warning("[%s] skipping not migrateable user db" % dbname) + if not _has_u1db_config_doc(db): + _report_missing_u1db_config_doc(dbname, db) continue logger.info("[%s] starting migration of user db" % dbname) try: -- cgit v1.2.3 From a39af0e003ba95c9b7ab554aa4a4c5ce316a43c7 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 18 Dec 2016 12:56:21 -0200 Subject: [bug] disallow all requests to "user-{uuid}/" --- server/src/leap/soledad/server/auth.py | 11 ++--------- testing/tests/server/test_server.py | 8 ++++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index b0764569..f3d9c8a8 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -100,7 +100,7 @@ class URLToAuthorization(object): /shared-db/docs | - /shared-db/doc/{any_id} | GET, PUT, DELETE /shared-db/sync-from/{source} | - - /user-db | GET, PUT, DELETE + /user-db | - /user-db/docs | - /user-db/doc/{id} | - /user-db/sync-from/{source} | GET, PUT, POST @@ -108,19 +108,12 @@ class URLToAuthorization(object): # auth info for global resource self._register('/', [self.HTTP_METHOD_GET]) # auth info for shared-db database resource - self._register( - '/%s' % SHARED_DB_NAME, - [self.HTTP_METHOD_GET]) + self._register('/%s' % SHARED_DB_NAME, [self.HTTP_METHOD_GET]) # auth info for shared-db doc resource self._register( '/%s/doc/{id:.*}' % SHARED_DB_NAME, [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) - # auth info for user-db database resource - self._register( - '/%s' % self._user_db_name, - [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, - self.HTTP_METHOD_DELETE]) # auth info for user-db sync resource self._register( '/%s/sync-from/{source_replica_uid}' % self._user_db_name, diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 6710caaf..cae2e75c 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -110,7 +110,7 @@ class ServerAuthorizationTestCase(BaseSoledadTest): /shared-db/docs | - /shared-db/doc/{id} | GET, PUT, DELETE /shared-db/sync-from/{source} | - - /user-db | GET, PUT, DELETE + /user-db | - /user-db/docs | - /user-db/doc/{id} | - /user-db/sync-from/{source} | GET, PUT, POST @@ -174,13 +174,13 @@ class ServerAuthorizationTestCase(BaseSoledadTest): authmap.is_authorized( self._make_environ('/shared/sync-from/x', 'POST'))) # test user-db database resource auth - self.assertTrue( + self.assertFalse( authmap.is_authorized( self._make_environ('/%s' % dbname, 'GET'))) - self.assertTrue( + self.assertFalse( authmap.is_authorized( self._make_environ('/%s' % dbname, 'PUT'))) - self.assertTrue( + self.assertFalse( authmap.is_authorized( self._make_environ('/%s' % dbname, 'DELETE'))) self.assertFalse( -- cgit v1.2.3 From e73d36621052a69aae327200c063ac1689bcf9e0 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 18 Dec 2016 14:21:54 -0200 Subject: [feat] reuse the url mapper instead of creating it for every request --- common/src/leap/soledad/common/__init__.py | 2 - server/src/leap/soledad/server/auth.py | 82 +++------- testing/tests/server/test_server.py | 251 +++++++++++++---------------- 3 files changed, 136 insertions(+), 199 deletions(-) diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index 1ba6ab89..4948ad20 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -30,8 +30,6 @@ Soledad routines common to client and server. # SHARED_DB_NAME = 'shared' -SHARED_DB_LOCK_DOC_ID_PREFIX = 'lock-' -USER_DB_PREFIX = 'user-' # diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index f3d9c8a8..c026a282 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -26,67 +26,31 @@ from routes.mapper import Mapper from leap.soledad.common.log import getLogger from leap.soledad.common.l2db import DBNAME_CONSTRAINTS, errors as u1db_errors from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.common import USER_DB_PREFIX logger = getLogger(__name__) -class URLToAuthorization(object): +class URLMapper(object): """ - Verify if actions can be performed by a user. + Maps the URLs users can access. """ - HTTP_METHOD_GET = 'GET' - HTTP_METHOD_PUT = 'PUT' - HTTP_METHOD_DELETE = 'DELETE' - HTTP_METHOD_POST = 'POST' - - def __init__(self, uuid): - """ - Initialize the mapper. - - The C{uuid} is used to create the rules that will either allow or - disallow the user to perform specific actions. - - @param uuid: The user uuid. - @type uuid: str - @param user_db_prefix: The string prefix of users' databases. - @type user_db_prefix: str - """ + def __init__(self): self._map = Mapper(controller_scan=None) - self._user_db_name = "%s%s" % (USER_DB_PREFIX, uuid) - self._uuid = uuid - self._register_auth_info() - - def is_authorized(self, environ): - """ - Return whether an HTTP request that produced the CGI C{environ} - corresponds to an authorized action. - - @param environ: Dictionary containing CGI variables. - @type environ: dict - - @return: Whether the action is authorized or not. - @rtype: bool - """ - return self._map.match(environ=environ) is not None + self._connect_urls() + self._map.create_regs() - def _register(self, pattern, http_methods): - """ - Register a C{pattern} in the mapper as valid for C{http_methods}. + def match(self, environ): + return self._map.match(environ=environ) - @param pattern: The URL pattern that corresponds to the user action. - @type pattern: str - @param http_methods: A list of authorized HTTP methods. - @type http_methods: list of str - """ + def _connect(self, pattern, http_methods): self._map.connect( None, pattern, http_methods=http_methods, conditions=dict(method=http_methods), requirements={'dbname': DBNAME_CONSTRAINTS}) - def _register_auth_info(self): + def _connect_urls(self): """ Register the authorization info in the mapper using C{SHARED_DB_NAME} as the user's database name. @@ -106,21 +70,15 @@ class URLToAuthorization(object): /user-db/sync-from/{source} | GET, PUT, POST """ # auth info for global resource - self._register('/', [self.HTTP_METHOD_GET]) + self._connect('/', ['GET']) # auth info for shared-db database resource - self._register('/%s' % SHARED_DB_NAME, [self.HTTP_METHOD_GET]) + self._connect('/%s' % SHARED_DB_NAME, ['GET']) # auth info for shared-db doc resource - self._register( - '/%s/doc/{id:.*}' % SHARED_DB_NAME, - [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, - self.HTTP_METHOD_DELETE]) + self._connect('/%s/doc/{id:.*}' % SHARED_DB_NAME, + ['GET', 'PUT', 'DELETE']) # auth info for user-db sync resource - self._register( - '/%s/sync-from/{source_replica_uid}' % self._user_db_name, - [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, - self.HTTP_METHOD_POST]) - # generate the regular expressions - self._map.create_regs() + self._connect('/user-{uuid}/sync-from/{source_replica_uid}', + ['GET', 'PUT', 'POST']) class SoledadAuthMiddleware(object): @@ -176,6 +134,7 @@ class SoledadAuthMiddleware(object): @type prefix: str """ self._app = app + self._mapper = URLMapper() def _error(self, start_response, status, description, message=None): """ @@ -310,14 +269,17 @@ class SoledadAuthMiddleware(object): @param environ: Dictionary containing CGI variables. @type environ: dict - @param uuid: The user's uuid. + @param uuid: The user's uuid from the Authorization header. @type uuid: str - @return: Whether the user is authorize to perform the requested action + @return: Whether the user is authorized to perform the requested action over the requested db. @rtype: bool """ - return URLToAuthorization(uuid).is_authorized(environ) + match = self._mapper.match(environ) + if not match: + return False + return uuid == match.get('uuid') @abstractmethod def _get_auth_error_string(self): diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index cae2e75c..09242736 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -45,7 +45,7 @@ from leap.soledad.client import _crypto from leap.soledad.client import Soledad from leap.soledad.server.config import load_configuration from leap.soledad.server.config import CONFIG_DEFAULTS -from leap.soledad.server.auth import URLToAuthorization +from leap.soledad.server.auth import URLMapper from leap.soledad.server.auth import SoledadTokenAuthMiddleware @@ -116,175 +116,152 @@ class ServerAuthorizationTestCase(BaseSoledadTest): /user-db/sync-from/{source} | GET, PUT, POST """ uuid = uuid4().hex - authmap = URLToAuthorization(uuid,) - dbname = authmap._user_db_name + urlmap = URLMapper() + dbname = 'user-%s' % uuid + # test global auth - self.assertTrue( - authmap.is_authorized(self._make_environ('/', 'GET'))) + match = urlmap.match(self._make_environ('/', 'GET')) + # test shared-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared', 'GET'))) - self.assertFalse( - authmap.is_authorized( + match = urlmap.match( + self._make_environ('/shared', 'GET')) + self.assertIsNotNone(match) + + self.assertIsNone( + urlmap.match( self._make_environ('/shared', 'PUT'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared', 'POST'))) + # test shared-db docs resource auth - self.assertFalse( - authmap.is_authorized( + self.assertIsNone( + urlmap.match( self._make_environ('/shared/docs', 'GET'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/docs', 'PUT'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/docs', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/docs', 'POST'))) + # test shared-db doc resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + match = urlmap.match( + self._make_environ('/shared/doc/x', 'GET')) + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = urlmap.match( + self._make_environ('/shared/doc/x', 'PUT')) + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = urlmap.match( + self._make_environ('/shared/doc/x', 'DELETE')) + self.assertEqual('x', match.get('id')) + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/doc/x', 'POST'))) + # test shared-db sync resource auth - self.assertFalse( - authmap.is_authorized( + self.assertIsNone( + urlmap.match( self._make_environ('/shared/sync-from/x', 'GET'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/sync-from/x', 'PUT'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/sync-from/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/shared/sync-from/x', 'POST'))) + # test user-db database resource auth - self.assertFalse( - authmap.is_authorized( + self.assertIsNone( + urlmap.match( self._make_environ('/%s' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s' % dbname, 'POST'))) + # test user-db docs resource auth - self.assertFalse( - authmap.is_authorized( + self.assertIsNone( + urlmap.match( self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s/docs' % dbname, 'POST'))) + # test user-db doc resource auth - self.assertFalse( - authmap.is_authorized( + self.assertIsNone( + urlmap.match( self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( + + self.assertIsNone( + urlmap.match( self._make_environ('/%s/doc/x' % dbname, 'POST'))) + # test user-db sync resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) - - def test_verify_action_with_wrong_dbnames(self): - """ - Test if authorization fails for a wrong dbname. - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid) - dbname = 'somedb' - # test wrong-db database resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test wrong-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test wrong-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test wrong-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) + match = urlmap.match( + self._make_environ('/%s/sync-from/x' % dbname, 'GET')) + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = urlmap.match( + self._make_environ('/%s/sync-from/x' % dbname, 'PUT')) + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = urlmap.match( + self._make_environ('/%s/sync-from/x' % dbname, 'DELETE')) + self.assertIsNone(match) + + match = urlmap.match( + self._make_environ('/%s/sync-from/x' % dbname, 'POST')) + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) @pytest.mark.usefixtures("method_tmpdir") -- cgit v1.2.3 From 260805b9967184841c4499f94713a9a48c49a813 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 18 Dec 2016 16:36:39 -0200 Subject: [feat] use twisted web http auth and creds --- common/src/leap/soledad/common/couch/state.py | 46 --- server/pkg/soledad-server | 2 +- server/src/leap/soledad/server/application.py | 16 +- server/src/leap/soledad/server/auth.py | 384 ++++++++------------------ server/src/leap/soledad/server/resource.py | 4 +- server/src/leap/soledad/server/session.py | 78 ++++++ testing/test_soledad/util.py | 4 +- testing/tests/client/test_http_client.py | 3 +- testing/tests/server/test_server.py | 164 +++-------- 9 files changed, 239 insertions(+), 462 deletions(-) create mode 100644 server/src/leap/soledad/server/session.py diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py index a7f5b7b6..a4841d0d 100644 --- a/common/src/leap/soledad/common/couch/state.py +++ b/common/src/leap/soledad/common/couch/state.py @@ -19,13 +19,10 @@ Server state using CouchDatabase as backend. """ import couchdb import re -import time from urlparse import urljoin -from hashlib import sha512 from leap.soledad.common.log import getLogger from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.couch import couch_server from leap.soledad.common.couch import CONFIG_DOC_ID from leap.soledad.common.couch import SCHEMA_VERSION from leap.soledad.common.couch import SCHEMA_VERSION_KEY @@ -59,12 +56,6 @@ class CouchServerState(ServerState): Inteface of the WSGI server with the CouchDB backend. """ - TOKENS_DB_PREFIX = "tokens_" - TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds - TOKENS_TYPE_KEY = "type" - TOKENS_TYPE_DEF = "Token" - TOKENS_USER_ID_KEY = "user_id" - def __init__(self, couch_url, create_cmd=None, check_schema_versions=False): """ @@ -164,40 +155,3 @@ class CouchServerState(ServerState): delete databases. """ raise Unauthorized() - - def verify_token(self, uuid, token): - """ - Query couchdb to decide if C{token} is valid for C{uuid}. - - @param uuid: The user uuid. - @type uuid: str - @param token: The token. - @type token: str - """ - with couch_server(self.couch_url) as server: - # the tokens db rotates every 30 days, and the current db name is - # "tokens_NNN", where NNN is the number of seconds since epoch - # divide dby the rotate period in seconds. When rotating, old and - # new tokens db coexist during a certain window of time and valid - # tokens are replicated from the old db to the new one. See: - # https://leap.se/code/issues/6785 - dbname = self._tokens_dbname() - db = server[dbname] - # lookup key is a hash of the token to prevent timing attacks. - token = db.get(sha512(token).hexdigest()) - if token is None: - return False - # we compare uuid hashes to avoid possible timing attacks that - # might exploit python's builtin comparison operator behaviour, - # which fails immediatelly when non-matching bytes are found. - couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest() - req_uuid_hash = sha512(uuid).digest() - if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ - or couch_uuid_hash != req_uuid_hash: - return False - return True - - def _tokens_dbname(self): - dbname = self.TOKENS_DB_PREFIX + \ - str(int(time.time() / self.TOKENS_DB_EXPIRE)) - return dbname diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server index 753a260b..92560fa8 100644 --- a/server/pkg/soledad-server +++ b/server/pkg/soledad-server @@ -11,7 +11,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/soledad.pid -RESOURCE_CLASS=leap.soledad.server.resource.SoledadResource +RESOURCE_CLASS=leap.soledad.server.session.SoledadSession HTTPS_PORT=2424 CONFDIR=/etc/soledad CERT_PATH="${CONFDIR}/soledad-server.pem" diff --git a/server/src/leap/soledad/server/application.py b/server/src/leap/soledad/server/application.py index 17296425..8fd343b3 100644 --- a/server/src/leap/soledad/server/application.py +++ b/server/src/leap/soledad/server/application.py @@ -24,7 +24,6 @@ Use it like this: from twisted.internet import reactor from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.server.config import load_configuration from leap.soledad.common.backend import SoledadBackend @@ -35,20 +34,25 @@ from leap.soledad.common.log import getLogger __all__ = ['wsgi_application'] -def _load_config(): - conf = load_configuration('/etc/soledad/soledad-server.conf') - return conf['soledad-server'] +_config = None + + +def get_config(): + global _config + if not _config: + _config = load_configuration('/etc/soledad/soledad-server.conf') + return _config['soledad-server'] def _get_couch_state(): - conf = _load_config() + conf = get_config() state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], check_schema_versions=True) SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) return state -_app = SoledadTokenAuthMiddleware(SoledadApp(None)) # delay state init +_app = SoledadApp(None) # delay state init wsgi_application = GzipMiddleware(_app) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index c026a282..f55b710e 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -15,20 +15,110 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Authentication facilities for Soledad Server. +Twisted http token auth. """ -import httplib -import json +import binascii +import time -from abc import ABCMeta, abstractmethod +from hashlib import sha512 from routes.mapper import Mapper +from zope.interface import implementer + +from twisted.cred import error +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.credentials import UsernamePassword +from twisted.cred.portal import IRealm +from twisted.cred.portal import Portal +from twisted.web.iweb import ICredentialFactory +from twisted.web.resource import IResource -from leap.soledad.common.log import getLogger -from leap.soledad.common.l2db import DBNAME_CONSTRAINTS, errors as u1db_errors from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.couch import couch_server +from leap.soledad.common.l2db import DBNAME_CONSTRAINTS +from leap.soledad.server.resource import SoledadResource +from leap.soledad.server.application import get_config + + +@implementer(IRealm) +class SoledadRealm(object): + + def requestAvatar(self, avatarId, mind, *interfaces): + if IResource in interfaces: + return (IResource, SoledadResource(avatarId), lambda: None) + raise NotImplementedError() + + +@implementer(ICredentialsChecker) +class TokenChecker(object): + + credentialInterfaces = [IUsernamePassword] + + TOKENS_DB_PREFIX = "tokens_" + TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds + TOKENS_TYPE_KEY = "type" + TOKENS_TYPE_DEF = "Token" + TOKENS_USER_ID_KEY = "user_id" + + def __init__(self): + config = get_config() + self.couch_url = config['couch_url'] + + def _tokens_dbname(self): + dbname = self.TOKENS_DB_PREFIX + \ + str(int(time.time() / self.TOKENS_DB_EXPIRE)) + return dbname + + def requestAvatarId(self, credentials): + uuid = credentials.username + token = credentials.password + with couch_server(self.couch_url) as server: + # the tokens db rotates every 30 days, and the current db name is + # "tokens_NNN", where NNN is the number of seconds since epoch + # divide dby the rotate period in seconds. When rotating, old and + # new tokens db coexist during a certain window of time and valid + # tokens are replicated from the old db to the new one. See: + # https://leap.se/code/issues/6785 + dbname = self._tokens_dbname() + db = server[dbname] + # lookup key is a hash of the token to prevent timing attacks. + token = db.get(sha512(token).hexdigest()) + if token is None: + return False + # we compare uuid hashes to avoid possible timing attacks that + # might exploit python's builtin comparison operator behaviour, + # which fails immediatelly when non-matching bytes are found. + couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest() + req_uuid_hash = sha512(uuid).digest() + if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ + or couch_uuid_hash != req_uuid_hash: + return False + return True -logger = getLogger(__name__) +@implementer(ICredentialFactory) +class TokenCredentialFactory(object): + + scheme = 'token' + + def getChallenge(self, request): + return {} + + def decode(self, response, request): + try: + creds = response.decode('base64') + except binascii.Error: + raise error.LoginFailed('Invalid credentials') + + creds = creds.split(b':', 1) + if len(creds) == 2: + return UsernamePassword(*creds) + else: + raise error.LoginFailed('Invalid credentials') + + +portal = Portal(SoledadRealm(), [TokenChecker()]) +credentialFactory = TokenCredentialFactory() class URLMapper(object): @@ -41,7 +131,8 @@ class URLMapper(object): self._connect_urls() self._map.create_regs() - def match(self, environ): + def match(self, path, method): + environ = {'PATH_INFO': path, 'REQUEST_METHOD': method} return self._map.match(environ=environ) def _connect(self, pattern, http_methods): @@ -81,272 +172,15 @@ class URLMapper(object): ['GET', 'PUT', 'POST']) -class SoledadAuthMiddleware(object): - """ - Soledad Authentication WSGI middleware. - - This class must be extended to implement specific authentication methods - (see SoledadTokenAuthMiddleware below). - - It expects an HTTP_AUTHORIZATION header containing the concatenation of - the following strings: - - 1. The authentication scheme. It will be verified by the - _verify_authentication_scheme() method. - - 2. A space character. - - 3. The base64 encoded string of the concatenation of the user uuid with - the authentication data, separated by a collon, like this: - - base64(":") - - After authentication check, the class performs an authorization check to - verify whether the user is authorized to perform the requested action. - - On client-side, 2 methods must be implemented so the soledad client knows - how to send authentication headers to server: - - * set__credentials: store authentication credentials in the - class. - - * _sign_request: format and include custom authentication data in - the HTTP_AUTHORIZATION header. - - See leap.soledad.auth and u1db.remote.http_client.HTTPClient to understand - how to do it. - """ - - __metaclass__ = ABCMeta - - HTTP_AUTH_KEY = "HTTP_AUTHORIZATION" - PATH_INFO_KEY = "PATH_INFO" - - CONTENT_TYPE_JSON = ('content-type', 'application/json') - - def __init__(self, app): - """ - Initialize the Soledad Authentication Middleware. - - @param app: The application to run on successfull authentication. - @type app: u1db.remote.http_app.HTTPApp - @param prefix: Auth app path prefix. - @type prefix: str - """ - self._app = app - self._mapper = URLMapper() - - def _error(self, start_response, status, description, message=None): - """ - Send a JSON serialized error to WSGI client. - - @param start_response: Callable of the form start_response(status, - response_headers, exc_info=None). - @type start_response: callable - @param status: Status string of the form "999 Message here" - @type status: str - @param response_headers: A list of (header_name, header_value) tuples - describing the HTTP response header. - @type response_headers: list - @param description: The error description. - @type description: str - @param message: The error message. - @type message: str - - @return: List with JSON serialized error message. - @rtype list - """ - start_response("%d %s" % (status, httplib.responses[status]), - [self.CONTENT_TYPE_JSON]) - err = {"error": description} - if message: - err['message'] = message - return [json.dumps(err)] - - def _unauthorized_error(self, start_response, message): - """ - Send a unauth error. - - @param message: The error message. - @type message: str - @param start_response: Callable of the form start_response(status, - response_headers, exc_info=None). - @type start_response: callable - - @return: List with JSON serialized error message. - @rtype list - """ - return self._error( - start_response, - 401, - "unauthorized", - message) - - def __call__(self, environ, start_response): - """ - Handle a WSGI call to the authentication application. - - @param environ: Dictionary containing CGI variables. - @type environ: dict - @param start_response: Callable of the form start_response(status, - response_headers, exc_info=None). - @type start_response: callable +@implementer(IResource) +class UnauthorizedResource(object): + isLeaf = True - @return: Target application results if authentication succeeds, an - error message otherwise. - @rtype: list - """ - # check for authentication header - auth = environ.get(self.HTTP_AUTH_KEY) - if not auth: - return self._unauthorized_error( - start_response, "Missing authentication header.") - - # get authentication data - scheme, encoded = auth.split(None, 1) - uuid, auth_data = encoded.decode('base64').split(':', 1) - if not self._verify_authentication_scheme(scheme): - return self._unauthorized_error( - start_response, "Wrong authentication scheme") - - # verify if user is athenticated - try: - if not self._verify_authentication_data(uuid, auth_data): - return self._unauthorized_error( - start_response, - self._get_auth_error_string()) - except u1db_errors.Unauthorized as e: - return self._error( - start_response, - 401, - e.wire_description) - - # verify if user is authorized to perform action - if not self._verify_authorization(environ, uuid): - return self._unauthorized_error( - start_response, - "Unauthorized action.") - - # move on to the real Soledad app - del environ[self.HTTP_AUTH_KEY] - return self._app(environ, start_response) - - @abstractmethod - def _verify_authentication_scheme(self, scheme): - """ - Verify if authentication scheme is valid. - - @param scheme: Auth scheme extracted from the HTTP_AUTHORIZATION - header. - @type scheme: str - - @return: Whether the authentitcation scheme is valid. - """ - return None - - @abstractmethod - def _verify_authentication_data(self, uuid, auth_data): - """ - Verify valid authenticatiion for this request. - - @param uuid: The user's uuid. - @type uuid: str - @param auth_data: Authentication data. - @type auth_data: str - - @return: Whether the token is valid for authenticating the request. - @rtype: bool - - @raise Unauthorized: Raised when C{auth_data} is not enough to - authenticate C{uuid}. - """ - return None + def render(self, request): + request.setResponseCode(401) + if request.method == b'HEAD': + return b'' + return b'Unauthorized' - def _verify_authorization(self, environ, uuid): - """ - Verify if the user is authorized to perform the requested action over - the requested database. - - @param environ: Dictionary containing CGI variables. - @type environ: dict - @param uuid: The user's uuid from the Authorization header. - @type uuid: str - - @return: Whether the user is authorized to perform the requested action - over the requested db. - @rtype: bool - """ - match = self._mapper.match(environ) - if not match: - return False - return uuid == match.get('uuid') - - @abstractmethod - def _get_auth_error_string(self): - """ - Return an error string specific for each kind of authentication method. - - @return: The error string. - """ - return None - - -class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): - """ - Token based authentication. - """ - - TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." - - def _get_state(self): - return self._app.state - - def _set_state(self, state): - self._app.state = state - - state = property(_get_state, _set_state) - - def _verify_authentication_scheme(self, scheme): - """ - Verify if authentication scheme is valid. - - @param scheme: Auth scheme extracted from the HTTP_AUTHORIZATION - header. - @type scheme: str - - @return: Whether the authentitcation scheme is valid. - """ - if scheme.lower() != 'token': - return False - return True - - def _verify_authentication_data(self, uuid, auth_data): - """ - Extract token from C{auth_data} and proceed with verification of - C{uuid} authentication. - - @param uuid: The user UID. - @type uuid: str - @param auth_data: Authentication data (i.e. the token). - @type auth_data: str - - @return: Whether the token is valid for authenticating the request. - @rtype: bool - - @raise Unauthorized: Raised when C{auth_data} is not enough to - authenticate C{uuid}. - """ - token = auth_data # we expect a cleartext token at this point - try: - return self.state.verify_token(uuid, token) - except Exception as e: - logger.error(e) - return False - - def _get_auth_error_string(self): - """ - Get the error string for token auth. - - @return: The error string. - """ - return self.TOKEN_AUTH_ERROR_STRING + def getChildWithDefault(self, path, request): + return self diff --git a/server/src/leap/soledad/server/resource.py b/server/src/leap/soledad/server/resource.py index dbb91b0a..9922c997 100644 --- a/server/src/leap/soledad/server/resource.py +++ b/server/src/leap/soledad/server/resource.py @@ -17,7 +17,6 @@ """ A twisted resource that serves the Soledad Server. """ - from twisted.web.resource import Resource from twisted.web.wsgi import WSGIResource from twisted.internet import reactor @@ -42,7 +41,8 @@ class SoledadResource(Resource): for the Soledad Server. """ - def __init__(self): + def __init__(self, uuid): + self._uuid = uuid self.children = {'': wsgi_resource} def getChild(self, path, request): diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py new file mode 100644 index 00000000..22e1d1fb --- /dev/null +++ b/server/src/leap/soledad/server/session.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# session.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Twisted resource containing an authenticated Soledad session. +""" +from zope.interface import implementer + +from twisted.cred import error +from twisted.python import log +from twisted.web import util +from twisted.web.guard import HTTPAuthSessionWrapper +from twisted.web.resource import ErrorPage +from twisted.web.resource import IResource + +from leap.soledad.server.auth import URLMapper +from leap.soledad.server.auth import portal +from leap.soledad.server.auth import credentialFactory +from leap.soledad.server.auth import UnauthorizedResource + + +@implementer(IResource) +class SoledadSession(HTTPAuthSessionWrapper): + + def __init__(self): + self._mapper = URLMapper() + self._portal = portal + self._credentialFactory = credentialFactory + + def _matchPath(self, request): + match = self._mapper.match(request.path, request.method) + return match + + def _parseHeader(self, header): + elements = header.split(b' ') + scheme = elements[0].lower() + if scheme == self._credentialFactory.scheme: + return (b' '.join(elements[1:])) + return None + + def _authorizedResource(self, request): + match = self._matchPath(request) + if not match: + return UnauthorizedResource() + + header = request.getHeader(b'authorization') + if not header: + return UnauthorizedResource() + + auth_data = self._parseHeader(header) + if not auth_data: + return UnauthorizedResource() + + try: + credentials = self._credentialFactory.decode(auth_data, request) + except error.LoginFailed: + return UnauthorizedResource() + except: + log.err(None, "Unexpected failure from credentials factory") + return ErrorPage(500, None, None) + else: + request_uuid = match.get('uuid') + if request_uuid and request_uuid != credentials.username: + return ErrorPage(500, None, None) + return util.DeferredResource(self._login(credentials)) diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py index 57f8199b..e44a165d 100644 --- a/testing/test_soledad/util.py +++ b/testing/test_soledad/util.py @@ -52,7 +52,6 @@ from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client._crypto import is_symmetrically_encrypted from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware PASSWORD = '123456' @@ -108,7 +107,7 @@ def make_soledad_app(state): def make_token_soledad_app(state): - app = SoledadApp(state) + application = SoledadApp(state) def _verify_authentication_data(uuid, auth_data): if uuid.startswith('user-') and auth_data == 'auth-token': @@ -119,7 +118,6 @@ def make_token_soledad_app(state): def _verify_authorization(uuid, environ): return True - application = SoledadTokenAuthMiddleware(app) application._verify_authentication_data = _verify_authentication_data application._verify_authorization = _verify_authorization return application diff --git a/testing/tests/client/test_http_client.py b/testing/tests/client/test_http_client.py index a107930a..691c7576 100644 --- a/testing/tests/client/test_http_client.py +++ b/testing/tests/client/test_http_client.py @@ -24,7 +24,6 @@ from testscenarios import TestWithScenarios from leap.soledad.client import auth from leap.soledad.common.l2db.remote import http_client from test_soledad.u1db_tests import test_http_client -from leap.soledad.server.auth import SoledadTokenAuthMiddleware # ----------------------------------------------------------------------------- @@ -67,7 +66,7 @@ class TestSoledadClientBase( return res # mime solead application here. if '/token' in environ['PATH_INFO']: - auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) + auth = environ.get('HTTP_AUTHORIZATION') if not auth: start_response("401 Unauthorized", [('Content-Type', 'application/json')]) diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 09242736..12f6fb20 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -18,11 +18,9 @@ Tests for server-related functionality. """ import binascii -import mock import os import pytest -from hashlib import sha512 from pkg_resources import resource_filename from urlparse import urljoin from uuid import uuid4 @@ -46,36 +44,6 @@ from leap.soledad.client import Soledad from leap.soledad.server.config import load_configuration from leap.soledad.server.config import CONFIG_DEFAULTS from leap.soledad.server.auth import URLMapper -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -class ServerAuthenticationMiddlewareTestCase(CouchDBTestCase): - - def setUp(self): - super(ServerAuthenticationMiddlewareTestCase, self).setUp() - app = mock.Mock() - self._state = CouchServerState(self.couch_url) - app.state = self._state - self.auth_middleware = SoledadTokenAuthMiddleware(app) - self._authorize('valid-uuid', 'valid-token') - - def _authorize(self, uuid, token): - token_doc = {} - token_doc['_id'] = sha512(token).hexdigest() - token_doc[self._state.TOKENS_USER_ID_KEY] = uuid - token_doc[self._state.TOKENS_TYPE_KEY] = \ - self._state.TOKENS_TYPE_DEF - dbname = self._state._tokens_dbname() - db = self.couch_server.create(dbname) - db.save(token_doc) - self.addCleanup(self.delete_db, db.name) - - def test_authorized_user(self): - is_authorized = self.auth_middleware._verify_authentication_data - self.assertTrue(is_authorized('valid-uuid', 'valid-token')) - self.assertFalse(is_authorized('valid-uuid', 'invalid-token')) - self.assertFalse(is_authorized('invalid-uuid', 'valid-token')) - self.assertFalse(is_authorized('eve', 'invalid-token')) class ServerAuthorizationTestCase(BaseSoledadTest): @@ -90,12 +58,6 @@ class ServerAuthorizationTestCase(BaseSoledadTest): def tearDown(self): pass - def _make_environ(self, path_info, request_method): - return { - 'PATH_INFO': path_info, - 'REQUEST_METHOD': request_method, - } - def test_verify_action_with_correct_dbnames(self): """ Test encrypting and decrypting documents. @@ -120,146 +82,94 @@ class ServerAuthorizationTestCase(BaseSoledadTest): dbname = 'user-%s' % uuid # test global auth - match = urlmap.match(self._make_environ('/', 'GET')) + match = urlmap.match('/', 'GET') + self.assertIsNotNone(match) # test shared-db database resource auth - match = urlmap.match( - self._make_environ('/shared', 'GET')) + match = urlmap.match('/shared', 'GET') self.assertIsNotNone(match) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared', 'PUT'))) + match = urlmap.match('/shared', 'PUT') + self.assertIsNone(match) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared', 'DELETE'))) + match = urlmap.match('/shared', 'DELETE') + self.assertIsNone(match) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared', 'POST'))) + match = urlmap.match('/shared', 'POST') + self.assertIsNone(match) # test shared-db docs resource auth - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/docs', 'GET'))) + self.assertIsNone(urlmap.match('/shared/docs', 'GET')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/docs', 'PUT'))) + self.assertIsNone(urlmap.match('/shared/docs', 'PUT')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/docs', 'DELETE'))) + self.assertIsNone(urlmap.match('/shared/docs', 'DELETE')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/docs', 'POST'))) + self.assertIsNone(urlmap.match('/shared/docs', 'POST')) # test shared-db doc resource auth - match = urlmap.match( - self._make_environ('/shared/doc/x', 'GET')) + match = urlmap.match('/shared/doc/x', 'GET') self.assertIsNotNone(match) self.assertEqual('x', match.get('id')) - match = urlmap.match( - self._make_environ('/shared/doc/x', 'PUT')) + match = urlmap.match('/shared/doc/x', 'PUT') self.assertIsNotNone(match) self.assertEqual('x', match.get('id')) - match = urlmap.match( - self._make_environ('/shared/doc/x', 'DELETE')) + match = urlmap.match('/shared/doc/x', 'DELETE') self.assertEqual('x', match.get('id')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/doc/x', 'POST'))) + self.assertIsNone(urlmap.match('/shared/doc/x', 'POST')) # test shared-db sync resource auth - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/sync-from/x', 'GET'))) + self.assertIsNone(urlmap.match('/shared/sync-from/x', 'GET')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/sync-from/x', 'PUT'))) + self.assertIsNone(urlmap.match('/shared/sync-from/x', 'PUT')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/sync-from/x', 'DELETE'))) + self.assertIsNone(urlmap.match('/shared/sync-from/x', 'DELETE')) - self.assertIsNone( - urlmap.match( - self._make_environ('/shared/sync-from/x', 'POST'))) + self.assertIsNone(urlmap.match('/shared/sync-from/x', 'POST')) # test user-db database resource auth - self.assertIsNone( - urlmap.match( - self._make_environ('/%s' % dbname, 'GET'))) + self.assertIsNone(urlmap.match('/%s' % dbname, 'GET')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s' % dbname, 'PUT'))) + self.assertIsNone(urlmap.match('/%s' % dbname, 'PUT')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s' % dbname, 'DELETE'))) + self.assertIsNone(urlmap.match('/%s' % dbname, 'DELETE')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s' % dbname, 'POST'))) + self.assertIsNone(urlmap.match('/%s' % dbname, 'POST')) # test user-db docs resource auth - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/docs' % dbname, 'GET'))) + self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'GET')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/docs' % dbname, 'PUT'))) + self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'PUT')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) + self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'DELETE')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/docs' % dbname, 'POST'))) + self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'POST')) # test user-db doc resource auth - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) + self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'GET')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) + self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'PUT')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) + self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'DELETE')) - self.assertIsNone( - urlmap.match( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) + self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'POST')) # test user-db sync resource auth - match = urlmap.match( - self._make_environ('/%s/sync-from/x' % dbname, 'GET')) + match = urlmap.match('/%s/sync-from/x' % dbname, 'GET') self.assertEqual(uuid, match.get('uuid')) self.assertEqual('x', match.get('source_replica_uid')) - match = urlmap.match( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT')) + match = urlmap.match('/%s/sync-from/x' % dbname, 'PUT') self.assertEqual(uuid, match.get('uuid')) self.assertEqual('x', match.get('source_replica_uid')) - match = urlmap.match( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE')) + match = urlmap.match('/%s/sync-from/x' % dbname, 'DELETE') self.assertIsNone(match) - match = urlmap.match( - self._make_environ('/%s/sync-from/x' % dbname, 'POST')) + match = urlmap.match('/%s/sync-from/x' % dbname, 'POST') self.assertEqual(uuid, match.get('uuid')) self.assertEqual('x', match.get('source_replica_uid')) -- cgit v1.2.3 From c39bde684da223c46368605368f63ac1beb8b6d4 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 18 Dec 2016 21:06:26 -0200 Subject: [feat] cache session data in server --- server/src/leap/soledad/server/auth.py | 2 +- server/src/leap/soledad/server/resource.py | 3 +- server/src/leap/soledad/server/session.py | 55 +++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index f55b710e..c5b90359 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -45,7 +45,7 @@ class SoledadRealm(object): def requestAvatar(self, avatarId, mind, *interfaces): if IResource in interfaces: - return (IResource, SoledadResource(avatarId), lambda: None) + return (IResource, SoledadResource(), lambda: None) raise NotImplementedError() diff --git a/server/src/leap/soledad/server/resource.py b/server/src/leap/soledad/server/resource.py index 9922c997..67e9ae32 100644 --- a/server/src/leap/soledad/server/resource.py +++ b/server/src/leap/soledad/server/resource.py @@ -41,8 +41,7 @@ class SoledadResource(Resource): for the Soledad Server. """ - def __init__(self, uuid): - self._uuid = uuid + def __init__(self): self.children = {'': wsgi_resource} def getChild(self, path, request): diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 22e1d1fb..75440089 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -21,15 +21,41 @@ from zope.interface import implementer from twisted.cred import error from twisted.python import log +from twisted.python.components import registerAdapter from twisted.web import util from twisted.web.guard import HTTPAuthSessionWrapper from twisted.web.resource import ErrorPage from twisted.web.resource import IResource +from twisted.web.server import Session +from zope.interface import Interface +from zope.interface import Attribute from leap.soledad.server.auth import URLMapper from leap.soledad.server.auth import portal from leap.soledad.server.auth import credentialFactory from leap.soledad.server.auth import UnauthorizedResource +from leap.soledad.server.resource import SoledadResource + + +class ISessionData(Interface): + username = Attribute('An uuid.') + password = Attribute('A token.') + + +@implementer(ISessionData) +class SessionData(object): + def __init__(self, session): + self.username = None + self.password = None + + +registerAdapter(SessionData, Session, ISessionData) + + +def _sessionData(request): + session = request.getSession() + data = ISessionData(session) + return data @implementer(IResource) @@ -71,8 +97,27 @@ class SoledadSession(HTTPAuthSessionWrapper): except: log.err(None, "Unexpected failure from credentials factory") return ErrorPage(500, None, None) - else: - request_uuid = match.get('uuid') - if request_uuid and request_uuid != credentials.username: - return ErrorPage(500, None, None) - return util.DeferredResource(self._login(credentials)) + + request_uuid = match.get('uuid') + if request_uuid and request_uuid != credentials.username: + return ErrorPage(500, None, None) + + # eventually return a cached resouce + sessionData = _sessionData(request) + if sessionData.username == credentials.username \ + and sessionData.password == credentials.password: + return SoledadResource() + + return util.DeferredResource(self._login(credentials, sessionData)) + + def _login(self, credentials, sessionData): + + def _cacheSessionData(res): + sessionData.username = credentials.username + sessionData.password = credentials.password + return res + + d = self._portal.login(credentials, None, IResource) + d.addCallback(_cacheSessionData) + d.addCallbacks(self._loginSucceeded, self._loginFailed) + return d -- cgit v1.2.3 From 994eaa79b274c3c37af42cb343c41b5dec6e8d19 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 18 Dec 2016 21:27:02 -0200 Subject: [feat] use cookies in the client syncer --- .../leap/soledad/client/http_target/__init__.py | 26 +++++++++++++++++----- client/src/leap/soledad/client/sqlcipher.py | 9 ++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 0e250bf1..590ae8f6 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -24,10 +24,14 @@ after receiving. import os -from leap.soledad.common.log import getLogger -from leap.common.certs import get_compatible_ssl_context_factory +from cookielib import CookieJar + from twisted.web.client import Agent +from twisted.web.client import CookieAgent from twisted.internet import reactor + +from leap.common.certs import get_compatible_ssl_context_factory +from leap.soledad.common.log import getLogger from leap.soledad.client.http_target.send import HTTPDocSender from leap.soledad.client.http_target.api import SyncTargetAPI from leap.soledad.client.http_target.fetch import HTTPDocFetcher @@ -43,6 +47,14 @@ if os.environ.get('SOLEDAD_STATS'): DO_STATS = True +def newCookieAgent(cert_file): + _factory = get_compatible_ssl_context_factory(cert_file) + _agent = Agent(reactor, _factory) + _cookieJar = CookieJar() + agent = CookieAgent(_agent, _cookieJar) + return agent + + class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): """ @@ -54,7 +66,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): the parsed documents that the remote send us, before being decrypted and written to the main database. """ - def __init__(self, url, source_replica_uid, creds, crypto, cert_file): + def __init__(self, url, source_replica_uid, creds, crypto, cert_file, + agent=None): """ Initialize the sync target. @@ -72,6 +85,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): the SSL certificate used by the remote soledad server. :type cert_file: str + :param agent: an http agent + :type agent: twisted.web.client.Agent """ if url.endswith("/"): url = url[:-1] @@ -86,8 +101,9 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): self._insert_doc_cb = None # Twisted default Agent with our own ssl context factory - self._http = Agent(reactor, - get_compatible_ssl_context_factory(cert_file)) + if not agent: + agent = newCookieAgent(cert_file) + self._http = agent if DO_STATS: self.sync_exchange_phase = [0] diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index c9a9444e..9b352bbf 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -59,6 +59,7 @@ from leap.soledad.common.l2db.backends import sqlite_backend from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client.http_target import SoledadHTTPSyncTarget +from leap.soledad.client.http_target import newCookieAgent from leap.soledad.client.sync import SoledadSynchronizer from leap.soledad.client import pragmas @@ -397,7 +398,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): ENCRYPT_LOOP_PERIOD = 1 def __init__(self, opts, soledad_crypto, replica_uid, cert_file): - self._opts = opts self._path = opts.path self._crypto = soledad_crypto @@ -407,6 +407,10 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # storage for the documents received during a sync self.received_docs = [] + # setup an http agent capable of storing cookies, so we can use + # server's session persistence feature + self._agent = newCookieAgent(cert_file) + self.running = False self._db_handle = None @@ -491,7 +495,8 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._replica_uid, creds=creds, crypto=self._crypto, - cert_file=self._cert_file)) + cert_file=self._cert_file, + agent=self._agent)) # # Symmetric encryption of syncing docs -- cgit v1.2.3 From 6043f7966b64d6922987bca9137a524fb06a3379 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 19 Dec 2016 09:43:03 -0200 Subject: [refactor] separate url mapper, avoid hanging tests Because the wsgi resource has its own threadpool, tests might get confused when shutting down and the reactor may get clogged waiting for the threadpool to be stopped. By refactoring the URLMapper to its own module, server tests can avoid loading the resource module, where the wsgi threadpool resides, so the threapool will not be started. --- server/src/leap/soledad/server/auth.py | 68 ------------------------- server/src/leap/soledad/server/session.py | 17 ++++++- server/src/leap/soledad/server/url_mapper.py | 74 ++++++++++++++++++++++++++++ testing/tests/server/test_server.py | 2 +- 4 files changed, 90 insertions(+), 71 deletions(-) create mode 100644 server/src/leap/soledad/server/url_mapper.py diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index c5b90359..13245cfe 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -21,7 +21,6 @@ import binascii import time from hashlib import sha512 -from routes.mapper import Mapper from zope.interface import implementer from twisted.cred import error @@ -33,9 +32,7 @@ from twisted.cred.portal import Portal from twisted.web.iweb import ICredentialFactory from twisted.web.resource import IResource -from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import couch_server -from leap.soledad.common.l2db import DBNAME_CONSTRAINTS from leap.soledad.server.resource import SoledadResource from leap.soledad.server.application import get_config @@ -119,68 +116,3 @@ class TokenCredentialFactory(object): portal = Portal(SoledadRealm(), [TokenChecker()]) credentialFactory = TokenCredentialFactory() - - -class URLMapper(object): - """ - Maps the URLs users can access. - """ - - def __init__(self): - self._map = Mapper(controller_scan=None) - self._connect_urls() - self._map.create_regs() - - def match(self, path, method): - environ = {'PATH_INFO': path, 'REQUEST_METHOD': method} - return self._map.match(environ=environ) - - def _connect(self, pattern, http_methods): - self._map.connect( - None, pattern, http_methods=http_methods, - conditions=dict(method=http_methods), - requirements={'dbname': DBNAME_CONSTRAINTS}) - - def _connect_urls(self): - """ - Register the authorization info in the mapper using C{SHARED_DB_NAME} - as the user's database name. - - This method sets up the following authorization rules: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{any_id} | GET, PUT, DELETE - /shared-db/sync-from/{source} | - - /user-db | - - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - # auth info for global resource - self._connect('/', ['GET']) - # auth info for shared-db database resource - self._connect('/%s' % SHARED_DB_NAME, ['GET']) - # auth info for shared-db doc resource - self._connect('/%s/doc/{id:.*}' % SHARED_DB_NAME, - ['GET', 'PUT', 'DELETE']) - # auth info for user-db sync resource - self._connect('/user-{uuid}/sync-from/{source_replica_uid}', - ['GET', 'PUT', 'POST']) - - -@implementer(IResource) -class UnauthorizedResource(object): - isLeaf = True - - def render(self, request): - request.setResponseCode(401) - if request.method == b'HEAD': - return b'' - return b'Unauthorized' - - def getChildWithDefault(self, path, request): - return self diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 75440089..1ef5b6a6 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -30,10 +30,9 @@ from twisted.web.server import Session from zope.interface import Interface from zope.interface import Attribute -from leap.soledad.server.auth import URLMapper from leap.soledad.server.auth import portal from leap.soledad.server.auth import credentialFactory -from leap.soledad.server.auth import UnauthorizedResource +from leap.soledad.server.urlmapper import URLMapper from leap.soledad.server.resource import SoledadResource @@ -58,6 +57,20 @@ def _sessionData(request): return data +@implementer(IResource) +class UnauthorizedResource(object): + isLeaf = True + + def render(self, request): + request.setResponseCode(401) + if request.method == b'HEAD': + return b'' + return b'Unauthorized' + + def getChildWithDefault(self, path, request): + return self + + @implementer(IResource) class SoledadSession(HTTPAuthSessionWrapper): diff --git a/server/src/leap/soledad/server/url_mapper.py b/server/src/leap/soledad/server/url_mapper.py new file mode 100644 index 00000000..483f7e87 --- /dev/null +++ b/server/src/leap/soledad/server/url_mapper.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# url_mapper.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +An URL mapper that represents authorized paths. +""" +from routes.mapper import Mapper + +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.l2db import DBNAME_CONSTRAINTS + + +class URLMapper(object): + """ + Maps the URLs users can access. + """ + + def __init__(self): + self._map = Mapper(controller_scan=None) + self._connect_urls() + self._map.create_regs() + + def match(self, path, method): + environ = {'PATH_INFO': path, 'REQUEST_METHOD': method} + return self._map.match(environ=environ) + + def _connect(self, pattern, http_methods): + self._map.connect( + None, pattern, http_methods=http_methods, + conditions=dict(method=http_methods), + requirements={'dbname': DBNAME_CONSTRAINTS}) + + def _connect_urls(self): + """ + Register the authorization info in the mapper using C{SHARED_DB_NAME} + as the user's database name. + + This method sets up the following authorization rules: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{any_id} | GET, PUT, DELETE + /shared-db/sync-from/{source} | - + /user-db | - + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST + """ + # auth info for global resource + self._connect('/', ['GET']) + # auth info for shared-db database resource + self._connect('/%s' % SHARED_DB_NAME, ['GET']) + # auth info for shared-db doc resource + self._connect('/%s/doc/{id:.*}' % SHARED_DB_NAME, + ['GET', 'PUT', 'DELETE']) + # auth info for user-db sync resource + self._connect('/user-{uuid}/sync-from/{source_replica_uid}', + ['GET', 'PUT', 'POST']) diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 12f6fb20..39d8e8c3 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -43,7 +43,7 @@ from leap.soledad.client import _crypto from leap.soledad.client import Soledad from leap.soledad.server.config import load_configuration from leap.soledad.server.config import CONFIG_DEFAULTS -from leap.soledad.server.auth import URLMapper +from leap.soledad.server.url_mapper import URLMapper class ServerAuthorizationTestCase(BaseSoledadTest): -- cgit v1.2.3 From c3c8afb68330bdfb8fe70efc7d055b55ca9c1c3a Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 22 Dec 2016 07:32:29 -0200 Subject: [refactor] remove leftover code from previous wsgi auth --- .../common/l2db/remote/basic_auth_middleware.py | 66 ------------- testing/tests/client/test_http_client.py | 107 --------------------- 2 files changed, 173 deletions(-) delete mode 100644 common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py delete mode 100644 testing/tests/client/test_http_client.py diff --git a/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py b/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py deleted file mode 100644 index 96d0d872..00000000 --- a/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . -"""U1DB Basic Auth authorisation WSGI middleware.""" -import httplib -import json - -from wsgiref.util import shift_path_info - - -class Unauthorized(Exception): - """User authorization failed.""" - - -class BasicAuthMiddleware(object): - """U1DB Basic Auth Authorisation WSGI middleware.""" - - def __init__(self, app, prefix): - self.app = app - self.prefix = prefix - - def _error(self, start_response, status, description, message=None): - start_response("%d %s" % (status, httplib.responses[status]), - [('content-type', 'application/json')]) - err = {"error": description} - if message: - err['message'] = message - return [json.dumps(err)] - - def __call__(self, environ, start_response): - if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): - return self._error(start_response, 400, "bad request") - auth = environ.get('HTTP_AUTHORIZATION') - if not auth: - return self._error(start_response, 401, "unauthorized", - "Missing Basic Authentication.") - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'basic': - return self._error( - start_response, 401, "unauthorized", - "Missing Basic Authentication") - user, password = encoded.decode('base64').split(':', 1) - try: - self.verify_user(environ, user, password) - except Unauthorized: - return self._error( - start_response, 401, "unauthorized", - "Incorrect password or login.") - del environ['HTTP_AUTHORIZATION'] - shift_path_info(environ) - return self.app(environ, start_response) - - def verify_user(self, environ, username, password): - raise NotImplementedError(self.verify_user) diff --git a/testing/tests/client/test_http_client.py b/testing/tests/client/test_http_client.py deleted file mode 100644 index 691c7576..00000000 --- a/testing/tests/client/test_http_client.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# test_http_client.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: sync target -""" -import json - -from testscenarios import TestWithScenarios - -from leap.soledad.client import auth -from leap.soledad.common.l2db.remote import http_client -from test_soledad.u1db_tests import test_http_client - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_client`. -# ----------------------------------------------------------------------------- - -class TestSoledadClientBase( - TestWithScenarios, - test_http_client.TestHTTPClientBase): - - """ - This class should be used to test Token auth. - """ - - def getClient(self, **kwds): - cli = self.getClientWithToken(**kwds) - if 'creds' not in kwds: - cli.set_token_credentials('user-uuid', 'auth-token') - return cli - - def getClientWithToken(self, **kwds): - self.startServer() - - class _HTTPClientWithToken( - http_client.HTTPClientBase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - return _HTTPClientWithToken(self.getURL('dbase'), **kwds) - - def app(self, environ, start_response): - res = test_http_client.TestHTTPClientBase.app( - self, environ, start_response) - if res is not None: - return res - # mime solead application here. - if '/token' in environ['PATH_INFO']: - auth = environ.get('HTTP_AUTHORIZATION') - if not auth: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [ - json.dumps( - {"error": "unauthorized", - "message": "no token found in environment"}) - ] - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'token': - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": "unknown scheme: %s" % scheme})] - uuid, token = encoded.decode('base64').split(':', 1) - if uuid != 'user-uuid' and token != 'auth-token': - return Exception("Incorrect address or token.") - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], uuid, token])] - - def test_token(self): - """ - Test if token is sent correctly. - """ - cli = self.getClientWithToken() - cli.set_token_credentials('user-uuid', 'auth-token') - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - def test_token_ctr_creds(self): - cli = self.getClientWithToken(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) -- cgit v1.2.3 From 9afcf45dc0e12073eaec7688061f03cc7e1dbd40 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 22 Dec 2016 07:33:11 -0200 Subject: [bug] fix name of module on import --- server/src/leap/soledad/server/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 1ef5b6a6..59424a7b 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -32,7 +32,7 @@ from zope.interface import Attribute from leap.soledad.server.auth import portal from leap.soledad.server.auth import credentialFactory -from leap.soledad.server.urlmapper import URLMapper +from leap.soledad.server.url_mapper import URLMapper from leap.soledad.server.resource import SoledadResource -- cgit v1.2.3 From 5058cae83227d4ba1b6390aa52a63b22a1acb11d Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 22 Dec 2016 07:56:01 -0200 Subject: [test] split url mapper test in many smaller tests --- testing/tests/server/test_server.py | 180 ++++++++++++++++-------------------- 1 file changed, 79 insertions(+), 101 deletions(-) diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 39d8e8c3..da69c423 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -36,7 +36,6 @@ from test_soledad.util import ( make_token_soledad_app, make_soledad_document_for_test, soledad_sync_target, - BaseSoledadTest, ) from leap.soledad.client import _crypto @@ -46,133 +45,112 @@ from leap.soledad.server.config import CONFIG_DEFAULTS from leap.soledad.server.url_mapper import URLMapper -class ServerAuthorizationTestCase(BaseSoledadTest): - +class URLMapperTestCase(unittest.TestCase): """ - Tests related to Soledad server authorization. + Test if the URLMapper behaves as expected. + + The following table lists the authorized actions among all possible + u1db remote actions: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{id} | - + /shared-db/sync-from/{source} | - + /user-db | - + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST """ def setUp(self): - pass - - def tearDown(self): - pass - - def test_verify_action_with_correct_dbnames(self): - """ - Test encrypting and decrypting documents. - - The following table lists the authorized actions among all possible - u1db remote actions: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{id} | GET, PUT, DELETE - /shared-db/sync-from/{source} | - - /user-db | - - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - uuid = uuid4().hex - urlmap = URLMapper() - dbname = 'user-%s' % uuid - - # test global auth - match = urlmap.match('/', 'GET') - self.assertIsNotNone(match) + self._uuid = uuid4().hex + self._urlmap = URLMapper() + self._dbname = 'user-%s' % self._uuid - # test shared-db database resource auth - match = urlmap.match('/shared', 'GET') + def test_root_authorized(self): + match = self._urlmap.match('/', 'GET') self.assertIsNotNone(match) - match = urlmap.match('/shared', 'PUT') - self.assertIsNone(match) + def test_shared_authorized(self): + self.assertIsNotNone(self._urlmap.match('/shared', 'GET')) - match = urlmap.match('/shared', 'DELETE') - self.assertIsNone(match) + def test_shared_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared', 'POST')) - match = urlmap.match('/shared', 'POST') - self.assertIsNone(match) + def test_shared_docs_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/docs', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'POST')) - # test shared-db docs resource auth - self.assertIsNone(urlmap.match('/shared/docs', 'GET')) - - self.assertIsNone(urlmap.match('/shared/docs', 'PUT')) - - self.assertIsNone(urlmap.match('/shared/docs', 'DELETE')) - - self.assertIsNone(urlmap.match('/shared/docs', 'POST')) - - # test shared-db doc resource auth - match = urlmap.match('/shared/doc/x', 'GET') + def test_shared_doc_authorized(self): + match = self._urlmap.match('/shared/doc/x', 'GET') self.assertIsNotNone(match) self.assertEqual('x', match.get('id')) - match = urlmap.match('/shared/doc/x', 'PUT') + match = self._urlmap.match('/shared/doc/x', 'PUT') self.assertIsNotNone(match) self.assertEqual('x', match.get('id')) - match = urlmap.match('/shared/doc/x', 'DELETE') + match = self._urlmap.match('/shared/doc/x', 'DELETE') + self.assertIsNotNone(match) self.assertEqual('x', match.get('id')) - self.assertIsNone(urlmap.match('/shared/doc/x', 'POST')) - - # test shared-db sync resource auth - self.assertIsNone(urlmap.match('/shared/sync-from/x', 'GET')) - - self.assertIsNone(urlmap.match('/shared/sync-from/x', 'PUT')) - - self.assertIsNone(urlmap.match('/shared/sync-from/x', 'DELETE')) - - self.assertIsNone(urlmap.match('/shared/sync-from/x', 'POST')) - - # test user-db database resource auth - self.assertIsNone(urlmap.match('/%s' % dbname, 'GET')) - - self.assertIsNone(urlmap.match('/%s' % dbname, 'PUT')) - - self.assertIsNone(urlmap.match('/%s' % dbname, 'DELETE')) - - self.assertIsNone(urlmap.match('/%s' % dbname, 'POST')) - - # test user-db docs resource auth - self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'GET')) - - self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'PUT')) - - self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'DELETE')) - - self.assertIsNone(urlmap.match('/%s/docs' % dbname, 'POST')) - - # test user-db doc resource auth - self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'GET')) - - self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'PUT')) - - self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'DELETE')) - - self.assertIsNone(urlmap.match('/%s/doc/x' % dbname, 'POST')) - - # test user-db sync resource auth - match = urlmap.match('/%s/sync-from/x' % dbname, 'GET') + def test_shared_doc_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/doc/x', 'POST')) + + def test_shared_sync_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'POST')) + + def test_user_db_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'POST')) + + def test_user_db_docs_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'POST')) + + def test_user_db_doc_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'POST')) + + def test_user_db_sync_authorized(self): + uuid = self._uuid + dbname = self._dbname + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'GET') self.assertEqual(uuid, match.get('uuid')) self.assertEqual('x', match.get('source_replica_uid')) - match = urlmap.match('/%s/sync-from/x' % dbname, 'PUT') + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'PUT') self.assertEqual(uuid, match.get('uuid')) self.assertEqual('x', match.get('source_replica_uid')) - match = urlmap.match('/%s/sync-from/x' % dbname, 'DELETE') - self.assertIsNone(match) - - match = urlmap.match('/%s/sync-from/x' % dbname, 'POST') + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'POST') self.assertEqual(uuid, match.get('uuid')) self.assertEqual('x', match.get('source_replica_uid')) + def test_user_db_sync_unauthorized(self): + dbname = self._dbname + self.assertIsNone( + self._urlmap.match('/%s/sync-from/x' % dbname, 'DELETE')) + @pytest.mark.usefixtures("method_tmpdir") class EncryptedSyncTestCase( -- cgit v1.2.3 From 4fce575de20effc9c4d934028f8ccdfbd97932e1 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 29 Dec 2016 09:28:10 -0200 Subject: [refactor] remove twisted session persistence The need for token caching in server is a matter of debate, as is the ideal way to do it. Twisted sessions store the session id in a cookie and use that session id to persist. It is not clear if that implementation is needed, works with future features (as multiple soledad servers) or represents a security problem in some way. Because of these, this commit removes it for now. The feature is left in git history so we can bring it back later if needed. --- .../leap/soledad/client/http_target/__init__.py | 21 ++-------- client/src/leap/soledad/client/sqlcipher.py | 8 +--- server/src/leap/soledad/server/session.py | 45 +--------------------- 3 files changed, 6 insertions(+), 68 deletions(-) diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 590ae8f6..b67d03f6 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -24,10 +24,7 @@ after receiving. import os -from cookielib import CookieJar - from twisted.web.client import Agent -from twisted.web.client import CookieAgent from twisted.internet import reactor from leap.common.certs import get_compatible_ssl_context_factory @@ -47,14 +44,6 @@ if os.environ.get('SOLEDAD_STATS'): DO_STATS = True -def newCookieAgent(cert_file): - _factory = get_compatible_ssl_context_factory(cert_file) - _agent = Agent(reactor, _factory) - _cookieJar = CookieJar() - agent = CookieAgent(_agent, _cookieJar) - return agent - - class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): """ @@ -66,8 +55,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): the parsed documents that the remote send us, before being decrypted and written to the main database. """ - def __init__(self, url, source_replica_uid, creds, crypto, cert_file, - agent=None): + def __init__(self, url, source_replica_uid, creds, crypto, cert_file): """ Initialize the sync target. @@ -85,8 +73,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): the SSL certificate used by the remote soledad server. :type cert_file: str - :param agent: an http agent - :type agent: twisted.web.client.Agent """ if url.endswith("/"): url = url[:-1] @@ -101,9 +87,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): self._insert_doc_cb = None # Twisted default Agent with our own ssl context factory - if not agent: - agent = newCookieAgent(cert_file) - self._http = agent + factory = get_compatible_ssl_context_factory(cert_file) + self._http = Agent(reactor, factory) if DO_STATS: self.sync_exchange_phase = [0] diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 9b352bbf..a3e45228 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -59,7 +59,6 @@ from leap.soledad.common.l2db.backends import sqlite_backend from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client.http_target import SoledadHTTPSyncTarget -from leap.soledad.client.http_target import newCookieAgent from leap.soledad.client.sync import SoledadSynchronizer from leap.soledad.client import pragmas @@ -407,10 +406,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # storage for the documents received during a sync self.received_docs = [] - # setup an http agent capable of storing cookies, so we can use - # server's session persistence feature - self._agent = newCookieAgent(cert_file) - self.running = False self._db_handle = None @@ -495,8 +490,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._replica_uid, creds=creds, crypto=self._crypto, - cert_file=self._cert_file, - agent=self._agent)) + cert_file=self._cert_file)) # # Symmetric encryption of syncing docs diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 59424a7b..4ed2721c 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -21,40 +21,14 @@ from zope.interface import implementer from twisted.cred import error from twisted.python import log -from twisted.python.components import registerAdapter from twisted.web import util from twisted.web.guard import HTTPAuthSessionWrapper from twisted.web.resource import ErrorPage from twisted.web.resource import IResource -from twisted.web.server import Session -from zope.interface import Interface -from zope.interface import Attribute from leap.soledad.server.auth import portal from leap.soledad.server.auth import credentialFactory from leap.soledad.server.url_mapper import URLMapper -from leap.soledad.server.resource import SoledadResource - - -class ISessionData(Interface): - username = Attribute('An uuid.') - password = Attribute('A token.') - - -@implementer(ISessionData) -class SessionData(object): - def __init__(self, session): - self.username = None - self.password = None - - -registerAdapter(SessionData, Session, ISessionData) - - -def _sessionData(request): - session = request.getSession() - data = ISessionData(session) - return data @implementer(IResource) @@ -115,22 +89,7 @@ class SoledadSession(HTTPAuthSessionWrapper): if request_uuid and request_uuid != credentials.username: return ErrorPage(500, None, None) - # eventually return a cached resouce - sessionData = _sessionData(request) - if sessionData.username == credentials.username \ - and sessionData.password == credentials.password: - return SoledadResource() - - return util.DeferredResource(self._login(credentials, sessionData)) - - def _login(self, credentials, sessionData): + return util.DeferredResource(self._login(credentials)) - def _cacheSessionData(res): - sessionData.username = credentials.username - sessionData.password = credentials.password - return res - d = self._portal.login(credentials, None, IResource) - d.addCallback(_cacheSessionData) - d.addCallbacks(self._loginSucceeded, self._loginFailed) - return d +soledadSession = SoledadSession() -- cgit v1.2.3 From b3e0af399b81b65d6665865d1e1b18aa589f5824 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 31 Jan 2017 23:51:42 -0200 Subject: [test] add tests for server auth --- server/src/leap/soledad/server/auth.py | 50 +++++++++++------ server/src/leap/soledad/server/session.py | 4 +- testing/tests/server/test_auth.py | 92 +++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 testing/tests/server/test_auth.py diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 13245cfe..bcef2e7c 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -29,6 +29,7 @@ from twisted.cred.credentials import IUsernamePassword from twisted.cred.credentials import UsernamePassword from twisted.cred.portal import IRealm from twisted.cred.portal import Portal +from twisted.internet import defer from twisted.web.iweb import ICredentialFactory from twisted.web.resource import IResource @@ -57,31 +58,45 @@ class TokenChecker(object): TOKENS_TYPE_DEF = "Token" TOKENS_USER_ID_KEY = "user_id" - def __init__(self): - config = get_config() - self.couch_url = config['couch_url'] + def __init__(self, server=None): + if server is None: + config = get_config() + couch_url = config['couch_url'] + server = couch_server(couch_url) + self._server = server + self._dbs = {} def _tokens_dbname(self): dbname = self.TOKENS_DB_PREFIX + \ str(int(time.time() / self.TOKENS_DB_EXPIRE)) return dbname + def _get_db(self, dbname): + if dbname not in self._dbs: + self._dbs[dbname] = self._server[dbname] + return self._dbs[dbname] + + def _tokens_db(self): + # the tokens db rotates every 30 days, and the current db name is + # "tokens_NNN", where NNN is the number of seconds since epoch + # divide dby the rotate period in seconds. When rotating, old and + # new tokens db coexist during a certain window of time and valid + # tokens are replicated from the old db to the new one. See: + # https://leap.se/code/issues/6785 + dbname = self._tokens_dbname() + db = self._get_db(dbname) + return db + def requestAvatarId(self, credentials): uuid = credentials.username token = credentials.password - with couch_server(self.couch_url) as server: - # the tokens db rotates every 30 days, and the current db name is - # "tokens_NNN", where NNN is the number of seconds since epoch - # divide dby the rotate period in seconds. When rotating, old and - # new tokens db coexist during a certain window of time and valid - # tokens are replicated from the old db to the new one. See: - # https://leap.se/code/issues/6785 - dbname = self._tokens_dbname() - db = server[dbname] + # lookup key is a hash of the token to prevent timing attacks. + db = self._tokens_db() token = db.get(sha512(token).hexdigest()) if token is None: - return False + return defer.fail(error.UnauthorizedLogin()) + # we compare uuid hashes to avoid possible timing attacks that # might exploit python's builtin comparison operator behaviour, # which fails immediatelly when non-matching bytes are found. @@ -89,8 +104,9 @@ class TokenChecker(object): req_uuid_hash = sha512(uuid).digest() if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ or couch_uuid_hash != req_uuid_hash: - return False - return True + return defer.fail(error.UnauthorizedLogin()) + + return defer.succeed(uuid) @implementer(ICredentialFactory) @@ -103,7 +119,7 @@ class TokenCredentialFactory(object): def decode(self, response, request): try: - creds = response.decode('base64') + creds = binascii.a2b_base64(response + b'===') except binascii.Error: raise error.LoginFailed('Invalid credentials') @@ -114,5 +130,5 @@ class TokenCredentialFactory(object): raise error.LoginFailed('Invalid credentials') -portal = Portal(SoledadRealm(), [TokenChecker()]) +get_portal = lambda: Portal(SoledadRealm(), [TokenChecker()]) credentialFactory = TokenCredentialFactory() diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 4ed2721c..f1626115 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -26,7 +26,7 @@ from twisted.web.guard import HTTPAuthSessionWrapper from twisted.web.resource import ErrorPage from twisted.web.resource import IResource -from leap.soledad.server.auth import portal +from leap.soledad.server.auth import get_portal from leap.soledad.server.auth import credentialFactory from leap.soledad.server.url_mapper import URLMapper @@ -50,7 +50,7 @@ class SoledadSession(HTTPAuthSessionWrapper): def __init__(self): self._mapper = URLMapper() - self._portal = portal + self._portal = get_portal() self._credentialFactory = credentialFactory def _matchPath(self, request): diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py new file mode 100644 index 00000000..3ff738f2 --- /dev/null +++ b/testing/tests/server/test_auth.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# test_auth.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for auth pieces. +""" +import collections + +from twisted.cred.credentials import UsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest +from twisted.web.resource import IResource + +from leap.soledad.server.auth import SoledadRealm +from leap.soledad.server.auth import TokenChecker +from leap.soledad.server.auth import TokenCredentialFactory +from leap.soledad.server.resource import SoledadResource + + +class SoledadRealmTestCase(unittest.TestCase): + + def test_returned_resource(self): + realm = SoledadRealm() + iface, avatar, logout = realm.requestAvatar('any', None, IResource) + self.assertIsInstance(avatar, SoledadResource) + self.assertIsNone(logout()) + + +class DummyServer(object): + """ + I fake the `couchdb.client.Server` GET api and always return the token + given on my creation. + """ + + def __init__(self, token): + self._token = token + + def get(self, _): + return self._token + + +class TokenCheckerTestCase(unittest.TestCase): + + @inlineCallbacks + def test_good_creds(self): + # set up a dummy server which always return a *valid* token document + token = {'user_id': 'user', 'type': 'Token'} + server = collections.defaultdict(lambda: DummyServer(token)) + # setup the checker with the custom server + checker = TokenChecker(server=server) + # assert the checker *can* verify the creds + creds = UsernamePassword('user', 'pass') + avatarId = yield checker.requestAvatarId(creds) + self.assertEqual('user', avatarId) + + @inlineCallbacks + def test_bad_creds(self): + # set up a dummy server which always return an *invalid* token document + token = None + server = collections.defaultdict(lambda: DummyServer(token)) + # setup the checker with the custom server + checker = TokenChecker(server=server) + # assert the checker *cannot* verify the creds + creds = UsernamePassword('user', '') + with self.assertRaises(UnauthorizedLogin): + yield checker.requestAvatarId(creds) + + +from twisted.web.test import test_httpauth + + +class TokenCredentialFactoryTestcase( + test_httpauth.RequestMixin, test_httpauth.BasicAuthTestsMixin, + unittest.TestCase): + + def setUp(self): + test_httpauth.BasicAuthTestsMixin.setUp(self) + self.credentialFactory = TokenCredentialFactory() -- cgit v1.2.3 From 911695e59ab60d2abaef3013330a6d41283cc733 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 1 Feb 2017 18:16:43 -0200 Subject: [test] add tests for server auth session --- server/src/leap/soledad/server/session.py | 22 +++- testing/tests/server/test_auth.py | 6 +- testing/tests/server/test_session.py | 209 ++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 testing/tests/server/test_session.py diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index f1626115..a56e4e97 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # session.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2017 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,6 +22,7 @@ from zope.interface import implementer from twisted.cred import error from twisted.python import log from twisted.web import util +from twisted.web._auth import wrapper from twisted.web.guard import HTTPAuthSessionWrapper from twisted.web.resource import ErrorPage from twisted.web.resource import IResource @@ -32,9 +33,12 @@ from leap.soledad.server.url_mapper import URLMapper @implementer(IResource) -class UnauthorizedResource(object): +class UnauthorizedResource(wrapper.UnauthorizedResource): isLeaf = True + def __init__(self): + pass + def render(self, request): request.setResponseCode(401) if request.method == b'HEAD': @@ -48,9 +52,12 @@ class UnauthorizedResource(object): @implementer(IResource) class SoledadSession(HTTPAuthSessionWrapper): - def __init__(self): + def __init__(self, portal=None): + if portal is None: + portal = get_portal() + self._mapper = URLMapper() - self._portal = get_portal() + self._portal = portal self._credentialFactory = credentialFactory def _matchPath(self, request): @@ -65,18 +72,22 @@ class SoledadSession(HTTPAuthSessionWrapper): return None def _authorizedResource(self, request): + # check whether the path of the request exists in the app match = self._matchPath(request) if not match: return UnauthorizedResource() + # get authorization header or fail header = request.getHeader(b'authorization') if not header: return UnauthorizedResource() + # parse the authorization header auth_data = self._parseHeader(header) if not auth_data: return UnauthorizedResource() + # decode the credentials from the parsed header try: credentials = self._credentialFactory.decode(auth_data, request) except error.LoginFailed: @@ -85,10 +96,13 @@ class SoledadSession(HTTPAuthSessionWrapper): log.err(None, "Unexpected failure from credentials factory") return ErrorPage(500, None, None) + # make sure the uuid given in path corresponds to the one given in + # the credentials request_uuid = match.get('uuid') if request_uuid and request_uuid != credentials.username: return ErrorPage(500, None, None) + # if all checks pass, try to login with credentials return util.DeferredResource(self._login(credentials)) diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index 3ff738f2..0e6baba3 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # test_auth.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2017 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ from twisted.cred.error import UnauthorizedLogin from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from twisted.web.resource import IResource +from twisted.web.test import test_httpauth from leap.soledad.server.auth import SoledadRealm from leap.soledad.server.auth import TokenChecker @@ -80,9 +81,6 @@ class TokenCheckerTestCase(unittest.TestCase): yield checker.requestAvatarId(creds) -from twisted.web.test import test_httpauth - - class TokenCredentialFactoryTestcase( test_httpauth.RequestMixin, test_httpauth.BasicAuthTestsMixin, unittest.TestCase): diff --git a/testing/tests/server/test_session.py b/testing/tests/server/test_session.py new file mode 100644 index 00000000..7883ef4a --- /dev/null +++ b/testing/tests/server/test_session.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# test_session.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server session entrypoint. +""" +from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse +from twisted.cred import portal +from twisted.web.test.test_httpauth import b64encode +from twisted.web.test.test_httpauth import Realm +from twisted.web.test.requesthelper import DummyRequest +from twisted.web.resource import getChildForRequest + +from twisted.web.resource import Resource + +from twisted.trial import unittest + +from leap.soledad.server.session import SoledadSession + +from twisted.web.static import Data +from twisted.web._auth.wrapper import UnauthorizedResource +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.checkers import ANONYMOUS, AllowAnonymousAccess + + +class SoledadSessionTestCase(unittest.TestCase): + """ + Tests adapted from for + L{twisted.web.test.test_httpauth.HTTPAuthSessionWrapper}. + """ + + def makeRequest(self, *args, **kwargs): + request = DummyRequest(*args, **kwargs) + request.path = '/' + return request + + def setUp(self): + self.username = b'foo bar' + self.password = b'bar baz' + self.avatarContent = b"contents of the avatar resource itself" + self.childName = b"foo-child" + self.childContent = b"contents of the foo child of the avatar" + self.checker = InMemoryUsernamePasswordDatabaseDontUse() + self.checker.addUser(self.username, self.password) + self.avatar = Data(self.avatarContent, 'text/plain') + self.avatar.putChild( + self.childName, Data(self.childContent, 'text/plain')) + self.avatars = {self.username: self.avatar} + self.realm = Realm(self.avatars.get) + self.portal = portal.Portal(self.realm, [self.checker]) + self.wrapper = SoledadSession(self.portal) + + def _authorizedTokenLogin(self, request): + authorization = b64encode( + self.username + b':' + self.password) + request.requestHeaders.addRawHeader(b'authorization', + b'Token ' + authorization) + return getChildForRequest(self.wrapper, request) + + def test_getChildWithDefault(self): + request = self.makeRequest([self.childName]) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(result): + self.assertEqual(request.responseCode, 401) + + d.addCallback(cbFinished) + request.render(child) + return d + + def _invalidAuthorizationTest(self, response): + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', response) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(result): + self.assertEqual(request.responseCode, 401) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_getChildWithDefaultUnauthorizedUser(self): + return self._invalidAuthorizationTest( + b'Basic ' + b64encode(b'foo:bar')) + + def test_getChildWithDefaultUnauthorizedPassword(self): + return self._invalidAuthorizationTest( + b'Basic ' + b64encode(self.username + b':bar')) + + def test_getChildWithDefaultUnrecognizedScheme(self): + return self._invalidAuthorizationTest(b'Quux foo bar baz') + + def test_getChildWithDefaultAuthorized(self): + request = self.makeRequest([self.childName]) + child = self._authorizedTokenLogin(request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [self.childContent]) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_renderAuthorized(self): + # Request it exactly, not any of its children. + request = self.makeRequest([]) + child = self._authorizedTokenLogin(request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [self.avatarContent]) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_decodeRaises(self): + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', + b'Token decode should fail') + child = getChildForRequest(self.wrapper, request) + self.assertIsInstance(child, UnauthorizedResource) + + def test_parseResponse(self): + basicAuthorization = b'Basic abcdef123456' + self.assertEqual( + self.wrapper._parseHeader(basicAuthorization), + None) + tokenAuthorization = b'Token abcdef123456' + self.assertEqual( + self.wrapper._parseHeader(tokenAuthorization), + b'abcdef123456') + + def test_unexpectedDecodeError(self): + + class UnexpectedException(Exception): + pass + + class BadFactory(object): + scheme = b'bad' + + def getChallenge(self, client): + return {} + + def decode(self, response, request): + raise UnexpectedException() + + self.wrapper._credentialFactory = BadFactory() + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', b'Bad abc') + child = getChildForRequest(self.wrapper, request) + request.render(child) + self.assertEqual(request.responseCode, 500) + + def test_unexpectedLoginError(self): + class UnexpectedException(Exception): + pass + + class BrokenChecker(object): + credentialInterfaces = (IUsernamePassword,) + + def requestAvatarId(self, credentials): + raise UnexpectedException() + + self.portal.registerChecker(BrokenChecker()) + request = self.makeRequest([self.childName]) + child = self._authorizedTokenLogin(request) + request.render(child) + self.assertEqual(request.responseCode, 500) + + def test_anonymousAccess(self): + """ + Anonymous requests are allowed if a L{Portal} has an anonymous checker + registered. + """ + unprotectedContents = b"contents of the unprotected child resource" + + self.avatars[ANONYMOUS] = Resource() + self.avatars[ANONYMOUS].putChild( + self.childName, Data(unprotectedContents, 'text/plain')) + self.portal.registerChecker(AllowAnonymousAccess()) + + request = self.makeRequest([self.childName]) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [unprotectedContents]) + + d.addCallback(cbFinished) + request.render(child) + return d -- cgit v1.2.3 From c9cb1a814b6bfaa40de3c35a590f39d5fb0ce18e Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 1 Feb 2017 19:44:03 -0200 Subject: [test] fix session and auth tests --- server/src/leap/soledad/server/auth.py | 18 +++++-------- server/src/leap/soledad/server/session.py | 3 --- testing/tests/server/test_auth.py | 11 ++++++-- testing/tests/server/test_session.py | 45 +++++++------------------------ 4 files changed, 26 insertions(+), 51 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index bcef2e7c..1f078bff 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -67,24 +67,20 @@ class TokenChecker(object): self._dbs = {} def _tokens_dbname(self): - dbname = self.TOKENS_DB_PREFIX + \ - str(int(time.time() / self.TOKENS_DB_EXPIRE)) - return dbname - - def _get_db(self, dbname): - if dbname not in self._dbs: - self._dbs[dbname] = self._server[dbname] - return self._dbs[dbname] - - def _tokens_db(self): # the tokens db rotates every 30 days, and the current db name is # "tokens_NNN", where NNN is the number of seconds since epoch # divide dby the rotate period in seconds. When rotating, old and # new tokens db coexist during a certain window of time and valid # tokens are replicated from the old db to the new one. See: # https://leap.se/code/issues/6785 + dbname = self.TOKENS_DB_PREFIX + \ + str(int(time.time() / self.TOKENS_DB_EXPIRE)) + return dbname + + def _tokens_db(self): dbname = self._tokens_dbname() - db = self._get_db(dbname) + with self._server as server: + db = server[dbname] return db def requestAvatarId(self, credentials): diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index a56e4e97..a2793bd3 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -104,6 +104,3 @@ class SoledadSession(HTTPAuthSessionWrapper): # if all checks pass, try to login with credentials return util.DeferredResource(self._login(credentials)) - - -soledadSession = SoledadSession() diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index 0e6baba3..5b215650 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -19,6 +19,8 @@ Tests for auth pieces. """ import collections +from contextlib import contextmanager + from twisted.cred.credentials import UsernamePassword from twisted.cred.error import UnauthorizedLogin from twisted.internet.defer import inlineCallbacks @@ -54,13 +56,18 @@ class DummyServer(object): return self._token +@contextmanager +def dummy_server(token): + yield collections.defaultdict(lambda: DummyServer(token)) + + class TokenCheckerTestCase(unittest.TestCase): @inlineCallbacks def test_good_creds(self): # set up a dummy server which always return a *valid* token document token = {'user_id': 'user', 'type': 'Token'} - server = collections.defaultdict(lambda: DummyServer(token)) + server = dummy_server(token) # setup the checker with the custom server checker = TokenChecker(server=server) # assert the checker *can* verify the creds @@ -72,7 +79,7 @@ class TokenCheckerTestCase(unittest.TestCase): def test_bad_creds(self): # set up a dummy server which always return an *invalid* token document token = None - server = collections.defaultdict(lambda: DummyServer(token)) + server = dummy_server(token) # setup the checker with the custom server checker = TokenChecker(server=server) # assert the checker *cannot* verify the creds diff --git a/testing/tests/server/test_session.py b/testing/tests/server/test_session.py index 7883ef4a..8131ddb3 100644 --- a/testing/tests/server/test_session.py +++ b/testing/tests/server/test_session.py @@ -17,24 +17,20 @@ """ Tests for server session entrypoint. """ -from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse +from twisted.trial import unittest + from twisted.cred import portal +from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse +from twisted.cred.credentials import IUsernamePassword +from twisted.web.resource import getChildForRequest +from twisted.web.static import Data +from twisted.web.test.requesthelper import DummyRequest from twisted.web.test.test_httpauth import b64encode from twisted.web.test.test_httpauth import Realm -from twisted.web.test.requesthelper import DummyRequest -from twisted.web.resource import getChildForRequest - -from twisted.web.resource import Resource - -from twisted.trial import unittest +from twisted.web._auth.wrapper import UnauthorizedResource from leap.soledad.server.session import SoledadSession -from twisted.web.static import Data -from twisted.web._auth.wrapper import UnauthorizedResource -from twisted.cred.credentials import IUsernamePassword -from twisted.cred.checkers import ANONYMOUS, AllowAnonymousAccess - class SoledadSessionTestCase(unittest.TestCase): """ @@ -168,6 +164,7 @@ class SoledadSessionTestCase(unittest.TestCase): child = getChildForRequest(self.wrapper, request) request.render(child) self.assertEqual(request.responseCode, 500) + self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) def test_unexpectedLoginError(self): class UnexpectedException(Exception): @@ -184,26 +181,4 @@ class SoledadSessionTestCase(unittest.TestCase): child = self._authorizedTokenLogin(request) request.render(child) self.assertEqual(request.responseCode, 500) - - def test_anonymousAccess(self): - """ - Anonymous requests are allowed if a L{Portal} has an anonymous checker - registered. - """ - unprotectedContents = b"contents of the unprotected child resource" - - self.avatars[ANONYMOUS] = Resource() - self.avatars[ANONYMOUS].putChild( - self.childName, Data(unprotectedContents, 'text/plain')) - self.portal.registerChecker(AllowAnonymousAccess()) - - request = self.makeRequest([self.childName]) - child = getChildForRequest(self.wrapper, request) - d = request.notifyFinish() - - def cbFinished(ignored): - self.assertEqual(request.written, [unprotectedContents]) - - d.addCallback(cbFinished) - request.render(child) - return d + self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) -- cgit v1.2.3 From 47c357213b4e39c6ced818de7eeb401e52bf92b9 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Jan 2017 18:37:45 -0200 Subject: [refactor] move wsgi sync setup to its own module Conflicts: server/src/leap/soledad/server/_wsgi.py server/src/leap/soledad/server/entrypoint.py server/src/leap/soledad/server/resource.py testing/tests/server/test__resource.py --- server/src/leap/soledad/server/_resource.py | 42 ++++++++++++++ server/src/leap/soledad/server/_wsgi.py | 83 +++++++++++++++++++++++++++ server/src/leap/soledad/server/application.py | 77 ------------------------- server/src/leap/soledad/server/auth.py | 5 +- server/src/leap/soledad/server/entrypoint.py | 39 +++++++++++++ server/src/leap/soledad/server/resource.py | 52 ----------------- testing/tests/conftest.py | 2 +- testing/tests/server/test_auth.py | 2 +- 8 files changed, 169 insertions(+), 133 deletions(-) create mode 100644 server/src/leap/soledad/server/_resource.py create mode 100644 server/src/leap/soledad/server/_wsgi.py delete mode 100644 server/src/leap/soledad/server/application.py create mode 100644 server/src/leap/soledad/server/entrypoint.py delete mode 100644 server/src/leap/soledad/server/resource.py diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py new file mode 100644 index 00000000..89e252d6 --- /dev/null +++ b/server/src/leap/soledad/server/_resource.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# resource.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +A twisted resource that serves the Soledad Server. +""" +from twisted.web.resource import Resource + +from ._wsgi import sync_resource + + +__all__ = ['SoledadResource'] + + +class SoledadResource(Resource): + """ + This is a dummy twisted resource, used only to allow different entry points + for the Soledad Server. + """ + + def __init__(self): + self.children = {'': sync_resource} + + def getChild(self, path, request): + # for now, just "rewind" the path and serve the wsgi resource for all + # requests. In the future, we might look into the request path to + # decide which child resources should serve each request. + request.postpath.insert(0, request.prepath.pop()) + return self.children[''] diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py new file mode 100644 index 00000000..13c8d13b --- /dev/null +++ b/server/src/leap/soledad/server/_wsgi.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# application.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +A WSGI application to serve as the root resource of the webserver. + +Use it like this: + + twistd web --wsgi=leap.soledad.server.application.wsgi_application +""" +from twisted.internet import reactor +from twisted.python import threadpool +from twisted.web.wsgi import WSGIResource + +from leap.soledad.server import SoledadApp +from leap.soledad.server.gzip_middleware import GzipMiddleware +from leap.soledad.server.config import load_configuration +from leap.soledad.common.backend import SoledadBackend +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.log import getLogger + + +__all__ = ['init_couch_state', 'sync_resource'] + + +_config = None + + +def get_config(): + global _config + if not _config: + _config = load_configuration('/etc/soledad/soledad-server.conf') + return _config['soledad-server'] + + +def _get_couch_state(): + conf = get_config() + state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], + check_schema_versions=True) + SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) + return state + + +_app = SoledadApp(None) # delay state init +wsgi_application = GzipMiddleware(_app) + + +# During its initialization, the couch state verifies if all user databases +# contain a config document with the correct couch schema version stored, and +# will log an error and raise an exception if that is not the case. +# +# If this verification made too early (i.e. before the reactor has started and +# the twistd web logging facilities have been setup), the logging will not +# work. Because of that, we delay couch state initialization until the reactor +# is running. + +def init_couch_state(_app): + try: + _app.state = _get_couch_state() + except Exception as e: + logger = getLogger() + logger.error(str(e)) + reactor.stop() + + +# setup a wsgi resource with its own threadpool +pool = threadpool.ThreadPool() +reactor.callWhenRunning(pool.start) +reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) +sync_resource = WSGIResource(reactor, pool, wsgi_application) diff --git a/server/src/leap/soledad/server/application.py b/server/src/leap/soledad/server/application.py deleted file mode 100644 index 8fd343b3..00000000 --- a/server/src/leap/soledad/server/application.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -# application.py -# Copyright (C) 2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -A WSGI application to serve as the root resource of the webserver. - -Use it like this: - - twistd web --wsgi=leap.soledad.server.application.wsgi_application -""" -from twisted.internet import reactor - -from leap.soledad.server import SoledadApp -from leap.soledad.server.gzip_middleware import GzipMiddleware -from leap.soledad.server.config import load_configuration -from leap.soledad.common.backend import SoledadBackend -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.log import getLogger - - -__all__ = ['wsgi_application'] - - -_config = None - - -def get_config(): - global _config - if not _config: - _config = load_configuration('/etc/soledad/soledad-server.conf') - return _config['soledad-server'] - - -def _get_couch_state(): - conf = get_config() - state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], - check_schema_versions=True) - SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) - return state - - -_app = SoledadApp(None) # delay state init -wsgi_application = GzipMiddleware(_app) - - -# During its initialization, the couch state verifies if all user databases -# contain a config document with the correct couch schema version stored, and -# will log an error and raise an exception if that is not the case. -# -# If this verification made too early (i.e. before the reactor has started and -# the twistd web logging facilities have been setup), the logging will not -# work. Because of that, we delay couch state initialization until the reactor -# is running. - -def _init_couch_state(_app): - try: - _app.state = _get_couch_state() - except Exception as e: - logger = getLogger() - logger.error(str(e)) - reactor.stop() - - -reactor.callWhenRunning(_init_couch_state, _app) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 1f078bff..e616a94f 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -34,8 +34,9 @@ from twisted.web.iweb import ICredentialFactory from twisted.web.resource import IResource from leap.soledad.common.couch import couch_server -from leap.soledad.server.resource import SoledadResource -from leap.soledad.server.application import get_config + +from ._resource import SoledadResource +from ._wsgi import get_config @implementer(IRealm) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py new file mode 100644 index 00000000..df2b8934 --- /dev/null +++ b/server/src/leap/soledad/server/entrypoint.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# entrypoint.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +The entrypoint for Soledad server. +""" +from twisted.internet import reactor + +from .config import load_configuration +from ._resource import SoledadResource +from ._wsgi import init_couch_state + + +# load configuration from file +conf = load_configuration('/etc/soledad/soledad-server.conf') + + +class SoledadEntrypoint(SoledadResource): + + def __init__(self): + SoledadResource.__init__(self, conf) + + +# see the comments in application.py recarding why couch state has to be +# initialized when the reactor is running +reactor.callWhenRunning(init_couch_state, conf['soledad-server']) diff --git a/server/src/leap/soledad/server/resource.py b/server/src/leap/soledad/server/resource.py deleted file mode 100644 index 67e9ae32..00000000 --- a/server/src/leap/soledad/server/resource.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# resource.py -# Copyright (C) 2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -A twisted resource that serves the Soledad Server. -""" -from twisted.web.resource import Resource -from twisted.web.wsgi import WSGIResource -from twisted.internet import reactor -from twisted.python import threadpool - -from leap.soledad.server.application import wsgi_application - - -__all__ = ['SoledadResource'] - - -# setup a wsgi resource with its own threadpool -pool = threadpool.ThreadPool() -reactor.callWhenRunning(pool.start) -reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) -wsgi_resource = WSGIResource(reactor, pool, wsgi_application) - - -class SoledadResource(Resource): - """ - This is a dummy twisted resource, used only to allow different entry points - for the Soledad Server. - """ - - def __init__(self): - self.children = {'': wsgi_resource} - - def getChild(self, path, request): - # for now, just "rewind" the path and serve the wsgi resource for all - # requests. In the future, we might look into the request path to - # decide which child resources should serve each request. - request.postpath.insert(0, request.prepath.pop()) - return self.children[''] diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 1ff1cbb7..decfb0a9 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -103,7 +103,7 @@ class SoledadServer(object): '--logfile=%s' % self._logfile, '--pidfile=%s' % self._pidfile, 'web', - '--wsgi=leap.soledad.server.application.wsgi_application', + '--class=leap.soledad.server.entrypoint.SoledadEntrypoint', '--port=2424' ]) diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index 5b215650..8ef35177 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -31,7 +31,7 @@ from twisted.web.test import test_httpauth from leap.soledad.server.auth import SoledadRealm from leap.soledad.server.auth import TokenChecker from leap.soledad.server.auth import TokenCredentialFactory -from leap.soledad.server.resource import SoledadResource +from leap.soledad.server._resource import SoledadResource class SoledadRealmTestCase(unittest.TestCase): -- cgit v1.2.3 From 47858d88ca4ca10ac363c71550b1bafe50f8f4ce Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Jan 2017 19:18:15 -0200 Subject: [refactor] allow passing threadpool pool for server sync resource Conflicts: server/src/leap/soledad/server/_resource.py testing/tests/server/test__resource.py --- server/pkg/soledad-server | 2 +- server/src/leap/soledad/server/_resource.py | 5 +++-- server/src/leap/soledad/server/_wsgi.py | 13 +++++++------ server/src/leap/soledad/server/auth.py | 13 +++++++++++-- server/src/leap/soledad/server/entrypoint.py | 11 +++++++---- testing/tests/server/test_auth.py | 5 ++++- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server index 92560fa8..450f2277 100644 --- a/server/pkg/soledad-server +++ b/server/pkg/soledad-server @@ -11,7 +11,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/soledad.pid -RESOURCE_CLASS=leap.soledad.server.session.SoledadSession +RESOURCE_CLASS=leap.soledad.server.entrypoint.SoledadEntrypoint HTTPS_PORT=2424 CONFDIR=/etc/soledad CERT_PATH="${CONFDIR}/soledad-server.pem" diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 89e252d6..4070d786 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -19,7 +19,7 @@ A twisted resource that serves the Soledad Server. """ from twisted.web.resource import Resource -from ._wsgi import sync_resource +from ._wsgi import get_sync_resource __all__ = ['SoledadResource'] @@ -31,7 +31,8 @@ class SoledadResource(Resource): for the Soledad Server. """ - def __init__(self): + def __init__(self, sync_pool=None): + sync_resource = get_sync_resource(sync_pool) self.children = {'': sync_resource} def getChild(self, path, request): diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index 13c8d13b..3e30d560 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -33,7 +33,7 @@ from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.log import getLogger -__all__ = ['init_couch_state', 'sync_resource'] +__all__ = ['init_couch_state', 'get_sync_resource'] _config = None @@ -76,8 +76,9 @@ def init_couch_state(_app): reactor.stop() -# setup a wsgi resource with its own threadpool -pool = threadpool.ThreadPool() -reactor.callWhenRunning(pool.start) -reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) -sync_resource = WSGIResource(reactor, pool, wsgi_application) +def get_sync_resource(pool=None): + if not pool: + pool = threadpool.ThreadPool() + reactor.callWhenRunning(pool.start) + reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) + return WSGIResource(reactor, pool, wsgi_application) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index e616a94f..a5d90c46 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -42,9 +42,13 @@ from ._wsgi import get_config @implementer(IRealm) class SoledadRealm(object): + def __init__(self, sync_pool=None): + self._sync_pool = sync_pool + def requestAvatar(self, avatarId, mind, *interfaces): if IResource in interfaces: - return (IResource, SoledadResource(), lambda: None) + resource = SoledadResource(sync_pool=self._sync_pool) + return (IResource, resource, lambda: None) raise NotImplementedError() @@ -127,5 +131,10 @@ class TokenCredentialFactory(object): raise error.LoginFailed('Invalid credentials') -get_portal = lambda: Portal(SoledadRealm(), [TokenChecker()]) +def get_portal(sync_pool=None): + realm = SoledadRealm(sync_pool=sync_pool) + checker = TokenChecker() + return Portal(realm, [checker]) + + credentialFactory = TokenCredentialFactory() diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index df2b8934..7501a447 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -20,7 +20,7 @@ The entrypoint for Soledad server. from twisted.internet import reactor from .config import load_configuration -from ._resource import SoledadResource +from ._session import SoledadSession from ._wsgi import init_couch_state @@ -28,10 +28,13 @@ from ._wsgi import init_couch_state conf = load_configuration('/etc/soledad/soledad-server.conf') -class SoledadEntrypoint(SoledadResource): +class SoledadEntrypoint(SoledadSession): - def __init__(self): - SoledadResource.__init__(self, conf) + # the purpose of the entrypoint is to avoid trying to load the + # configuration file during tests. This class will be more useful when we + # add the blobs feature toggle. For now, the whole entrypoint + + pass # see the comments in application.py recarding why couch state has to be diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index 8ef35177..f7fe0f25 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -23,6 +23,7 @@ from contextlib import contextmanager from twisted.cred.credentials import UsernamePassword from twisted.cred.error import UnauthorizedLogin +from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from twisted.web.resource import IResource @@ -37,7 +38,9 @@ from leap.soledad.server._resource import SoledadResource class SoledadRealmTestCase(unittest.TestCase): def test_returned_resource(self): - realm = SoledadRealm() + # we have to pass a pool to the realm , otherwise tests will hang + pool = reactor.getThreadPool() + realm = SoledadRealm(sync_pool=pool) iface, avatar, logout = realm.requestAvatar('any', None, IResource) self.assertIsInstance(avatar, SoledadResource) self.assertIsNone(logout()) -- cgit v1.2.3 From 02764109fa1145474c24b73d537280e3a5652f78 Mon Sep 17 00:00:00 2001 From: Thais Siqueira Date: Wed, 11 Jan 2017 15:26:11 -0200 Subject: [bug] Fix import for load_configuration on migration script --- scripts/migration/0.9.0/migrate.py | 4 ++-- server/src/leap/soledad/server/__init__.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/migration/0.9.0/migrate.py b/scripts/migration/0.9.0/migrate.py index 9a29d8b5..7b2ec005 100755 --- a/scripts/migration/0.9.0/migrate.py +++ b/scripts/migration/0.9.0/migrate.py @@ -27,14 +27,14 @@ import os from argparse import ArgumentParser -from leap.soledad.server.config import load_configuration +from leap.soledad.server import get_config from migrate_couch_schema import migrate TARGET_VERSION = '0.8.2' DEFAULT_COUCH_URL = 'http://127.0.0.1:5984' -CONF = load_configuration('/etc/soledad/soledad-server.conf') +CONF = get_config() NETRC_PATH = CONF['soledad-server']['admin_netrc'] diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 039bef75..5bed22c9 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -88,15 +88,17 @@ import sys from leap.soledad.common.l2db.remote import http_app, utils from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.server.sync import SyncResource -from leap.soledad.server.sync import MAX_REQUEST_SIZE -from leap.soledad.server.sync import MAX_ENTRY_SIZE +from .sync import SyncResource +from .sync import MAX_REQUEST_SIZE +from .sync import MAX_ENTRY_SIZE from ._version import get_versions +from ._config import get_config __all__ = [ 'SoledadApp', + 'get_config', '__version__', ] -- cgit v1.2.3 From 0e12cd3eb1e20bb867f34e0bf60f280d93b6182d Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 18 Jan 2017 17:28:01 -0200 Subject: [feature] add server config option for blobs --- server/src/leap/soledad/server/_blobs.py | 34 ++++++++++++ server/src/leap/soledad/server/_config.py | 81 ++++++++++++++++++++++++++++ server/src/leap/soledad/server/_resource.py | 22 ++++++-- server/src/leap/soledad/server/_wsgi.py | 18 ++----- server/src/leap/soledad/server/auth.py | 2 +- server/src/leap/soledad/server/config.py | 67 ----------------------- server/src/leap/soledad/server/entrypoint.py | 4 +- testing/tests/server/test_server.py | 3 +- 8 files changed, 140 insertions(+), 91 deletions(-) create mode 100644 server/src/leap/soledad/server/_blobs.py create mode 100644 server/src/leap/soledad/server/_config.py delete mode 100644 server/src/leap/soledad/server/config.py diff --git a/server/src/leap/soledad/server/_blobs.py b/server/src/leap/soledad/server/_blobs.py new file mode 100644 index 00000000..0424aae0 --- /dev/null +++ b/server/src/leap/soledad/server/_blobs.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# _blobs.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Blobs Server implementation. +""" +from twisted.web import resource + + +__all__ = ['blobs_resource'] + + +class BlobsResource(resource.Resource): + + isLeaf = True + + def render_GET(self, request): + return 'blobs is not implemented yet!' + + +blobs_resource = BlobsResource() diff --git a/server/src/leap/soledad/server/_config.py b/server/src/leap/soledad/server/_config.py new file mode 100644 index 00000000..2c7f530d --- /dev/null +++ b/server/src/leap/soledad/server/_config.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# config.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import configparser + + +__all__ = ['get_config'] + + +CONFIG_DEFAULTS = { + 'soledad-server': { + 'couch_url': 'http://localhost:5984', + 'create_cmd': None, + 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', + 'batching': True, + 'blobs': False, + }, + 'database-security': { + 'members': ['soledad'], + 'members_roles': [], + 'admins': [], + 'admins_roles': [] + } +} + + +_config = None + + +def get_config(): + global _config + if not _config: + _config = _load_config('/etc/soledad/soledad-server.conf') + return _config['soledad-server'] + + +def _load_config(file_path): + """ + Load server configuration from file. + + @param file_path: The path to the configuration file. + @type file_path: str + + @return: A dictionary with the configuration. + @rtype: dict + """ + conf = dict(CONFIG_DEFAULTS) + config = configparser.SafeConfigParser() + config.read(file_path) + for section in conf: + if not config.has_section(section): + continue + for key, value in conf[section].items(): + if not config.has_option(section, key): + continue + elif type(value) == bool: + conf[section][key] = config.getboolean(section, key) + elif type(value) == list: + values = config.get(section, key).split(',') + values = [v.strip() for v in values] + conf[section][key] = values + else: + conf[section][key] = config.get(section, key) + # TODO: implement basic parsing/sanitization of options comming from + # config file. + return conf diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 4070d786..4f8e98df 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -17,8 +17,10 @@ """ A twisted resource that serves the Soledad Server. """ +from twisted.web.error import Error from twisted.web.resource import Resource +from ._blobs import blobs_resource from ._wsgi import get_sync_resource @@ -33,11 +35,21 @@ class SoledadResource(Resource): def __init__(self, sync_pool=None): sync_resource = get_sync_resource(sync_pool) - self.children = {'': sync_resource} + self.children = { + 'sync': sync_resource, + 'blobs': blobs_resource, + } def getChild(self, path, request): - # for now, just "rewind" the path and serve the wsgi resource for all - # requests. In the future, we might look into the request path to - # decide which child resources should serve each request. + """ + Decide which child resource to serve based on the given path. + """ + if path == 'blobs': + if not self._blobs_enabled: + msg = 'Blobs feature is disabled in this server.' + raise Error(403, message=msg) + return self.children['blobs'] + + # rewind the path and serve the wsgi sync resource request.postpath.insert(0, request.prepath.pop()) - return self.children[''] + return self.children['sync'] diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index 3e30d560..c00d00ae 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -27,7 +27,6 @@ from twisted.web.wsgi import WSGIResource from leap.soledad.server import SoledadApp from leap.soledad.server.gzip_middleware import GzipMiddleware -from leap.soledad.server.config import load_configuration from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.log import getLogger @@ -36,18 +35,7 @@ from leap.soledad.common.log import getLogger __all__ = ['init_couch_state', 'get_sync_resource'] -_config = None - - -def get_config(): - global _config - if not _config: - _config = load_configuration('/etc/soledad/soledad-server.conf') - return _config['soledad-server'] - - -def _get_couch_state(): - conf = get_config() +def _get_couch_state(conf): state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], check_schema_versions=True) SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) @@ -67,9 +55,9 @@ wsgi_application = GzipMiddleware(_app) # work. Because of that, we delay couch state initialization until the reactor # is running. -def init_couch_state(_app): +def init_couch_state(conf): try: - _app.state = _get_couch_state() + _app.state = _get_couch_state(conf) except Exception as e: logger = getLogger() logger.error(str(e)) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index a5d90c46..d7ccdeb9 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -36,7 +36,7 @@ from twisted.web.resource import IResource from leap.soledad.common.couch import couch_server from ._resource import SoledadResource -from ._wsgi import get_config +from ._config import get_config @implementer(IRealm) diff --git a/server/src/leap/soledad/server/config.py b/server/src/leap/soledad/server/config.py deleted file mode 100644 index 3c17ec19..00000000 --- a/server/src/leap/soledad/server/config.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -# config.py -# Copyright (C) 2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import configparser - - -CONFIG_DEFAULTS = { - 'soledad-server': { - 'couch_url': 'http://localhost:5984', - 'create_cmd': None, - 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', - 'batching': True - }, - 'database-security': { - 'members': ['soledad'], - 'members_roles': [], - 'admins': [], - 'admins_roles': [] - } -} - - -def load_configuration(file_path): - """ - Load server configuration from file. - - @param file_path: The path to the configuration file. - @type file_path: str - - @return: A dictionary with the configuration. - @rtype: dict - """ - defaults = dict(CONFIG_DEFAULTS) - config = configparser.SafeConfigParser() - config.read(file_path) - for section in defaults: - if not config.has_section(section): - continue - for key, value in defaults[section].items(): - if not config.has_option(section, key): - continue - elif type(value) == bool: - defaults[section][key] = config.getboolean(section, key) - elif type(value) == list: - values = config.get(section, key).split(',') - values = [v.strip() for v in values] - defaults[section][key] = values - else: - defaults[section][key] = config.get(section, key) - # TODO: implement basic parsing/sanitization of options comming from - # config file. - return defaults diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index 7501a447..714490ae 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -19,13 +19,13 @@ The entrypoint for Soledad server. """ from twisted.internet import reactor -from .config import load_configuration +from ._config import get_config from ._session import SoledadSession from ._wsgi import init_couch_state # load configuration from file -conf = load_configuration('/etc/soledad/soledad-server.conf') +conf = get_config class SoledadEntrypoint(SoledadSession): diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index da69c423..866d9eab 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -379,5 +379,6 @@ class ConfigurationParsingTest(unittest.TestCase): 'sudo -u soledad-admin /usr/bin/create-user-db', 'admin_netrc': '/etc/couchdb/couchdb-soledad-admin.netrc', - 'batching': False} + 'batching': False, + 'blobs': False} self.assertDictEqual(expected, config['soledad-server']) -- cgit v1.2.3 From e05e9566c55e20833a2c45fe2ca67e60df009aac Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 2 Feb 2017 12:15:45 -0200 Subject: [test] move server auth tests to its own file --- testing/tests/server/test_config.py | 68 +++++++++++++++++++++++++++++++++++++ testing/tests/server/test_server.py | 46 ------------------------- 2 files changed, 68 insertions(+), 46 deletions(-) create mode 100644 testing/tests/server/test_config.py diff --git a/testing/tests/server/test_config.py b/testing/tests/server/test_config.py new file mode 100644 index 00000000..1241472b --- /dev/null +++ b/testing/tests/server/test_config.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# test_config.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server configuration. +""" + +from twisted.trial import unittest +from pkg_resources import resource_filename + +from leap.soledad.server._config import _load_config +from leap.soledad.server._config import CONFIG_DEFAULTS + + +class ConfigurationParsingTest(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_use_defaults_on_failure(self): + config = _load_config('this file will never exist') + expected = CONFIG_DEFAULTS + self.assertEquals(expected, config) + + def test_security_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = _load_config(config_path) + + # then + expected = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['user3', 'user4'], + 'admins_roles': ['role3', 'role3']} + self.assertDictEqual(expected, config['database-security']) + + def test_server_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = _load_config(config_path) + + # then + expected = {'couch_url': + 'http://soledad:passwd@localhost:5984', + 'create_cmd': + 'sudo -u soledad-admin /usr/bin/create-user-db', + 'admin_netrc': + '/etc/couchdb/couchdb-soledad-admin.netrc', + 'batching': False, + 'blobs': False} + self.assertDictEqual(expected, config['soledad-server']) diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 866d9eab..ec0fc31d 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -21,7 +21,6 @@ import binascii import os import pytest -from pkg_resources import resource_filename from urlparse import urljoin from uuid import uuid4 @@ -40,8 +39,6 @@ from test_soledad.util import ( from leap.soledad.client import _crypto from leap.soledad.client import Soledad -from leap.soledad.server.config import load_configuration -from leap.soledad.server.config import CONFIG_DEFAULTS from leap.soledad.server.url_mapper import URLMapper @@ -339,46 +336,3 @@ class EncryptedSyncTestCase( Test if Soledad can sync many smallfiles. """ return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) - - -class ConfigurationParsingTest(unittest.TestCase): - - def setUp(self): - self.maxDiff = None - - def test_use_defaults_on_failure(self): - config = load_configuration('this file will never exist') - expected = CONFIG_DEFAULTS - self.assertEquals(expected, config) - - def test_security_values_configuration(self): - # given - config_path = resource_filename('test_soledad', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['user3', 'user4'], - 'admins_roles': ['role3', 'role3']} - self.assertDictEqual(expected, config['database-security']) - - def test_server_values_configuration(self): - # given - config_path = resource_filename('test_soledad', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'couch_url': - 'http://soledad:passwd@localhost:5984', - 'create_cmd': - 'sudo -u soledad-admin /usr/bin/create-user-db', - 'admin_netrc': - '/etc/couchdb/couchdb-soledad-admin.netrc', - 'batching': False, - 'blobs': False} - self.assertDictEqual(expected, config['soledad-server']) -- cgit v1.2.3 From ce0aed6d832f2f49c9306c8ccb6dd128cca1c511 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 18 Jan 2017 17:28:01 -0200 Subject: [feature] add server config option for blobs --- server/src/leap/soledad/server/_resource.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 4f8e98df..3e307e44 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -21,6 +21,7 @@ from twisted.web.error import Error from twisted.web.resource import Resource from ._blobs import blobs_resource +from ._config import get_config from ._wsgi import get_sync_resource @@ -33,8 +34,11 @@ class SoledadResource(Resource): for the Soledad Server. """ + _conf = get_config() + def __init__(self, sync_pool=None): sync_resource = get_sync_resource(sync_pool) + self._blobs_enabled = self._conf['soledad-server']['blobs'] self.children = { 'sync': sync_resource, 'blobs': blobs_resource, -- cgit v1.2.3 From 6793ff931a5fea247d5bd1634d48a789f5e7d845 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 22 Jan 2017 19:36:42 -0200 Subject: [refactor] rename server auth classes --- server/src/leap/soledad/server/_wsgi.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index c00d00ae..37a03ced 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -15,11 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -A WSGI application to serve as the root resource of the webserver. - -Use it like this: - - twistd web --wsgi=leap.soledad.server.application.wsgi_application +A WSGI application that serves Soledad synchronization. """ from twisted.internet import reactor from twisted.python import threadpool -- cgit v1.2.3 From accd15d5842640ba4cd9a5a1e6024a2ef5f7eb06 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 23 Jan 2017 18:53:16 -0200 Subject: [feature] announce server blobs capabilities - add a new ServerInfo resource for / - move entrypoint to its own module --- server/src/leap/soledad/server/_resource.py | 16 +++++++--- server/src/leap/soledad/server/_server_info.py | 41 ++++++++++++++++++++++++++ server/src/leap/soledad/server/entrypoint.py | 7 ++--- 3 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 server/src/leap/soledad/server/_server_info.py diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 3e307e44..1c4edade 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -22,6 +22,7 @@ from twisted.web.resource import Resource from ._blobs import blobs_resource from ._config import get_config +from ._server_info import ServerInfo from ._wsgi import get_sync_resource @@ -34,26 +35,33 @@ class SoledadResource(Resource): for the Soledad Server. """ - _conf = get_config() - def __init__(self, sync_pool=None): sync_resource = get_sync_resource(sync_pool) - self._blobs_enabled = self._conf['soledad-server']['blobs'] + conf = get_config() + self._blobs_enabled = conf['soledad-server']['blobs'] + server_info = ServerInfo(self._blobs_enabled) self.children = { + '': server_info, 'sync': sync_resource, 'blobs': blobs_resource, + 'sync': sync_resource, } def getChild(self, path, request): """ Decide which child resource to serve based on the given path. """ + # requests to / return server information + if path == '': + return self.children[''] + + # requests to /blobs will serve blobs if enabled if path == 'blobs': if not self._blobs_enabled: msg = 'Blobs feature is disabled in this server.' raise Error(403, message=msg) return self.children['blobs'] - # rewind the path and serve the wsgi sync resource + # other requesta are routed to legacy sync resource request.postpath.insert(0, request.prepath.pop()) return self.children['sync'] diff --git a/server/src/leap/soledad/server/_server_info.py b/server/src/leap/soledad/server/_server_info.py new file mode 100644 index 00000000..a1dd3555 --- /dev/null +++ b/server/src/leap/soledad/server/_server_info.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# _server_info.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Resource that announces information about the server. +""" +import json + +from twisted.web.resource import Resource + + +__all__ = ['ServerInfo'] + + +class ServerInfo(Resource): + """ + Return information about the server. + """ + + isLeaf = True + + def __init__(self, blobs_enabled): + self._info = { + "blobs": blobs_enabled, + } + + def render_GET(self, request): + return json.dumps(self._info) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index 714490ae..cfc557a3 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -30,11 +30,8 @@ conf = get_config class SoledadEntrypoint(SoledadSession): - # the purpose of the entrypoint is to avoid trying to load the - # configuration file during tests. This class will be more useful when we - # add the blobs feature toggle. For now, the whole entrypoint - - pass + def __init__(self): + SoledadSession.__init__(self, conf) # see the comments in application.py recarding why couch state has to be -- cgit v1.2.3 From 30c47eea2b10f10204b5d61ecb550edd24e59067 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Jan 2017 08:24:39 -0200 Subject: [test] add tests for server resource and server info --- testing/tests/server/test__resource.py | 66 +++++++++++++++++++++++++++++++ testing/tests/server/test__server_info.py | 39 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 testing/tests/server/test__resource.py create mode 100644 testing/tests/server/test__server_info.py diff --git a/testing/tests/server/test__resource.py b/testing/tests/server/test__resource.py new file mode 100644 index 00000000..0ea46d6a --- /dev/null +++ b/testing/tests/server/test__resource.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# test__resource.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for Soledad server main resource. +""" +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest +from twisted.web.error import Error +from twisted.web.wsgi import WSGIResource + +from leap.soledad.server._resource import SoledadResource +from leap.soledad.server._server_info import ServerInfo +from leap.soledad.server._blobs import BlobsResource + + +conf_blobs_false = {'soledad-server': {'blobs': False}} + + +class SoledadResourceTestCase(unittest.TestCase): + + def test_get_root(self): + conf = {'soledad-server': {'blobs': None}} # doesn't matter + resource = SoledadResource(conf) + path = '' + request = DummyRequest([]) + child = resource.getChild(path, request) + self.assertIsInstance(child, ServerInfo) + + def test_get_blobs_enabled(self): + conf = {'soledad-server': {'blobs': True}} + resource = SoledadResource(conf) + path = 'blobs' + request = DummyRequest([]) + child = resource.getChild(path, request) + self.assertIsInstance(child, BlobsResource) + + def test_get_blobs_disabled(self): + conf = {'soledad-server': {'blobs': False}} + resource = SoledadResource(conf) + path = 'blobs' + request = DummyRequest([]) + with self.assertRaises(Error): + resource.getChild(path, request) + + def test_get_sync(self): + conf = {'soledad-server': {'blobs': None}} # doesn't matter + resource = SoledadResource(conf) + path = 'sync' # if not 'blobs' or '', should be routed to sync + request = DummyRequest([]) + request.prepath = ['user-db'] + child = resource.getChild(path, request) + self.assertIsInstance(child, WSGIResource) diff --git a/testing/tests/server/test__server_info.py b/testing/tests/server/test__server_info.py new file mode 100644 index 00000000..80770721 --- /dev/null +++ b/testing/tests/server/test__server_info.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# test__server_info.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Tests for Soledad server information announcement. +""" +import json + +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest + +from leap.soledad.server._server_info import ServerInfo + + +class ServerInfoTestCase(unittest.TestCase): + + def test_blobs_enabled(self): + resource = ServerInfo(True) + response = resource.render(DummyRequest([''])) + self.assertEquals(json.loads(response), {'blobs': True}) + + def test_blobs_disabled(self): + resource = ServerInfo(False) + response = resource.render(DummyRequest([''])) + self.assertEquals(json.loads(response), {'blobs': False}) -- cgit v1.2.3 From 7432b72471fa3de03d13b9d35f6004d14deae63d Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Jan 2017 18:37:45 -0200 Subject: [refactor] move wsgi sync setup to its own module --- testing/tests/server/test__resource.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/tests/server/test__resource.py b/testing/tests/server/test__resource.py index 0ea46d6a..b5285195 100644 --- a/testing/tests/server/test__resource.py +++ b/testing/tests/server/test__resource.py @@ -25,6 +25,7 @@ from twisted.web.wsgi import WSGIResource from leap.soledad.server._resource import SoledadResource from leap.soledad.server._server_info import ServerInfo from leap.soledad.server._blobs import BlobsResource +from leap.soledad.server.gzip_middleware import GzipMiddleware conf_blobs_false = {'soledad-server': {'blobs': False}} @@ -64,3 +65,4 @@ class SoledadResourceTestCase(unittest.TestCase): request.prepath = ['user-db'] child = resource.getChild(path, request) self.assertIsInstance(child, WSGIResource) + self.assertIsInstance(child._application, GzipMiddleware) -- cgit v1.2.3 From 1f1a9847117d23a767e99fe80484baf232375d36 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Jan 2017 19:18:15 -0200 Subject: [refactor] allow passing threadpool pool for server sync resource --- server/src/leap/soledad/server/_resource.py | 2 +- testing/tests/server/test__resource.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 1c4edade..046e20d1 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -36,10 +36,10 @@ class SoledadResource(Resource): """ def __init__(self, sync_pool=None): - sync_resource = get_sync_resource(sync_pool) conf = get_config() self._blobs_enabled = conf['soledad-server']['blobs'] server_info = ServerInfo(self._blobs_enabled) + sync_resource = get_sync_resource(sync_pool) self.children = { '': server_info, 'sync': sync_resource, diff --git a/testing/tests/server/test__resource.py b/testing/tests/server/test__resource.py index b5285195..2a387416 100644 --- a/testing/tests/server/test__resource.py +++ b/testing/tests/server/test__resource.py @@ -21,6 +21,7 @@ from twisted.trial import unittest from twisted.web.test.test_web import DummyRequest from twisted.web.error import Error from twisted.web.wsgi import WSGIResource +from twisted.internet import reactor from leap.soledad.server._resource import SoledadResource from leap.soledad.server._server_info import ServerInfo @@ -28,14 +29,14 @@ from leap.soledad.server._blobs import BlobsResource from leap.soledad.server.gzip_middleware import GzipMiddleware -conf_blobs_false = {'soledad-server': {'blobs': False}} +_pool = reactor.getThreadPool() class SoledadResourceTestCase(unittest.TestCase): def test_get_root(self): conf = {'soledad-server': {'blobs': None}} # doesn't matter - resource = SoledadResource(conf) + resource = SoledadResource(conf, sync_pool=_pool) path = '' request = DummyRequest([]) child = resource.getChild(path, request) @@ -43,7 +44,7 @@ class SoledadResourceTestCase(unittest.TestCase): def test_get_blobs_enabled(self): conf = {'soledad-server': {'blobs': True}} - resource = SoledadResource(conf) + resource = SoledadResource(conf, sync_pool=_pool) path = 'blobs' request = DummyRequest([]) child = resource.getChild(path, request) @@ -51,7 +52,7 @@ class SoledadResourceTestCase(unittest.TestCase): def test_get_blobs_disabled(self): conf = {'soledad-server': {'blobs': False}} - resource = SoledadResource(conf) + resource = SoledadResource(conf, sync_pool=_pool) path = 'blobs' request = DummyRequest([]) with self.assertRaises(Error): @@ -59,7 +60,7 @@ class SoledadResourceTestCase(unittest.TestCase): def test_get_sync(self): conf = {'soledad-server': {'blobs': None}} # doesn't matter - resource = SoledadResource(conf) + resource = SoledadResource(conf, sync_pool=_pool) path = 'sync' # if not 'blobs' or '', should be routed to sync request = DummyRequest([]) request.prepath = ['user-db'] -- cgit v1.2.3 From 53b5a6788ad8416f78b24cc9880d02da73c52d70 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 27 Jan 2017 20:30:02 -0200 Subject: [refacor] make proper use of twisted web dyamic resources in server --- server/src/leap/soledad/server/_resource.py | 42 +++++++++++------------------ testing/tests/server/test__resource.py | 27 +++++++++---------- 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 046e20d1..fec290a2 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -17,11 +17,9 @@ """ A twisted resource that serves the Soledad Server. """ -from twisted.web.error import Error from twisted.web.resource import Resource from ._blobs import blobs_resource -from ._config import get_config from ._server_info import ServerInfo from ._wsgi import get_sync_resource @@ -35,33 +33,25 @@ class SoledadResource(Resource): for the Soledad Server. """ - def __init__(self, sync_pool=None): - conf = get_config() - self._blobs_enabled = conf['soledad-server']['blobs'] - server_info = ServerInfo(self._blobs_enabled) - sync_resource = get_sync_resource(sync_pool) - self.children = { - '': server_info, - 'sync': sync_resource, - 'blobs': blobs_resource, - 'sync': sync_resource, - } + def __init__(self, conf, sync_pool=None): + Resource.__init__(self) + + blobs_enabled = conf['soledad-server']['blobs'] - def getChild(self, path, request): - """ - Decide which child resource to serve based on the given path. - """ # requests to / return server information - if path == '': - return self.children[''] + server_info = ServerInfo(blobs_enabled) + self.putChild('', server_info) # requests to /blobs will serve blobs if enabled - if path == 'blobs': - if not self._blobs_enabled: - msg = 'Blobs feature is disabled in this server.' - raise Error(403, message=msg) - return self.children['blobs'] + if blobs_enabled: + self.putChild('blobs', blobs_resource) - # other requesta are routed to legacy sync resource + # other requests are routed to legacy sync resource + self._sync_resource = get_sync_resource(sync_pool) + + def getChild(self, path, request): + """ + Route requests to legacy WSGI sync resource dynamically. + """ request.postpath.insert(0, request.prepath.pop()) - return self.children['sync'] + return self._sync_resource diff --git a/testing/tests/server/test__resource.py b/testing/tests/server/test__resource.py index 2a387416..30ef782d 100644 --- a/testing/tests/server/test__resource.py +++ b/testing/tests/server/test__resource.py @@ -19,8 +19,8 @@ Tests for Soledad server main resource. """ from twisted.trial import unittest from twisted.web.test.test_web import DummyRequest -from twisted.web.error import Error from twisted.web.wsgi import WSGIResource +from twisted.web.resource import getChildForRequest from twisted.internet import reactor from leap.soledad.server._resource import SoledadResource @@ -37,33 +37,30 @@ class SoledadResourceTestCase(unittest.TestCase): def test_get_root(self): conf = {'soledad-server': {'blobs': None}} # doesn't matter resource = SoledadResource(conf, sync_pool=_pool) - path = '' - request = DummyRequest([]) - child = resource.getChild(path, request) + request = DummyRequest(['']) + child = getChildForRequest(resource, request) self.assertIsInstance(child, ServerInfo) def test_get_blobs_enabled(self): conf = {'soledad-server': {'blobs': True}} resource = SoledadResource(conf, sync_pool=_pool) - path = 'blobs' - request = DummyRequest([]) - child = resource.getChild(path, request) + request = DummyRequest(['blobs']) + child = getChildForRequest(resource, request) self.assertIsInstance(child, BlobsResource) def test_get_blobs_disabled(self): conf = {'soledad-server': {'blobs': False}} resource = SoledadResource(conf, sync_pool=_pool) - path = 'blobs' - request = DummyRequest([]) - with self.assertRaises(Error): - resource.getChild(path, request) + request = DummyRequest(['blobs']) + child = getChildForRequest(resource, request) + # if blobs is disabled, the request should be routed to sync + self.assertIsInstance(child, WSGIResource) + self.assertIsInstance(child._application, GzipMiddleware) def test_get_sync(self): conf = {'soledad-server': {'blobs': None}} # doesn't matter resource = SoledadResource(conf, sync_pool=_pool) - path = 'sync' # if not 'blobs' or '', should be routed to sync - request = DummyRequest([]) - request.prepath = ['user-db'] - child = resource.getChild(path, request) + request = DummyRequest(['user-db', 'sync-from', 'source-id']) + child = getChildForRequest(resource, request) self.assertIsInstance(child, WSGIResource) self.assertIsInstance(child._application, GzipMiddleware) -- cgit v1.2.3 From a2f041de7f1ea653f078ac9cd532e2d2b774248f Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 2 Feb 2017 11:45:57 -0200 Subject: [refactor] parametrize blobs toggling in soledad server resource --- server/src/leap/soledad/server/_resource.py | 17 ++++++++++++----- server/src/leap/soledad/server/auth.py | 10 ++++++++-- testing/tests/server/test__resource.py | 16 ++++++++-------- testing/tests/server/test_auth.py | 3 ++- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index fec290a2..156e18aa 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -33,17 +33,24 @@ class SoledadResource(Resource): for the Soledad Server. """ - def __init__(self, conf, sync_pool=None): - Resource.__init__(self) + def __init__(self, enable_blobs=False, sync_pool=None): + """ + Initialize the Soledad resource. + + :param enable_blobs: Whether the blobs feature should be enabled. + :type enable_blobs: bool - blobs_enabled = conf['soledad-server']['blobs'] + :param sync_pool: A pool to pass to the WSGI sync resource. + :type sync_pool: twisted.python.threadpool.ThreadPool + """ + Resource.__init__(self) # requests to / return server information - server_info = ServerInfo(blobs_enabled) + server_info = ServerInfo(enable_blobs) self.putChild('', server_info) # requests to /blobs will serve blobs if enabled - if blobs_enabled: + if enable_blobs: self.putChild('blobs', blobs_resource) # other requests are routed to legacy sync resource diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index d7ccdeb9..0ec890ca 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -42,12 +42,18 @@ from ._config import get_config @implementer(IRealm) class SoledadRealm(object): - def __init__(self, sync_pool=None): + def __init__(self, conf=None, sync_pool=None): + if not conf: + conf = get_config() + self._conf = conf self._sync_pool = sync_pool def requestAvatar(self, avatarId, mind, *interfaces): if IResource in interfaces: - resource = SoledadResource(sync_pool=self._sync_pool) + enable_blobs = self._conf['soledad-server']['blobs'] + resource = SoledadResource( + enable_blobs=enable_blobs, + sync_pool=self._sync_pool) return (IResource, resource, lambda: None) raise NotImplementedError() diff --git a/testing/tests/server/test__resource.py b/testing/tests/server/test__resource.py index 30ef782d..c066435e 100644 --- a/testing/tests/server/test__resource.py +++ b/testing/tests/server/test__resource.py @@ -35,22 +35,22 @@ _pool = reactor.getThreadPool() class SoledadResourceTestCase(unittest.TestCase): def test_get_root(self): - conf = {'soledad-server': {'blobs': None}} # doesn't matter - resource = SoledadResource(conf, sync_pool=_pool) + enable_blobs = None # doesn't matter + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) request = DummyRequest(['']) child = getChildForRequest(resource, request) self.assertIsInstance(child, ServerInfo) def test_get_blobs_enabled(self): - conf = {'soledad-server': {'blobs': True}} - resource = SoledadResource(conf, sync_pool=_pool) + enable_blobs = True + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) request = DummyRequest(['blobs']) child = getChildForRequest(resource, request) self.assertIsInstance(child, BlobsResource) def test_get_blobs_disabled(self): - conf = {'soledad-server': {'blobs': False}} - resource = SoledadResource(conf, sync_pool=_pool) + enable_blobs = False + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) request = DummyRequest(['blobs']) child = getChildForRequest(resource, request) # if blobs is disabled, the request should be routed to sync @@ -58,8 +58,8 @@ class SoledadResourceTestCase(unittest.TestCase): self.assertIsInstance(child._application, GzipMiddleware) def test_get_sync(self): - conf = {'soledad-server': {'blobs': None}} # doesn't matter - resource = SoledadResource(conf, sync_pool=_pool) + enable_blobs = None # doesn't matter + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) request = DummyRequest(['user-db', 'sync-from', 'source-id']) child = getChildForRequest(resource, request) self.assertIsInstance(child, WSGIResource) diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index f7fe0f25..00d416d5 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -39,8 +39,9 @@ class SoledadRealmTestCase(unittest.TestCase): def test_returned_resource(self): # we have to pass a pool to the realm , otherwise tests will hang + conf = {'soledad-server': {'blobs': False}} pool = reactor.getThreadPool() - realm = SoledadRealm(sync_pool=pool) + realm = SoledadRealm(conf=conf, sync_pool=pool) iface, avatar, logout = realm.requestAvatar('any', None, IResource) self.assertIsInstance(avatar, SoledadResource) self.assertIsNone(logout()) -- cgit v1.2.3 From 4ae57257fa2d40ceeba1558d995d3514e6f6d6fa Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 2 Feb 2017 12:28:19 -0200 Subject: [test] move server url mapper tests to its own file --- testing/tests/server/test_config.py | 2 +- testing/tests/server/test_server.py | 109 -------------------------- testing/tests/server/test_url_mapper.py | 131 ++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 110 deletions(-) create mode 100644 testing/tests/server/test_url_mapper.py diff --git a/testing/tests/server/test_config.py b/testing/tests/server/test_config.py index 1241472b..133057f5 100644 --- a/testing/tests/server/test_config.py +++ b/testing/tests/server/test_config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # test_config.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2017 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index ec0fc31d..647ef5a8 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -25,7 +25,6 @@ from urlparse import urljoin from uuid import uuid4 from twisted.internet import defer -from twisted.trial import unittest from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.couch import CouchDatabase @@ -39,114 +38,6 @@ from test_soledad.util import ( from leap.soledad.client import _crypto from leap.soledad.client import Soledad -from leap.soledad.server.url_mapper import URLMapper - - -class URLMapperTestCase(unittest.TestCase): - """ - Test if the URLMapper behaves as expected. - - The following table lists the authorized actions among all possible - u1db remote actions: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{id} | - - /shared-db/sync-from/{source} | - - /user-db | - - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - - def setUp(self): - self._uuid = uuid4().hex - self._urlmap = URLMapper() - self._dbname = 'user-%s' % self._uuid - - def test_root_authorized(self): - match = self._urlmap.match('/', 'GET') - self.assertIsNotNone(match) - - def test_shared_authorized(self): - self.assertIsNotNone(self._urlmap.match('/shared', 'GET')) - - def test_shared_unauthorized(self): - self.assertIsNone(self._urlmap.match('/shared', 'PUT')) - self.assertIsNone(self._urlmap.match('/shared', 'DELETE')) - self.assertIsNone(self._urlmap.match('/shared', 'POST')) - - def test_shared_docs_unauthorized(self): - self.assertIsNone(self._urlmap.match('/shared/docs', 'GET')) - self.assertIsNone(self._urlmap.match('/shared/docs', 'PUT')) - self.assertIsNone(self._urlmap.match('/shared/docs', 'DELETE')) - self.assertIsNone(self._urlmap.match('/shared/docs', 'POST')) - - def test_shared_doc_authorized(self): - match = self._urlmap.match('/shared/doc/x', 'GET') - self.assertIsNotNone(match) - self.assertEqual('x', match.get('id')) - - match = self._urlmap.match('/shared/doc/x', 'PUT') - self.assertIsNotNone(match) - self.assertEqual('x', match.get('id')) - - match = self._urlmap.match('/shared/doc/x', 'DELETE') - self.assertIsNotNone(match) - self.assertEqual('x', match.get('id')) - - def test_shared_doc_unauthorized(self): - self.assertIsNone(self._urlmap.match('/shared/doc/x', 'POST')) - - def test_shared_sync_unauthorized(self): - self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'GET')) - self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'PUT')) - self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'DELETE')) - self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'POST')) - - def test_user_db_unauthorized(self): - dbname = self._dbname - self.assertIsNone(self._urlmap.match('/%s' % dbname, 'GET')) - self.assertIsNone(self._urlmap.match('/%s' % dbname, 'PUT')) - self.assertIsNone(self._urlmap.match('/%s' % dbname, 'DELETE')) - self.assertIsNone(self._urlmap.match('/%s' % dbname, 'POST')) - - def test_user_db_docs_unauthorized(self): - dbname = self._dbname - self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'GET')) - self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'PUT')) - self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'DELETE')) - self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'POST')) - - def test_user_db_doc_unauthorized(self): - dbname = self._dbname - self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'GET')) - self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'PUT')) - self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'DELETE')) - self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'POST')) - - def test_user_db_sync_authorized(self): - uuid = self._uuid - dbname = self._dbname - match = self._urlmap.match('/%s/sync-from/x' % dbname, 'GET') - self.assertEqual(uuid, match.get('uuid')) - self.assertEqual('x', match.get('source_replica_uid')) - - match = self._urlmap.match('/%s/sync-from/x' % dbname, 'PUT') - self.assertEqual(uuid, match.get('uuid')) - self.assertEqual('x', match.get('source_replica_uid')) - - match = self._urlmap.match('/%s/sync-from/x' % dbname, 'POST') - self.assertEqual(uuid, match.get('uuid')) - self.assertEqual('x', match.get('source_replica_uid')) - - def test_user_db_sync_unauthorized(self): - dbname = self._dbname - self.assertIsNone( - self._urlmap.match('/%s/sync-from/x' % dbname, 'DELETE')) @pytest.mark.usefixtures("method_tmpdir") diff --git a/testing/tests/server/test_url_mapper.py b/testing/tests/server/test_url_mapper.py new file mode 100644 index 00000000..fa99cae7 --- /dev/null +++ b/testing/tests/server/test_url_mapper.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# test_url_mapper.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server-related functionality. +""" + +from twisted.trial import unittest +from uuid import uuid4 + +from leap.soledad.server.url_mapper import URLMapper + + +class URLMapperTestCase(unittest.TestCase): + """ + Test if the URLMapper behaves as expected. + + The following table lists the authorized actions among all possible + u1db remote actions: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{id} | - + /shared-db/sync-from/{source} | - + /user-db | - + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST + """ + + def setUp(self): + self._uuid = uuid4().hex + self._urlmap = URLMapper() + self._dbname = 'user-%s' % self._uuid + + def test_root_authorized(self): + match = self._urlmap.match('/', 'GET') + self.assertIsNotNone(match) + + def test_shared_authorized(self): + self.assertIsNotNone(self._urlmap.match('/shared', 'GET')) + + def test_shared_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared', 'POST')) + + def test_shared_docs_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/docs', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'POST')) + + def test_shared_doc_authorized(self): + match = self._urlmap.match('/shared/doc/x', 'GET') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = self._urlmap.match('/shared/doc/x', 'PUT') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = self._urlmap.match('/shared/doc/x', 'DELETE') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + def test_shared_doc_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/doc/x', 'POST')) + + def test_shared_sync_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'POST')) + + def test_user_db_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'POST')) + + def test_user_db_docs_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'POST')) + + def test_user_db_doc_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'POST')) + + def test_user_db_sync_authorized(self): + uuid = self._uuid + dbname = self._dbname + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'GET') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'PUT') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'POST') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + def test_user_db_sync_unauthorized(self): + dbname = self._dbname + self.assertIsNone( + self._urlmap.match('/%s/sync-from/x' % dbname, 'DELETE')) -- cgit v1.2.3 From b6359ed68d611cfa11b11e0591eb6186351793bd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Feb 2017 18:01:40 +0100 Subject: [pkg] add systemd service file to master this is to ease the packaging flow used in some environments like Pixelated, that use a debian branch against different branches. - Resolves: #8762 --- server/pkg/soledad-server.service | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 server/pkg/soledad-server.service diff --git a/server/pkg/soledad-server.service b/server/pkg/soledad-server.service new file mode 100644 index 00000000..ccd03b97 --- /dev/null +++ b/server/pkg/soledad-server.service @@ -0,0 +1,31 @@ +[Unit] +Description=Soledad Server + +[Service] +Environment=PATH=/sbin:/bin:/usr/sbin:/usr/bin +Environment=CLASS=leap.soledad.server.resource.SoledadResource +Environment=HTTPS_PORT=2424 +Environment=CERT_PATH=/etc/soledad/soledad-server.pem +Environment=PRIVKEY_PATH=/etc/soledad/soledad-server.key +Environment=HOME=/var/lib/soledad/ +Environment=SSL_METHOD=SSLv23_METHOD +EnvironmentFile=-/etc/default/soledad + +ExecStart=/usr/bin/twistd \ + --nodaemon \ + --pidfile= \ + --syslog \ + --prefix=soledad-server \ + web \ + --class=${CLASS} \ + --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD} + +WorkingDirectory=/var/lib/soledad/ + +User=soledad +Group=soledad + +Restart=always + +[Install] +WantedBy=multi-user.target -- cgit v1.2.3 From 71010be44aaa1086f6417198b46c22266b3d605a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Feb 2017 18:05:00 +0100 Subject: [docs] add changelog-next file --- docs/changelog-next.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/changelog-next.rst diff --git a/docs/changelog-next.rst b/docs/changelog-next.rst new file mode 100644 index 00000000..c5430b54 --- /dev/null +++ b/docs/changelog-next.rst @@ -0,0 +1,28 @@ +0.x.x - ++++++++++++++++++++++++++++++++ + +Please add lines to this file, they will be moved to the CHANGELOG.rst during +the next release. + +There are two template lines for each category, use them as reference. + +I've added a new category `Misc` so we can track doc/style/packaging stuff. + +Features +~~~~~~~~ +- `#1234 `_: Description of the new feature corresponding with issue #1234. +- New feature without related issue number. + +Bugfixes +~~~~~~~~ +- `#1235 `_: Description for the fixed stuff corresponding with issue #1235. +- Bugfix without related issue number. + +Other +~~~~~ +- #8762: Add a systemd service file +- `#1236 `_: Description of the new feature corresponding with issue #1236. +- Some change without issue number. + +Known Issues +~~~~~~~~~~~~ -- cgit v1.2.3 From cfc0dadc7cc927dd9ee6df8c07c1094ccfdc53c5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Feb 2017 00:15:25 +0100 Subject: [bug] fix typo in the resource path --- server/pkg/soledad-server.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pkg/soledad-server.service b/server/pkg/soledad-server.service index ccd03b97..85a156b5 100644 --- a/server/pkg/soledad-server.service +++ b/server/pkg/soledad-server.service @@ -3,7 +3,7 @@ Description=Soledad Server [Service] Environment=PATH=/sbin:/bin:/usr/sbin:/usr/bin -Environment=CLASS=leap.soledad.server.resource.SoledadResource +Environment=CLASS=leap.soledad.server._resource.SoledadResource Environment=HTTPS_PORT=2424 Environment=CERT_PATH=/etc/soledad/soledad-server.pem Environment=PRIVKEY_PATH=/etc/soledad/soledad-server.key -- cgit v1.2.3 From c5388e464907ab0b7192d269173d632af14ae4f1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Feb 2017 00:16:56 +0100 Subject: [bug] fix import for the session module --- server/src/leap/soledad/server/entrypoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index cfc557a3..0bb1c854 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -20,8 +20,8 @@ The entrypoint for Soledad server. from twisted.internet import reactor from ._config import get_config -from ._session import SoledadSession from ._wsgi import init_couch_state +from .session import SoledadSession # load configuration from file -- cgit v1.2.3 From 8a1cbb7bd1ea79408e9ca8f777277d1155f47504 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Feb 2017 01:57:26 +0100 Subject: [bug] effectively load the configuration for the app the code for passing the configuration to the couch initialization was never called. it seems the entrypoint module wasn't finally hooked as expected. I think this fixes the problem, but further review is needed here: either the entrypoint module is to be used, or it better is removed. in the first case, this workaround probably needs to be reverted. --- server/src/leap/soledad/server/_wsgi.py | 10 ++++++++++ server/src/leap/soledad/server/entrypoint.py | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index 37a03ced..f0961eaf 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -27,6 +27,7 @@ from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.log import getLogger +from ._config import get_config __all__ = ['init_couch_state', 'get_sync_resource'] @@ -66,3 +67,12 @@ def get_sync_resource(pool=None): reactor.callWhenRunning(pool.start) reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) return WSGIResource(reactor, pool, wsgi_application) + + +# load configuration from file +conf = get_config() + +# see the comments in application.py recarding why couch state has to be +# initialized when the reactor is running + +reactor.callWhenRunning(init_couch_state, conf) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index 0bb1c854..9cc1f97b 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -17,6 +17,7 @@ """ The entrypoint for Soledad server. """ + from twisted.internet import reactor from ._config import get_config @@ -34,6 +35,12 @@ class SoledadEntrypoint(SoledadSession): SoledadSession.__init__(self, conf) +# XXX FIXME ---------------------------- +# this is not executed from anywhere. +# what's the plan for this module? +# use me, or delete me. +# -------------------------------------- # see the comments in application.py recarding why couch state has to be # initialized when the reactor is running + reactor.callWhenRunning(init_couch_state, conf['soledad-server']) -- cgit v1.2.3 From 3b41a4de9c0d0f32815fbd11df3b379bc8688069 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Feb 2017 22:45:40 +0100 Subject: [pkg] use the entrypoint from the systemd script too --- server/pkg/soledad-server.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pkg/soledad-server.service b/server/pkg/soledad-server.service index 85a156b5..30c4bf88 100644 --- a/server/pkg/soledad-server.service +++ b/server/pkg/soledad-server.service @@ -3,7 +3,7 @@ Description=Soledad Server [Service] Environment=PATH=/sbin:/bin:/usr/sbin:/usr/bin -Environment=CLASS=leap.soledad.server._resource.SoledadResource +Environment=CLASS=leap.soledad.server.entrypoint.SoledadEntrypoint Environment=HTTPS_PORT=2424 Environment=CERT_PATH=/etc/soledad/soledad-server.pem Environment=PRIVKEY_PATH=/etc/soledad/soledad-server.key -- cgit v1.2.3 From e095cd3f00a06bf0e3705b28d01b49b61388ef85 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Feb 2017 22:46:15 +0100 Subject: [bug] revert loading from the wsgi entrypoint --- server/src/leap/soledad/server/_wsgi.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index f0961eaf..37a03ced 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -27,7 +27,6 @@ from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.log import getLogger -from ._config import get_config __all__ = ['init_couch_state', 'get_sync_resource'] @@ -67,12 +66,3 @@ def get_sync_resource(pool=None): reactor.callWhenRunning(pool.start) reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) return WSGIResource(reactor, pool, wsgi_application) - - -# load configuration from file -conf = get_config() - -# see the comments in application.py recarding why couch state has to be -# initialized when the reactor is running - -reactor.callWhenRunning(init_couch_state, conf) -- cgit v1.2.3 From 05b604a7134866c23aa98069cfc8542feaa08404 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Feb 2017 22:47:03 +0100 Subject: [bug] fix config handling after refactor --- server/src/leap/soledad/server/auth.py | 2 +- server/src/leap/soledad/server/entrypoint.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 0ec890ca..c5954c60 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -50,7 +50,7 @@ class SoledadRealm(object): def requestAvatar(self, avatarId, mind, *interfaces): if IResource in interfaces: - enable_blobs = self._conf['soledad-server']['blobs'] + enable_blobs = self._conf['blobs'] resource = SoledadResource( enable_blobs=enable_blobs, sync_pool=self._sync_pool) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index 9cc1f97b..8078a54a 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -16,6 +16,9 @@ # along with this program. If not, see . """ The entrypoint for Soledad server. + +This is the entrypoint for the application that is loaded from the initscript +or the systemd script. """ from twisted.internet import reactor @@ -26,21 +29,16 @@ from .session import SoledadSession # load configuration from file -conf = get_config +conf = get_config() class SoledadEntrypoint(SoledadSession): def __init__(self): - SoledadSession.__init__(self, conf) + SoledadSession.__init__(self) -# XXX FIXME ---------------------------- -# this is not executed from anywhere. -# what's the plan for this module? -# use me, or delete me. -# -------------------------------------- # see the comments in application.py recarding why couch state has to be # initialized when the reactor is running -reactor.callWhenRunning(init_couch_state, conf['soledad-server']) +reactor.callWhenRunning(init_couch_state, conf) -- cgit v1.2.3 From 6cabe46e4671627c22d5eed9ebb3bdc751948414 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 12 Feb 2017 20:33:48 +0100 Subject: [refactor] update create-user-db script to use config refactor --- server/pkg/create-user-db | 7 ++++--- server/src/leap/soledad/server/_config.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 9e2b6b50..fe1a99a0 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -38,8 +38,9 @@ parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') parser.add_argument('--migrate-all', action='store_true', help="recreate all design docs for all existing account") -CONF = load_configuration('/etc/soledad/soledad-server.conf') -NETRC_PATH = CONF['soledad-server']['admin_netrc'] +CONF = get_config() +DBCONF = get_config(section='database-security') +NETRC_PATH = CONF['admin_netrc'] def url_for_db(dbname): @@ -78,7 +79,7 @@ def ensure_database(dbname): print ("Invalid name! %s" % dbname) sys.exit(1) url = url_for_db(dbname) - db_security = CONF['database-security'] + db_security = DBCONF db = CouchDatabase.open_database(url=url, create=True, replica_uid=None, database_security=db_security) diff --git a/server/src/leap/soledad/server/_config.py b/server/src/leap/soledad/server/_config.py index 2c7f530d..1818c38d 100644 --- a/server/src/leap/soledad/server/_config.py +++ b/server/src/leap/soledad/server/_config.py @@ -42,11 +42,11 @@ CONFIG_DEFAULTS = { _config = None -def get_config(): +def get_config(section='soledad-server'): global _config if not _config: _config = _load_config('/etc/soledad/soledad-server.conf') - return _config['soledad-server'] + return _config[section] def _load_config(file_path): -- cgit v1.2.3 From 1a5b292bd1fbd57b0b3127857e74bdf1ac22a7c6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 12 Feb 2017 20:40:01 +0100 Subject: [bug] get a new server instance on each request to the tokens db --- server/src/leap/soledad/server/auth.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index c5954c60..7112aa35 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -69,13 +69,11 @@ class TokenChecker(object): TOKENS_TYPE_DEF = "Token" TOKENS_USER_ID_KEY = "user_id" - def __init__(self, server=None): - if server is None: - config = get_config() - couch_url = config['couch_url'] - server = couch_server(couch_url) - self._server = server - self._dbs = {} + def __init__(self): + self._couch_url = get_config().get('couch_url') + + def _get_server(self): + return couch_server(self._couch_url) def _tokens_dbname(self): # the tokens db rotates every 30 days, and the current db name is @@ -90,7 +88,11 @@ class TokenChecker(object): def _tokens_db(self): dbname = self._tokens_dbname() - with self._server as server: + + # TODO -- leaking abstraction here: this module shouldn't need + # to known anything about the context manager. hide that in the couch + # module + with self._get_server() as server: db = server[dbname] return db @@ -99,11 +101,14 @@ class TokenChecker(object): token = credentials.password # lookup key is a hash of the token to prevent timing attacks. + # TODO cache the tokens already! + db = self._tokens_db() token = db.get(sha512(token).hexdigest()) if token is None: return defer.fail(error.UnauthorizedLogin()) + # TODO -- use cryptography constant time builtin comparison. # we compare uuid hashes to avoid possible timing attacks that # might exploit python's builtin comparison operator behaviour, # which fails immediatelly when non-matching bytes are found. -- cgit v1.2.3 From 3fb32423b75898f4eb0812c8927dd7f06839e756 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 12 Feb 2017 23:14:55 -0300 Subject: [tests] server parameter is gone, patch the module --- testing/tests/server/test_auth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index 00d416d5..c9b941f9 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -29,6 +29,7 @@ from twisted.trial import unittest from twisted.web.resource import IResource from twisted.web.test import test_httpauth +import leap.soledad.server.auth as auth_module from leap.soledad.server.auth import SoledadRealm from leap.soledad.server.auth import TokenChecker from leap.soledad.server.auth import TokenCredentialFactory @@ -73,7 +74,8 @@ class TokenCheckerTestCase(unittest.TestCase): token = {'user_id': 'user', 'type': 'Token'} server = dummy_server(token) # setup the checker with the custom server - checker = TokenChecker(server=server) + checker = TokenChecker() + auth_module.couch_server = lambda url: server # assert the checker *can* verify the creds creds = UsernamePassword('user', 'pass') avatarId = yield checker.requestAvatarId(creds) @@ -85,7 +87,8 @@ class TokenCheckerTestCase(unittest.TestCase): token = None server = dummy_server(token) # setup the checker with the custom server - checker = TokenChecker(server=server) + checker = TokenChecker() + auth_module.couch_server = lambda url: server # assert the checker *cannot* verify the creds creds = UsernamePassword('user', '') with self.assertRaises(UnauthorizedLogin): -- cgit v1.2.3 From 45c58edc99009fc0f2ac044a08a74fd742326a58 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 12 Feb 2017 23:15:30 -0300 Subject: [tests] conf format changed, no more nesting --- testing/tests/server/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py index c9b941f9..6eb647ee 100644 --- a/testing/tests/server/test_auth.py +++ b/testing/tests/server/test_auth.py @@ -40,7 +40,7 @@ class SoledadRealmTestCase(unittest.TestCase): def test_returned_resource(self): # we have to pass a pool to the realm , otherwise tests will hang - conf = {'soledad-server': {'blobs': False}} + conf = {'blobs': False} pool = reactor.getThreadPool() realm = SoledadRealm(conf=conf, sync_pool=pool) iface, avatar, logout = realm.requestAvatar('any', None, IResource) -- cgit v1.2.3 From 12e14b761d4669a2f654d870d444356623ffc56f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Feb 2017 14:19:49 +0100 Subject: [bug] fix import typo on create-user-db --- server/pkg/create-user-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index fe1a99a0..5cc1754b 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -22,7 +22,7 @@ import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch.state import is_db_name_valid from leap.soledad.common.couch import list_users_dbs -from leap.soledad.server.config import load_configuration +from leap.soledad.server.config import get_config BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False) -- cgit v1.2.3 From 19dfe7e9d1f1862026e768301da2ee1273ec7f9f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Feb 2017 14:28:07 +0100 Subject: [bug] fix typo, again --- server/pkg/create-user-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 5cc1754b..5e0ef5e2 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -22,7 +22,7 @@ import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch.state import is_db_name_valid from leap.soledad.common.couch import list_users_dbs -from leap.soledad.server.config import get_config +from leap.soledad.server._config import get_config BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False) -- cgit v1.2.3 From e600a7f57c177fa45c598d24248ef937f1d02e4b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Feb 2017 14:13:14 +0100 Subject: [pkg] add script for deploying from git - Resolves: #8765 --- scripts/deploy/deploy_soledad_from_git | 51 ++++++++++++++++++++++++++++++++++ server/pkg/soledad-sudoers | 2 ++ 2 files changed, 53 insertions(+) create mode 100755 scripts/deploy/deploy_soledad_from_git create mode 100644 server/pkg/soledad-sudoers diff --git a/scripts/deploy/deploy_soledad_from_git b/scripts/deploy/deploy_soledad_from_git new file mode 100755 index 00000000..7e4ca29c --- /dev/null +++ b/scripts/deploy/deploy_soledad_from_git @@ -0,0 +1,51 @@ +#!/bin/bash +###################################################### +# Deploy soledad-server from a given remote and branch +# valid remotes are: origin shyba drebs kali +# DO NOT USE IN PRODUCTION OR I'LL SEND NINJAS TO YOUR +# HOUSE!!!! +# (c) LEAP, 2017 +###################################################### + +REMOTE=$1 +BRANCH=$2 + +if [ "$#" -lt 2 ]; then + echo "USAGE: $0 REMOTE BRANCH" + exit 1 +fi + +SOLEDADPATH="/usr/lib/python2.7/dist-packages/leap/soledad/server" +REPO="https://0xacab.org/leap/soledad" +LOCALREPO="$HOME/soledad" +SYSTEMDINIT="/lib/systemd/system/soledad-server.service" + +apt remove --yes soledad-server + +if [ ! -d "$LOCALREPO" ]; then + echo "soledad repo not found, cloning..." + cd $HOME + git clone $REPO + cd $LOCALREPO + git remote add shyba https://0xacab.org/shyba/soledad.git + git remote add drebs https://0xacab.org/drebs/soledad.git + git remote add kali https://0xacab.org/kali/soledad.git +fi + +cd $LOCALREPO && git checkout -- . && git fetch $REMOTE && git checkout $REMOTE/$BRANCH +rm -rf $SOLEDADPATH + +# copy over some stuff that we'll need -- stolen from debian package +cp -r $LOCALREPO/server/src/leap/soledad/server $SOLEDADPATH +cp $LOCALREPO/server/pkg/soledad-server.service $SYSTEMDINIT +cp $LOCALREPO/server/pkg/create-user-db /usr/bin/ +cp $LOCALREPO/server/pkg/soledad-sudoers /etc/sudoers.d/ + +# Let's append the branch info to the version string! So that nobody is lost +cd $LOCALREPO && echo "__version__ = '"`git describe`"~"`git status | head -n 1 | cut -d' ' -f 4`"'" >> $SOLEDADPATH/__init__.py + +# restart the daemon +systemctl --system daemon-reload >/dev/null || true +deb-systemd-invoke start soledad-server.service >/dev/null || true + +tail -n 20 /var/log/syslog diff --git a/server/pkg/soledad-sudoers b/server/pkg/soledad-sudoers new file mode 100644 index 00000000..642497f8 --- /dev/null +++ b/server/pkg/soledad-sudoers @@ -0,0 +1,2 @@ +Cmnd_Alias SOLEDAD_CREATE_DB = /usr/bin/create-user-db +soledad ALL=(soledad-admin) NOPASSWD: SOLEDAD_CREATE_DB -- cgit v1.2.3 From 1c002ae9bc2a5c38f078550d993c229c32586ea8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Feb 2017 14:44:28 +0100 Subject: [pkg] fail on errors --- scripts/deploy/deploy_soledad_from_git | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/deploy/deploy_soledad_from_git b/scripts/deploy/deploy_soledad_from_git index 7e4ca29c..0f914199 100755 --- a/scripts/deploy/deploy_soledad_from_git +++ b/scripts/deploy/deploy_soledad_from_git @@ -6,6 +6,7 @@ # HOUSE!!!! # (c) LEAP, 2017 ###################################################### +set -e REMOTE=$1 BRANCH=$2 -- cgit v1.2.3 From 6c7e7294da800febc3266cf37066b2e59917eb67 Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 4 Feb 2017 08:07:44 -0200 Subject: [pkg] add build-deb.sh script --- scripts/packaging/build-deb.sh | 32 +++++++++++++++++++++++++++++++ scripts/packaging/build_debian_package.sh | 32 ------------------------------- 2 files changed, 32 insertions(+), 32 deletions(-) create mode 100755 scripts/packaging/build-deb.sh delete mode 100755 scripts/packaging/build_debian_package.sh diff --git a/scripts/packaging/build-deb.sh b/scripts/packaging/build-deb.sh new file mode 100755 index 00000000..f7dd22a0 --- /dev/null +++ b/scripts/packaging/build-deb.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# This script generates a debian package from your current repository tree +# (including modified and unstaged files), using the debian directory from the +# latest debian/platform-X.Y branch. +# +# In order to achieve that, what it does is: +# +# - copy the current repository into a temporary directory. +# - find what is the latest "debian/platform-X.Y" branch. +# - checkout the "debian/" directory from that branch. +# - update the "debian/changelog" file with dummy information. +# - run "debuild -uc -us". + +debemail="Leap Automatic Deb Builder " +scriptdir=$(dirname "${0}") +gitroot=$(git -C "${scriptdir}" rev-parse --show-toplevel) +deb_branch=$(git -C "${gitroot}" branch | grep "debian/platform" | sort | tail -n 1 | xargs) +reponame=$(basename "${gitroot}") +tempdir=$(mktemp -d) +targetdir="${tempdir}/${reponame}" + +cp -r "${gitroot}" "${tempdir}/${reponame}" +git -C "${targetdir}" checkout "${deb_branch}" -- debian + +(cd "${targetdir}" && DEBEMAIL="${debemail}" dch -b "Automatic build.") +(cd "${targetdir}" && debuild -uc -us) + +echo "****************************************" +echo "Packages can be found in: ${tempdir}" +ls "${tempdir}" +echo "****************************************" diff --git a/scripts/packaging/build_debian_package.sh b/scripts/packaging/build_debian_package.sh deleted file mode 100755 index b9fb93a9..00000000 --- a/scripts/packaging/build_debian_package.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -# This script generates Soledad Debian packages. -# -# When invoking this script, you should pass a git repository URL and the name -# of the branch that contains the code you wish to build the packages from. -# -# The script will clone the given branch from the given repo, as well as the -# main Soledad repo in github which contains the most up-to-date debian -# branch. It will then merge the desired branch into the debian branch and -# build the packages. - -if [ $# -ne 2 ]; then - echo "Usage: ${0} " - exit 1 -fi - -SOLEDAD_MAIN_REPO=git://github.com/leapcode/soledad.git - -url=$1 -branch=$2 -workdir=`mktemp -d` - -git clone -b ${branch} ${url} ${workdir}/soledad -export GIT_DIR=${workdir}/soledad/.git -export GIT_WORK_TREE=${workdir}/soledad -git remote add leapcode ${SOLEDAD_MAIN_REPO} -git fetch leapcode -git checkout -b debian/experimental leapcode/debian/experimental -git merge --no-edit ${branch} -(cd ${workdir}/soledad && debuild -uc -us) -echo "Packages generated in ${workdir}" -- cgit v1.2.3 From 4d26c3c6ee43f110f30e0fb1c14d93f847c3081b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Feb 2017 14:53:25 +0100 Subject: [bug] add expected attribute to authentication wrapper in entrypoint the authentication wrapper is goin to look for the _credentialFactories attribute. it will raise an exception if not found - Resolves: #8766 --- server/src/leap/soledad/server/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index 8078a54a..b10bfed6 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -35,6 +35,7 @@ conf = get_config() class SoledadEntrypoint(SoledadSession): def __init__(self): + self._credentialFactories = [] SoledadSession.__init__(self) -- cgit v1.2.3 From e4a9127914dac8cf2cd8b8f7a4e48bbcf1de7a5d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 14 Feb 2017 23:31:36 +0100 Subject: [feature] add version to the banner --- server/src/leap/soledad/server/_server_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/leap/soledad/server/_server_info.py b/server/src/leap/soledad/server/_server_info.py index a1dd3555..50659338 100644 --- a/server/src/leap/soledad/server/_server_info.py +++ b/server/src/leap/soledad/server/_server_info.py @@ -21,6 +21,8 @@ import json from twisted.web.resource import Resource +from leap.soledad.server import __version__ + __all__ = ['ServerInfo'] @@ -35,6 +37,7 @@ class ServerInfo(Resource): def __init__(self, blobs_enabled): self._info = { "blobs": blobs_enabled, + "version": __version__ } def render_GET(self, request): -- cgit v1.2.3 From ed85f545a9965eb637c1544b3973e3d68e5a602c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 14 Feb 2017 23:52:13 +0100 Subject: [pkg] restart service instead of starting it --- scripts/deploy/deploy_soledad_from_git | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy/deploy_soledad_from_git b/scripts/deploy/deploy_soledad_from_git index 0f914199..3c4ac8be 100755 --- a/scripts/deploy/deploy_soledad_from_git +++ b/scripts/deploy/deploy_soledad_from_git @@ -47,6 +47,6 @@ cd $LOCALREPO && echo "__version__ = '"`git describe`"~"`git status | head -n 1 # restart the daemon systemctl --system daemon-reload >/dev/null || true -deb-systemd-invoke start soledad-server.service >/dev/null || true +deb-systemd-invoke restart soledad-server.service >/dev/null || true tail -n 20 /var/log/syslog -- cgit v1.2.3 From ccb280703ba851265702b8a92cdedb294cc93608 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 14 Feb 2017 23:40:29 +0100 Subject: [feature] authenticate as anonymous if no token in header and serve / banner and robots to anon users. instead of returning 401 for all cases, I treat the unauthenticated case as a special case, and switch the service tree apart. this allows to serve a different resource tree to unauthenticated users. the new URLs are registered with the mapper. I don't really like that dependency, could be handled by twisted alone, but meh. - Resolves: #8764 --- docs/changelog-next.rst | 3 +++ server/src/leap/soledad/server/_resource.py | 19 ++++++++++++++++++- server/src/leap/soledad/server/auth.py | 26 +++++++++++++++++++++++--- server/src/leap/soledad/server/session.py | 8 ++++++-- server/src/leap/soledad/server/url_mapper.py | 3 +++ 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/docs/changelog-next.rst b/docs/changelog-next.rst index c5430b54..9d5f6297 100644 --- a/docs/changelog-next.rst +++ b/docs/changelog-next.rst @@ -10,6 +10,9 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff. Features ~~~~~~~~ +- Refactor authentication code to use twisted credential system. +- `#8764 `_: Allow unauthenticated + users to retrieve the capabilties banner. - `#1234 `_: Description of the new feature corresponding with issue #1234. - New feature without related issue number. diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index 156e18aa..e04c0708 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -24,7 +24,24 @@ from ._server_info import ServerInfo from ._wsgi import get_sync_resource -__all__ = ['SoledadResource'] +__all__ = ['SoledadResource', 'SoledadAnonResource'] + + +class Robots(Resource): + def render_GET(self, request): + return 'robots, go away! please!' + + +class SoledadAnonResource(Resource): + """ + The parts of Soledad Server that unauthenticated users can see + """ + + def __init__(self, enable_blobs=False): + Resource.__init__(self) + server_info = ServerInfo(enable_blobs) + self.putChild('', server_info) + self.putChild('robots.txt', Robots()) class SoledadResource(Resource): diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 7112aa35..6ce11e71 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -26,19 +26,25 @@ from zope.interface import implementer from twisted.cred import error from twisted.cred.checkers import ICredentialsChecker from twisted.cred.credentials import IUsernamePassword +from twisted.cred.credentials import IAnonymous +from twisted.cred.credentials import Anonymous from twisted.cred.credentials import UsernamePassword from twisted.cred.portal import IRealm from twisted.cred.portal import Portal +from twisted.logger import Logger from twisted.internet import defer from twisted.web.iweb import ICredentialFactory from twisted.web.resource import IResource from leap.soledad.common.couch import couch_server -from ._resource import SoledadResource +from ._resource import SoledadResource, SoledadAnonResource from ._config import get_config +log = Logger() + + @implementer(IRealm) class SoledadRealm(object): @@ -49,8 +55,17 @@ class SoledadRealm(object): self._sync_pool = sync_pool def requestAvatar(self, avatarId, mind, *interfaces): + log.warn('avatarId {0}'.format(avatarId)) + enable_blobs = self._conf['blobs'] + + # Anonymous access + if IAnonymous.providedBy(avatarId): + resource = SoledadAnonResource( + enable_blobs=enable_blobs) + return (IResource, resource, lambda: None) + + # Authenticated users if IResource in interfaces: - enable_blobs = self._conf['blobs'] resource = SoledadResource( enable_blobs=enable_blobs, sync_pool=self._sync_pool) @@ -61,7 +76,7 @@ class SoledadRealm(object): @implementer(ICredentialsChecker) class TokenChecker(object): - credentialInterfaces = [IUsernamePassword] + credentialInterfaces = [IUsernamePassword, IAnonymous] TOKENS_DB_PREFIX = "tokens_" TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds @@ -97,6 +112,10 @@ class TokenChecker(object): return db def requestAvatarId(self, credentials): + if IAnonymous.providedBy(credentials): + log.warn('we are anon') + return defer.succeed(Anonymous()) + uuid = credentials.username token = credentials.password @@ -106,6 +125,7 @@ class TokenChecker(object): db = self._tokens_db() token = db.get(sha512(token).hexdigest()) if token is None: + log.warn('token is none') return defer.fail(error.UnauthorizedLogin()) # TODO -- use cryptography constant time builtin comparison. diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index a2793bd3..70e4a35b 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -19,8 +19,9 @@ Twisted resource containing an authenticated Soledad session. """ from zope.interface import implementer +from twisted.cred.credentials import Anonymous from twisted.cred import error -from twisted.python import log +from twisted.logger import Logger from twisted.web import util from twisted.web._auth import wrapper from twisted.web.guard import HTTPAuthSessionWrapper @@ -32,6 +33,9 @@ from leap.soledad.server.auth import credentialFactory from leap.soledad.server.url_mapper import URLMapper +log = Logger() + + @implementer(IResource) class UnauthorizedResource(wrapper.UnauthorizedResource): isLeaf = True @@ -80,7 +84,7 @@ class SoledadSession(HTTPAuthSessionWrapper): # get authorization header or fail header = request.getHeader(b'authorization') if not header: - return UnauthorizedResource() + return util.DeferredResource(self._login(Anonymous())) # parse the authorization header auth_data = self._parseHeader(header) diff --git a/server/src/leap/soledad/server/url_mapper.py b/server/src/leap/soledad/server/url_mapper.py index 483f7e87..a0edeaca 100644 --- a/server/src/leap/soledad/server/url_mapper.py +++ b/server/src/leap/soledad/server/url_mapper.py @@ -53,6 +53,7 @@ class URLMapper(object): URL path | Authorized actions -------------------------------------------------- / | GET + /robots.txt | GET /shared-db | GET /shared-db/docs | - /shared-db/doc/{any_id} | GET, PUT, DELETE @@ -64,6 +65,8 @@ class URLMapper(object): """ # auth info for global resource self._connect('/', ['GET']) + # robots + self._connect('/robots.txt', ['GET']) # auth info for shared-db database resource self._connect('/%s' % SHARED_DB_NAME, ['GET']) # auth info for shared-db doc resource -- cgit v1.2.3 From 87bfc0ec7d7faae9dceea3717611a1a2851ad93f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Feb 2017 00:37:04 +0100 Subject: [feature] add robots.txt -Resolves: #6178 --- server/src/leap/soledad/server/_resource.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py index e04c0708..7a00ad9a 100644 --- a/server/src/leap/soledad/server/_resource.py +++ b/server/src/leap/soledad/server/_resource.py @@ -27,21 +27,27 @@ from ._wsgi import get_sync_resource __all__ = ['SoledadResource', 'SoledadAnonResource'] -class Robots(Resource): +class _Robots(Resource): def render_GET(self, request): - return 'robots, go away! please!' + return ( + 'User-agent: *\n' + 'Disallow: /\n' + '# you are not a robot, are you???') class SoledadAnonResource(Resource): + """ - The parts of Soledad Server that unauthenticated users can see + The parts of Soledad Server that unauthenticated users can see. + This is nice because this means that a non-authenticated user will get 404 + for anything that is not in this minimal resource tree. """ def __init__(self, enable_blobs=False): Resource.__init__(self) server_info = ServerInfo(enable_blobs) self.putChild('', server_info) - self.putChild('robots.txt', Robots()) + self.putChild('robots.txt', _Robots()) class SoledadResource(Resource): -- cgit v1.2.3 From 5c6fe9dc71d2e47f4698acf550b9fd16ce86637b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Feb 2017 00:48:40 +0100 Subject: [docs] add a to-do about perf --- server/src/leap/soledad/server/auth.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 6ce11e71..aea003ff 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -31,8 +31,8 @@ from twisted.cred.credentials import Anonymous from twisted.cred.credentials import UsernamePassword from twisted.cred.portal import IRealm from twisted.cred.portal import Portal -from twisted.logger import Logger from twisted.internet import defer +from twisted.logger import Logger from twisted.web.iweb import ICredentialFactory from twisted.web.resource import IResource @@ -65,6 +65,11 @@ class SoledadRealm(object): return (IResource, resource, lambda: None) # Authenticated users + + # XXX review this... we're creating a Resource tree + # for each request, for every user. + # What are the perf implications of this?? + if IResource in interfaces: resource = SoledadResource( enable_blobs=enable_blobs, @@ -113,7 +118,6 @@ class TokenChecker(object): def requestAvatarId(self, credentials): if IAnonymous.providedBy(credentials): - log.warn('we are anon') return defer.succeed(Anonymous()) uuid = credentials.username @@ -125,7 +129,6 @@ class TokenChecker(object): db = self._tokens_db() token = db.get(sha512(token).hexdigest()) if token is None: - log.warn('token is none') return defer.fail(error.UnauthorizedLogin()) # TODO -- use cryptography constant time builtin comparison. -- cgit v1.2.3 From 6d7dd39fb3d4f138595f885d19315008d13f8907 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Feb 2017 01:53:01 +0100 Subject: [tests] fix tests --- server/src/leap/soledad/server/auth.py | 4 ++-- server/src/leap/soledad/server/entrypoint.py | 1 - server/src/leap/soledad/server/session.py | 8 +++++--- testing/tests/server/test__server_info.py | 8 ++++++-- testing/tests/server/test_session.py | 5 ++++- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index aea003ff..c52370cb 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -55,7 +55,6 @@ class SoledadRealm(object): self._sync_pool = sync_pool def requestAvatar(self, avatarId, mind, *interfaces): - log.warn('avatarId {0}'.format(avatarId)) enable_blobs = self._conf['blobs'] # Anonymous access @@ -66,7 +65,8 @@ class SoledadRealm(object): # Authenticated users - # XXX review this... we're creating a Resource tree + # TODO review this: #8770 ---------------- + # we're creating a Resource tree # for each request, for every user. # What are the perf implications of this?? diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index b10bfed6..8078a54a 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -35,7 +35,6 @@ conf = get_config() class SoledadEntrypoint(SoledadSession): def __init__(self): - self._credentialFactories = [] SoledadSession.__init__(self) diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 70e4a35b..515fd9b3 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -22,6 +22,7 @@ from zope.interface import implementer from twisted.cred.credentials import Anonymous from twisted.cred import error from twisted.logger import Logger +from twisted.python import log from twisted.web import util from twisted.web._auth import wrapper from twisted.web.guard import HTTPAuthSessionWrapper @@ -33,9 +34,6 @@ from leap.soledad.server.auth import credentialFactory from leap.soledad.server.url_mapper import URLMapper -log = Logger() - - @implementer(IResource) class UnauthorizedResource(wrapper.UnauthorizedResource): isLeaf = True @@ -63,6 +61,8 @@ class SoledadSession(HTTPAuthSessionWrapper): self._mapper = URLMapper() self._portal = portal self._credentialFactory = credentialFactory + # expected by the contract of the parent class + self._credentialFactories = [credentialFactory] def _matchPath(self, request): match = self._mapper.match(request.path, request.method) @@ -97,6 +97,8 @@ class SoledadSession(HTTPAuthSessionWrapper): except error.LoginFailed: return UnauthorizedResource() except: + # If you port this to the newer log facility, be aware that + # the tests rely on the error to be logged. log.err(None, "Unexpected failure from credentials factory") return ErrorPage(500, None, None) diff --git a/testing/tests/server/test__server_info.py b/testing/tests/server/test__server_info.py index 80770721..40567ef1 100644 --- a/testing/tests/server/test__server_info.py +++ b/testing/tests/server/test__server_info.py @@ -31,9 +31,13 @@ class ServerInfoTestCase(unittest.TestCase): def test_blobs_enabled(self): resource = ServerInfo(True) response = resource.render(DummyRequest([''])) - self.assertEquals(json.loads(response), {'blobs': True}) + _info = json.loads(response) + self.assertEquals(_info['blobs'], True) + self.assertTrue(isinstance(_info['version'], basestring)) def test_blobs_disabled(self): resource = ServerInfo(False) response = resource.render(DummyRequest([''])) - self.assertEquals(json.loads(response), {'blobs': False}) + _info = json.loads(response) + self.assertEquals(_info['blobs'], False) + self.assertTrue(isinstance(_info['version'], basestring)) diff --git a/testing/tests/server/test_session.py b/testing/tests/server/test_session.py index 8131ddb3..b00ab858 100644 --- a/testing/tests/server/test_session.py +++ b/testing/tests/server/test_session.py @@ -156,6 +156,7 @@ class SoledadSessionTestCase(unittest.TestCase): return {} def decode(self, response, request): + print "decode raised" raise UnexpectedException() self.wrapper._credentialFactory = BadFactory() @@ -164,7 +165,9 @@ class SoledadSessionTestCase(unittest.TestCase): child = getChildForRequest(self.wrapper, request) request.render(child) self.assertEqual(request.responseCode, 500) - self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) + #self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) + errors = self.flushLoggedErrors(UnexpectedException) + self.assertEqual(len(errors), 1) def test_unexpectedLoginError(self): class UnexpectedException(Exception): -- cgit v1.2.3 From 13fdd28cd7448b11a35c794e69e5c64e1c9cd154 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 15 Feb 2017 05:17:43 -0300 Subject: [style] pep8 --- server/src/leap/soledad/server/session.py | 1 - testing/tests/server/test_session.py | 1 - 2 files changed, 2 deletions(-) diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index 515fd9b3..c1ceb340 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -21,7 +21,6 @@ from zope.interface import implementer from twisted.cred.credentials import Anonymous from twisted.cred import error -from twisted.logger import Logger from twisted.python import log from twisted.web import util from twisted.web._auth import wrapper diff --git a/testing/tests/server/test_session.py b/testing/tests/server/test_session.py index b00ab858..ebb94476 100644 --- a/testing/tests/server/test_session.py +++ b/testing/tests/server/test_session.py @@ -165,7 +165,6 @@ class SoledadSessionTestCase(unittest.TestCase): child = getChildForRequest(self.wrapper, request) request.render(child) self.assertEqual(request.responseCode, 500) - #self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) errors = self.flushLoggedErrors(UnexpectedException) self.assertEqual(len(errors), 1) -- cgit v1.2.3 From e6ed77ce83a37dd4fffb8ac560ae34fbee8acc22 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 15 Feb 2017 17:43:17 -0300 Subject: [tests] add tests for preamble encoding --- client/src/leap/soledad/client/_crypto.py | 4 ++-- testing/tests/client/test_crypto.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index 4bbdd044..9f403cc9 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -192,7 +192,7 @@ class BlobEncryptor(object): sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret) self._aes = AESWriter(sym_key) - self._aes.authenticate(self._make_preamble()) + self._aes.authenticate(self._encode_preamble()) @property def iv(self): @@ -214,7 +214,7 @@ class BlobEncryptor(object): d.addCallback(lambda _: self._end_crypto_stream()) return d - def _make_preamble(self): + def _encode_preamble(self): current_time = int(time.time()) return PACMAN.pack( diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 5411a4e8..852448e0 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -235,6 +235,37 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): _crypto.decrypt_sym(cyphertext, wrongkey, iv) +class PreambleTestCase(unittest.TestCase): + class doc_info: + doc_id = 'D-deadbeef' + rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + + def setUp(self): + inf = BytesIO(snowden1) + self.blob = _crypto.BlobEncryptor( + self.doc_info, inf, + secret='A' * 96) + + def test_preamble_starts_with_magic_signature(self): + preamble = self.blob._encode_preamble() + assert preamble.startswith(_crypto.BLOB_SIGNATURE_MAGIC) + + def test_preamble_has_cipher_metadata(self): + preamble = self.blob._encode_preamble() + unpacked = _crypto.PACMAN.unpack(preamble) + encryption_scheme, encryption_method = unpacked[1:3] + assert encryption_scheme in _crypto.ENC_SCHEME + assert encryption_method in _crypto.ENC_METHOD + assert unpacked[4] == self.blob.iv + + def test_preamble_has_document_sync_metadata(self): + preamble = self.blob._encode_preamble() + unpacked = _crypto.PACMAN.unpack(preamble) + doc_id, doc_rev = unpacked[5:] + assert doc_id == self.doc_info.doc_id + assert doc_rev == self.doc_info.rev + + def _aes_encrypt(key, iv, data): backend = default_backend() cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend) -- cgit v1.2.3 From d2ef605af73a592ea21c5bae005f53f483e310a6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 16 Feb 2017 04:48:58 -0300 Subject: [feature] add doc size to preamble That's necessary for blobs-io. Current code includes backwards compatibility branching and tests, which shall be removed on next releases. --- client/src/leap/soledad/client/_crypto.py | 30 ++++++++++++++--------- testing/tests/client/test_crypto.py | 40 +++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index 9f403cc9..c4c6d336 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -49,7 +49,8 @@ SECRET_LENGTH = 64 CRYPTO_BACKEND = MultiBackend([OpenSSLBackend()]) -PACMAN = struct.Struct('2sbbQ16s255p255p') +PACMAN = struct.Struct('2sbbQ16s255p255pQ') +LEGACY_PACMAN = struct.Struct('2sbbQ16s255p255p') BLOB_SIGNATURE_MAGIC = '\x13\x37' @@ -188,10 +189,13 @@ class BlobEncryptor(object): self.doc_id = doc_info.doc_id self.rev = doc_info.rev self._content_fd = content_fd + content_fd.seek(0, os.SEEK_END) + self._content_size = content_fd.tell() + content_fd.seek(0) self._producer = FileBodyProducer(content_fd, readSize=2**16) - sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret) - self._aes = AESWriter(sym_key) + self.sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret) + self._aes = AESWriter(self.sym_key) self._aes.authenticate(self._encode_preamble()) @property @@ -224,7 +228,8 @@ class BlobEncryptor(object): current_time, self.iv, str(self.doc_id), - str(self.rev)) + str(self.rev), + self._content_size) def _end_crypto_stream(self): preamble, encrypted = self._aes.end() @@ -271,14 +276,17 @@ class BlobDecryptor(object): raise InvalidBlob ciphertext_fd.close() - if len(preamble) != PACMAN.size: - raise InvalidBlob - try: - unpacked_data = PACMAN.unpack(preamble) - magic, sch, meth, ts, iv, doc_id, rev = unpacked_data - except struct.error: - raise InvalidBlob + if len(preamble) == LEGACY_PACMAN.size: + unpacked_data = LEGACY_PACMAN.unpack(preamble) + magic, sch, meth, ts, iv, doc_id, rev = unpacked_data + elif len(preamble) == PACMAN.size: + unpacked_data = PACMAN.unpack(preamble) + magic, sch, meth, ts, iv, doc_id, rev, doc_size = unpacked_data + else: + raise InvalidBlob("Unexpected preamble size %d", len(preamble)) + except struct.error, e: + raise InvalidBlob(e) if magic != BLOB_SIGNATURE_MAGIC: raise InvalidBlob diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 852448e0..2d4d827b 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -110,7 +110,7 @@ class BlobTestCase(unittest.TestCase): assert len(preamble) == _crypto.PACMAN.size unpacked_data = _crypto.PACMAN.unpack(preamble) - magic, sch, meth, ts, iv, doc_id, rev = unpacked_data + magic, sch, meth, ts, iv, doc_id, rev, _ = unpacked_data assert magic == _crypto.BLOB_SIGNATURE_MAGIC assert sch == 1 assert meth == _crypto.ENC_METHOD.aes_256_gcm @@ -241,9 +241,9 @@ class PreambleTestCase(unittest.TestCase): rev = '397932e0c77f45fcb7c3732930e7e9b2:1' def setUp(self): - inf = BytesIO(snowden1) + self.cleartext = BytesIO(snowden1) self.blob = _crypto.BlobEncryptor( - self.doc_info, inf, + self.doc_info, self.cleartext, secret='A' * 96) def test_preamble_starts_with_magic_signature(self): @@ -261,15 +261,45 @@ class PreambleTestCase(unittest.TestCase): def test_preamble_has_document_sync_metadata(self): preamble = self.blob._encode_preamble() unpacked = _crypto.PACMAN.unpack(preamble) - doc_id, doc_rev = unpacked[5:] + doc_id, doc_rev = unpacked[5:7] assert doc_id == self.doc_info.doc_id assert doc_rev == self.doc_info.rev + def test_preamble_has_document_size(self): + preamble = self.blob._encode_preamble() + unpacked = _crypto.PACMAN.unpack(preamble) + size = unpacked[7] + assert size == len(snowden1) -def _aes_encrypt(key, iv, data): + @defer.inlineCallbacks + def test_preamble_can_come_without_size(self): + # XXX: This test case is here only to test backwards compatibility! + preamble = self.blob._encode_preamble() + # repack preamble using legacy format, without doc size + unpacked = _crypto.PACMAN.unpack(preamble) + preamble_without_size = _crypto.LEGACY_PACMAN.pack(*unpacked[0:7]) + # encrypt it manually for custom tag + ciphertext, tag = _aes_encrypt(self.blob.sym_key, self.blob.iv, + self.cleartext.getvalue(), + aead=preamble_without_size) + ciphertext = ciphertext + tag + # encode it + ciphertext = base64.urlsafe_b64encode(ciphertext) + preamble_without_size = base64.urlsafe_b64encode(preamble_without_size) + # decrypt it + ciphertext = preamble_without_size + ' ' + ciphertext + cleartext = yield _crypto.BlobDecryptor( + self.doc_info, BytesIO(ciphertext), + secret='A' * 96).decrypt() + assert cleartext == self.cleartext.getvalue() + + +def _aes_encrypt(key, iv, data, aead=''): backend = default_backend() cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend) encryptor = cipher.encryptor() + if aead: + encryptor.authenticate_additional_data(aead) return encryptor.update(data) + encryptor.finalize(), encryptor.tag -- cgit v1.2.3 From ac6d87e83f91ed61b160e7cdd968f4a6f3d68f34 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 16 Feb 2017 17:18:06 -0300 Subject: [style] add deprecation warning on legacy decoder --- client/src/leap/soledad/client/_crypto.py | 5 +++++ testing/tests/client/test_crypto.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index c4c6d336..f91084a4 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -22,6 +22,7 @@ Cryptographic operations for the soledad client import binascii import base64 import hashlib +import warnings import hmac import os import re @@ -278,6 +279,10 @@ class BlobDecryptor(object): try: if len(preamble) == LEGACY_PACMAN.size: + warnings.warn("Decrypting a legacy document without size. " + + "This will be deprecated in 0.12. Doc was: " + + "doc_id: %s rev: %s" % (self.doc_id, self.rev), + Warning) unpacked_data = LEGACY_PACMAN.unpack(preamble) magic, sch, meth, ts, iv, doc_id, rev = unpacked_data elif len(preamble) == PACMAN.size: diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 2d4d827b..399fdc99 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -292,6 +292,9 @@ class PreambleTestCase(unittest.TestCase): self.doc_info, BytesIO(ciphertext), secret='A' * 96).decrypt() assert cleartext == self.cleartext.getvalue() + warnings = self.flushWarnings() + assert len(warnings) == 1 + assert 'legacy document without size' in warnings[0]['message'] def _aes_encrypt(key, iv, data, aead=''): -- cgit v1.2.3 From 11416de633f355cbed1380404ef2807fd82b1e19 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Feb 2017 23:03:53 +0100 Subject: [docs] add crypto preamble to changelog --- docs/changelog-next.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog-next.rst b/docs/changelog-next.rst index 9d5f6297..1da32789 100644 --- a/docs/changelog-next.rst +++ b/docs/changelog-next.rst @@ -1,4 +1,4 @@ -0.x.x - +0.9.3 - +++++++++++++++++++++++++++++++ Please add lines to this file, they will be moved to the CHANGELOG.rst during @@ -13,6 +13,7 @@ Features - Refactor authentication code to use twisted credential system. - `#8764 `_: Allow unauthenticated users to retrieve the capabilties banner. +- `#8758 `_: Add blob size to the crypto preamble - `#1234 `_: Description of the new feature corresponding with issue #1234. - New feature without related issue number. -- cgit v1.2.3 From bab34cde11bdeb2810cc9f5d223957af26b2b6d3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 17 Feb 2017 16:15:10 +0100 Subject: [refactor] create resources only once it doesn't make sense to create the resources for every request, we can reuse the same resource and create it in the constructor. - Resolves: #8770 --- server/src/leap/soledad/server/_wsgi.py | 3 +++ server/src/leap/soledad/server/auth.py | 35 ++++++++++++++------------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index 37a03ced..a719aacb 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -27,6 +27,8 @@ from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.log import getLogger +from twisted.logger import Logger +log = Logger() __all__ = ['init_couch_state', 'get_sync_resource'] @@ -62,6 +64,7 @@ def init_couch_state(conf): def get_sync_resource(pool=None): if not pool: + log.warn("NO POOL PASSED, CREATING----------") pool = threadpool.ThreadPool() reactor.callWhenRunning(pool.start) reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index c52370cb..e064341b 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -51,30 +51,25 @@ class SoledadRealm(object): def __init__(self, conf=None, sync_pool=None): if not conf: conf = get_config() - self._conf = conf - self._sync_pool = sync_pool + blobs = conf['blobs'] + self.anon_resource = SoledadAnonResource( + enable_blobs=blobs) + self.auth_resource = SoledadResource( + enable_blobs=blobs, + sync_pool=sync_pool) def requestAvatar(self, avatarId, mind, *interfaces): - enable_blobs = self._conf['blobs'] # Anonymous access if IAnonymous.providedBy(avatarId): - resource = SoledadAnonResource( - enable_blobs=enable_blobs) - return (IResource, resource, lambda: None) - - # Authenticated users - - # TODO review this: #8770 ---------------- - # we're creating a Resource tree - # for each request, for every user. - # What are the perf implications of this?? - - if IResource in interfaces: - resource = SoledadResource( - enable_blobs=enable_blobs, - sync_pool=self._sync_pool) - return (IResource, resource, lambda: None) + return (IResource, self.anon_resource, + lambda: None) + + # Authenticated access + else: + if IResource in interfaces: + return (IResource, self.auth_resource, + lambda: None) raise NotImplementedError() @@ -165,7 +160,7 @@ class TokenCredentialFactory(object): raise error.LoginFailed('Invalid credentials') -def get_portal(sync_pool=None): +def portalFactory(sync_pool=None): realm = SoledadRealm(sync_pool=sync_pool) checker = TokenChecker() return Portal(realm, [checker]) -- cgit v1.2.3 From 193573cf8d44a3b6a7d8ae0e43988cffb38a428a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 17 Feb 2017 16:52:07 +0100 Subject: [bug] reuse wsgi threadpool it seems evident that the functions were thought to pass a threadpool along, but it finally wasn't properly passed and so there was a new threadpool created to handle every resource. I have removed the creation from the factory because I don't think it makes sense to create a threadpool on the fly, it's prone to errors. - Resolves: #8774 --- server/src/leap/soledad/server/_wsgi.py | 8 +------- server/src/leap/soledad/server/auth.py | 7 ++++--- server/src/leap/soledad/server/entrypoint.py | 10 ++++++++-- server/src/leap/soledad/server/session.py | 6 +----- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/server/src/leap/soledad/server/_wsgi.py b/server/src/leap/soledad/server/_wsgi.py index a719aacb..f6ff6b26 100644 --- a/server/src/leap/soledad/server/_wsgi.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -18,7 +18,6 @@ A WSGI application that serves Soledad synchronization. """ from twisted.internet import reactor -from twisted.python import threadpool from twisted.web.wsgi import WSGIResource from leap.soledad.server import SoledadApp @@ -62,10 +61,5 @@ def init_couch_state(conf): reactor.stop() -def get_sync_resource(pool=None): - if not pool: - log.warn("NO POOL PASSED, CREATING----------") - pool = threadpool.ThreadPool() - reactor.callWhenRunning(pool.start) - reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) +def get_sync_resource(pool): return WSGIResource(reactor, pool, wsgi_application) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index e064341b..b5744fe9 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -48,8 +48,9 @@ log = Logger() @implementer(IRealm) class SoledadRealm(object): - def __init__(self, conf=None, sync_pool=None): - if not conf: + def __init__(self, sync_pool, conf=None): + assert sync_pool is not None + if conf is None: conf = get_config() blobs = conf['blobs'] self.anon_resource = SoledadAnonResource( @@ -160,7 +161,7 @@ class TokenCredentialFactory(object): raise error.LoginFailed('Invalid credentials') -def portalFactory(sync_pool=None): +def portalFactory(sync_pool): realm = SoledadRealm(sync_pool=sync_pool) checker = TokenChecker() return Portal(realm, [checker]) diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py index 8078a54a..c06b740e 100644 --- a/server/src/leap/soledad/server/entrypoint.py +++ b/server/src/leap/soledad/server/entrypoint.py @@ -22,10 +22,12 @@ or the systemd script. """ from twisted.internet import reactor +from twisted.python import threadpool +from .auth import portalFactory +from .session import SoledadSession from ._config import get_config from ._wsgi import init_couch_state -from .session import SoledadSession # load configuration from file @@ -35,7 +37,11 @@ conf = get_config() class SoledadEntrypoint(SoledadSession): def __init__(self): - SoledadSession.__init__(self) + pool = threadpool.ThreadPool(name='wsgi') + reactor.callWhenRunning(pool.start) + reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) + portal = portalFactory(pool) + SoledadSession.__init__(self, portal) # see the comments in application.py recarding why couch state has to be diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py index c1ceb340..1c1b5345 100644 --- a/server/src/leap/soledad/server/session.py +++ b/server/src/leap/soledad/server/session.py @@ -28,7 +28,6 @@ from twisted.web.guard import HTTPAuthSessionWrapper from twisted.web.resource import ErrorPage from twisted.web.resource import IResource -from leap.soledad.server.auth import get_portal from leap.soledad.server.auth import credentialFactory from leap.soledad.server.url_mapper import URLMapper @@ -53,10 +52,7 @@ class UnauthorizedResource(wrapper.UnauthorizedResource): @implementer(IResource) class SoledadSession(HTTPAuthSessionWrapper): - def __init__(self, portal=None): - if portal is None: - portal = get_portal() - + def __init__(self, portal): self._mapper = URLMapper() self._portal = portal self._credentialFactory = credentialFactory -- cgit v1.2.3 From 1ab75f97d77f9883c4d83d43513d9d47dfb397d9 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Feb 2017 18:06:07 -0300 Subject: [refactor] remove creds from client api --- client/src/leap/soledad/client/api.py | 44 ++++++++--------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 54cbcd9d..6a508d05 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -187,7 +187,7 @@ class Soledad(object): global SOLEDAD_CERT SOLEDAD_CERT = cert_file - self._set_token(auth_token) + self.set_token(auth_token) self._init_config_with_defaults() self._init_working_dirs() @@ -249,9 +249,10 @@ class Soledad(object): """ Initialize Soledad secrets. """ + creds = {'token': {'uuid': self.uuid, 'token': self.token}} self._secrets = Secrets( self._uuid, self._passphrase, self._server_url, self._secrets_path, - self._creds, self.userid, shared_db=shared_db) + creds, self.userid, shared_db=shared_db) def _init_u1db_sqlcipher_backend(self): """ @@ -675,9 +676,8 @@ class Soledad(object): sync_url = urlparse.urljoin(self._server_url, 'user-%s' % self.uuid) if not self._dbsyncer: return - d = self._dbsyncer.sync( - sync_url, - creds=self._creds) + creds = {'token': {'uuid': self.uuid, 'token': self.token}} + d = self._dbsyncer.sync(sync_url, creds=creds) def _sync_callback(local_gen): self._last_received_docs = docs = self._dbsyncer.received_docs @@ -734,37 +734,13 @@ class Soledad(object): """ return self.sync_lock.locked - def _set_token(self, token): - """ - Set the authentication token for remote database access. - - Internally, this builds the credentials dictionary with the following - format: + def set_token(self, token): + self._token = token - { - 'token': { - 'uuid': '' - 'token': '' - } - } - - :param token: The authentication token. - :type token: str - """ - self._creds = { - 'token': { - 'uuid': self.uuid, - 'token': token, - } - } - - def _get_token(self): - """ - Return current token from credentials dictionary. - """ - return self._creds['token']['token'] + def get_token(self): + return self._token - token = property(_get_token, _set_token, doc='The authentication Token.') + token = property(get_token, set_token, doc='The authentication Token.') # # ISecretsStorage -- cgit v1.2.3 From 459c5efd59e2ebaa21e5c461c275d866a43534e8 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Feb 2017 18:11:49 -0300 Subject: [refactor] add EmitMixin for a cleaner emitting experience --- client/src/leap/soledad/client/_secrets/__init__.py | 7 ++++--- client/src/leap/soledad/client/_secrets/storage.py | 12 ++---------- client/src/leap/soledad/client/_secrets/util.py | 7 +++++++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index 42fe5a2d..26294364 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -24,7 +24,7 @@ from leap.soledad.common.log import getLogger from leap.soledad.client._secrets.storage import SecretsStorage from leap.soledad.client._secrets.crypto import SecretsCrypto -from leap.soledad.client._secrets.util import emit +from leap.soledad.client._secrets.util import emit, EmitMixin logger = getLogger(__name__) @@ -33,7 +33,7 @@ logger = getLogger(__name__) SecretLength = namedtuple('SecretLength', 'name length') -class Secrets(object): +class Secrets(EmitMixin): # remote secret is used @@ -45,9 +45,10 @@ class Secrets(object): def __init__(self, uuid, passphrase, url, local_path, creds, userid, shared_db=None): + self._uuid = uuid self._passphrase = passphrase + self._userid = userid self._secrets = {} - self._user_data = {'uuid': uuid, 'userid': userid} self.crypto = SecretsCrypto(self.get_passphrase) self.storage = SecretsStorage( uuid, self.get_passphrase, url, local_path, creds, userid, diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py index da3aa9d7..730926ee 100644 --- a/client/src/leap/soledad/client/_secrets/storage.py +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -25,13 +25,13 @@ from leap.soledad.common.log import getLogger from leap.soledad.common.document import SoledadDocument from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client._secrets.util import emit +from leap.soledad.client._secrets.util import emit, EmitMixin logger = getLogger(__name__) -class SecretsStorage(object): +class SecretsStorage(EmitMixin): def __init__(self, uuid, get_pass, url, local_path, creds, userid, shared_db=None): @@ -43,14 +43,6 @@ class SecretsStorage(object): self._shared_db = shared_db or self._init_shared_db(url, creds) self.__remote_doc = None - # - # properties - # - - @property - def _user_data(self): - return {'uuid': self._uuid, 'userid': self._userid} - # # local storage # diff --git a/client/src/leap/soledad/client/_secrets/util.py b/client/src/leap/soledad/client/_secrets/util.py index f75b2bb6..0dcdd3af 100644 --- a/client/src/leap/soledad/client/_secrets/util.py +++ b/client/src/leap/soledad/client/_secrets/util.py @@ -23,6 +23,13 @@ class SecretsError(Exception): pass +class EmitMixin(object): + + @property + def _user_data(self): + return {'uuid': self._uuid, 'userid': self._userid} + + def emit(verb): def _decorator(method): def _decorated(self, *args, **kwargs): -- cgit v1.2.3 From 2422561268aa0214d8521ef5d2f456318391711d Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Feb 2017 18:18:39 -0300 Subject: [refactor] use get_token in client secrets api --- client/src/leap/soledad/client/_secrets/__init__.py | 4 ++-- client/src/leap/soledad/client/_secrets/storage.py | 9 +++++++-- client/src/leap/soledad/client/api.py | 3 +-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index 26294364..69c9141f 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -43,7 +43,7 @@ class Secrets(EmitMixin): 'local_secret': 448, # local_secret to derive a local_key for storage } - def __init__(self, uuid, passphrase, url, local_path, creds, userid, + def __init__(self, uuid, passphrase, url, local_path, get_token, userid, shared_db=None): self._uuid = uuid self._passphrase = passphrase @@ -51,7 +51,7 @@ class Secrets(EmitMixin): self._secrets = {} self.crypto = SecretsCrypto(self.get_passphrase) self.storage = SecretsStorage( - uuid, self.get_passphrase, url, local_path, creds, userid, + uuid, self.get_passphrase, url, local_path, get_token, userid, shared_db=shared_db) self._bootstrap() diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py index 730926ee..5fde8988 100644 --- a/client/src/leap/soledad/client/_secrets/storage.py +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -33,16 +33,21 @@ logger = getLogger(__name__) class SecretsStorage(EmitMixin): - def __init__(self, uuid, get_pass, url, local_path, creds, userid, + def __init__(self, uuid, get_pass, url, local_path, get_token, userid, shared_db=None): self._uuid = uuid self._get_pass = get_pass self._local_path = local_path + self._get_token = get_token self._userid = userid - self._shared_db = shared_db or self._init_shared_db(url, creds) + self._shared_db = shared_db or self._init_shared_db(url, self._creds) self.__remote_doc = None + @property + def _creds(self): + return {'token': {'uuid': self._uuid, 'token': self._get_token()}} + # # local storage # diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 6a508d05..e84ba848 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -249,10 +249,9 @@ class Soledad(object): """ Initialize Soledad secrets. """ - creds = {'token': {'uuid': self.uuid, 'token': self.token}} self._secrets = Secrets( self._uuid, self._passphrase, self._server_url, self._secrets_path, - creds, self.userid, shared_db=shared_db) + self.get_token, self.userid, shared_db=shared_db) def _init_u1db_sqlcipher_backend(self): """ -- cgit v1.2.3 From 75e4d4e7dafe4472ee55fa0159eeb8270df2dd49 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Feb 2017 18:40:02 -0300 Subject: [feat] avoid client sync if no token is set --- client/src/leap/soledad/client/_secrets/__init__.py | 2 ++ client/src/leap/soledad/client/api.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index 69c9141f..78cfae5e 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -81,6 +81,8 @@ class Secrets(EmitMixin): self._secrets = secrets if encrypted['version'] < self.crypto.VERSION or force_storage: + # TODO: what should we do if it's the first run and remote save + # fails? self.storage.save_local(encrypted) self.storage.save_remote(encrypted) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index e84ba848..07eb8e9e 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -39,7 +39,7 @@ from itertools import chain from StringIO import StringIO from collections import defaultdict -from twisted.internet.defer import DeferredLock, returnValue, inlineCallbacks +from twisted.internet import defer from zope.interface import implements from leap.common.config import get_path_prefix @@ -124,7 +124,7 @@ class Soledad(object): same database replica. The dictionary indexes are the paths to each local db, so we guarantee that only one sync happens for a local db at a time. """ - _sync_lock = defaultdict(DeferredLock) + _sync_lock = defaultdict(defer.DeferredLock) def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, @@ -660,6 +660,11 @@ class Soledad(object): generation before the synchronization was performed. :rtype: twisted.internet.defer.Deferred """ + # bypass sync if there's no token set + if not self.token: + generation = self._dbsyncer.get_generation() + return defer.succeed(generation) + d = self.sync_lock.run( self._sync) return d @@ -788,7 +793,7 @@ class Soledad(object): # Service authentication # - @inlineCallbacks + @defer.inlineCallbacks def get_or_create_service_token(self, service): """ Return the stored token for a given service, or generates and stores a @@ -803,11 +808,11 @@ class Soledad(object): docs = yield self._get_token_for_service(service) if docs: doc = docs[0] - returnValue(doc.content['token']) + defer.returnValue(doc.content['token']) else: token = str(uuid.uuid4()).replace('-', '')[-24:] yield self._set_token_for_service(service, token) - returnValue(token) + defer.returnValue(token) def _get_token_for_service(self, service): return self.get_from_index('by-servicetoken', 'servicetoken', service) -- cgit v1.2.3 From 67fc01d1b055b4a0ff3742ba4c3d32bbef3e1b87 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Feb 2017 19:01:17 -0300 Subject: [feature] add offline status to soledad client api --- client/src/leap/soledad/client/api.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 07eb8e9e..16569ec2 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -128,7 +128,7 @@ class Soledad(object): def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, - auth_token=None): + auth_token=None, offline=False): """ Initialize configuration, cryptographic keys and dbs. @@ -165,10 +165,11 @@ class Soledad(object): Authorization token for accessing remote databases. :type auth_token: str - :param syncable: - If set to ``False``, this database will not attempt to synchronize - with remote replicas (default is ``True``) - :type syncable: bool + :param offline: + If set to ``True``, this database will not attempt to save/load + secrets to/from server or synchronize with remote replicas (default + is ``False``) + :type offline: bool :raise BootstrapSequenceError: Raised when the secret initialization sequence (i.e. retrieval @@ -182,6 +183,7 @@ class Soledad(object): self._server_url = server_url self._secrets_path = None self._dbsyncer = None + self._offline = offline # configure SSL certificate global SOLEDAD_CERT @@ -212,6 +214,14 @@ class Soledad(object): self._dbpool.close() raise + def _get_offline(self): + return self._offline + + def _set_offline(self, offline): + self._offline = offline + + offline = property(_get_offline, _set_offline) + # # initialization/destruction methods # @@ -660,8 +670,8 @@ class Soledad(object): generation before the synchronization was performed. :rtype: twisted.internet.defer.Deferred """ - # bypass sync if there's no token set - if not self.token: + # maybe bypass sync + if self.offline or not self.token: generation = self._dbsyncer.get_generation() return defer.succeed(generation) -- cgit v1.2.3 From a2e29d8bebbe7809dc10195982a205ed3709459c Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Feb 2017 19:08:02 -0300 Subject: [refactor] remove syncable property from shared db --- client/src/leap/soledad/client/shared_db.py | 15 +-------------- testing/test_soledad/util.py | 1 - 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 52b226b9..b429d2cb 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -55,10 +55,6 @@ class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): # TODO: prevent client from messing with the shared DB. # TODO: define and document API. - # If syncable is False, the database will not attempt to sync against - # a remote replica. Default is True. - syncable = True - # # Token auth methods. # @@ -95,7 +91,7 @@ class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, uuid, creds=None, syncable=True): + def open_database(url, uuid, creds=None): """ Open a Soledad shared database. @@ -106,20 +102,11 @@ class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): :param creds: A tuple containing the authentication method and credentials. :type creds: tuple - :param syncable: - If syncable is False, the database will not attempt to sync against - a remote replica. - :type syncable: bool :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ - # XXX fix below, doesn't work with tests. - # if syncable and not url.startswith('https://'): - # raise ImproperlyConfiguredError( - # "Remote soledad server must be an https URI") db = SoledadSharedDatabase(url, uuid, creds=creds) - db.syncable = syncable return db @staticmethod diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py index e44a165d..83a27016 100644 --- a/testing/test_soledad/util.py +++ b/testing/test_soledad/util.py @@ -180,7 +180,6 @@ class MockedSharedDBTest(object): put_doc = Mock(side_effect=put_doc_side_effect) open = Mock(return_value=None) close = Mock(return_value=None) - syncable = True def __call__(self): return self -- cgit v1.2.3 From b433c1ed736f5d4c19da4cdb21108a02459ca7fd Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 25 Feb 2017 08:53:38 -0300 Subject: [refactor] pass soledad object to client secrets api In order to be able to change passphrase, token and offline status of soledad from the bitmask client api, the secrets api also has to be able to use up-to-date values when encrypting/decrypting secrets and uploading/downloading them to the server. This commit makes public some soledad attributes that were previously "private" (i.e. used to start with "_" and were not meant to be accessed from outside), and passes the whole soledad object to the client secrets api. This makes the code cleaner and also allows for always getting newest values of soledad attributes. --- .../src/leap/soledad/client/_secrets/__init__.py | 25 +++------- client/src/leap/soledad/client/_secrets/crypto.py | 7 +-- client/src/leap/soledad/client/_secrets/storage.py | 40 +++++++-------- client/src/leap/soledad/client/_secrets/util.py | 4 +- client/src/leap/soledad/client/api.py | 57 ++++++---------------- testing/tests/client/test_aux_methods.py | 12 ++--- testing/tests/client/test_deprecated_crypto.py | 2 +- testing/tests/client/test_secrets.py | 10 ++-- testing/tests/server/test_server.py | 2 +- 9 files changed, 60 insertions(+), 99 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index 78cfae5e..43541e16 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -43,16 +43,11 @@ class Secrets(EmitMixin): 'local_secret': 448, # local_secret to derive a local_key for storage } - def __init__(self, uuid, passphrase, url, local_path, get_token, userid, - shared_db=None): - self._uuid = uuid - self._passphrase = passphrase - self._userid = userid + def __init__(self, soledad): + self._soledad = soledad self._secrets = {} - self.crypto = SecretsCrypto(self.get_passphrase) - self.storage = SecretsStorage( - uuid, self.get_passphrase, url, local_path, get_token, userid, - shared_db=shared_db) + self.crypto = SecretsCrypto(soledad) + self.storage = SecretsStorage(soledad) self._bootstrap() # @@ -83,6 +78,8 @@ class Secrets(EmitMixin): if encrypted['version'] < self.crypto.VERSION or force_storage: # TODO: what should we do if it's the first run and remote save # fails? + # TODO: we have to actually update the encrypted version before + # saving, we are currently not doing it. self.storage.save_local(encrypted) self.storage.save_remote(encrypted) @@ -112,15 +109,7 @@ class Secrets(EmitMixin): data = {'secret': encrypted, 'version': 2} return data - def get_passphrase(self): - return self._passphrase.encode('utf-8') - - @property - def passphrase(self): - return self.get_passphrase() - - def change_passphrase(self, new_passphrase): - self._passphrase = new_passphrase + def store_secrets(self): encrypted = self.crypto.encrypt(self._secrets) self.storage.save_local(encrypted) self.storage.save_remote(encrypted) diff --git a/client/src/leap/soledad/client/_secrets/crypto.py b/client/src/leap/soledad/client/_secrets/crypto.py index 02d7dc02..fa7aaca0 100644 --- a/client/src/leap/soledad/client/_secrets/crypto.py +++ b/client/src/leap/soledad/client/_secrets/crypto.py @@ -34,11 +34,12 @@ class SecretsCrypto(object): VERSION = 2 - def __init__(self, get_pass): - self._get_pass = get_pass + def __init__(self, soledad): + self._soledad = soledad def _get_key(self, salt): - key = scrypt.hash(self._get_pass(), salt, buflen=32) + passphrase = self._soledad.passphrase.encode('utf8') + key = scrypt.hash(passphrase, salt, buflen=32) return key # diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py index 5fde8988..bb74dba3 100644 --- a/client/src/leap/soledad/client/_secrets/storage.py +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -33,29 +33,26 @@ logger = getLogger(__name__) class SecretsStorage(EmitMixin): - def __init__(self, uuid, get_pass, url, local_path, get_token, userid, - shared_db=None): - self._uuid = uuid - self._get_pass = get_pass - self._local_path = local_path - self._get_token = get_token - self._userid = userid - - self._shared_db = shared_db or self._init_shared_db(url, self._creds) + def __init__(self, soledad): + self._soledad = soledad + self._shared_db = self._soledad.shared_db or self._init_shared_db() self.__remote_doc = None @property def _creds(self): - return {'token': {'uuid': self._uuid, 'token': self._get_token()}} + uuid = self._soledad.uuid + token = self._soledad.token + return {'token': {'uuid': uuid, 'token': token}} # # local storage # def load_local(self): - logger.info("trying to load secrets from disk: %s" % self._local_path) + path = self._soledad.secrets_path + logger.info("trying to load secrets from disk: %s" % path) try: - with open(self._local_path, 'r') as f: + with open(path, 'r') as f: encrypted = json.loads(f.read()) logger.info("secrets loaded successfully from disk") return encrypted @@ -64,23 +61,26 @@ class SecretsStorage(EmitMixin): return None def save_local(self, encrypted): + path = self._soledad.secrets_path json_data = json.dumps(encrypted) - with open(self._local_path, 'w') as f: + with open(path, 'w') as f: f.write(json_data) # # remote storage # - def _init_shared_db(self, url, creds): - url = urlparse.urljoin(url, SHARED_DB_NAME) - db = SoledadSharedDatabase.open_database( - url, self._uuid, creds=creds) - self._shared_db = db + def _init_shared_db(self): + url = urlparse.urljoin(self._soledad.server_url, SHARED_DB_NAME) + uuid = self._soledad.uuid + creds = self._creds + db = SoledadSharedDatabase.open_database(url, uuid, creds) + return db def _remote_doc_id(self): - passphrase = self._get_pass() - text = '%s%s' % (passphrase, self._uuid) + passphrase = self._soledad.passphrase.encode('utf8') + uuid = self._soledad.uuid + text = '%s%s' % (passphrase, uuid) digest = sha256(text).hexdigest() return digest diff --git a/client/src/leap/soledad/client/_secrets/util.py b/client/src/leap/soledad/client/_secrets/util.py index 0dcdd3af..75418518 100644 --- a/client/src/leap/soledad/client/_secrets/util.py +++ b/client/src/leap/soledad/client/_secrets/util.py @@ -27,7 +27,9 @@ class EmitMixin(object): @property def _user_data(self): - return {'uuid': self._uuid, 'userid': self._userid} + uuid = self._soledad.uuid + userid = self._soledad.userid + return {'uuid': uuid, 'userid': userid} def emit(verb): diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 16569ec2..4be38cf1 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -177,27 +177,25 @@ class Soledad(object): some reason. """ # store config params - self._uuid = uuid - self._passphrase = passphrase + self.uuid = uuid + self.passphrase = passphrase + self.secrets_path = secrets_path self._local_db_path = local_db_path - self._server_url = server_url - self._secrets_path = None + self.server_url = server_url + self.shared_db = shared_db + self.token = auth_token + self.offline = offline + self._dbsyncer = None - self._offline = offline # configure SSL certificate global SOLEDAD_CERT SOLEDAD_CERT = cert_file - self.set_token(auth_token) - self._init_config_with_defaults() self._init_working_dirs() - self._secrets_path = secrets_path - - self._init_secrets(shared_db=shared_db) - + self._secrets = Secrets(self) self._crypto = SoledadCrypto(self._secrets.remote_secret) try: @@ -214,14 +212,6 @@ class Soledad(object): self._dbpool.close() raise - def _get_offline(self): - return self._offline - - def _set_offline(self, offline): - self._offline = offline - - offline = property(_get_offline, _set_offline) - # # initialization/destruction methods # @@ -230,7 +220,7 @@ class Soledad(object): """ Initialize configuration using default values for missing params. """ - soledad_assert_type(self._passphrase, unicode) + soledad_assert_type(self.passphrase, unicode) def initialize(attr, val): return ((getattr(self, attr, None) is None) and @@ -241,7 +231,7 @@ class Soledad(object): initialize("_local_db_path", os.path.join( self.default_prefix, self.local_db_file_name)) # initialize server_url - soledad_assert(self._server_url is not None, + soledad_assert(self.server_url is not None, 'Missing URL for Soledad server.') def _init_working_dirs(self): @@ -255,14 +245,6 @@ class Soledad(object): for path in paths: create_path_if_not_exists(path) - def _init_secrets(self, shared_db=None): - """ - Initialize Soledad secrets. - """ - self._secrets = Secrets( - self._uuid, self._passphrase, self._server_url, self._secrets_path, - self.get_token, self.userid, shared_db=shared_db) - def _init_u1db_sqlcipher_backend(self): """ Initialize the U1DB SQLCipher database for local storage. @@ -646,10 +628,6 @@ class Soledad(object): def local_db_path(self): return self._local_db_path - @property - def uuid(self): - return self._uuid - @property def userid(self): return self.uuid @@ -687,7 +665,7 @@ class Soledad(object): generation before the synchronization was performed. :rtype: twisted.internet.defer.Deferred """ - sync_url = urlparse.urljoin(self._server_url, 'user-%s' % self.uuid) + sync_url = urlparse.urljoin(self.server_url, 'user-%s' % self.uuid) if not self._dbsyncer: return creds = {'token': {'uuid': self.uuid, 'token': self.token}} @@ -748,14 +726,6 @@ class Soledad(object): """ return self.sync_lock.locked - def set_token(self, token): - self._token = token - - def get_token(self): - return self._token - - token = property(get_token, set_token, doc='The authentication Token.') - # # ISecretsStorage # @@ -779,7 +749,8 @@ class Soledad(object): :raise NoStorageSecret: Raised if there's no storage secret available. """ - self._secrets.change_passphrase(new_passphrase) + self.passphrase = new_passphrase + self._secrets.store_secrets() # # Raw SQLCIPHER Queries diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py index a08f7d36..729aa28a 100644 --- a/testing/tests/client/test_aux_methods.py +++ b/testing/tests/client/test_aux_methods.py @@ -33,7 +33,7 @@ class AuxMethodsTestCase(BaseSoledadTest): def test__init_dirs(self): sol = self._soledad_instance(prefix='_init_dirs') local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets.storage._local_path) + secrets_path = os.path.dirname(sol.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) @@ -63,8 +63,8 @@ class AuxMethodsTestCase(BaseSoledadTest): # instantiate without initializing so we just test # _init_config_with_defaults() sol = SoledadMock() - sol._passphrase = u'' - sol._server_url = '' + sol.passphrase = u'' + sol.server_url = '' sol._init_config_with_defaults() # assert value of local_db_path self.assertEquals( @@ -84,11 +84,11 @@ class AuxMethodsTestCase(BaseSoledadTest): cert_file=None) self.assertEqual( os.path.join(self.tempdir, 'value_3'), - sol.secrets.storage._local_path) + sol.secrets_path) self.assertEqual( os.path.join(self.tempdir, 'value_2'), sol.local_db_path) - self.assertEqual('value_1', sol._server_url) + self.assertEqual('value_1', sol.server_url) sol.close() @inlineCallbacks @@ -128,5 +128,5 @@ class AuxMethodsTestCase(BaseSoledadTest): Assert passphrase getter works fine. """ sol = self._soledad_instance() - self.assertEqual('123', sol._passphrase) + self.assertEqual('123', sol.passphrase) sol.close() diff --git a/testing/tests/client/test_deprecated_crypto.py b/testing/tests/client/test_deprecated_crypto.py index 8c711c22..1af1a130 100644 --- a/testing/tests/client/test_deprecated_crypto.py +++ b/testing/tests/client/test_deprecated_crypto.py @@ -51,7 +51,7 @@ class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): self._soledad_instance(user=user, server_url=server_url)) self.make_app() - remote = self.request_state._create_database(replica_uid=client._uuid) + remote = self.request_state._create_database(replica_uid=client.uuid) remote = CouchDatabase.open_database( urljoin(self.couch_url, 'user-' + user), create=True) diff --git a/testing/tests/client/test_secrets.py b/testing/tests/client/test_secrets.py index bbeb1fc2..18ff458b 100644 --- a/testing/tests/client/test_secrets.py +++ b/testing/tests/client/test_secrets.py @@ -121,12 +121,10 @@ class SecretsCryptoTestCase(unittest.TestCase): } def setUp(self): - def _get_pass(): - return '123' - self._crypto = SecretsCrypto(_get_pass) - - def test__get_pass(self): - self.assertEqual(self._crypto._get_pass(), '123') + class Soledad(object): + passphrase = '123' + soledad = Soledad() + self._crypto = SecretsCrypto(soledad) def test__get_key(self): salt = 'abc' diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 647ef5a8..4a5ec43f 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -135,7 +135,7 @@ class EncryptedSyncTestCase( user=user, prefix='x', auth_token='auth-token', - secrets_path=sol1._secrets_path, + secrets_path=sol1.secrets_path, passphrase=passphrase) # ensure remote db exists before syncing -- cgit v1.2.3 From 425bfe42ca3758cfa4cda4589ebb42530313850b Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 25 Feb 2017 09:11:44 -0300 Subject: [doc] improve doc and rename EmitMixin to UserDataMixin --- client/src/leap/soledad/client/_secrets/__init__.py | 4 ++-- client/src/leap/soledad/client/_secrets/storage.py | 4 ++-- client/src/leap/soledad/client/_secrets/util.py | 10 +++++++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index 43541e16..79b6844a 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -24,7 +24,7 @@ from leap.soledad.common.log import getLogger from leap.soledad.client._secrets.storage import SecretsStorage from leap.soledad.client._secrets.crypto import SecretsCrypto -from leap.soledad.client._secrets.util import emit, EmitMixin +from leap.soledad.client._secrets.util import emit, UserDataMixin logger = getLogger(__name__) @@ -33,7 +33,7 @@ logger = getLogger(__name__) SecretLength = namedtuple('SecretLength', 'name length') -class Secrets(EmitMixin): +class Secrets(UserDataMixin): # remote secret is used diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py index bb74dba3..89b44266 100644 --- a/client/src/leap/soledad/client/_secrets/storage.py +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -25,13 +25,13 @@ from leap.soledad.common.log import getLogger from leap.soledad.common.document import SoledadDocument from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client._secrets.util import emit, EmitMixin +from leap.soledad.client._secrets.util import emit, UserDataMixin logger = getLogger(__name__) -class SecretsStorage(EmitMixin): +class SecretsStorage(UserDataMixin): def __init__(self, soledad): self._soledad = soledad diff --git a/client/src/leap/soledad/client/_secrets/util.py b/client/src/leap/soledad/client/_secrets/util.py index 75418518..6401889b 100644 --- a/client/src/leap/soledad/client/_secrets/util.py +++ b/client/src/leap/soledad/client/_secrets/util.py @@ -23,12 +23,20 @@ class SecretsError(Exception): pass -class EmitMixin(object): +class UserDataMixin(object): + """ + When emitting an event, we have to pass a dictionary containing user data. + This class only defines a property so we don't have to define it in + multiple places. + """ @property def _user_data(self): uuid = self._soledad.uuid userid = self._soledad.userid + # TODO: seems that uuid and userid hold the same value! We should check + # whether we should pass something different or if the events api + # really needs two different values. return {'uuid': uuid, 'userid': userid} -- cgit v1.2.3 From a0029b3c7beb8682c8aa3691a5d67003168c3e07 Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 25 Feb 2017 09:42:38 -0300 Subject: [refactor] improve secret bootstrap code and doc --- .../src/leap/soledad/client/_secrets/__init__.py | 50 ++++++++++------------ 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index 79b6844a..f8da792d 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -55,33 +55,29 @@ class Secrets(UserDataMixin): # def _bootstrap(self): - force_storage = False - # attempt to load secrets from local storage encrypted = self.storage.load_local() - # if not found, attempt to load secrets from remote storage if not encrypted: + # we have not found a secret stored locally, so this is a first run + # of soledad for this user in this device. It is mandatory that we + # check if there's a secret stored in server. encrypted = self.storage.load_remote() - if not encrypted: - # if not found, generate new secrets - secrets = self._generate() - encrypted = self.crypto.encrypt(secrets) - force_storage = True + if encrypted: + # we found a secret either in local or in remote storage, so we + # have to decrypt it. + self._secrets = self.crypto.decrypt(encrypted) + if encrypted['version'] < self.crypto.VERSION: + # there is a format version for secret storage that is newer + # than the one we found (either in local or remote storage), so + # we re-encrypt and store with the newest version. + self.store_secrets() else: - # decrypt secrets found either in local or remote storage - secrets = self.crypto.decrypt(encrypted) - - self._secrets = secrets - - if encrypted['version'] < self.crypto.VERSION or force_storage: - # TODO: what should we do if it's the first run and remote save - # fails? - # TODO: we have to actually update the encrypted version before - # saving, we are currently not doing it. - self.storage.save_local(encrypted) - self.storage.save_remote(encrypted) + # we have *not* found a secret neither in local nor in remote + # storage, so we have to generate a new one, and store it. + self._secrets = self._generate() + self.store_secrets() # # generation @@ -101,15 +97,13 @@ class Secrets(UserDataMixin): # crypto # - def _encrypt(self): - # encrypt secrets - secrets = self._secrets - encrypted = self.crypto.encrypt(secrets) - # create the recovery document - data = {'secret': encrypted, 'version': 2} - return data - def store_secrets(self): + # TODO: we have to improve the logic here, as we want to make sure that + # whatever is stored locally should only be used after remote storage + # is successful. Otherwise, this soledad could start encrypting with a + # secret while another soledad in another device could start encrypting + # with another secret, which would lead to decryption failures during + # sync. encrypted = self.crypto.encrypt(self._secrets) self.storage.save_local(encrypted) self.storage.save_remote(encrypted) -- cgit v1.2.3 From 87b65c731bb32bb9f0953d23b750ac8e8fda9eab Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 25 Feb 2017 18:06:03 -0300 Subject: [bug] remove unused named tuple from client secrets --- client/src/leap/soledad/client/_secrets/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index f8da792d..bb8e9086 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -18,8 +18,6 @@ import os import scrypt -from collections import namedtuple - from leap.soledad.common.log import getLogger from leap.soledad.client._secrets.storage import SecretsStorage @@ -30,13 +28,8 @@ from leap.soledad.client._secrets.util import emit, UserDataMixin logger = getLogger(__name__) -SecretLength = namedtuple('SecretLength', 'name length') - - class Secrets(UserDataMixin): - # remote secret is used - lengths = { 'remote_secret': 512, # remote_secret is used to encrypt remote data. 'local_salt': 64, # local_salt is used in conjunction with -- cgit v1.2.3 From a96801e7f3f4e6aeeb08355f7bac4f47b2454dac Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 25 Feb 2017 18:17:18 -0300 Subject: [bug] save client secret downloaded from remote storage After refactor, the client secret bootstrap logic was flawed, and remote secret was not being saved properly. This commit fixed that and tries to improve the bootstrap code to make it more clear. --- .../src/leap/soledad/client/_secrets/__init__.py | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py index bb8e9086..b6c81cda 100644 --- a/client/src/leap/soledad/client/_secrets/__init__.py +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -48,29 +48,29 @@ class Secrets(UserDataMixin): # def _bootstrap(self): + # attempt to load secrets from local storage encrypted = self.storage.load_local() - - if not encrypted: - # we have not found a secret stored locally, so this is a first run - # of soledad for this user in this device. It is mandatory that we - # check if there's a secret stored in server. - encrypted = self.storage.load_remote() - if encrypted: - # we found a secret either in local or in remote storage, so we - # have to decrypt it. self._secrets = self.crypto.decrypt(encrypted) + # maybe update the format of storage of local secret. if encrypted['version'] < self.crypto.VERSION: - # there is a format version for secret storage that is newer - # than the one we found (either in local or remote storage), so - # we re-encrypt and store with the newest version. self.store_secrets() - else: - # we have *not* found a secret neither in local nor in remote - # storage, so we have to generate a new one, and store it. - self._secrets = self._generate() + return + + # no secret was found in local storage, so this is a first run of + # soledad for this user in this device. It is mandatory that we check + # if there's a secret stored in server. + encrypted = self.storage.load_remote() + if encrypted: + self._secrets = self.crypto.decrypt(encrypted) self.store_secrets() + return + + # we have *not* found a secret neither in local nor in remote storage, + # so we have to generate a new one, and then store it. + self._secrets = self._generate() + self.store_secrets() # # generation -- cgit v1.2.3 From 7eb1ffa1d49a7e0016c5980da71151e715abc77a Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 21 Feb 2017 14:16:26 -0300 Subject: [feat] add configurable blobs path in server - Resolves: #8777 --- server/src/leap/soledad/server/_blobs.py | 10 ++++++++++ server/src/leap/soledad/server/_config.py | 1 + testing/tests/server/test_config.py | 3 ++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/leap/soledad/server/_blobs.py b/server/src/leap/soledad/server/_blobs.py index 0424aae0..ae09f409 100644 --- a/server/src/leap/soledad/server/_blobs.py +++ b/server/src/leap/soledad/server/_blobs.py @@ -19,14 +19,24 @@ Blobs Server implementation. """ from twisted.web import resource +from ._config import get_config + __all__ = ['blobs_resource'] +_config = get_config() +DEFAULT_BLOBS_PATH = _config['blobs_path'] + + class BlobsResource(resource.Resource): isLeaf = True + def __init__(self, blobs_path=DEFAULT_BLOBS_PATH): + resource.Resource.__init__(self) + self._blobs_path = blobs_path + def render_GET(self, request): return 'blobs is not implemented yet!' diff --git a/server/src/leap/soledad/server/_config.py b/server/src/leap/soledad/server/_config.py index 1818c38d..e89e70d6 100644 --- a/server/src/leap/soledad/server/_config.py +++ b/server/src/leap/soledad/server/_config.py @@ -29,6 +29,7 @@ CONFIG_DEFAULTS = { 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', 'batching': True, 'blobs': False, + 'blobs_path': '/srv/leap/soledad/blobs', }, 'database-security': { 'members': ['soledad'], diff --git a/testing/tests/server/test_config.py b/testing/tests/server/test_config.py index 133057f5..d2a8a9de 100644 --- a/testing/tests/server/test_config.py +++ b/testing/tests/server/test_config.py @@ -64,5 +64,6 @@ class ConfigurationParsingTest(unittest.TestCase): 'admin_netrc': '/etc/couchdb/couchdb-soledad-admin.netrc', 'batching': False, - 'blobs': False} + 'blobs': False, + 'blobs_path': '/srv/leap/soledad/blobs'} self.assertDictEqual(expected, config['soledad-server']) -- cgit v1.2.3 From 5139e95a65cf6094711ebf12aca01fb6e9b47c8c Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 27 Feb 2017 16:10:36 -0300 Subject: [style] move path config closer to blobs resource instantiation --- server/src/leap/soledad/server/_blobs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/leap/soledad/server/_blobs.py b/server/src/leap/soledad/server/_blobs.py index ae09f409..cacabbdf 100644 --- a/server/src/leap/soledad/server/_blobs.py +++ b/server/src/leap/soledad/server/_blobs.py @@ -22,18 +22,14 @@ from twisted.web import resource from ._config import get_config -__all__ = ['blobs_resource'] - - -_config = get_config() -DEFAULT_BLOBS_PATH = _config['blobs_path'] +__all__ = ['BlobsResource', 'blobs_resource'] class BlobsResource(resource.Resource): isLeaf = True - def __init__(self, blobs_path=DEFAULT_BLOBS_PATH): + def __init__(self, blobs_path): resource.Resource.__init__(self) self._blobs_path = blobs_path @@ -41,4 +37,8 @@ class BlobsResource(resource.Resource): return 'blobs is not implemented yet!' -blobs_resource = BlobsResource() +# provide a configured instance of the resource +_config = get_config() +_path = _config['blobs_path'] + +blobs_resource = BlobsResource(_path) -- cgit v1.2.3 From d92f1d1840d28ee6c2058556b047db75ed737f77 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 2 Mar 2017 13:58:49 -0300 Subject: [bug] fix shared database initialization --- client/src/leap/soledad/client/_secrets/storage.py | 3 +-- client/src/leap/soledad/client/shared_db.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py index 89b44266..056c4322 100644 --- a/client/src/leap/soledad/client/_secrets/storage.py +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -72,9 +72,8 @@ class SecretsStorage(UserDataMixin): def _init_shared_db(self): url = urlparse.urljoin(self._soledad.server_url, SHARED_DB_NAME) - uuid = self._soledad.uuid creds = self._creds - db = SoledadSharedDatabase.open_database(url, uuid, creds) + db = SoledadSharedDatabase.open_database(url, creds) return db def _remote_doc_id(self): diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index b429d2cb..4f70c74b 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -91,14 +91,12 @@ class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, uuid, creds=None): + def open_database(url, creds=None): """ Open a Soledad shared database. :param url: URL of the remote database. :type url: str - :param uuid: The user's unique id. - :type uuid: str :param creds: A tuple containing the authentication method and credentials. :type creds: tuple @@ -106,7 +104,7 @@ class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ - db = SoledadSharedDatabase(url, uuid, creds=creds) + db = SoledadSharedDatabase(url, creds=creds) return db @staticmethod -- cgit v1.2.3 From 0b60027dacf331c103f764b7b1b5dee8b6123938 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 11 Jan 2017 18:31:34 -0300 Subject: [bug] handle error once Handle it only if self.deferred wasnt called yet, otherwise that's just an out-of-sync call from a scheduled deferred. Since it was already logged, it's ok to ignore. --- client/src/leap/soledad/client/http_target/fetch_protocol.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/client/src/leap/soledad/client/http_target/fetch_protocol.py b/client/src/leap/soledad/client/http_target/fetch_protocol.py index fa6b1969..c7eabe2b 100644 --- a/client/src/leap/soledad/client/http_target/fetch_protocol.py +++ b/client/src/leap/soledad/client/http_target/fetch_protocol.py @@ -63,6 +63,8 @@ class DocStreamReceiver(ReadBodyProtocol): Deliver the accumulated response bytes to the waiting L{Deferred}, if the response body has been completely received without error. """ + if self.deferred.called: + return try: if reason.check(ResponseDone): self.dataBuffer = self.metadata @@ -125,11 +127,7 @@ class DocStreamReceiver(ReadBodyProtocol): else: d = self._doc_reader( self.current_doc, line.strip() or None, self.total) - d.addErrback(self._error) - - def _error(self, reason): - logger.error(reason) - self.transport.loseConnection() + d.addErrback(self.deferred.errback) def finish(self): """ -- cgit v1.2.3 From 9a73db4ca31c86bb8c657360e891d55329d65283 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 11 Jan 2017 18:33:05 -0300 Subject: [test] fix test_sync_target for error handling It needs a defer.Deferred --- testing/tests/sync/test_sync_target.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 34702362..25136ba1 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -63,7 +63,8 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ def parse(self, stream): - parser = DocStreamReceiver(None, None, lambda *_: defer.succeed(42)) + parser = DocStreamReceiver(None, defer.Deferred(), + lambda *_: defer.succeed(42)) parser.dataReceived(stream) parser.finish() -- cgit v1.2.3 From d285f07b365645a0c327a8d41402d710456583b6 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 27 Feb 2017 11:22:11 -0300 Subject: [test] use new way of indicating the tcp port for twisted web --- testing/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index decfb0a9..1d3125d5 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -104,7 +104,7 @@ class SoledadServer(object): '--pidfile=%s' % self._pidfile, 'web', '--class=leap.soledad.server.entrypoint.SoledadEntrypoint', - '--port=2424' + '--port=tcp:2424' ]) def _create_conf_file(self): -- cgit v1.2.3 From 5af57250f2b1cd174fa75c559bac6ea179c43d09 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 28 Feb 2017 10:22:06 -0300 Subject: [test] bugfix: actually use an empty local db in download benchmarks We were previously not using an empty local db for download benchmark tests, so there was actually nothing to sync. This commit fixes that by adding a way to force an empty local db on soledad client instantiation. --- testing/tests/benchmarks/test_sync.py | 2 +- testing/tests/conftest.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index 1501d74b..ee2e2960 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -41,7 +41,7 @@ def create_download(downloads, size): # ensures we are dealing with properly encrypted docs def setup(): - return soledad_client() + return soledad_client(force_fresh_db=True) def sync(clean_client): return clean_client.sync() diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 1d3125d5..49cc35f6 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -191,9 +191,19 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): soledad_dbs(default_uuid) # get a soledad instance - def create(): - secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % default_uuid) - local_db_path = os.path.join(tmpdir.strpath, '%s.db' % default_uuid) + def create(force_fresh_db=False): + secrets_file = '%s.secret' % default_uuid + secrets_path = os.path.join(tmpdir.strpath, secrets_file) + + # in some tests we might want to use the same user and remote database + # but with a clean/empty local database (i.e. download benchmarks), so + # here we provide a way to do that. + db_file = '%s.db' % default_uuid + if force_fresh_db: + prefix = uuid4().hex + db_file = prefix + '-' + db_file + local_db_path = os.path.join(tmpdir.strpath, db_file) + soledad_client = Soledad( default_uuid, unicode(passphrase), -- cgit v1.2.3 From 6deadf8adeb8c6b4a98e2d69e2bf3b1447a6b4ca Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 1 Mar 2017 13:39:04 -0300 Subject: [test] mark benchmark tests using their group names --- testing/tests/benchmarks/conftest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py index a9cc3464..1b99d96e 100644 --- a/testing/tests/benchmarks/conftest.py +++ b/testing/tests/benchmarks/conftest.py @@ -12,12 +12,30 @@ from leap.common.events import server server.ensure_server() +# +# pytest customizations +# + def pytest_addoption(parser): parser.addoption( "--num-docs", type="int", default=100, help="the number of documents to use in performance tests") +# mark benchmark tests using their group names (thanks ionelmc! :) +def pytest_collection_modifyitems(items): + for item in items: + bench = item.get_marker("benchmark") + if bench and bench.kwargs.get('group'): + group = bench.kwargs['group'] + marker = getattr(pytest.mark, 'benchmark_' + group) + item.add_marker(marker) + + +# +# benchmark fixtures +# + @pytest.fixture() def payload(): def generate(size): -- cgit v1.2.3 From 32c950d16e7bea55bb7af3f4650af857be9027e0 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 1 Mar 2017 21:02:41 -0300 Subject: [test] improve twistd startup and termination - use subprocess.check_call() to ensure any errors during twistd startup will properly show up on test reports. - use SIGTERM instead of SIGKILL to gracefully terminate twistd. --- testing/tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 49cc35f6..c077828f 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -6,7 +6,7 @@ import signal import time from hashlib import sha512 -from subprocess import call +from subprocess import check_call from urlparse import urljoin from uuid import uuid4 @@ -98,7 +98,7 @@ class SoledadServer(object): def start(self): self._create_conf_file() # start the server - call([ + check_call([ 'twistd', '--logfile=%s' % self._logfile, '--pidfile=%s' % self._pidfile, @@ -118,7 +118,7 @@ class SoledadServer(object): def stop(self): pid = get_pid(self._pidfile) - os.kill(pid, signal.SIGKILL) + os.kill(pid, signal.SIGTERM) @pytest.fixture(scope='module') -- cgit v1.2.3 From 32c6d2eae998ceec302995e52c70a0636ca1e463 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 2 Mar 2017 15:47:04 -0300 Subject: [test] add comments explaining behaviour of upload/download benchmark --- testing/tests/benchmarks/test_sync.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index ee2e2960..fcfab998 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -11,6 +11,12 @@ def load_up(client, amount, payload): yield gatherResults(deferreds) +# Each test created with this function will: +# +# - get a fresh client. +# - iterate: +# - setup: create N docs of a certain size +# - benchmark: sync() -- uploads N docs. def create_upload(uploads, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_upload") @@ -29,6 +35,14 @@ test_upload_100_100k = create_upload(100, 100 * 1000) test_upload_1000_10k = create_upload(1000, 10 * 1000) +# Each test created with this function will: +# +# - get a fresh client. +# - create N docs of a certain size +# - sync (uploads those docs) +# - iterate: +# - setup: get a fresh client with empty local db +# - benchmark: sync() -- downloads N docs. def create_download(downloads, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_download") -- cgit v1.2.3 From 3eefcb7d138ef41932a748ae729bfa0b629758d2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 6 Mar 2017 00:43:51 +0100 Subject: [pkg] write changelog for 0.9.3 --- CHANGELOG.rst | 25 +++++++++++++++++++++++++ docs/changelog-next.rst | 7 +------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f47749d1..9570b70b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,28 @@ +0.9.3 - 06 March, 2017 ++++++++++++++++++++++++++++++++ + + +Server +~~~~~~ +- Refactor authentication code to use twisted credential system. +- Announce server blobs capabilities +- `#8764 `_: Allow unauthenticated users to retrieve the capabilties banner. +- `#6178 `_: Add robots.txt +- #8762: Add a systemd service file +- Add script to deploy from git + +Client +~~~~~~~~ +- `#8758 `_: Add blob size to the crypto preamble +- Improve secrets generation and storage code +- Add offline status to soledad client api. +- Remove syncable property + +Misc +~~~~ +- Improvements in performance benchmarks. + + 0.9.2 - 22 December, 2016 +++++++++++++++++++++++++ diff --git a/docs/changelog-next.rst b/docs/changelog-next.rst index 1da32789..52540f8e 100644 --- a/docs/changelog-next.rst +++ b/docs/changelog-next.rst @@ -1,4 +1,4 @@ -0.9.3 - +0.9.4 - +++++++++++++++++++++++++++++++ Please add lines to this file, they will be moved to the CHANGELOG.rst during @@ -10,10 +10,6 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff. Features ~~~~~~~~ -- Refactor authentication code to use twisted credential system. -- `#8764 `_: Allow unauthenticated - users to retrieve the capabilties banner. -- `#8758 `_: Add blob size to the crypto preamble - `#1234 `_: Description of the new feature corresponding with issue #1234. - New feature without related issue number. @@ -24,7 +20,6 @@ Bugfixes Other ~~~~~ -- #8762: Add a systemd service file - `#1236 `_: Description of the new feature corresponding with issue #1236. - Some change without issue number. -- cgit v1.2.3