diff options
| author | drebs <drebs@leap.se> | 2017-03-11 14:38:51 +0100 |
|---|---|---|
| committer | drebs <drebs@leap.se> | 2017-03-11 14:38:51 +0100 |
| commit | 3ec21a7a6b39b4fe8885f3050ab75402e6812a1f (patch) | |
| tree | 300a2d8da67bbc7c6484f9ee136a4a18c41c48c6 | |
| parent | c379a58d84fbf061b8d046057e45089f0e3c65f6 (diff) | |
| parent | 3eefcb7d138ef41932a748ae729bfa0b629758d2 (diff) | |
Merge tag '0.9.3' into debian/platform-0.9
0.9.3
Conflicts:
server/pkg/soledad-server
server/pkg/soledad-server.service
55 files changed, 2260 insertions, 2250 deletions
diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f47749d1..9570b70b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,28 @@ +0.9.3 - 06 March, 2017 ++++++++++++++++++++++++++++++++ + + +Server +~~~~~~ +- Refactor authentication code to use twisted credential system. +- Announce server blobs capabilities +- `#8764 <https://0xacab.org/leap/soledad/issues/8764>`_: Allow unauthenticated users to retrieve the capabilties banner. +- `#6178 <https://0xacab.org/leap/soledad/issues/6178>`_: Add robots.txt +- #8762: Add a systemd service file +- Add script to deploy from git + +Client +~~~~~~~~ +- `#8758 <https://0xacab.org/leap/soledad/issues/8758>`_: Add blob size to the crypto preamble +- Improve secrets generation and storage code +- Add offline status to soledad client api. +- Remove syncable property + +Misc +~~~~ +- Improvements in performance benchmarks. + + 0.9.2 - 22 December, 2016 +++++++++++++++++++++++++ diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index 4bbdd044..f91084a4 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -22,6 +22,7 @@ Cryptographic operations for the soledad client import binascii import base64 import hashlib +import warnings import hmac import os import re @@ -49,7 +50,8 @@ SECRET_LENGTH = 64 CRYPTO_BACKEND = MultiBackend([OpenSSLBackend()]) -PACMAN = struct.Struct('2sbbQ16s255p255p') +PACMAN = struct.Struct('2sbbQ16s255p255pQ') +LEGACY_PACMAN = struct.Struct('2sbbQ16s255p255p') BLOB_SIGNATURE_MAGIC = '\x13\x37' @@ -188,11 +190,14 @@ class BlobEncryptor(object): self.doc_id = doc_info.doc_id self.rev = doc_info.rev self._content_fd = content_fd + content_fd.seek(0, os.SEEK_END) + self._content_size = content_fd.tell() + content_fd.seek(0) self._producer = FileBodyProducer(content_fd, readSize=2**16) - sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret) - self._aes = AESWriter(sym_key) - self._aes.authenticate(self._make_preamble()) + self.sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret) + self._aes = AESWriter(self.sym_key) + self._aes.authenticate(self._encode_preamble()) @property def iv(self): @@ -214,7 +219,7 @@ class BlobEncryptor(object): d.addCallback(lambda _: self._end_crypto_stream()) return d - def _make_preamble(self): + def _encode_preamble(self): current_time = int(time.time()) return PACMAN.pack( @@ -224,7 +229,8 @@ class BlobEncryptor(object): current_time, self.iv, str(self.doc_id), - str(self.rev)) + str(self.rev), + self._content_size) def _end_crypto_stream(self): preamble, encrypted = self._aes.end() @@ -271,14 +277,21 @@ class BlobDecryptor(object): raise InvalidBlob ciphertext_fd.close() - if len(preamble) != PACMAN.size: - raise InvalidBlob - try: - unpacked_data = PACMAN.unpack(preamble) - magic, sch, meth, ts, iv, doc_id, rev = unpacked_data - except struct.error: - raise InvalidBlob + if len(preamble) == LEGACY_PACMAN.size: + warnings.warn("Decrypting a legacy document without size. " + + "This will be deprecated in 0.12. Doc was: " + + "doc_id: %s rev: %s" % (self.doc_id, self.rev), + Warning) + unpacked_data = LEGACY_PACMAN.unpack(preamble) + magic, sch, meth, ts, iv, doc_id, rev = unpacked_data + elif len(preamble) == PACMAN.size: + unpacked_data = PACMAN.unpack(preamble) + magic, sch, meth, ts, iv, doc_id, rev, doc_size = unpacked_data + else: + raise InvalidBlob("Unexpected preamble size %d", len(preamble)) + except struct.error, e: + raise InvalidBlob(e) if magic != BLOB_SIGNATURE_MAGIC: raise InvalidBlob diff --git a/client/src/leap/soledad/client/_secrets/__init__.py b/client/src/leap/soledad/client/_secrets/__init__.py new file mode 100644 index 00000000..b6c81cda --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/__init__.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +# _secrets/__init__.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import scrypt + +from leap.soledad.common.log import getLogger + +from leap.soledad.client._secrets.storage import SecretsStorage +from leap.soledad.client._secrets.crypto import SecretsCrypto +from leap.soledad.client._secrets.util import emit, UserDataMixin + + +logger = getLogger(__name__) + + +class Secrets(UserDataMixin): + + lengths = { + 'remote_secret': 512, # remote_secret is used to encrypt remote data. + 'local_salt': 64, # local_salt is used in conjunction with + 'local_secret': 448, # local_secret to derive a local_key for storage + } + + def __init__(self, soledad): + self._soledad = soledad + self._secrets = {} + self.crypto = SecretsCrypto(soledad) + self.storage = SecretsStorage(soledad) + self._bootstrap() + + # + # bootstrap + # + + def _bootstrap(self): + + # attempt to load secrets from local storage + encrypted = self.storage.load_local() + if encrypted: + self._secrets = self.crypto.decrypt(encrypted) + # maybe update the format of storage of local secret. + if encrypted['version'] < self.crypto.VERSION: + self.store_secrets() + return + + # no secret was found in local storage, so this is a first run of + # soledad for this user in this device. It is mandatory that we check + # if there's a secret stored in server. + encrypted = self.storage.load_remote() + if encrypted: + self._secrets = self.crypto.decrypt(encrypted) + self.store_secrets() + return + + # we have *not* found a secret neither in local nor in remote storage, + # so we have to generate a new one, and then store it. + self._secrets = self._generate() + self.store_secrets() + + # + # generation + # + + @emit('creating') + def _generate(self): + logger.info("generating new set of secrets...") + secrets = {} + for name, length in self.lengths.iteritems(): + secret = os.urandom(length) + secrets[name] = secret + logger.info("new set of secrets successfully generated") + return secrets + + # + # crypto + # + + def store_secrets(self): + # TODO: we have to improve the logic here, as we want to make sure that + # whatever is stored locally should only be used after remote storage + # is successful. Otherwise, this soledad could start encrypting with a + # secret while another soledad in another device could start encrypting + # with another secret, which would lead to decryption failures during + # sync. + encrypted = self.crypto.encrypt(self._secrets) + self.storage.save_local(encrypted) + self.storage.save_remote(encrypted) + + # + # secrets + # + + @property + def remote_secret(self): + return self._secrets.get('remote_secret') + + @property + def local_salt(self): + return self._secrets.get('local_salt') + + @property + def local_secret(self): + return self._secrets.get('local_secret') + + @property + def local_key(self): + # local storage key is scrypt-derived from `local_secret` and + # `local_salt` above + secret = scrypt.hash( + password=self.local_secret, + salt=self.local_salt, + buflen=32, # we need a key with 256 bits (32 bytes) + ) + return secret diff --git a/client/src/leap/soledad/client/_secrets/crypto.py b/client/src/leap/soledad/client/_secrets/crypto.py new file mode 100644 index 00000000..fa7aaca0 --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/crypto.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# _secrets/crypto.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import binascii +import json +import os +import scrypt + +from leap.soledad.common import soledad_assert +from leap.soledad.common.log import getLogger + +from leap.soledad.client._crypto import encrypt_sym, decrypt_sym, ENC_METHOD +from leap.soledad.client._secrets.util import SecretsError + + +logger = getLogger(__name__) + + +class SecretsCrypto(object): + + VERSION = 2 + + def __init__(self, soledad): + self._soledad = soledad + + def _get_key(self, salt): + passphrase = self._soledad.passphrase.encode('utf8') + key = scrypt.hash(passphrase, salt, buflen=32) + return key + + # + # encryption + # + + def encrypt(self, secrets): + encoded = {} + for name, value in secrets.iteritems(): + encoded[name] = binascii.b2a_base64(value) + plaintext = json.dumps(encoded) + salt = os.urandom(64) # TODO: get salt length from somewhere else + key = self._get_key(salt) + iv, ciphertext = encrypt_sym(plaintext, key, + method=ENC_METHOD.aes_256_gcm) + encrypted = { + 'version': self.VERSION, + 'kdf': 'scrypt', + 'kdf_salt': binascii.b2a_base64(salt), + 'kdf_length': len(key), + 'cipher': ENC_METHOD.aes_256_gcm, + 'length': len(plaintext), + 'iv': str(iv), + 'secrets': binascii.b2a_base64(ciphertext), + } + return encrypted + + # + # decryption + # + + def decrypt(self, data): + version = data.get('version') + method = getattr(self, '_decrypt_v%d' % version) + try: + return method(data) + except Exception as e: + logger.error('error decrypting secrets: %r' % e) + raise SecretsError(e) + + def _decrypt_v1(self, data): + # get encrypted secret from dictionary: the old format allowed for + # storage of more than one secret, but this feature was never used and + # soledad has been using only one secret so far. As there is a corner + # case where the old 'active_secret' key might not be set, we just + # ignore it and pop the only secret found in the 'storage_secrets' key. + secret_id = data['storage_secrets'].keys().pop() + encrypted = data['storage_secrets'][secret_id] + + # assert that we know how to decrypt the secret + soledad_assert('cipher' in encrypted) + cipher = encrypted['cipher'] + if cipher == 'aes256': + cipher = ENC_METHOD.aes_256_ctr + soledad_assert(cipher in ENC_METHOD) + + # decrypt + salt = binascii.a2b_base64(encrypted['kdf_salt']) + key = self._get_key(salt) + separator = ':' + iv, ciphertext = encrypted['secret'].split(separator, 1) + ciphertext = binascii.a2b_base64(ciphertext) + plaintext = self._decrypt(key, iv, ciphertext, encrypted, cipher) + + # create secrets dictionary + secrets = { + 'remote_secret': plaintext[0:512], + 'local_salt': plaintext[512:576], + 'local_secret': plaintext[576:1024], + } + return secrets + + def _decrypt_v2(self, encrypted): + cipher = encrypted['cipher'] + soledad_assert(cipher in ENC_METHOD) + + salt = binascii.a2b_base64(encrypted['kdf_salt']) + key = self._get_key(salt) + iv = encrypted['iv'] + ciphertext = binascii.a2b_base64(encrypted['secrets']) + plaintext = self._decrypt( + key, iv, ciphertext, encrypted, cipher) + encoded = json.loads(plaintext) + secrets = {} + for name, value in encoded.iteritems(): + secrets[name] = binascii.a2b_base64(value) + return secrets + + def _decrypt(self, key, iv, ciphertext, encrypted, method): + # assert some properties of the stored secret + soledad_assert(encrypted['kdf'] == 'scrypt') + soledad_assert(encrypted['kdf_length'] == len(key)) + # decrypt + plaintext = decrypt_sym(ciphertext, key, iv, method) + soledad_assert(encrypted['length'] == len(plaintext)) + return plaintext diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py new file mode 100644 index 00000000..056c4322 --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# _secrets/storage.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import json +import urlparse + +from hashlib import sha256 + +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.log import getLogger + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client._secrets.util import emit, UserDataMixin + + +logger = getLogger(__name__) + + +class SecretsStorage(UserDataMixin): + + def __init__(self, soledad): + self._soledad = soledad + self._shared_db = self._soledad.shared_db or self._init_shared_db() + self.__remote_doc = None + + @property + def _creds(self): + uuid = self._soledad.uuid + token = self._soledad.token + return {'token': {'uuid': uuid, 'token': token}} + + # + # local storage + # + + def load_local(self): + path = self._soledad.secrets_path + logger.info("trying to load secrets from disk: %s" % path) + try: + with open(path, 'r') as f: + encrypted = json.loads(f.read()) + logger.info("secrets loaded successfully from disk") + return encrypted + except IOError: + logger.warn("secrets not found in disk") + return None + + def save_local(self, encrypted): + path = self._soledad.secrets_path + json_data = json.dumps(encrypted) + with open(path, 'w') as f: + f.write(json_data) + + # + # remote storage + # + + def _init_shared_db(self): + url = urlparse.urljoin(self._soledad.server_url, SHARED_DB_NAME) + creds = self._creds + db = SoledadSharedDatabase.open_database(url, creds) + return db + + def _remote_doc_id(self): + passphrase = self._soledad.passphrase.encode('utf8') + uuid = self._soledad.uuid + text = '%s%s' % (passphrase, uuid) + digest = sha256(text).hexdigest() + return digest + + @property + def _remote_doc(self): + if not self.__remote_doc and self._shared_db: + doc = self._get_remote_doc() + self.__remote_doc = doc + return self.__remote_doc + + @emit('downloading') + def _get_remote_doc(self): + logger.info('trying to load secrets from server...') + doc = self._shared_db.get_doc(self._remote_doc_id()) + if doc: + logger.info('secrets loaded successfully from server') + else: + logger.warn('secrets not found in server') + return doc + + def load_remote(self): + doc = self._remote_doc + if not doc: + return None + encrypted = doc.content + return encrypted + + @emit('uploading') + def save_remote(self, encrypted): + doc = self._remote_doc + if not doc: + doc = SoledadDocument(doc_id=self._remote_doc_id()) + doc.content = encrypted + db = self._shared_db + if not db: + logger.warn('no shared db found') + return + db.put_doc(doc) diff --git a/client/src/leap/soledad/client/_secrets/util.py b/client/src/leap/soledad/client/_secrets/util.py new file mode 100644 index 00000000..6401889b --- /dev/null +++ b/client/src/leap/soledad/client/_secrets/util.py @@ -0,0 +1,63 @@ +# -*- coding:utf-8 -*- +# _secrets/util.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +from leap.soledad.client import events + + +class SecretsError(Exception): + pass + + +class UserDataMixin(object): + """ + When emitting an event, we have to pass a dictionary containing user data. + This class only defines a property so we don't have to define it in + multiple places. + """ + + @property + def _user_data(self): + uuid = self._soledad.uuid + userid = self._soledad.userid + # TODO: seems that uuid and userid hold the same value! We should check + # whether we should pass something different or if the events api + # really needs two different values. + return {'uuid': uuid, 'userid': userid} + + +def emit(verb): + def _decorator(method): + def _decorated(self, *args, **kwargs): + + # emit starting event + user_data = self._user_data + name = 'SOLEDAD_' + verb.upper() + '_KEYS' + event = getattr(events, name) + events.emit_async(event, user_data) + + # run the method + result = method(self, *args, **kwargs) + + # emit a finished event + name = 'SOLEDAD_DONE_' + verb.upper() + '_KEYS' + event = getattr(events, name) + events.emit_async(event, user_data) + + return result + return _decorated + return _decorator diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index da6eec66..4be38cf1 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -39,13 +39,12 @@ from itertools import chain from StringIO import StringIO from collections import defaultdict -from twisted.internet.defer import DeferredLock, returnValue, inlineCallbacks +from twisted.internet import defer from zope.interface import implements from leap.common.config import get_path_prefix from leap.common.plugins import collect_plugins -from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type from leap.soledad.common.log import getLogger @@ -57,8 +56,7 @@ from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events from leap.soledad.client import interfaces as soledad_interfaces from leap.soledad.client import sqlcipher -from leap.soledad.client.secrets import SoledadSecrets -from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client._secrets import Secrets from leap.soledad.client._crypto import SoledadCrypto logger = getLogger(__name__) @@ -126,11 +124,11 @@ class Soledad(object): same database replica. The dictionary indexes are the paths to each local db, so we guarantee that only one sync happens for a local db at a time. """ - _sync_lock = defaultdict(DeferredLock) + _sync_lock = defaultdict(defer.DeferredLock) def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, - auth_token=None, syncable=True): + auth_token=None, offline=False): """ Initialize configuration, cryptographic keys and dbs. @@ -167,10 +165,11 @@ class Soledad(object): Authorization token for accessing remote databases. :type auth_token: str - :param syncable: - If set to ``False``, this database will not attempt to synchronize - with remote replicas (default is ``True``) - :type syncable: bool + :param offline: + If set to ``True``, this database will not attempt to save/load + secrets to/from server or synchronize with remote replicas (default + is ``False``) + :type offline: bool :raise BootstrapSequenceError: Raised when the secret initialization sequence (i.e. retrieval @@ -178,41 +177,32 @@ class Soledad(object): some reason. """ # store config params - self._uuid = uuid - self._passphrase = passphrase + self.uuid = uuid + self.passphrase = passphrase + self.secrets_path = secrets_path self._local_db_path = local_db_path - self._server_url = server_url - self._secrets_path = None - self._dbsyncer = None - + self.server_url = server_url self.shared_db = shared_db + self.token = auth_token + self.offline = offline + + self._dbsyncer = None # configure SSL certificate global SOLEDAD_CERT SOLEDAD_CERT = cert_file - self._set_token(auth_token) - self._init_config_with_defaults() self._init_working_dirs() - self._secrets_path = secrets_path - - # Initialize shared recovery database - self.init_shared_db(server_url, uuid, self._creds, syncable=syncable) - - # The following can raise BootstrapSequenceError, that will be - # propagated upwards. - self._init_secrets() - - self._crypto = SoledadCrypto(self._secrets.remote_storage_secret) + self._secrets = Secrets(self) + self._crypto = SoledadCrypto(self._secrets.remote_secret) try: # initialize database access, trap any problems so we can shutdown # smoothly. self._init_u1db_sqlcipher_backend() - if syncable: - self._init_u1db_syncer() + self._init_u1db_syncer() except DatabaseAccessError: # oops! something went wrong with backend initialization. We # have to close any thread-related stuff we have already opened @@ -230,7 +220,7 @@ class Soledad(object): """ Initialize configuration using default values for missing params. """ - soledad_assert_type(self._passphrase, unicode) + soledad_assert_type(self.passphrase, unicode) def initialize(attr, val): return ((getattr(self, attr, None) is None) and @@ -241,7 +231,7 @@ class Soledad(object): initialize("_local_db_path", os.path.join( self.default_prefix, self.local_db_file_name)) # initialize server_url - soledad_assert(self._server_url is not None, + soledad_assert(self.server_url is not None, 'Missing URL for Soledad server.') def _init_working_dirs(self): @@ -255,15 +245,6 @@ class Soledad(object): for path in paths: create_path_if_not_exists(path) - def _init_secrets(self): - """ - Initialize Soledad secrets. - """ - self._secrets = SoledadSecrets( - self.uuid, self._passphrase, self._secrets_path, - self.shared_db, userid=self.userid) - self._secrets.bootstrap() - def _init_u1db_sqlcipher_backend(self): """ Initialize the U1DB SQLCipher database for local storage. @@ -279,7 +260,7 @@ class Soledad(object): """ tohex = binascii.b2a_hex # sqlcipher only accepts the hex version - key = tohex(self._secrets.get_local_storage_key()) + key = tohex(self._secrets.local_key) opts = sqlcipher.SQLCipherOptions( self._local_db_path, key, @@ -648,10 +629,6 @@ class Soledad(object): return self._local_db_path @property - def uuid(self): - return self._uuid - - @property def userid(self): return self.uuid @@ -659,21 +636,6 @@ class Soledad(object): # ISyncableStorage # - def set_syncable(self, syncable): - """ - Toggle the syncable state for this database. - - This can be used to start a database with offline state and switch it - online afterwards. Or the opposite: stop syncs when connection is lost. - - :param syncable: new status for syncable. - :type syncable: bool - """ - # TODO should check that we've got a token! - self.shared_db.syncable = syncable - if syncable and not self._dbsyncer: - self._init_u1db_syncer() - def sync(self): """ Synchronize documents with the server replica. @@ -686,6 +648,11 @@ class Soledad(object): generation before the synchronization was performed. :rtype: twisted.internet.defer.Deferred """ + # maybe bypass sync + if self.offline or not self.token: + generation = self._dbsyncer.get_generation() + return defer.succeed(generation) + d = self.sync_lock.run( self._sync) return d @@ -698,12 +665,11 @@ class Soledad(object): generation before the synchronization was performed. :rtype: twisted.internet.defer.Deferred """ - sync_url = urlparse.urljoin(self._server_url, 'user-%s' % self.uuid) + sync_url = urlparse.urljoin(self.server_url, 'user-%s' % self.uuid) if not self._dbsyncer: return - d = self._dbsyncer.sync( - sync_url, - creds=self._creds) + creds = {'token': {'uuid': self.uuid, 'token': self.token}} + d = self._dbsyncer.sync(sync_url, creds=creds) def _sync_callback(local_gen): self._last_received_docs = docs = self._dbsyncer.received_docs @@ -760,101 +726,17 @@ class Soledad(object): """ return self.sync_lock.locked - @property - def syncable(self): - if self.shared_db: - return self.shared_db.syncable - else: - return False - - def _set_token(self, token): - """ - Set the authentication token for remote database access. - - Internally, this builds the credentials dictionary with the following - format: - - { - 'token': { - 'uuid': '<uuid>' - 'token': '<token>' - } - } - - :param token: The authentication token. - :type token: str - """ - self._creds = { - 'token': { - 'uuid': self.uuid, - 'token': token, - } - } - - def _get_token(self): - """ - Return current token from credentials dictionary. - """ - return self._creds['token']['token'] - - token = property(_get_token, _set_token, doc='The authentication Token.') - # # ISecretsStorage # - def init_shared_db(self, server_url, uuid, creds, syncable=True): - """ - Initialize the shared database. - - :param server_url: URL of the remote database. - :type server_url: str - :param uuid: The user's unique id. - :type uuid: str - :param creds: A tuple containing the authentication method and - credentials. - :type creds: tuple - :param syncable: - If syncable is False, the database will not attempt to sync against - a remote replica. - :type syncable: bool - """ - # only case this is False is for testing purposes - if self.shared_db is None: - shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) - self.shared_db = SoledadSharedDatabase.open_database( - shared_db_url, - uuid, - creds=creds, - syncable=syncable) - - @property - def storage_secret(self): - """ - Return the secret used for local storage encryption. - - :return: The secret used for local storage encryption. - :rtype: str - """ - return self._secrets.storage_secret - - @property - def remote_storage_secret(self): - """ - Return the secret used for encryption of remotely stored data. - - :return: The secret used for remote storage encryption. - :rtype: str - """ - return self._secrets.remote_storage_secret - @property def secrets(self): """ Return the secrets object. :return: The secrets object. - :rtype: SoledadSecrets + :rtype: Secrets """ return self._secrets @@ -867,7 +749,8 @@ class Soledad(object): :raise NoStorageSecret: Raised if there's no storage secret available. """ - self._secrets.change_passphrase(new_passphrase) + self.passphrase = new_passphrase + self._secrets.store_secrets() # # Raw SQLCIPHER Queries @@ -891,7 +774,7 @@ class Soledad(object): # Service authentication # - @inlineCallbacks + @defer.inlineCallbacks def get_or_create_service_token(self, service): """ Return the stored token for a given service, or generates and stores a @@ -906,11 +789,11 @@ class Soledad(object): docs = yield self._get_token_for_service(service) if docs: doc = docs[0] - returnValue(doc.content['token']) + defer.returnValue(doc.content['token']) else: token = str(uuid.uuid4()).replace('-', '')[-24:] yield self._set_token_for_service(service, token) - returnValue(token) + defer.returnValue(token) def _get_token_for_service(self, service): return self.get_from_index('by-servicetoken', 'servicetoken', service) diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 0e250bf1..b67d03f6 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -24,10 +24,11 @@ after receiving. import os -from leap.soledad.common.log import getLogger -from leap.common.certs import get_compatible_ssl_context_factory from twisted.web.client import Agent from twisted.internet import reactor + +from leap.common.certs import get_compatible_ssl_context_factory +from leap.soledad.common.log import getLogger from leap.soledad.client.http_target.send import HTTPDocSender from leap.soledad.client.http_target.api import SyncTargetAPI from leap.soledad.client.http_target.fetch import HTTPDocFetcher @@ -86,8 +87,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): self._insert_doc_cb = None # Twisted default Agent with our own ssl context factory - self._http = Agent(reactor, - get_compatible_ssl_context_factory(cert_file)) + factory = get_compatible_ssl_context_factory(cert_file) + self._http = Agent(reactor, factory) if DO_STATS: self.sync_exchange_phase = [0] diff --git a/client/src/leap/soledad/client/http_target/fetch_protocol.py b/client/src/leap/soledad/client/http_target/fetch_protocol.py index fa6b1969..c7eabe2b 100644 --- a/client/src/leap/soledad/client/http_target/fetch_protocol.py +++ b/client/src/leap/soledad/client/http_target/fetch_protocol.py @@ -63,6 +63,8 @@ class DocStreamReceiver(ReadBodyProtocol): Deliver the accumulated response bytes to the waiting L{Deferred}, if the response body has been completely received without error. """ + if self.deferred.called: + return try: if reason.check(ResponseDone): self.dataBuffer = self.metadata @@ -125,11 +127,7 @@ class DocStreamReceiver(ReadBodyProtocol): else: d = self._doc_reader( self.current_doc, line.strip() or None, self.total) - d.addErrback(self._error) - - def _error(self, reason): - logger.error(reason) - self.transport.loseConnection() + d.addErrback(self.deferred.errback) def finish(self): """ diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py index 82927ff4..1be47df7 100644 --- a/client/src/leap/soledad/client/interfaces.py +++ b/client/src/leap/soledad/client/interfaces.py @@ -351,28 +351,12 @@ class ISecretsStorage(Interface): secrets_file_name = Attribute( "The name of the file where the storage secrets will be stored") - storage_secret = Attribute("") - remote_storage_secret = Attribute("") - shared_db = Attribute("The shared db object") - # XXX this used internally from secrets, so it might be good to preserve # as a public boundary with other components. # We should also probably document its interface. secrets = Attribute("A SoledadSecrets object containing access to secrets") - def init_shared_db(self, server_url, uuid, creds): - """ - Initialize the shared recovery database. - - :param server_url: - :type server_url: - :param uuid: - :type uuid: - :param creds: - :type creds: - """ - def change_passphrase(self, new_passphrase): """ Change the passphrase that encrypts the storage secret. @@ -382,7 +366,3 @@ class ISecretsStorage(Interface): :raise NoStorageSecret: Raised if there's no storage secret available. """ - - # XXX not in use. Uncomment if we ever decide to allow - # multiple secrets. - # secret_id = Attribute("The id of the storage secret to be used") diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py deleted file mode 100644 index 3fe98c64..00000000 --- a/client/src/leap/soledad/client/secrets.py +++ /dev/null @@ -1,794 +0,0 @@ -# -*- coding: utf-8 -*- -# secrets.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Soledad secrets handling. -""" - - -import os -import scrypt -import binascii -import errno -import json - -from hashlib import sha256 - -from leap.soledad.common import soledad_assert -from leap.soledad.common import soledad_assert_type -from leap.soledad.common import document -from leap.soledad.common.log import getLogger -from leap.soledad.client import events -from leap.soledad.client import _crypto - - -logger = getLogger(__name__) - - -# -# Exceptions -# - - -class SecretsException(Exception): - - """ - Generic exception type raised by this module. - """ - - -class NoStorageSecret(SecretsException): - - """ - Raised when trying to use a storage secret but none is available. - """ - pass - - -class PassphraseTooShort(SecretsException): - - """ - Raised when trying to change the passphrase but the provided passphrase is - too short. - """ - - -class BootstrapSequenceError(SecretsException): - - """ - Raised when an attempt to generate a secret and store it in a recovery - document on server failed. - """ - - -# -# Secrets handler -# - - -class SoledadSecrets(object): - - """ - Soledad secrets handler. - - The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage - secret are used for remote storage encryption. We use the next - C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage. - From these bytes, the first C{self.SALT_LENGTH} bytes are used as the - salt and the rest as the password for the scrypt hashing. - """ - - LOCAL_STORAGE_SECRET_LENGTH = 512 - """ - The length, in bytes, of the secret used to derive a passphrase for the - SQLCipher database. - """ - - REMOTE_STORAGE_SECRET_LENGTH = 512 - """ - The length, in bytes, of the secret used to derive an encryption key for - remote storage. - """ - - SALT_LENGTH = 64 - """ - The length, in bytes, of the salt used to derive the key for the storage - secret encryption. - """ - - GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \ - + REMOTE_STORAGE_SECRET_LENGTH \ - + SALT_LENGTH # for sync db - """ - The length, in bytes, of the secret to be generated. This includes local - and remote secrets, and the salt for deriving the sync db secret. - """ - - MINIMUM_PASSPHRASE_LENGTH = 6 - """ - The minimum length, in bytes, for a passphrase. The passphrase length is - only checked when the user changes her passphrase, not when she - instantiates Soledad. - """ - - SEPARATOR = ":" - """ - A separator used for storing the encryption initial value prepended to the - ciphertext. - """ - - UUID_KEY = 'uuid' - STORAGE_SECRETS_KEY = 'storage_secrets' - ACTIVE_SECRET_KEY = 'active_secret' - SECRET_KEY = 'secret' - CIPHER_KEY = 'cipher' - LENGTH_KEY = 'length' - KDF_KEY = 'kdf' - KDF_SALT_KEY = 'kdf_salt' - KDF_LENGTH_KEY = 'kdf_length' - KDF_SCRYPT = 'scrypt' - CIPHER_AES256 = 'aes256' # deprecated, AES-CTR - CIPHER_AES256_GCM = _crypto.ENC_METHOD.aes_256_gcm - RECOVERY_DOC_VERSION_KEY = 'version' - RECOVERY_DOC_VERSION = 1 - """ - Keys used to access storage secrets in recovery documents. - """ - - def __init__(self, uuid, passphrase, secrets_path, shared_db, userid=None): - """ - Initialize the secrets manager. - - :param uuid: User's unique id. - :type uuid: str - :param passphrase: The passphrase for locking and unlocking encryption - secrets for local and remote storage. - :type passphrase: unicode - :param secrets_path: Path for storing encrypted key used for - symmetric encryption. - :type secrets_path: str - :param shared_db: The shared database that stores user secrets. - :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase - """ - self._uuid = uuid - self._userid = userid - self._passphrase = passphrase - self._secrets_path = secrets_path - self._shared_db = shared_db - self._secrets = {} - self._secret_id = None - - def bootstrap(self): - """ - Bootstrap secrets. - - Soledad secrets bootstrap is the following sequence of stages: - - * stage 1 - local secret loading: - - if secrets exist locally, load them. - * stage 2 - remote secret loading: - - else, if secrets exist in server, download them. - * stage 3 - secret generation: - - else, generate a new secret and store in server. - - This method decides which bootstrap stages have already been performed - and performs the missing ones in order. - - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed for some reason. - """ - # STAGE 1 - verify if secrets exist locally - try: - logger.info("trying to load secrets from local storage...") - version = self._load_secrets_from_local_file() - # eventually migrate local and remote stored documents from old - # format version - if version < self.RECOVERY_DOC_VERSION: - self._store_secrets() - self._upload_crypto_secrets() - logger.info("found secrets in local storage") - return - - except NoStorageSecret: - logger.info("could not find secrets in local storage") - - # STAGE 2 - there are no secrets in local storage and this is the - # first time we are running soledad with the specified - # secrets_path. Try to fetch encrypted secrets from - # server. - try: - logger.info('trying to fetch secrets from remote storage...') - version = self._download_crypto_secrets() - self._store_secrets() - # eventually migrate remote stored document from old format - # version - if version < self.RECOVERY_DOC_VERSION: - self._upload_crypto_secrets() - logger.info('found secrets in remote storage.') - return - except NoStorageSecret: - logger.info("could not find secrets in remote storage.") - - # STAGE 3 - there are no secrets in server also, so we want to - # generate the secrets and store them in the remote - # db. - logger.info("generating secrets...") - self._gen_crypto_secrets() - logger.info("uploading secrets...") - self._upload_crypto_secrets() - - def _has_secret(self): - """ - Return whether there is a storage secret available for use or not. - - :return: Whether there's a storage secret for symmetric encryption. - :rtype: bool - """ - return self.storage_secret is not None - - def _maybe_set_active_secret(self, active_secret): - """ - If no secret_id is already set, choose the passed active secret, or - just choose first secret available if none. - """ - if not self._secret_id: - if not active_secret: - active_secret = self._secrets.items()[0][0] - self.set_secret_id(active_secret) - - def _load_secrets_from_local_file(self): - """ - Load storage secrets from local file. - - :return version: The version of the locally stored recovery document. - - :raise NoStorageSecret: Raised if there are no secrets available in - local storage. - """ - # check if secrets file exists and we can read it - if not os.path.isfile(self._secrets_path): - raise NoStorageSecret - - # read storage secrets from file - content = None - with open(self._secrets_path, 'r') as f: - content = json.loads(f.read()) - _, active_secret, version = self._import_recovery_document(content) - - self._maybe_set_active_secret(active_secret) - - return version - - def _download_crypto_secrets(self): - """ - Download crypto secrets. - - :return version: The version of the remotelly stored recovery document. - - :raise NoStorageSecret: Raised if there are no secrets available in - remote storage. - """ - doc = None - if self._shared_db.syncable: - doc = self._get_secrets_from_shared_db() - - if doc is None: - raise NoStorageSecret - - _, active_secret, version = self._import_recovery_document(doc.content) - self._maybe_set_active_secret(active_secret) - - return version - - def _gen_crypto_secrets(self): - """ - Generate the crypto secrets. - """ - logger.info('no cryptographic secrets found, creating new secrets...') - secret_id = self._gen_secret() - self.set_secret_id(secret_id) - - def _upload_crypto_secrets(self): - """ - Send crypto secrets to shared db. - - :raises BootstrapSequenceError: Raised when unable to store secrets in - shared database. - """ - if self._shared_db.syncable: - try: - self._put_secrets_in_shared_db() - except Exception as ex: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. - try: - os.unlink(self._secrets_path) - except OSError as e: - if e.errno != errno.ENOENT: - # no such file or directory - logger.exception(e) - logger.exception(ex) - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') - - # - # Shared DB related methods - # - - def _shared_db_doc_id(self): - """ - Calculate the doc_id of the document in the shared db that stores key - material. - - :return: the hash - :rtype: str - """ - return sha256( - '%s%s' % - (self._passphrase_as_string(), self._uuid)).hexdigest() - - def _export_recovery_document(self, cipher=None): - """ - Export the storage secrets. - - Current format of recovery document has the following structure: - - { - 'storage_secrets': { - '<storage_secret id>': { - 'cipher': 'aes256', - 'length': <secret length>, - 'secret': '<encrypted storage_secret>', - }, - }, - 'active_secret': '<secret_id>', - 'version': '<recovery document format version>', - } - - Note that multiple storage secrets might be stored in one recovery - document. - - :param cipher: (Optional) The ciper to use. Defaults to AES256 - :type cipher: str - - :return: The recovery document. - :rtype: dict - """ - # encrypt secrets - encrypted_secrets = {} - for secret_id in self._secrets: - encrypted_secrets[secret_id] = self._encrypt_storage_secret( - self._secrets[secret_id], doc_cipher=cipher) - # create the recovery document - data = { - self.STORAGE_SECRETS_KEY: encrypted_secrets, - self.ACTIVE_SECRET_KEY: self._secret_id, - self.RECOVERY_DOC_VERSION_KEY: self.RECOVERY_DOC_VERSION, - } - return data - - def _import_recovery_document(self, data): - """ - Import storage secrets for symmetric encryption from a recovery - document. - - Note that this method does not store the imported data on disk. For - that, use C{self._store_secrets()}. - - :param data: The recovery document. - :type data: dict - - :return: A tuple containing the number of imported secrets, the - secret_id of the last active secret, and the recovery - document format version. - :rtype: (int, str, int) - """ - soledad_assert(self.STORAGE_SECRETS_KEY in data) - version = data.get(self.RECOVERY_DOC_VERSION_KEY, 1) - meth = getattr(self, '_import_recovery_document_version_%d' % version) - secret_count, active_secret = meth(data) - return secret_count, active_secret, version - - def _import_recovery_document_version_1(self, data): - """ - Import storage secrets for symmetric encryption from a recovery - document with format version 1. - - Version 1 of recovery document has the following structure: - - { - 'storage_secrets': { - '<storage_secret id>': { - 'cipher': 'aes256', - 'length': <secret length>, - 'secret': '<encrypted storage_secret>', - }, - }, - 'active_secret': '<secret_id>', - 'version': '<recovery document format version>', - } - - :param data: The recovery document. - :type data: dict - - :return: A tuple containing the number of imported secrets, the - secret_id of the last active secret, and the recovery - document format version. - :rtype: (int, str, int) - """ - # include secrets in the secret pool. - secret_count = 0 - secrets = data[self.STORAGE_SECRETS_KEY].items() - active_secret = None - # XXX remove check for existence of key (included for backwards - # compatibility) - if self.ACTIVE_SECRET_KEY in data: - active_secret = data[self.ACTIVE_SECRET_KEY] - for secret_id, encrypted_secret in secrets: - if secret_id not in self._secrets: - try: - self._secrets[secret_id] = \ - self._decrypt_storage_secret_version_1( - encrypted_secret) - secret_count += 1 - except SecretsException as e: - logger.error("failed to decrypt storage secret: %s" - % str(e)) - raise e - return secret_count, active_secret - - def _get_secrets_from_shared_db(self): - """ - Retrieve the document with encrypted key material from the shared - database. - - :return: a document with encrypted key material in its contents - :rtype: document.SoledadDocument - """ - user_data = self._get_user_data() - events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, user_data) - db = self._shared_db - if not db: - logger.warn('no shared db found') - return - doc = db.get_doc(self._shared_db_doc_id()) - user_data = {'userid': self._userid, 'uuid': self._uuid} - events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) - return doc - - def _put_secrets_in_shared_db(self): - """ - Assert local keys are the same as shared db's ones. - - Try to fetch keys from shared recovery database. If they already exist - in the remote db, assert that that data is the same as local data. - Otherwise, upload keys to shared recovery database. - """ - soledad_assert( - self._has_secret(), - 'Tried to send keys to server but they don\'t exist in local ' - 'storage.') - # try to get secrets doc from server, otherwise create it - doc = self._get_secrets_from_shared_db() - if doc is None: - doc = document.SoledadDocument( - doc_id=self._shared_db_doc_id()) - # fill doc with encrypted secrets - doc.content = self._export_recovery_document() - # upload secrets to server - user_data = self._get_user_data() - events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data) - db = self._shared_db - if not db: - logger.warn('no shared db found') - return - db.put_doc(doc) - events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data) - - # - # Management of secret for symmetric encryption. - # - - def _decrypt_storage_secret_version_1(self, encrypted_secret_dict): - """ - Decrypt the storage secret. - - Storage secret is encrypted before being stored. This method decrypts - and returns the decrypted storage secret. - - Version 1 of storage secret format has the following structure: - - '<storage_secret id>': { - 'cipher': 'aes256', - 'length': <secret length>, - 'secret': '<encrypted storage_secret>', - }, - - :param encrypted_secret_dict: The encrypted storage secret. - :type encrypted_secret_dict: dict - - :return: The decrypted storage secret. - :rtype: str - - :raise SecretsException: Raised in case the decryption of the storage - secret fails for some reason. - """ - # calculate the encryption key - if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT: - raise SecretsException("Unknown KDF in stored secret.") - key = scrypt.hash( - self._passphrase_as_string(), - # the salt is stored base64 encoded - binascii.a2b_base64( - encrypted_secret_dict[self.KDF_SALT_KEY]), - buflen=32, # we need a key with 256 bits (32 bytes). - ) - if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key): - raise SecretsException("Wrong length of decryption key.") - supported_ciphers = [self.CIPHER_AES256, self.CIPHER_AES256_GCM] - doc_cipher = encrypted_secret_dict[self.CIPHER_KEY] - if doc_cipher not in supported_ciphers: - raise SecretsException("Unknown cipher in stored secret.") - # recover the initial value and ciphertext - iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split( - self.SEPARATOR, 1) - ciphertext = binascii.a2b_base64(ciphertext) - try: - decrypted_secret = _crypto.decrypt_sym( - ciphertext, key, iv, doc_cipher) - except Exception as e: - logger.error(e) - raise SecretsException("Unable to decrypt secret.") - if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret): - raise SecretsException("Wrong length of decrypted secret.") - return decrypted_secret - - def _encrypt_storage_secret(self, decrypted_secret, doc_cipher=None): - """ - Encrypt the storage secret. - - An encrypted secret has the following structure: - - { - '<secret_id>': { - 'kdf': 'scrypt', - 'kdf_salt': '<b64 repr of salt>' - 'kdf_length': <key length> - 'cipher': 'aes256', - 'length': <secret length>, - 'secret': '<encrypted b64 repr of storage_secret>', - } - } - - :param decrypted_secret: The decrypted storage secret. - :type decrypted_secret: str - :param cipher: (Optional) The ciper to use. Defaults to AES256 - :type cipher: str - - :return: The encrypted storage secret. - :rtype: dict - """ - # generate random salt - salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - doc_cipher = doc_cipher or self.CIPHER_AES256_GCM - iv, ciphertext = _crypto.encrypt_sym(decrypted_secret, key, doc_cipher) - ciphertext = binascii.b2a_base64(ciphertext) - encrypted_secret_dict = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: doc_cipher, - self.LENGTH_KEY: len(decrypted_secret), - self.SECRET_KEY: self.SEPARATOR.join([str(iv), ciphertext]) - } - return encrypted_secret_dict - - @property - def storage_secret(self): - """ - Return the storage secret. - - :return: The decrypted storage secret. - :rtype: str - """ - return self._secrets.get(self._secret_id) - - def set_secret_id(self, secret_id): - """ - Define the id of the storage secret to be used. - - This method will also replace the secret in the crypto object. - - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - """ - self._secret_id = secret_id - - def _gen_secret(self): - """ - Generate a secret for symmetric encryption and store in a local - encrypted file. - - This method emits the following events.signals: - - * SOLEDAD_CREATING_KEYS - * SOLEDAD_DONE_CREATING_KEYS - - :return: The id of the generated secret. - :rtype: str - """ - user_data = self._get_user_data() - events.emit_async(events.SOLEDAD_CREATING_KEYS, user_data) - # generate random secret - secret = os.urandom(self.GEN_SECRET_LENGTH) - secret_id = sha256(secret).hexdigest() - self._secrets[secret_id] = secret - self._store_secrets() - events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, user_data) - return secret_id - - def _store_secrets(self): - """ - Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. - """ - with open(self._secrets_path, 'w') as f: - f.write( - json.dumps( - self._export_recovery_document())) - - def change_passphrase(self, new_passphrase): - """ - Change the passphrase that encrypts the storage secret. - - :param new_passphrase: The new passphrase. - :type new_passphrase: unicode - - :raise NoStorageSecret: Raised if there's no storage secret available. - """ - # TODO: maybe we want to add more checks to guarantee passphrase is - # reasonable? - soledad_assert_type(new_passphrase, unicode) - if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: - raise PassphraseTooShort( - 'Passphrase must be at least %d characters long!' % - self.MINIMUM_PASSPHRASE_LENGTH) - # ensure there's a secret for which the passphrase will be changed. - if not self._has_secret(): - raise NoStorageSecret() - self._passphrase = new_passphrase - self._store_secrets() - self._put_secrets_in_shared_db() - - # - # Setters and getters - # - - @property - def secret_id(self): - return self._secret_id - - def _get_secrets_path(self): - return self._secrets_path - - def _set_secrets_path(self, secrets_path): - self._secrets_path = secrets_path - - secrets_path = property( - _get_secrets_path, - _set_secrets_path, - doc='The path for the file containing the encrypted symmetric secret.') - - @property - def passphrase(self): - """ - Return the passphrase for locking and unlocking encryption secrets for - local and remote storage. - """ - return self._passphrase - - def _passphrase_as_string(self): - return self._passphrase.encode('utf-8') - - # - # remote storage secret - # - - @property - def remote_storage_secret(self): - """ - Return the secret for remote storage. - """ - key_start = 0 - key_end = self.REMOTE_STORAGE_SECRET_LENGTH - return self.storage_secret[key_start:key_end] - - # - # local storage key - # - - def _get_local_storage_secret(self): - """ - Return the local storage secret. - - :return: The local storage secret. - :rtype: str - """ - secret_len = self.REMOTE_STORAGE_SECRET_LENGTH - lsecret_len = self.LOCAL_STORAGE_SECRET_LENGTH - pwd_start = secret_len + self.SALT_LENGTH - pwd_end = secret_len + lsecret_len - return self.storage_secret[pwd_start:pwd_end] - - def _get_local_storage_salt(self): - """ - Return the local storage salt. - - :return: The local storage salt. - :rtype: str - """ - salt_start = self.REMOTE_STORAGE_SECRET_LENGTH - salt_end = salt_start + self.SALT_LENGTH - return self.storage_secret[salt_start:salt_end] - - def get_local_storage_key(self): - """ - Return the local storage key derived from the local storage secret. - - :return: The key for protecting the local database. - :rtype: str - """ - return scrypt.hash( - password=self._get_local_storage_secret(), - salt=self._get_local_storage_salt(), - buflen=32, # we need a key with 256 bits (32 bytes) - ) - - # - # sync db key - # - - def _get_sync_db_salt(self): - """ - Return the salt for sync db. - """ - salt_start = self.LOCAL_STORAGE_SECRET_LENGTH \ - + self.REMOTE_STORAGE_SECRET_LENGTH - salt_end = salt_start + self.SALT_LENGTH - return self.storage_secret[salt_start:salt_end] - - def get_sync_db_key(self): - """ - Return the key for protecting the sync database. - - :return: The key for protecting the sync database. - :rtype: str - """ - return scrypt.hash( - password=self._get_local_storage_secret(), - salt=self._get_sync_db_salt(), - buflen=32, # we need a key with 256 bits (32 bytes) - ) - - def _get_user_data(self): - return {'uuid': self._uuid, 'userid': self._userid} diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index d43db045..4f70c74b 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -17,7 +17,7 @@ """ A shared database for storing/retrieving encrypted key material. """ -from leap.soledad.common.l2db.remote import http_database +from leap.soledad.common.l2db.remote.http_database import HTTPDatabase from leap.soledad.client.auth import TokenBasedAuth @@ -47,7 +47,7 @@ class ImproperlyConfiguredError(Exception): """ -class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): +class SoledadSharedDatabase(HTTPDatabase, TokenBasedAuth): """ This is a shared recovery database that enables users to store their encryption secrets in the server and retrieve them afterwards. @@ -55,10 +55,6 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # TODO: prevent client from messing with the shared DB. # TODO: define and document API. - # If syncable is False, the database will not attempt to sync against - # a remote replica. Default is True. - syncable = True - # # Token auth methods. # @@ -95,31 +91,20 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, uuid, creds=None, syncable=True): + def open_database(url, creds=None): """ Open a Soledad shared database. :param url: URL of the remote database. :type url: str - :param uuid: The user's unique id. - :type uuid: str :param creds: A tuple containing the authentication method and credentials. :type creds: tuple - :param syncable: - If syncable is False, the database will not attempt to sync against - a remote replica. - :type syncable: bool :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ - # XXX fix below, doesn't work with tests. - # if syncable and not url.startswith('https://'): - # raise ImproperlyConfiguredError( - # "Remote soledad server must be an https URI") - db = SoledadSharedDatabase(url, uuid, creds=creds) - db.syncable = syncable + db = SoledadSharedDatabase(url, creds=creds) return db @staticmethod @@ -134,20 +119,16 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): """ raise Unauthorized("Can't delete shared database.") - def __init__(self, url, uuid, document_factory=None, creds=None): + def __init__(self, url, document_factory=None, creds=None): """ Initialize database with auth token and encryption powers. :param url: URL of the remote database. :type url: str - :param uuid: The user's unique id. - :type uuid: str :param document_factory: A factory for U1BD documents. :type document_factory: u1db.Document :param creds: A tuple containing the authentication method and credentials. :type creds: tuple """ - http_database.HTTPDatabase.__init__(self, url, document_factory, - creds) - self._uuid = uuid + HTTPDatabase.__init__(self, url, document_factory, creds) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index c9a9444e..a3e45228 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -397,7 +397,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): ENCRYPT_LOOP_PERIOD = 1 def __init__(self, opts, soledad_crypto, replica_uid, cert_file): - self._opts = opts self._path = opts.path self._crypto = soledad_crypto diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index 1ba6ab89..4948ad20 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -30,8 +30,6 @@ Soledad routines common to client and server. # SHARED_DB_NAME = 'shared' -SHARED_DB_LOCK_DOC_ID_PREFIX = 'lock-' -USER_DB_PREFIX = 'user-' # diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py index a7f5b7b6..a4841d0d 100644 --- a/common/src/leap/soledad/common/couch/state.py +++ b/common/src/leap/soledad/common/couch/state.py @@ -19,13 +19,10 @@ Server state using CouchDatabase as backend. """ import couchdb import re -import time from urlparse import urljoin -from hashlib import sha512 from leap.soledad.common.log import getLogger from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.couch import couch_server from leap.soledad.common.couch import CONFIG_DOC_ID from leap.soledad.common.couch import SCHEMA_VERSION from leap.soledad.common.couch import SCHEMA_VERSION_KEY @@ -59,12 +56,6 @@ class CouchServerState(ServerState): Inteface of the WSGI server with the CouchDB backend. """ - TOKENS_DB_PREFIX = "tokens_" - TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds - TOKENS_TYPE_KEY = "type" - TOKENS_TYPE_DEF = "Token" - TOKENS_USER_ID_KEY = "user_id" - def __init__(self, couch_url, create_cmd=None, check_schema_versions=False): """ @@ -164,40 +155,3 @@ class CouchServerState(ServerState): delete databases. """ raise Unauthorized() - - def verify_token(self, uuid, token): - """ - Query couchdb to decide if C{token} is valid for C{uuid}. - - @param uuid: The user uuid. - @type uuid: str - @param token: The token. - @type token: str - """ - with couch_server(self.couch_url) as server: - # the tokens db rotates every 30 days, and the current db name is - # "tokens_NNN", where NNN is the number of seconds since epoch - # divide dby the rotate period in seconds. When rotating, old and - # new tokens db coexist during a certain window of time and valid - # tokens are replicated from the old db to the new one. See: - # https://leap.se/code/issues/6785 - dbname = self._tokens_dbname() - db = server[dbname] - # lookup key is a hash of the token to prevent timing attacks. - token = db.get(sha512(token).hexdigest()) - if token is None: - return False - # we compare uuid hashes to avoid possible timing attacks that - # might exploit python's builtin comparison operator behaviour, - # which fails immediatelly when non-matching bytes are found. - couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest() - req_uuid_hash = sha512(uuid).digest() - if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ - or couch_uuid_hash != req_uuid_hash: - return False - return True - - def _tokens_dbname(self): - dbname = self.TOKENS_DB_PREFIX + \ - str(int(time.time() / self.TOKENS_DB_EXPIRE)) - return dbname diff --git a/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py b/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py deleted file mode 100644 index 96d0d872..00000000 --- a/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. -"""U1DB Basic Auth authorisation WSGI middleware.""" -import httplib -import json - -from wsgiref.util import shift_path_info - - -class Unauthorized(Exception): - """User authorization failed.""" - - -class BasicAuthMiddleware(object): - """U1DB Basic Auth Authorisation WSGI middleware.""" - - def __init__(self, app, prefix): - self.app = app - self.prefix = prefix - - def _error(self, start_response, status, description, message=None): - start_response("%d %s" % (status, httplib.responses[status]), - [('content-type', 'application/json')]) - err = {"error": description} - if message: - err['message'] = message - return [json.dumps(err)] - - def __call__(self, environ, start_response): - if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): - return self._error(start_response, 400, "bad request") - auth = environ.get('HTTP_AUTHORIZATION') - if not auth: - return self._error(start_response, 401, "unauthorized", - "Missing Basic Authentication.") - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'basic': - return self._error( - start_response, 401, "unauthorized", - "Missing Basic Authentication") - user, password = encoded.decode('base64').split(':', 1) - try: - self.verify_user(environ, user, password) - except Unauthorized: - return self._error( - start_response, 401, "unauthorized", - "Incorrect password or login.") - del environ['HTTP_AUTHORIZATION'] - shift_path_info(environ) - return self.app(environ, start_response) - - def verify_user(self, environ, username, password): - raise NotImplementedError(self.verify_user) diff --git a/docs/changelog-next.rst b/docs/changelog-next.rst new file mode 100644 index 00000000..52540f8e --- /dev/null +++ b/docs/changelog-next.rst @@ -0,0 +1,27 @@ +0.9.4 - ++++++++++++++++++++++++++++++++ + +Please add lines to this file, they will be moved to the CHANGELOG.rst during +the next release. + +There are two template lines for each category, use them as reference. + +I've added a new category `Misc` so we can track doc/style/packaging stuff. + +Features +~~~~~~~~ +- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234. +- New feature without related issue number. + +Bugfixes +~~~~~~~~ +- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235. +- Bugfix without related issue number. + +Other +~~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236. +- Some change without issue number. + +Known Issues +~~~~~~~~~~~~ diff --git a/scripts/deploy/deploy_soledad_from_git b/scripts/deploy/deploy_soledad_from_git new file mode 100755 index 00000000..3c4ac8be --- /dev/null +++ b/scripts/deploy/deploy_soledad_from_git @@ -0,0 +1,52 @@ +#!/bin/bash +###################################################### +# Deploy soledad-server from a given remote and branch +# valid remotes are: origin shyba drebs kali +# DO NOT USE IN PRODUCTION OR I'LL SEND NINJAS TO YOUR +# HOUSE!!!! +# (c) LEAP, 2017 +###################################################### +set -e + +REMOTE=$1 +BRANCH=$2 + +if [ "$#" -lt 2 ]; then + echo "USAGE: $0 REMOTE BRANCH" + exit 1 +fi + +SOLEDADPATH="/usr/lib/python2.7/dist-packages/leap/soledad/server" +REPO="https://0xacab.org/leap/soledad" +LOCALREPO="$HOME/soledad" +SYSTEMDINIT="/lib/systemd/system/soledad-server.service" + +apt remove --yes soledad-server + +if [ ! -d "$LOCALREPO" ]; then + echo "soledad repo not found, cloning..." + cd $HOME + git clone $REPO + cd $LOCALREPO + git remote add shyba https://0xacab.org/shyba/soledad.git + git remote add drebs https://0xacab.org/drebs/soledad.git + git remote add kali https://0xacab.org/kali/soledad.git +fi + +cd $LOCALREPO && git checkout -- . && git fetch $REMOTE && git checkout $REMOTE/$BRANCH +rm -rf $SOLEDADPATH + +# copy over some stuff that we'll need -- stolen from debian package +cp -r $LOCALREPO/server/src/leap/soledad/server $SOLEDADPATH +cp $LOCALREPO/server/pkg/soledad-server.service $SYSTEMDINIT +cp $LOCALREPO/server/pkg/create-user-db /usr/bin/ +cp $LOCALREPO/server/pkg/soledad-sudoers /etc/sudoers.d/ + +# Let's append the branch info to the version string! So that nobody is lost +cd $LOCALREPO && echo "__version__ = '"`git describe`"~"`git status | head -n 1 | cut -d' ' -f 4`"'" >> $SOLEDADPATH/__init__.py + +# restart the daemon +systemctl --system daemon-reload >/dev/null || true +deb-systemd-invoke restart soledad-server.service >/dev/null || true + +tail -n 20 /var/log/syslog diff --git a/scripts/migration/0.9.0/migrate.py b/scripts/migration/0.9.0/migrate.py index 6ad5bc2d..7b2ec005 100755 --- a/scripts/migration/0.9.0/migrate.py +++ b/scripts/migration/0.9.0/migrate.py @@ -27,14 +27,14 @@ import os from argparse import ArgumentParser -from leap.soledad.server import load_configuration +from leap.soledad.server import get_config from migrate_couch_schema import migrate TARGET_VERSION = '0.8.2' DEFAULT_COUCH_URL = 'http://127.0.0.1:5984' -CONF = load_configuration('/etc/soledad/soledad-server.conf') +CONF = get_config() NETRC_PATH = CONF['soledad-server']['admin_netrc'] diff --git a/scripts/migration/0.9.0/migrate_couch_schema/__init__.py b/scripts/migration/0.9.0/migrate_couch_schema/__init__.py index f0b456e4..d1560c59 100644 --- a/scripts/migration/0.9.0/migrate_couch_schema/__init__.py +++ b/scripts/migration/0.9.0/migrate_couch_schema/__init__.py @@ -30,7 +30,7 @@ def _get_couch_server(couch_url): return Server(couch_url) -def _is_migrateable(db): +def _has_u1db_config_doc(db): config_doc = db.get('u1db_config') return bool(config_doc) @@ -63,6 +63,35 @@ def _get_user_dbs(server): # migration main functions # +def _report_missing_u1db_config_doc(dbname, db): + config_doc = db.get(CONFIG_DOC_ID) + if not config_doc: + logger.warning( + "[%s] no '%s' or '%s' documents found, possibly an empty db? I " + "don't know what to do with this db, so I am skipping it." + % (dbname, 'u1db_config', CONFIG_DOC_ID)) + else: + if SCHEMA_VERSION_KEY in config_doc: + version = config_doc[SCHEMA_VERSION_KEY] + if version == SCHEMA_VERSION: + logger.info( + "[%s] '%s' document exists, and schema versions match " + "(expected %r and found %r). This database reports to be " + "using the new schema version, so I am skipping it." + % (dbname, CONFIG_DOC_ID)) + else: + logger.error( + "[%s] '%s' document exists, but schema versions don't " + "match (expected %r, found %r instead). I don't know " + "how to migrate such a db, so I am skipping it." + % (dbname, CONFIG_DOC_ID, SCHEMA_VERSION, version)) + else: + logger.error( + "[%s] '%s' document exists, but has no schema version " + "information in it. I don't know how to migrate such a db, " + "so I am skipping it." % (dbname, CONFIG_DOC_ID)) + + def migrate(args, target_version): server = _get_couch_server(args.couch_url) logger.info('starting couch schema migration to %s' % target_version) @@ -71,8 +100,8 @@ def migrate(args, target_version): user_dbs = _get_user_dbs(server) for dbname in user_dbs: db = server[dbname] - if not _is_migrateable(db): - logger.warning("[%s] skipping not migrateable user db" % dbname) + if not _has_u1db_config_doc(db): + _report_missing_u1db_config_doc(dbname, db) continue logger.info("[%s] starting migration of user db" % dbname) try: diff --git a/scripts/packaging/build-deb.sh b/scripts/packaging/build-deb.sh new file mode 100755 index 00000000..f7dd22a0 --- /dev/null +++ b/scripts/packaging/build-deb.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# This script generates a debian package from your current repository tree +# (including modified and unstaged files), using the debian directory from the +# latest debian/platform-X.Y branch. +# +# In order to achieve that, what it does is: +# +# - copy the current repository into a temporary directory. +# - find what is the latest "debian/platform-X.Y" branch. +# - checkout the "debian/" directory from that branch. +# - update the "debian/changelog" file with dummy information. +# - run "debuild -uc -us". + +debemail="Leap Automatic Deb Builder <deb@leap.se>" +scriptdir=$(dirname "${0}") +gitroot=$(git -C "${scriptdir}" rev-parse --show-toplevel) +deb_branch=$(git -C "${gitroot}" branch | grep "debian/platform" | sort | tail -n 1 | xargs) +reponame=$(basename "${gitroot}") +tempdir=$(mktemp -d) +targetdir="${tempdir}/${reponame}" + +cp -r "${gitroot}" "${tempdir}/${reponame}" +git -C "${targetdir}" checkout "${deb_branch}" -- debian + +(cd "${targetdir}" && DEBEMAIL="${debemail}" dch -b "Automatic build.") +(cd "${targetdir}" && debuild -uc -us) + +echo "****************************************" +echo "Packages can be found in: ${tempdir}" +ls "${tempdir}" +echo "****************************************" diff --git a/scripts/packaging/build_debian_package.sh b/scripts/packaging/build_debian_package.sh deleted file mode 100755 index b9fb93a9..00000000 --- a/scripts/packaging/build_debian_package.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -# This script generates Soledad Debian packages. -# -# When invoking this script, you should pass a git repository URL and the name -# of the branch that contains the code you wish to build the packages from. -# -# The script will clone the given branch from the given repo, as well as the -# main Soledad repo in github which contains the most up-to-date debian -# branch. It will then merge the desired branch into the debian branch and -# build the packages. - -if [ $# -ne 2 ]; then - echo "Usage: ${0} <url> <branch>" - exit 1 -fi - -SOLEDAD_MAIN_REPO=git://github.com/leapcode/soledad.git - -url=$1 -branch=$2 -workdir=`mktemp -d` - -git clone -b ${branch} ${url} ${workdir}/soledad -export GIT_DIR=${workdir}/soledad/.git -export GIT_WORK_TREE=${workdir}/soledad -git remote add leapcode ${SOLEDAD_MAIN_REPO} -git fetch leapcode -git checkout -b debian/experimental leapcode/debian/experimental -git merge --no-edit ${branch} -(cd ${workdir}/soledad && debuild -uc -us) -echo "Packages generated in ${workdir}" diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 9e2b6b50..5e0ef5e2 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -22,7 +22,7 @@ import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch.state import is_db_name_valid from leap.soledad.common.couch import list_users_dbs -from leap.soledad.server.config import load_configuration +from leap.soledad.server._config import get_config BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False) @@ -38,8 +38,9 @@ parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') parser.add_argument('--migrate-all', action='store_true', help="recreate all design docs for all existing account") -CONF = load_configuration('/etc/soledad/soledad-server.conf') -NETRC_PATH = CONF['soledad-server']['admin_netrc'] +CONF = get_config() +DBCONF = get_config(section='database-security') +NETRC_PATH = CONF['admin_netrc'] def url_for_db(dbname): @@ -78,7 +79,7 @@ def ensure_database(dbname): print ("Invalid name! %s" % dbname) sys.exit(1) url = url_for_db(dbname) - db_security = CONF['database-security'] + db_security = DBCONF db = CouchDatabase.open_database(url=url, create=True, replica_uid=None, database_security=db_security) diff --git a/server/pkg/soledad-server.service b/server/pkg/soledad-server.service index ccd03b97..30c4bf88 100644 --- a/server/pkg/soledad-server.service +++ b/server/pkg/soledad-server.service @@ -3,7 +3,7 @@ Description=Soledad Server [Service] Environment=PATH=/sbin:/bin:/usr/sbin:/usr/bin -Environment=CLASS=leap.soledad.server.resource.SoledadResource +Environment=CLASS=leap.soledad.server.entrypoint.SoledadEntrypoint Environment=HTTPS_PORT=2424 Environment=CERT_PATH=/etc/soledad/soledad-server.pem Environment=PRIVKEY_PATH=/etc/soledad/soledad-server.key diff --git a/server/pkg/soledad-sudoers b/server/pkg/soledad-sudoers new file mode 100644 index 00000000..642497f8 --- /dev/null +++ b/server/pkg/soledad-sudoers @@ -0,0 +1,2 @@ +Cmnd_Alias SOLEDAD_CREATE_DB = /usr/bin/create-user-db +soledad ALL=(soledad-admin) NOPASSWD: SOLEDAD_CREATE_DB diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 039bef75..5bed22c9 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -88,15 +88,17 @@ import sys from leap.soledad.common.l2db.remote import http_app, utils from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.server.sync import SyncResource -from leap.soledad.server.sync import MAX_REQUEST_SIZE -from leap.soledad.server.sync import MAX_ENTRY_SIZE +from .sync import SyncResource +from .sync import MAX_REQUEST_SIZE +from .sync import MAX_ENTRY_SIZE from ._version import get_versions +from ._config import get_config __all__ = [ 'SoledadApp', + 'get_config', '__version__', ] diff --git a/server/src/leap/soledad/server/_blobs.py b/server/src/leap/soledad/server/_blobs.py new file mode 100644 index 00000000..cacabbdf --- /dev/null +++ b/server/src/leap/soledad/server/_blobs.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# _blobs.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Blobs Server implementation. +""" +from twisted.web import resource + +from ._config import get_config + + +__all__ = ['BlobsResource', 'blobs_resource'] + + +class BlobsResource(resource.Resource): + + isLeaf = True + + def __init__(self, blobs_path): + resource.Resource.__init__(self) + self._blobs_path = blobs_path + + def render_GET(self, request): + return 'blobs is not implemented yet!' + + +# provide a configured instance of the resource +_config = get_config() +_path = _config['blobs_path'] + +blobs_resource = BlobsResource(_path) diff --git a/server/src/leap/soledad/server/config.py b/server/src/leap/soledad/server/_config.py index 3c17ec19..e89e70d6 100644 --- a/server/src/leap/soledad/server/config.py +++ b/server/src/leap/soledad/server/_config.py @@ -19,12 +19,17 @@ import configparser +__all__ = ['get_config'] + + CONFIG_DEFAULTS = { 'soledad-server': { 'couch_url': 'http://localhost:5984', 'create_cmd': None, 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', - 'batching': True + 'batching': True, + 'blobs': False, + 'blobs_path': '/srv/leap/soledad/blobs', }, 'database-security': { 'members': ['soledad'], @@ -35,7 +40,17 @@ CONFIG_DEFAULTS = { } -def load_configuration(file_path): +_config = None + + +def get_config(section='soledad-server'): + global _config + if not _config: + _config = _load_config('/etc/soledad/soledad-server.conf') + return _config[section] + + +def _load_config(file_path): """ Load server configuration from file. @@ -45,23 +60,23 @@ def load_configuration(file_path): @return: A dictionary with the configuration. @rtype: dict """ - defaults = dict(CONFIG_DEFAULTS) + conf = dict(CONFIG_DEFAULTS) config = configparser.SafeConfigParser() config.read(file_path) - for section in defaults: + for section in conf: if not config.has_section(section): continue - for key, value in defaults[section].items(): + for key, value in conf[section].items(): if not config.has_option(section, key): continue elif type(value) == bool: - defaults[section][key] = config.getboolean(section, key) + conf[section][key] = config.getboolean(section, key) elif type(value) == list: values = config.get(section, key).split(',') values = [v.strip() for v in values] - defaults[section][key] = values + conf[section][key] = values else: - defaults[section][key] = config.get(section, key) + conf[section][key] = config.get(section, key) # TODO: implement basic parsing/sanitization of options comming from # config file. - return defaults + return conf diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py new file mode 100644 index 00000000..7a00ad9a --- /dev/null +++ b/server/src/leap/soledad/server/_resource.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# resource.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +A twisted resource that serves the Soledad Server. +""" +from twisted.web.resource import Resource + +from ._blobs import blobs_resource +from ._server_info import ServerInfo +from ._wsgi import get_sync_resource + + +__all__ = ['SoledadResource', 'SoledadAnonResource'] + + +class _Robots(Resource): + def render_GET(self, request): + return ( + 'User-agent: *\n' + 'Disallow: /\n' + '# you are not a robot, are you???') + + +class SoledadAnonResource(Resource): + + """ + The parts of Soledad Server that unauthenticated users can see. + This is nice because this means that a non-authenticated user will get 404 + for anything that is not in this minimal resource tree. + """ + + def __init__(self, enable_blobs=False): + Resource.__init__(self) + server_info = ServerInfo(enable_blobs) + self.putChild('', server_info) + self.putChild('robots.txt', _Robots()) + + +class SoledadResource(Resource): + """ + This is a dummy twisted resource, used only to allow different entry points + for the Soledad Server. + """ + + def __init__(self, enable_blobs=False, sync_pool=None): + """ + Initialize the Soledad resource. + + :param enable_blobs: Whether the blobs feature should be enabled. + :type enable_blobs: bool + + :param sync_pool: A pool to pass to the WSGI sync resource. + :type sync_pool: twisted.python.threadpool.ThreadPool + """ + Resource.__init__(self) + + # requests to / return server information + server_info = ServerInfo(enable_blobs) + self.putChild('', server_info) + + # requests to /blobs will serve blobs if enabled + if enable_blobs: + self.putChild('blobs', blobs_resource) + + # other requests are routed to legacy sync resource + self._sync_resource = get_sync_resource(sync_pool) + + def getChild(self, path, request): + """ + Route requests to legacy WSGI sync resource dynamically. + """ + request.postpath.insert(0, request.prepath.pop()) + return self._sync_resource diff --git a/server/src/leap/soledad/server/_server_info.py b/server/src/leap/soledad/server/_server_info.py new file mode 100644 index 00000000..50659338 --- /dev/null +++ b/server/src/leap/soledad/server/_server_info.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# _server_info.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Resource that announces information about the server. +""" +import json + +from twisted.web.resource import Resource + +from leap.soledad.server import __version__ + + +__all__ = ['ServerInfo'] + + +class ServerInfo(Resource): + """ + Return information about the server. + """ + + isLeaf = True + + def __init__(self, blobs_enabled): + self._info = { + "blobs": blobs_enabled, + "version": __version__ + } + + def render_GET(self, request): + return json.dumps(self._info) diff --git a/server/src/leap/soledad/server/application.py b/server/src/leap/soledad/server/_wsgi.py index 17296425..f6ff6b26 100644 --- a/server/src/leap/soledad/server/application.py +++ b/server/src/leap/soledad/server/_wsgi.py @@ -15,40 +15,31 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -A WSGI application to serve as the root resource of the webserver. - -Use it like this: - - twistd web --wsgi=leap.soledad.server.application.wsgi_application +A WSGI application that serves Soledad synchronization. """ from twisted.internet import reactor +from twisted.web.wsgi import WSGIResource from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware -from leap.soledad.server.config import load_configuration from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.log import getLogger +from twisted.logger import Logger +log = Logger() -__all__ = ['wsgi_application'] - - -def _load_config(): - conf = load_configuration('/etc/soledad/soledad-server.conf') - return conf['soledad-server'] +__all__ = ['init_couch_state', 'get_sync_resource'] -def _get_couch_state(): - conf = _load_config() +def _get_couch_state(conf): state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], check_schema_versions=True) SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) return state -_app = SoledadTokenAuthMiddleware(SoledadApp(None)) # delay state init +_app = SoledadApp(None) # delay state init wsgi_application = GzipMiddleware(_app) @@ -61,13 +52,14 @@ wsgi_application = GzipMiddleware(_app) # work. Because of that, we delay couch state initialization until the reactor # is running. -def _init_couch_state(_app): +def init_couch_state(conf): try: - _app.state = _get_couch_state() + _app.state = _get_couch_state(conf) except Exception as e: logger = getLogger() logger.error(str(e)) reactor.stop() -reactor.callWhenRunning(_init_couch_state, _app) +def get_sync_resource(pool): + return WSGIResource(reactor, pool, wsgi_application) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index b0764569..b5744fe9 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -15,383 +15,156 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Authentication facilities for Soledad Server. +Twisted http token auth. """ -import httplib -import json - -from abc import ABCMeta, abstractmethod -from routes.mapper import Mapper - -from leap.soledad.common.log import getLogger -from leap.soledad.common.l2db import DBNAME_CONSTRAINTS, errors as u1db_errors -from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.common import USER_DB_PREFIX - - -logger = getLogger(__name__) - - -class URLToAuthorization(object): - """ - Verify if actions can be performed by a user. - """ - - HTTP_METHOD_GET = 'GET' - HTTP_METHOD_PUT = 'PUT' - HTTP_METHOD_DELETE = 'DELETE' - HTTP_METHOD_POST = 'POST' - - def __init__(self, uuid): - """ - Initialize the mapper. - - The C{uuid} is used to create the rules that will either allow or - disallow the user to perform specific actions. - - @param uuid: The user uuid. - @type uuid: str - @param user_db_prefix: The string prefix of users' databases. - @type user_db_prefix: str - """ - self._map = Mapper(controller_scan=None) - self._user_db_name = "%s%s" % (USER_DB_PREFIX, uuid) - self._uuid = uuid - self._register_auth_info() - - def is_authorized(self, environ): - """ - Return whether an HTTP request that produced the CGI C{environ} - corresponds to an authorized action. - - @param environ: Dictionary containing CGI variables. - @type environ: dict - - @return: Whether the action is authorized or not. - @rtype: bool - """ - return self._map.match(environ=environ) is not None - - def _register(self, pattern, http_methods): - """ - Register a C{pattern} in the mapper as valid for C{http_methods}. - - @param pattern: The URL pattern that corresponds to the user action. - @type pattern: str - @param http_methods: A list of authorized HTTP methods. - @type http_methods: list of str - """ - self._map.connect( - None, pattern, http_methods=http_methods, - conditions=dict(method=http_methods), - requirements={'dbname': DBNAME_CONSTRAINTS}) - - def _register_auth_info(self): - """ - Register the authorization info in the mapper using C{SHARED_DB_NAME} - as the user's database name. - - This method sets up the following authorization rules: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{any_id} | GET, PUT, DELETE - /shared-db/sync-from/{source} | - - /user-db | GET, PUT, DELETE - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - # auth info for global resource - self._register('/', [self.HTTP_METHOD_GET]) - # auth info for shared-db database resource - self._register( - '/%s' % SHARED_DB_NAME, - [self.HTTP_METHOD_GET]) - # auth info for shared-db doc resource - self._register( - '/%s/doc/{id:.*}' % SHARED_DB_NAME, - [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, - self.HTTP_METHOD_DELETE]) - # auth info for user-db database resource - self._register( - '/%s' % self._user_db_name, - [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, - self.HTTP_METHOD_DELETE]) - # auth info for user-db sync resource - self._register( - '/%s/sync-from/{source_replica_uid}' % self._user_db_name, - [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, - self.HTTP_METHOD_POST]) - # generate the regular expressions - self._map.create_regs() - - -class SoledadAuthMiddleware(object): - """ - Soledad Authentication WSGI middleware. - - This class must be extended to implement specific authentication methods - (see SoledadTokenAuthMiddleware below). - - It expects an HTTP_AUTHORIZATION header containing the concatenation of - the following strings: - - 1. The authentication scheme. It will be verified by the - _verify_authentication_scheme() method. - - 2. A space character. - - 3. The base64 encoded string of the concatenation of the user uuid with - the authentication data, separated by a collon, like this: - - base64("<uuid>:<auth_data>") - - After authentication check, the class performs an authorization check to - verify whether the user is authorized to perform the requested action. - - On client-side, 2 methods must be implemented so the soledad client knows - how to send authentication headers to server: - - * set_<method>_credentials: store authentication credentials in the - class. - - * _sign_request: format and include custom authentication data in - the HTTP_AUTHORIZATION header. - - See leap.soledad.auth and u1db.remote.http_client.HTTPClient to understand - how to do it. - """ - - __metaclass__ = ABCMeta - - HTTP_AUTH_KEY = "HTTP_AUTHORIZATION" - PATH_INFO_KEY = "PATH_INFO" - - CONTENT_TYPE_JSON = ('content-type', 'application/json') - - def __init__(self, app): - """ - Initialize the Soledad Authentication Middleware. - - @param app: The application to run on successfull authentication. - @type app: u1db.remote.http_app.HTTPApp - @param prefix: Auth app path prefix. - @type prefix: str - """ - self._app = app - - def _error(self, start_response, status, description, message=None): - """ - Send a JSON serialized error to WSGI client. - - @param start_response: Callable of the form start_response(status, - response_headers, exc_info=None). - @type start_response: callable - @param status: Status string of the form "999 Message here" - @type status: str - @param response_headers: A list of (header_name, header_value) tuples - describing the HTTP response header. - @type response_headers: list - @param description: The error description. - @type description: str - @param message: The error message. - @type message: str - - @return: List with JSON serialized error message. - @rtype list - """ - start_response("%d %s" % (status, httplib.responses[status]), - [self.CONTENT_TYPE_JSON]) - err = {"error": description} - if message: - err['message'] = message - return [json.dumps(err)] - - def _unauthorized_error(self, start_response, message): - """ - Send a unauth error. - - @param message: The error message. - @type message: str - @param start_response: Callable of the form start_response(status, - response_headers, exc_info=None). - @type start_response: callable - - @return: List with JSON serialized error message. - @rtype list - """ - return self._error( - start_response, - 401, - "unauthorized", - message) - - def __call__(self, environ, start_response): - """ - Handle a WSGI call to the authentication application. - - @param environ: Dictionary containing CGI variables. - @type environ: dict - @param start_response: Callable of the form start_response(status, - response_headers, exc_info=None). - @type start_response: callable - - @return: Target application results if authentication succeeds, an - error message otherwise. - @rtype: list - """ - # check for authentication header - auth = environ.get(self.HTTP_AUTH_KEY) - if not auth: - return self._unauthorized_error( - start_response, "Missing authentication header.") - - # get authentication data - scheme, encoded = auth.split(None, 1) - uuid, auth_data = encoded.decode('base64').split(':', 1) - if not self._verify_authentication_scheme(scheme): - return self._unauthorized_error( - start_response, "Wrong authentication scheme") - - # verify if user is athenticated - try: - if not self._verify_authentication_data(uuid, auth_data): - return self._unauthorized_error( - start_response, - self._get_auth_error_string()) - except u1db_errors.Unauthorized as e: - return self._error( - start_response, - 401, - e.wire_description) - - # verify if user is authorized to perform action - if not self._verify_authorization(environ, uuid): - return self._unauthorized_error( - start_response, - "Unauthorized action.") - - # move on to the real Soledad app - del environ[self.HTTP_AUTH_KEY] - return self._app(environ, start_response) - - @abstractmethod - def _verify_authentication_scheme(self, scheme): - """ - Verify if authentication scheme is valid. - - @param scheme: Auth scheme extracted from the HTTP_AUTHORIZATION - header. - @type scheme: str - - @return: Whether the authentitcation scheme is valid. - """ - return None - - @abstractmethod - def _verify_authentication_data(self, uuid, auth_data): - """ - Verify valid authenticatiion for this request. - - @param uuid: The user's uuid. - @type uuid: str - @param auth_data: Authentication data. - @type auth_data: str - - @return: Whether the token is valid for authenticating the request. - @rtype: bool - - @raise Unauthorized: Raised when C{auth_data} is not enough to - authenticate C{uuid}. - """ - return None - - def _verify_authorization(self, environ, uuid): - """ - Verify if the user is authorized to perform the requested action over - the requested database. - - @param environ: Dictionary containing CGI variables. - @type environ: dict - @param uuid: The user's uuid. - @type uuid: str - - @return: Whether the user is authorize to perform the requested action - over the requested db. - @rtype: bool - """ - return URLToAuthorization(uuid).is_authorized(environ) - - @abstractmethod - def _get_auth_error_string(self): - """ - Return an error string specific for each kind of authentication method. - - @return: The error string. - """ - return None - - -class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): - """ - Token based authentication. - """ - - TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." - - def _get_state(self): - return self._app.state - - def _set_state(self, state): - self._app.state = state - - state = property(_get_state, _set_state) - - def _verify_authentication_scheme(self, scheme): - """ - Verify if authentication scheme is valid. - - @param scheme: Auth scheme extracted from the HTTP_AUTHORIZATION - header. - @type scheme: str - - @return: Whether the authentitcation scheme is valid. - """ - if scheme.lower() != 'token': - return False - return True - - def _verify_authentication_data(self, uuid, auth_data): - """ - Extract token from C{auth_data} and proceed with verification of - C{uuid} authentication. - - @param uuid: The user UID. - @type uuid: str - @param auth_data: Authentication data (i.e. the token). - @type auth_data: str - - @return: Whether the token is valid for authenticating the request. - @rtype: bool - - @raise Unauthorized: Raised when C{auth_data} is not enough to - authenticate C{uuid}. - """ - token = auth_data # we expect a cleartext token at this point +import binascii +import time + +from hashlib import sha512 +from zope.interface import implementer + +from twisted.cred import error +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.credentials import IAnonymous +from twisted.cred.credentials import Anonymous +from twisted.cred.credentials import UsernamePassword +from twisted.cred.portal import IRealm +from twisted.cred.portal import Portal +from twisted.internet import defer +from twisted.logger import Logger +from twisted.web.iweb import ICredentialFactory +from twisted.web.resource import IResource + +from leap.soledad.common.couch import couch_server + +from ._resource import SoledadResource, SoledadAnonResource +from ._config import get_config + + +log = Logger() + + +@implementer(IRealm) +class SoledadRealm(object): + + def __init__(self, sync_pool, conf=None): + assert sync_pool is not None + if conf is None: + conf = get_config() + blobs = conf['blobs'] + self.anon_resource = SoledadAnonResource( + enable_blobs=blobs) + self.auth_resource = SoledadResource( + enable_blobs=blobs, + sync_pool=sync_pool) + + def requestAvatar(self, avatarId, mind, *interfaces): + + # Anonymous access + if IAnonymous.providedBy(avatarId): + return (IResource, self.anon_resource, + lambda: None) + + # Authenticated access + else: + if IResource in interfaces: + return (IResource, self.auth_resource, + lambda: None) + raise NotImplementedError() + + +@implementer(ICredentialsChecker) +class TokenChecker(object): + + credentialInterfaces = [IUsernamePassword, IAnonymous] + + TOKENS_DB_PREFIX = "tokens_" + TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds + TOKENS_TYPE_KEY = "type" + TOKENS_TYPE_DEF = "Token" + TOKENS_USER_ID_KEY = "user_id" + + def __init__(self): + self._couch_url = get_config().get('couch_url') + + def _get_server(self): + return couch_server(self._couch_url) + + def _tokens_dbname(self): + # the tokens db rotates every 30 days, and the current db name is + # "tokens_NNN", where NNN is the number of seconds since epoch + # divide dby the rotate period in seconds. When rotating, old and + # new tokens db coexist during a certain window of time and valid + # tokens are replicated from the old db to the new one. See: + # https://leap.se/code/issues/6785 + dbname = self.TOKENS_DB_PREFIX + \ + str(int(time.time() / self.TOKENS_DB_EXPIRE)) + return dbname + + def _tokens_db(self): + dbname = self._tokens_dbname() + + # TODO -- leaking abstraction here: this module shouldn't need + # to known anything about the context manager. hide that in the couch + # module + with self._get_server() as server: + db = server[dbname] + return db + + def requestAvatarId(self, credentials): + if IAnonymous.providedBy(credentials): + return defer.succeed(Anonymous()) + + uuid = credentials.username + token = credentials.password + + # lookup key is a hash of the token to prevent timing attacks. + # TODO cache the tokens already! + + db = self._tokens_db() + token = db.get(sha512(token).hexdigest()) + if token is None: + return defer.fail(error.UnauthorizedLogin()) + + # TODO -- use cryptography constant time builtin comparison. + # we compare uuid hashes to avoid possible timing attacks that + # might exploit python's builtin comparison operator behaviour, + # which fails immediatelly when non-matching bytes are found. + couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest() + req_uuid_hash = sha512(uuid).digest() + if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ + or couch_uuid_hash != req_uuid_hash: + return defer.fail(error.UnauthorizedLogin()) + + return defer.succeed(uuid) + + +@implementer(ICredentialFactory) +class TokenCredentialFactory(object): + + scheme = 'token' + + def getChallenge(self, request): + return {} + + def decode(self, response, request): try: - return self.state.verify_token(uuid, token) - except Exception as e: - logger.error(e) - return False - - def _get_auth_error_string(self): - """ - Get the error string for token auth. - - @return: The error string. - """ - return self.TOKEN_AUTH_ERROR_STRING + creds = binascii.a2b_base64(response + b'===') + except binascii.Error: + raise error.LoginFailed('Invalid credentials') + + creds = creds.split(b':', 1) + if len(creds) == 2: + return UsernamePassword(*creds) + else: + raise error.LoginFailed('Invalid credentials') + + +def portalFactory(sync_pool): + realm = SoledadRealm(sync_pool=sync_pool) + checker = TokenChecker() + return Portal(realm, [checker]) + + +credentialFactory = TokenCredentialFactory() diff --git a/server/src/leap/soledad/server/entrypoint.py b/server/src/leap/soledad/server/entrypoint.py new file mode 100644 index 00000000..c06b740e --- /dev/null +++ b/server/src/leap/soledad/server/entrypoint.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# entrypoint.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +The entrypoint for Soledad server. + +This is the entrypoint for the application that is loaded from the initscript +or the systemd script. +""" + +from twisted.internet import reactor +from twisted.python import threadpool + +from .auth import portalFactory +from .session import SoledadSession +from ._config import get_config +from ._wsgi import init_couch_state + + +# load configuration from file +conf = get_config() + + +class SoledadEntrypoint(SoledadSession): + + def __init__(self): + pool = threadpool.ThreadPool(name='wsgi') + reactor.callWhenRunning(pool.start) + reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) + portal = portalFactory(pool) + SoledadSession.__init__(self, portal) + + +# see the comments in application.py recarding why couch state has to be +# initialized when the reactor is running + +reactor.callWhenRunning(init_couch_state, conf) diff --git a/server/src/leap/soledad/server/resource.py b/server/src/leap/soledad/server/resource.py deleted file mode 100644 index dbb91b0a..00000000 --- a/server/src/leap/soledad/server/resource.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# resource.py -# Copyright (C) 2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -A twisted resource that serves the Soledad Server. -""" - -from twisted.web.resource import Resource -from twisted.web.wsgi import WSGIResource -from twisted.internet import reactor -from twisted.python import threadpool - -from leap.soledad.server.application import wsgi_application - - -__all__ = ['SoledadResource'] - - -# setup a wsgi resource with its own threadpool -pool = threadpool.ThreadPool() -reactor.callWhenRunning(pool.start) -reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) -wsgi_resource = WSGIResource(reactor, pool, wsgi_application) - - -class SoledadResource(Resource): - """ - This is a dummy twisted resource, used only to allow different entry points - for the Soledad Server. - """ - - def __init__(self): - self.children = {'': wsgi_resource} - - def getChild(self, path, request): - # for now, just "rewind" the path and serve the wsgi resource for all - # requests. In the future, we might look into the request path to - # decide which child resources should serve each request. - request.postpath.insert(0, request.prepath.pop()) - return self.children[''] diff --git a/server/src/leap/soledad/server/session.py b/server/src/leap/soledad/server/session.py new file mode 100644 index 00000000..1c1b5345 --- /dev/null +++ b/server/src/leap/soledad/server/session.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# session.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Twisted resource containing an authenticated Soledad session. +""" +from zope.interface import implementer + +from twisted.cred.credentials import Anonymous +from twisted.cred import error +from twisted.python import log +from twisted.web import util +from twisted.web._auth import wrapper +from twisted.web.guard import HTTPAuthSessionWrapper +from twisted.web.resource import ErrorPage +from twisted.web.resource import IResource + +from leap.soledad.server.auth import credentialFactory +from leap.soledad.server.url_mapper import URLMapper + + +@implementer(IResource) +class UnauthorizedResource(wrapper.UnauthorizedResource): + isLeaf = True + + def __init__(self): + pass + + def render(self, request): + request.setResponseCode(401) + if request.method == b'HEAD': + return b'' + return b'Unauthorized' + + def getChildWithDefault(self, path, request): + return self + + +@implementer(IResource) +class SoledadSession(HTTPAuthSessionWrapper): + + def __init__(self, portal): + self._mapper = URLMapper() + self._portal = portal + self._credentialFactory = credentialFactory + # expected by the contract of the parent class + self._credentialFactories = [credentialFactory] + + def _matchPath(self, request): + match = self._mapper.match(request.path, request.method) + return match + + def _parseHeader(self, header): + elements = header.split(b' ') + scheme = elements[0].lower() + if scheme == self._credentialFactory.scheme: + return (b' '.join(elements[1:])) + return None + + def _authorizedResource(self, request): + # check whether the path of the request exists in the app + match = self._matchPath(request) + if not match: + return UnauthorizedResource() + + # get authorization header or fail + header = request.getHeader(b'authorization') + if not header: + return util.DeferredResource(self._login(Anonymous())) + + # parse the authorization header + auth_data = self._parseHeader(header) + if not auth_data: + return UnauthorizedResource() + + # decode the credentials from the parsed header + try: + credentials = self._credentialFactory.decode(auth_data, request) + except error.LoginFailed: + return UnauthorizedResource() + except: + # If you port this to the newer log facility, be aware that + # the tests rely on the error to be logged. + log.err(None, "Unexpected failure from credentials factory") + return ErrorPage(500, None, None) + + # make sure the uuid given in path corresponds to the one given in + # the credentials + request_uuid = match.get('uuid') + if request_uuid and request_uuid != credentials.username: + return ErrorPage(500, None, None) + + # if all checks pass, try to login with credentials + return util.DeferredResource(self._login(credentials)) diff --git a/server/src/leap/soledad/server/url_mapper.py b/server/src/leap/soledad/server/url_mapper.py new file mode 100644 index 00000000..a0edeaca --- /dev/null +++ b/server/src/leap/soledad/server/url_mapper.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# url_mapper.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +An URL mapper that represents authorized paths. +""" +from routes.mapper import Mapper + +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.l2db import DBNAME_CONSTRAINTS + + +class URLMapper(object): + """ + Maps the URLs users can access. + """ + + def __init__(self): + self._map = Mapper(controller_scan=None) + self._connect_urls() + self._map.create_regs() + + def match(self, path, method): + environ = {'PATH_INFO': path, 'REQUEST_METHOD': method} + return self._map.match(environ=environ) + + def _connect(self, pattern, http_methods): + self._map.connect( + None, pattern, http_methods=http_methods, + conditions=dict(method=http_methods), + requirements={'dbname': DBNAME_CONSTRAINTS}) + + def _connect_urls(self): + """ + Register the authorization info in the mapper using C{SHARED_DB_NAME} + as the user's database name. + + This method sets up the following authorization rules: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /robots.txt | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{any_id} | GET, PUT, DELETE + /shared-db/sync-from/{source} | - + /user-db | - + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST + """ + # auth info for global resource + self._connect('/', ['GET']) + # robots + self._connect('/robots.txt', ['GET']) + # auth info for shared-db database resource + self._connect('/%s' % SHARED_DB_NAME, ['GET']) + # auth info for shared-db doc resource + self._connect('/%s/doc/{id:.*}' % SHARED_DB_NAME, + ['GET', 'PUT', 'DELETE']) + # auth info for user-db sync resource + self._connect('/user-{uuid}/sync-from/{source_replica_uid}', + ['GET', 'PUT', 'POST']) diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py index 57f8199b..83a27016 100644 --- a/testing/test_soledad/util.py +++ b/testing/test_soledad/util.py @@ -52,7 +52,6 @@ from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client._crypto import is_symmetrically_encrypted from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware PASSWORD = '123456' @@ -108,7 +107,7 @@ def make_soledad_app(state): def make_token_soledad_app(state): - app = SoledadApp(state) + application = SoledadApp(state) def _verify_authentication_data(uuid, auth_data): if uuid.startswith('user-') and auth_data == 'auth-token': @@ -119,7 +118,6 @@ def make_token_soledad_app(state): def _verify_authorization(uuid, environ): return True - application = SoledadTokenAuthMiddleware(app) application._verify_authentication_data = _verify_authentication_data application._verify_authorization = _verify_authorization return application @@ -182,7 +180,6 @@ class MockedSharedDBTest(object): put_doc = Mock(side_effect=put_doc_side_effect) open = Mock(return_value=None) close = Mock(return_value=None) - syncable = True def __call__(self): return self diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py index a9cc3464..1b99d96e 100644 --- a/testing/tests/benchmarks/conftest.py +++ b/testing/tests/benchmarks/conftest.py @@ -12,12 +12,30 @@ from leap.common.events import server server.ensure_server() +# +# pytest customizations +# + def pytest_addoption(parser): parser.addoption( "--num-docs", type="int", default=100, help="the number of documents to use in performance tests") +# mark benchmark tests using their group names (thanks ionelmc! :) +def pytest_collection_modifyitems(items): + for item in items: + bench = item.get_marker("benchmark") + if bench and bench.kwargs.get('group'): + group = bench.kwargs['group'] + marker = getattr(pytest.mark, 'benchmark_' + group) + item.add_marker(marker) + + +# +# benchmark fixtures +# + @pytest.fixture() def payload(): def generate(size): diff --git a/testing/tests/benchmarks/test_sync.py b/testing/tests/benchmarks/test_sync.py index 1501d74b..fcfab998 100644 --- a/testing/tests/benchmarks/test_sync.py +++ b/testing/tests/benchmarks/test_sync.py @@ -11,6 +11,12 @@ def load_up(client, amount, payload): yield gatherResults(deferreds) +# Each test created with this function will: +# +# - get a fresh client. +# - iterate: +# - setup: create N docs of a certain size +# - benchmark: sync() -- uploads N docs. def create_upload(uploads, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_upload") @@ -29,6 +35,14 @@ test_upload_100_100k = create_upload(100, 100 * 1000) test_upload_1000_10k = create_upload(1000, 10 * 1000) +# Each test created with this function will: +# +# - get a fresh client. +# - create N docs of a certain size +# - sync (uploads those docs) +# - iterate: +# - setup: get a fresh client with empty local db +# - benchmark: sync() -- downloads N docs. def create_download(downloads, size): @pytest.inlineCallbacks @pytest.mark.benchmark(group="test_download") @@ -41,7 +55,7 @@ def create_download(downloads, size): # ensures we are dealing with properly encrypted docs def setup(): - return soledad_client() + return soledad_client(force_fresh_db=True) def sync(clean_client): return clean_client.sync() diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py index 9b4a175f..729aa28a 100644 --- a/testing/tests/client/test_aux_methods.py +++ b/testing/tests/client/test_aux_methods.py @@ -19,12 +19,11 @@ Tests for general Soledad functionality. """ import os -from twisted.internet import defer +from pytest import inlineCallbacks from leap.soledad.client import Soledad from leap.soledad.client.adbapi import U1DBConnectionPool -from leap.soledad.client.secrets import PassphraseTooShort -from leap.soledad.client.secrets import SecretsException +from leap.soledad.client._secrets.util import SecretsError from test_soledad.util import BaseSoledadTest @@ -34,7 +33,7 @@ class AuxMethodsTestCase(BaseSoledadTest): def test__init_dirs(self): sol = self._soledad_instance(prefix='_init_dirs') local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets.secrets_path) + secrets_path = os.path.dirname(sol.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) @@ -64,8 +63,8 @@ class AuxMethodsTestCase(BaseSoledadTest): # instantiate without initializing so we just test # _init_config_with_defaults() sol = SoledadMock() - sol._passphrase = u'' - sol._server_url = '' + sol.passphrase = u'' + sol.server_url = '' sol._init_config_with_defaults() # assert value of local_db_path self.assertEquals( @@ -85,14 +84,14 @@ class AuxMethodsTestCase(BaseSoledadTest): cert_file=None) self.assertEqual( os.path.join(self.tempdir, 'value_3'), - sol.secrets.secrets_path) + sol.secrets_path) self.assertEqual( os.path.join(self.tempdir, 'value_2'), sol.local_db_path) - self.assertEqual('value_1', sol._server_url) + self.assertEqual('value_1', sol.server_url) sol.close() - @defer.inlineCallbacks + @inlineCallbacks def test_change_passphrase(self): """ Test if passphrase can be changed. @@ -108,7 +107,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol.change_passphrase(u'654321') sol.close() - with self.assertRaises(SecretsException): + with self.assertRaises(SecretsError): self._soledad_instance( 'leap@leap.se', passphrase=u'123', @@ -124,24 +123,10 @@ class AuxMethodsTestCase(BaseSoledadTest): sol2.close() - def test_change_passphrase_with_short_passphrase_raises(self): - """ - Test if attempt to change passphrase passing a short passphrase - raises. - """ - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123') - # check that soledad complains about new passphrase length - self.assertRaises( - PassphraseTooShort, - sol.change_passphrase, u'54321') - sol.close() - def test_get_passphrase(self): """ Assert passphrase getter works fine. """ sol = self._soledad_instance() - self.assertEqual('123', sol._passphrase) + self.assertEqual('123', sol.passphrase) sol.close() diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index 49a61438..399fdc99 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -19,7 +19,6 @@ Tests for cryptographic related stuff. """ import binascii import base64 -import hashlib import json import os @@ -111,7 +110,7 @@ class BlobTestCase(unittest.TestCase): assert len(preamble) == _crypto.PACMAN.size unpacked_data = _crypto.PACMAN.unpack(preamble) - magic, sch, meth, ts, iv, doc_id, rev = unpacked_data + magic, sch, meth, ts, iv, doc_id, rev, _ = unpacked_data assert magic == _crypto.BLOB_SIGNATURE_MAGIC assert sch == 1 assert meth == _crypto.ENC_METHOD.aes_256_gcm @@ -186,99 +185,13 @@ class BlobTestCase(unittest.TestCase): yield crypto.decrypt_doc(doc2) -class RecoveryDocumentTestCase(BaseSoledadTest): - - def test_export_recovery_document_raw(self): - rd = self._soledad.secrets._export_recovery_document() - secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] - # assert exported secret is the same - secret = self._soledad.secrets._decrypt_storage_secret_version_1( - rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) - self.assertEqual(secret_id, self._soledad.secrets._secret_id) - self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) - # assert recovery document structure - encrypted_secret = rd[ - self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] - self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) - self.assertEquals( - _crypto.ENC_METHOD.aes_256_gcm, - encrypted_secret[self._soledad.secrets.CIPHER_KEY]) - self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) - self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) - - def test_import_recovery_document(self, cipher='aes256'): - rd = self._soledad.secrets._export_recovery_document(cipher) - s = self._soledad_instance() - s.secrets._import_recovery_document(rd) - s.secrets.set_secret_id(self._soledad.secrets._secret_id) - self.assertEqual(self._soledad.storage_secret, - s.storage_secret, - 'Failed settinng secret for symmetric encryption.') - s.close() - - def test_import_GCM_recovery_document(self): - cipher = self._soledad.secrets.CIPHER_AES256_GCM - self.test_import_recovery_document(cipher) - - def test_import_legacy_CTR_recovery_document(self): - cipher = self._soledad.secrets.CIPHER_AES256 - self.test_import_recovery_document(cipher) - - class SoledadSecretsTestCase(BaseSoledadTest): - def test_new_soledad_instance_generates_one_secret(self): - self.assertTrue( - self._soledad.storage_secret is not None, - "Expected secret to be something different than None") - number_of_secrets = len(self._soledad.secrets._secrets) - self.assertTrue( - number_of_secrets == 1, - "Expected exactly 1 secret, got %d instead." % number_of_secrets) - - def test_generated_secret_is_of_correct_type(self): - expected_type = str - self.assertIsInstance( - self._soledad.storage_secret, expected_type, - "Expected secret to be of type %s" % expected_type) - - def test_generated_secret_has_correct_lengt(self): - expected_length = self._soledad.secrets.GEN_SECRET_LENGTH - actual_length = len(self._soledad.storage_secret) - self.assertTrue( - expected_length == actual_length, - "Expected secret with length %d, got %d instead." - % (expected_length, actual_length)) - - def test_generated_secret_id_is_sha256_hash_of_secret(self): - generated = self._soledad.secrets.secret_id - expected = hashlib.sha256(self._soledad.storage_secret).hexdigest() - self.assertTrue( - generated == expected, - "Expeceted generated secret id to be sha256 hash, got something " - "else instead.") - - def test_generate_new_secret_generates_different_secret_id(self): - # generate new secret - secret_id_1 = self._soledad.secrets.secret_id - secret_id_2 = self._soledad.secrets._gen_secret() - self.assertTrue( - len(self._soledad.secrets._secrets) == 2, - "Expected exactly 2 secrets.") - self.assertTrue( - secret_id_1 != secret_id_2, - "Expected IDs of secrets to be distinct.") - self.assertTrue( - secret_id_1 in self._soledad.secrets._secrets, - "Expected to find ID of first secret in Soledad Secrets.") - self.assertTrue( - secret_id_2 in self._soledad.secrets._secrets, - "Expected to find ID of second secret in Soledad Secrets.") - - def test__has_secret(self): - self.assertTrue( - self._soledad._secrets._has_secret(), - "Should have a secret at this point") + def test_generated_secrets_have_correct_length(self): + expected = self._soledad.secrets.lengths + for name, length in expected.iteritems(): + secret = getattr(self._soledad.secrets, name) + self.assertEqual(length, len(secret)) class SoledadCryptoAESTestCase(BaseSoledadTest): @@ -322,10 +235,74 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): _crypto.decrypt_sym(cyphertext, wrongkey, iv) -def _aes_encrypt(key, iv, data): +class PreambleTestCase(unittest.TestCase): + class doc_info: + doc_id = 'D-deadbeef' + rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + + def setUp(self): + self.cleartext = BytesIO(snowden1) + self.blob = _crypto.BlobEncryptor( + self.doc_info, self.cleartext, + secret='A' * 96) + + def test_preamble_starts_with_magic_signature(self): + preamble = self.blob._encode_preamble() + assert preamble.startswith(_crypto.BLOB_SIGNATURE_MAGIC) + + def test_preamble_has_cipher_metadata(self): + preamble = self.blob._encode_preamble() + unpacked = _crypto.PACMAN.unpack(preamble) + encryption_scheme, encryption_method = unpacked[1:3] + assert encryption_scheme in _crypto.ENC_SCHEME + assert encryption_method in _crypto.ENC_METHOD + assert unpacked[4] == self.blob.iv + + def test_preamble_has_document_sync_metadata(self): + preamble = self.blob._encode_preamble() + unpacked = _crypto.PACMAN.unpack(preamble) + doc_id, doc_rev = unpacked[5:7] + assert doc_id == self.doc_info.doc_id + assert doc_rev == self.doc_info.rev + + def test_preamble_has_document_size(self): + preamble = self.blob._encode_preamble() + unpacked = _crypto.PACMAN.unpack(preamble) + size = unpacked[7] + assert size == len(snowden1) + + @defer.inlineCallbacks + def test_preamble_can_come_without_size(self): + # XXX: This test case is here only to test backwards compatibility! + preamble = self.blob._encode_preamble() + # repack preamble using legacy format, without doc size + unpacked = _crypto.PACMAN.unpack(preamble) + preamble_without_size = _crypto.LEGACY_PACMAN.pack(*unpacked[0:7]) + # encrypt it manually for custom tag + ciphertext, tag = _aes_encrypt(self.blob.sym_key, self.blob.iv, + self.cleartext.getvalue(), + aead=preamble_without_size) + ciphertext = ciphertext + tag + # encode it + ciphertext = base64.urlsafe_b64encode(ciphertext) + preamble_without_size = base64.urlsafe_b64encode(preamble_without_size) + # decrypt it + ciphertext = preamble_without_size + ' ' + ciphertext + cleartext = yield _crypto.BlobDecryptor( + self.doc_info, BytesIO(ciphertext), + secret='A' * 96).decrypt() + assert cleartext == self.cleartext.getvalue() + warnings = self.flushWarnings() + assert len(warnings) == 1 + assert 'legacy document without size' in warnings[0]['message'] + + +def _aes_encrypt(key, iv, data, aead=''): backend = default_backend() cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend) encryptor = cipher.encryptor() + if aead: + encryptor.authenticate_additional_data(aead) return encryptor.update(data) + encryptor.finalize(), encryptor.tag diff --git a/testing/tests/client/test_deprecated_crypto.py b/testing/tests/client/test_deprecated_crypto.py index 8ee3735c..1af1a130 100644 --- a/testing/tests/client/test_deprecated_crypto.py +++ b/testing/tests/client/test_deprecated_crypto.py @@ -1,5 +1,7 @@ import json -from twisted.internet import defer + +from pytest import inlineCallbacks + from uuid import uuid4 from urlparse import urljoin @@ -39,7 +41,7 @@ class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): def make_app_with_state(state): return make_token_soledad_app(state) - @defer.inlineCallbacks + @inlineCallbacks def test_touch_updates_remote_representation(self): self.startTwistedServer() user = 'user-' + uuid4().hex @@ -49,7 +51,7 @@ class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): self._soledad_instance(user=user, server_url=server_url)) self.make_app() - remote = self.request_state._create_database(replica_uid=client._uuid) + remote = self.request_state._create_database(replica_uid=client.uuid) remote = CouchDatabase.open_database( urljoin(self.couch_url, 'user-' + user), create=True) diff --git a/testing/tests/client/test_http_client.py b/testing/tests/client/test_http_client.py deleted file mode 100644 index a107930a..00000000 --- a/testing/tests/client/test_http_client.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -# test_http_client.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: sync target -""" -import json - -from testscenarios import TestWithScenarios - -from leap.soledad.client import auth -from leap.soledad.common.l2db.remote import http_client -from test_soledad.u1db_tests import test_http_client -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_client`. -# ----------------------------------------------------------------------------- - -class TestSoledadClientBase( - TestWithScenarios, - test_http_client.TestHTTPClientBase): - - """ - This class should be used to test Token auth. - """ - - def getClient(self, **kwds): - cli = self.getClientWithToken(**kwds) - if 'creds' not in kwds: - cli.set_token_credentials('user-uuid', 'auth-token') - return cli - - def getClientWithToken(self, **kwds): - self.startServer() - - class _HTTPClientWithToken( - http_client.HTTPClientBase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - return _HTTPClientWithToken(self.getURL('dbase'), **kwds) - - def app(self, environ, start_response): - res = test_http_client.TestHTTPClientBase.app( - self, environ, start_response) - if res is not None: - return res - # mime solead application here. - if '/token' in environ['PATH_INFO']: - auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) - if not auth: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [ - json.dumps( - {"error": "unauthorized", - "message": "no token found in environment"}) - ] - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'token': - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": "unknown scheme: %s" % scheme})] - uuid, token = encoded.decode('base64').split(':', 1) - if uuid != 'user-uuid' and token != 'auth-token': - return Exception("Incorrect address or token.") - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], uuid, token])] - - def test_token(self): - """ - Test if token is sent correctly. - """ - cli = self.getClientWithToken() - cli.set_token_credentials('user-uuid', 'auth-token') - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - def test_token_ctr_creds(self): - cli = self.getClientWithToken(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) diff --git a/testing/tests/client/test_secrets.py b/testing/tests/client/test_secrets.py new file mode 100644 index 00000000..18ff458b --- /dev/null +++ b/testing/tests/client/test_secrets.py @@ -0,0 +1,159 @@ +# -*- CODing: utf-8 -*- +# test_secrets.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for secrets encryption and decryption. +""" +import scrypt + +from twisted.trial import unittest + +from leap.soledad.client._crypto import ENC_METHOD +from leap.soledad.client._secrets import SecretsCrypto + + +class SecretsCryptoTestCase(unittest.TestCase): + + SECRETS = { + 'remote_secret': 'a' * 512, + 'local_salt': 'b' * 64, + 'local_secret': 'c' * 448 + } + ENCRYPTED_V2 = { + 'cipher': 2, + 'length': 1437, + 'kdf_salt': 'TSgNLeAGFeITeSgNzmYZHh+mzmkZPOqao7CAV/tx3KZCLwsrT0HmWtVK3' + 'TyWHWNgVdeamMZYRuvZavE2sp0DGw==\n', + 'iv': 'TKZQKIlRgdnXFhJf08qswg==', + 'secrets': 'ZNZRi72VDtwZqyuU+uf3yzZt23vCtMS3Ki2bnZyeHUOSGVweJeDadF4oqE' + 'BW87NN00j9E49BzyzLr9SNgwZjPp0wlUm7kt+s8EUfJUdH8nxaQ+9iqGXM' + 'cCHmBM8L8DRN2m3BrPGx7m+QGlN9sbrRpl7fqc46RWcYuTEpm4upjdtI7O' + 'jDd0JG3C0rUzIuKJn9w4rEpX3tLEKXVdZfLvRXS5roR0cauazsDO69E13q' + 'a01vDuY+UJ+buLQ3FluPnnk8QE7ztPVUmRJJ76yAIhjVX9owiwlp9GnUJY' + 'sETRCqdRSTwUcHIkzVR0zAvtxTX7eGTitzf4gCYEC4T9v5N/jHxEfPdx28' + 'MM4KShWN2nFxNFQLQUpMN2OrM7UyUw+DQ3ydqBeBPKPHRN5s05kIK7P/Ra' + 'aLNcrJWa7DopLbgLlei0Jd7S4sjv1ufaRY7v0qJaVkhh/VaCylTSVw1rv5' + 'YzSWcHHcLuC0R8xLadz6T+EpsVYxgPYCS7w5xoE82zwNQzw/EBxLIcyLPl' + 'ipKnr2dttrmm3KXUOT1IdbSbI5elF6yQTAusdqiXuypey+MDqHYWEYWkCn' + 'e9/uGM9FjklDLE0RtPEDxhq64tw6u2Xu7RzDzyQDI8EIoTdU+4zEMTnelZ' + 'fKEwdG58EDxTXfUk6IDcRUupz3YuToSMhIOkqgXnbWl/nrK0O9v4JMhQjI' + 'r+oPICYfFr14kvJXBsfntILTJCxzbqTQcNba3jc8rGqCZ6gM0u4PndwTG2' + 'UiCqPU2HMnWvVGQOXeLdQY+EqqXQiRDi0DrDmkVwFf+27dPXxmZ43C48W3' + 'lMhTKXl0rdBFnOD5jiMh0X6q/KYXonyEtMZMsjT7dFePcCy4wQRhuut+ac' + '/TJWyrr+/IB45E+LZbhV7xCy1dYsbdb52jTRJFpaQ83sj6Iv6SYdiqqXzL' + 'F5JGMyuovTjwAoIIQzpLv36xY2wGGAH1V8c7QmDR2qumXrHD9R68WjBoSY' + '7IFM0TFAGZNun56y/zQ4r8yOMSAId+j4kuRH0fENEi0FJ+SpmSdHfpvBhE' + 'MdGh927E9enEYWmUQMmkxXIw6E+O3cmOWt2hsMbUAikDCpQOnVP2BD55HT' + '6FfbW7ITVwPINHYexmy2Xcm8H5zzGFSp+uYIyPBYDKA+VJ+QQI8bud9K+T' + 'NBybUv9u6LbB6BsLpwLoxMPJu0WsN2HpmLYgrg2ML1huMF1OtaGRuUr2PL' + 'NBaZaL6VOztYrVtQG1+tNyRxn8XQTtx0l6n+EihGVe9Sk5XF6DJA9ZN7uO' + 'svTUFJ5qG3Erf4AmbUJWoOR/NvapBtifidM7gPZZ6NqBs6v72rU1pGy+p7' + 'o84KrmB2MNf3yJ0BvKxPvFmltF3Dc7LB5TN8ycbmFM6hgrLvvhPxiHEnG/' + '8Qcrg0nUXOipFGNgZEU7t7Mz6RJ189Z2Kx1HVGrkAzEgqwZYqijAPlsgzO' + 'bg6DwzwC7stolQWGCDQUtJVlE8FZ/Up8zFYYZKn52WzjmSN4/hHhEvdkck' + 'Nez/JVev6fMcVrgdrTZ+uCwxjN/4xPdgog2HV470ea1bvIkSNOOrhm194M' + '40GmvmBhiSSMjdRQCQtM0t9bUuSQLPDzEiCA9QaLyygtlz9uRR/dXgkEg0' + 'J4YNpZvhE0wbyp4GHytbPaAmrcd7im9+buTuMwhXpZl0stmfkJxVHJSZ8Y' + 'IHakHs3W1fdYyI3wxGpel/9eYO3ISukolwrHXESP65wVNKfBwbqVJzQmts' + 'pyDBOI6DcLKZfE1EVg0+uwQ/5PKZbn0TwlXO1YE3NL3mAply3zQR9hyBrY' + '6f1jkHVD3irIlWkSiPJsP8sW+nrK8c/Ha8F+dua6DTZmg594OIaQj8mPiY' + 'GcIusiARWocR5/MmSjupGOgFx4HtmckTJtAta3XP4elOx04teH/P9Cgr1x' + 'XYf+cEX6gp92L9rTo0FCz3Hw==\n', + 'version': 2, + 'kdf': 'scrypt', + 'kdf_length': 32 + } + + ENCRYPTED_V1 = { + 'version': 1, + 'active_secret': 'secret_id', + 'storage_secrets': { + 'secret_id': { + 'kdf': 'scrypt', + 'secret': 'u31ObvxNU8jB0HgMj3TVwQ==:JQwlYq6sAQmHYS3x2CJzObT9h' + 'j1iiHthvrMh887qedNCcOfJyCA3jpRkc0vjd2Qk/2HSJ+JxM2F' + 'MrPzzx5O34EHlgF2scen34guZRRIf42WpnMy+PrL4cnMlZLgCh' + 'H1Jz6wcIMEpU9LQ8OaCShk1/yJ6qcVHOV4DDt3mTF7ttiqI5cp' + 'msaVtxxYCcpxFiWSeSCEgr0h4/Ih1qHuM6vk+CQjf/zg1f/7HR' + 'imIyNYXit9Fw3YTkxBen1wG3f5L7OAODRTuqnWpkQFOmclx050' + 'k0frKRcX6UWhIOWpW2mqJXnvzDtQQVGzqIdSgGTGtUDGQ7Onnc' + 'NkUlSnuVC7PkDNNRuwit3pCB9YWBWyPAQgs0kLqoV4YcuSctz6' + 'SAf76ozdcK5/SrOzutOfyPag4V3AYKMv6rCKALJ10OnFJ61FL9' + 'kd6JZam7WOlEUXyO7Gdgvz+eKiQMTZXbtO2kAKqel513MedPXC' + 'dzajUe1U2JaGg86UdiDWoPYOiWxnAPwfNJk+1QuNy5NZ7PaMtF' + 'IKT3/Xema2U8mufS0FbvJyK2flP1VUWcCzHKTSqX6+kU7UpoWa' + 'hYa7PlO40El+putTQLBmNaEeaWFngO+XB4TReICHSiCdcAb3pw' + 'sabjtxt+OpK4vbj3yBSfpiZTpVbEjt9U/tUpVp/T2M66lMi3ZC' + 'oHLlhu45Zo0aEq3UmQ/WBXu6EkO2eLYz2br9YQwRbSJ6z5CHmu' + 'hjKBQlpvGNfZYObx5lY4o6Ab4f/N8gyukskjmAFAf7Fr8cEog/' + 'oxmbagoCtUGRYJp2paooqH8L6xXp0Y8+23g7WJaAIr1i4V4aKS' + 'r9x7iUK6prcZTtMJZEHCswkLN/+DU6/FX3YZcOjseC+Qv3P+9v' + 'zIDp/92KJzqVqITGwrsc6ZsglMW37qxs6albtw3lMWSHlkcLbj' + 'Xf/iHPeKnb2WNLdkFNQ1J5OaTJR+E1CrXN+pm1JtB6XaUbaLGV' + 'CGUo13lAPVDtXcPbo64kMrQtQu4m9m8X8t8tfuJmINfwBnrKzk' + 'O6pl+LwimFaFEArV6wcaMxmwi0lM7mt4U1u9OIQjghQ/dEmOyV' + 'dZBnvyG7T/oRuLdUyZ/QGXZMlPQ3lAZ0ONn1Mk4bmKToW8ToE8' + 'ylld3rLlWDjjoQP8mP05Izg3mguLHXUhikUL8MD5NdYyeZJ1XZ' + '0OZ5S9uncurYj2ABWJoVaq/tFCdCEo9bbjWsePei26GZjaM3Fx' + 'RkAICXe/bt6/uLgaPZtO+sdARDuU3DRKMIdgM9NBaIn0kC7Wk4' + 'bnYShZ/rbhVt2/ds5XinnDBZsxSR3s553DixJ9v6w9Db++9Stw' + '4DgePd9lLy+6WuVBlKmcNflx9zg7US0AOarX2UNiQ==', + 'kdf_length': 32, + 'kdf_salt': 'MYH68QH48nRFMWH44piFWqBnKtU8KCz6Ajh24otrvzJlqPgB' + 'v6bvFJjRvjRp/0/v1j2nt40RZ6H5hfoKmore0g==\n', + 'length': 1024, + 'cipher': 'aes256', + } + } + } + + def setUp(self): + class Soledad(object): + passphrase = '123' + soledad = Soledad() + self._crypto = SecretsCrypto(soledad) + + def test__get_key(self): + salt = 'abc' + expected = scrypt.hash('123', salt, buflen=32) + key = self._crypto._get_key(salt) + self.assertEqual(expected, key) + + def test_encrypt(self): + info = self._crypto.encrypt(self.SECRETS) + self.assertEqual(8, len(info)) + for key, value in [ + ('kdf', 'scrypt'), + ('kdf_salt', None), + ('kdf_length', None), + ('cipher', ENC_METHOD.aes_256_gcm), + ('length', None), + ('iv', None), + ('secrets', None), + ('version', 2)]: + self.assertTrue(key in info) + if value: + self.assertEqual(info[key], value) + + def test__decrypt_v2(self): + encrypted = self.ENCRYPTED_V2 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) + + def test__decrypt_v1(self): + encrypted = self.ENCRYPTED_V1 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) diff --git a/testing/tests/client/test_shared_db.py b/testing/tests/client/test_shared_db.py index aac766c2..b045e524 100644 --- a/testing/tests/client/test_shared_db.py +++ b/testing/tests/client/test_shared_db.py @@ -2,7 +2,6 @@ from leap.soledad.common.document import SoledadDocument from leap.soledad.client.shared_db import SoledadSharedDatabase from test_soledad.util import BaseSoledadTest -from test_soledad.util import ADDRESS class SoledadSharedDBTestCase(BaseSoledadTest): @@ -14,37 +13,28 @@ class SoledadSharedDBTestCase(BaseSoledadTest): def setUp(self): BaseSoledadTest.setUp(self) self._shared_db = SoledadSharedDatabase( - 'https://provider/', ADDRESS, document_factory=SoledadDocument, + 'https://provider/', document_factory=SoledadDocument, creds=None) def tearDown(self): BaseSoledadTest.tearDown(self) - def test__get_secrets_from_shared_db(self): + def test__get_remote_doc(self): """ Ensure the shared db is queried with the correct doc_id. """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._get_secrets_from_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - - def test__put_secrets_in_shared_db(self): + doc_id = self._soledad.secrets.storage._remote_doc_id() + self._soledad.secrets.storage._get_remote_doc() + self._soledad.secrets.storage._shared_db.get_doc.assert_called_with( + doc_id) + + def test_save_remote(self): """ Ensure recovery document is put into shared recover db. """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._put_secrets_in_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - self.assertTrue( - self._soledad.shared_db.put_doc.assert_called_with( - self._doc_put) is None, - 'Wrong document when putting recovery document.') - self.assertTrue( - self._doc_put.doc_id == doc_id, - 'Wrong doc_id when putting recovery document.') + doc_id = self._soledad.secrets.storage._remote_doc_id() + storage = self._soledad.secrets.storage + storage.save_remote({'content': 'blah'}) + storage._shared_db.get_doc.assert_called_with(doc_id) + storage._shared_db.put_doc.assert_called_with(self._doc_put) + self.assertTrue(self._doc_put.doc_id == doc_id) diff --git a/testing/tests/client/test_signals.py b/testing/tests/client/test_signals.py index 4e9ebfd0..c7609a74 100644 --- a/testing/tests/client/test_signals.py +++ b/testing/tests/client/test_signals.py @@ -20,7 +20,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def setUp(self): # mock signaling soledad.client.signal = Mock() - soledad.client.secrets.events.emit_async = Mock() + soledad.client._secrets.util.events.emit_async = Mock() # run parent's setUp BaseSoledadTest.setUp(self) @@ -42,55 +42,36 @@ class SoledadSignalingTestCase(BaseSoledadTest): - downloading keys / done downloading keys. - uploading keys / done uploading keys. """ - soledad.client.secrets.events.emit_async.reset_mock() + soledad.client._secrets.util.events.emit_async.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage3.json', local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() + soledad.client._secrets.util.events.emit_async.mock_calls.reverse() + soledad.client._secrets.util.events.emit_async.call_args = \ + soledad.client._secrets.util.events.emit_async.call_args_list[0] + soledad.client._secrets.util.events.emit_async.call_args_list.reverse() user_data = {'userid': ADDRESS, 'uuid': ADDRESS} - # downloading keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # creating keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_CREATING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_CREATING_KEYS, user_data - ) - # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # uploading keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_UPLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data - ) + def _assert(*args, **kwargs): + mocked = soledad.client._secrets.util.events.emit_async + mocked.assert_called_with(*args) + pop = kwargs.get('pop') + if pop or pop is None: + self._pop_mock_call(mocked) + + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_CREATING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_CREATING_KEYS, user_data) + _assert(catalog.SOLEDAD_UPLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data, pop=False) + sol.close() def test_stage2_bootstrap_signals(self): @@ -101,11 +82,11 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get existing instance so we have access to keys sol = self._soledad_instance() # create a document with secrets - doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) - doc.content = sol.secrets._export_recovery_document() + doc = SoledadDocument(doc_id=sol.secrets.storage._remote_doc_id()) + doc.content = sol.secrets.crypto.encrypt(sol.secrets._secrets) sol.close() # reset mock - soledad.client.secrets.events.emit_async.reset_mock() + soledad.client._secrets.util.events.emit_async.reset_mock() # get a fresh instance so it emits all bootstrap signals shared_db = self.get_default_shared_mock(get_doc_return_value=doc) sol = self._soledad_instance( @@ -114,20 +95,23 @@ class SoledadSignalingTestCase(BaseSoledadTest): shared_db_class=shared_db) # reverse call order so we can verify in the order the signals were # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() + mocked = soledad.client._secrets.util.events.emit_async + mocked.mock_calls.reverse() + mocked.call_args = mocked.call_args_list[0] + mocked.call_args_list.reverse() + + def _assert(*args, **kwargs): + mocked = soledad.client._secrets.util.events.emit_async + mocked.assert_called_with(*args) + pop = kwargs.get('pop') + if pop or pop is None: + self._pop_mock_call(mocked) + # assert download keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS} - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS}, - ) + user_data = {'userid': ADDRESS, 'uuid': ADDRESS} + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data, pop=False) + sol.close() def test_stage1_bootstrap_signals(self): diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 1ff1cbb7..c077828f 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -6,7 +6,7 @@ import signal import time from hashlib import sha512 -from subprocess import call +from subprocess import check_call from urlparse import urljoin from uuid import uuid4 @@ -98,13 +98,13 @@ class SoledadServer(object): def start(self): self._create_conf_file() # start the server - call([ + check_call([ 'twistd', '--logfile=%s' % self._logfile, '--pidfile=%s' % self._pidfile, 'web', - '--wsgi=leap.soledad.server.application.wsgi_application', - '--port=2424' + '--class=leap.soledad.server.entrypoint.SoledadEntrypoint', + '--port=tcp:2424' ]) def _create_conf_file(self): @@ -118,7 +118,7 @@ class SoledadServer(object): def stop(self): pid = get_pid(self._pidfile) - os.kill(pid, signal.SIGKILL) + os.kill(pid, signal.SIGTERM) @pytest.fixture(scope='module') @@ -191,9 +191,19 @@ def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): soledad_dbs(default_uuid) # get a soledad instance - def create(): - secrets_path = os.path.join(tmpdir.strpath, '%s.secret' % default_uuid) - local_db_path = os.path.join(tmpdir.strpath, '%s.db' % default_uuid) + def create(force_fresh_db=False): + secrets_file = '%s.secret' % default_uuid + secrets_path = os.path.join(tmpdir.strpath, secrets_file) + + # in some tests we might want to use the same user and remote database + # but with a clean/empty local database (i.e. download benchmarks), so + # here we provide a way to do that. + db_file = '%s.db' % default_uuid + if force_fresh_db: + prefix = uuid4().hex + db_file = prefix + '-' + db_file + local_db_path = os.path.join(tmpdir.strpath, db_file) + soledad_client = Soledad( default_uuid, unicode(passphrase), diff --git a/testing/tests/server/test__resource.py b/testing/tests/server/test__resource.py new file mode 100644 index 00000000..c066435e --- /dev/null +++ b/testing/tests/server/test__resource.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# test__resource.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for Soledad server main resource. +""" +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest +from twisted.web.wsgi import WSGIResource +from twisted.web.resource import getChildForRequest +from twisted.internet import reactor + +from leap.soledad.server._resource import SoledadResource +from leap.soledad.server._server_info import ServerInfo +from leap.soledad.server._blobs import BlobsResource +from leap.soledad.server.gzip_middleware import GzipMiddleware + + +_pool = reactor.getThreadPool() + + +class SoledadResourceTestCase(unittest.TestCase): + + def test_get_root(self): + enable_blobs = None # doesn't matter + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) + request = DummyRequest(['']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, ServerInfo) + + def test_get_blobs_enabled(self): + enable_blobs = True + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) + request = DummyRequest(['blobs']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, BlobsResource) + + def test_get_blobs_disabled(self): + enable_blobs = False + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) + request = DummyRequest(['blobs']) + child = getChildForRequest(resource, request) + # if blobs is disabled, the request should be routed to sync + self.assertIsInstance(child, WSGIResource) + self.assertIsInstance(child._application, GzipMiddleware) + + def test_get_sync(self): + enable_blobs = None # doesn't matter + resource = SoledadResource(enable_blobs=enable_blobs, sync_pool=_pool) + request = DummyRequest(['user-db', 'sync-from', 'source-id']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, WSGIResource) + self.assertIsInstance(child._application, GzipMiddleware) diff --git a/testing/tests/server/test__server_info.py b/testing/tests/server/test__server_info.py new file mode 100644 index 00000000..40567ef1 --- /dev/null +++ b/testing/tests/server/test__server_info.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# test__server_info.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Tests for Soledad server information announcement. +""" +import json + +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest + +from leap.soledad.server._server_info import ServerInfo + + +class ServerInfoTestCase(unittest.TestCase): + + def test_blobs_enabled(self): + resource = ServerInfo(True) + response = resource.render(DummyRequest([''])) + _info = json.loads(response) + self.assertEquals(_info['blobs'], True) + self.assertTrue(isinstance(_info['version'], basestring)) + + def test_blobs_disabled(self): + resource = ServerInfo(False) + response = resource.render(DummyRequest([''])) + _info = json.loads(response) + self.assertEquals(_info['blobs'], False) + self.assertTrue(isinstance(_info['version'], basestring)) diff --git a/testing/tests/server/test_auth.py b/testing/tests/server/test_auth.py new file mode 100644 index 00000000..6eb647ee --- /dev/null +++ b/testing/tests/server/test_auth.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# test_auth.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for auth pieces. +""" +import collections + +from contextlib import contextmanager + +from twisted.cred.credentials import UsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest +from twisted.web.resource import IResource +from twisted.web.test import test_httpauth + +import leap.soledad.server.auth as auth_module +from leap.soledad.server.auth import SoledadRealm +from leap.soledad.server.auth import TokenChecker +from leap.soledad.server.auth import TokenCredentialFactory +from leap.soledad.server._resource import SoledadResource + + +class SoledadRealmTestCase(unittest.TestCase): + + def test_returned_resource(self): + # we have to pass a pool to the realm , otherwise tests will hang + conf = {'blobs': False} + pool = reactor.getThreadPool() + realm = SoledadRealm(conf=conf, sync_pool=pool) + iface, avatar, logout = realm.requestAvatar('any', None, IResource) + self.assertIsInstance(avatar, SoledadResource) + self.assertIsNone(logout()) + + +class DummyServer(object): + """ + I fake the `couchdb.client.Server` GET api and always return the token + given on my creation. + """ + + def __init__(self, token): + self._token = token + + def get(self, _): + return self._token + + +@contextmanager +def dummy_server(token): + yield collections.defaultdict(lambda: DummyServer(token)) + + +class TokenCheckerTestCase(unittest.TestCase): + + @inlineCallbacks + def test_good_creds(self): + # set up a dummy server which always return a *valid* token document + token = {'user_id': 'user', 'type': 'Token'} + server = dummy_server(token) + # setup the checker with the custom server + checker = TokenChecker() + auth_module.couch_server = lambda url: server + # assert the checker *can* verify the creds + creds = UsernamePassword('user', 'pass') + avatarId = yield checker.requestAvatarId(creds) + self.assertEqual('user', avatarId) + + @inlineCallbacks + def test_bad_creds(self): + # set up a dummy server which always return an *invalid* token document + token = None + server = dummy_server(token) + # setup the checker with the custom server + checker = TokenChecker() + auth_module.couch_server = lambda url: server + # assert the checker *cannot* verify the creds + creds = UsernamePassword('user', '') + with self.assertRaises(UnauthorizedLogin): + yield checker.requestAvatarId(creds) + + +class TokenCredentialFactoryTestcase( + test_httpauth.RequestMixin, test_httpauth.BasicAuthTestsMixin, + unittest.TestCase): + + def setUp(self): + test_httpauth.BasicAuthTestsMixin.setUp(self) + self.credentialFactory = TokenCredentialFactory() diff --git a/testing/tests/server/test_config.py b/testing/tests/server/test_config.py new file mode 100644 index 00000000..d2a8a9de --- /dev/null +++ b/testing/tests/server/test_config.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# test_config.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for server configuration. +""" + +from twisted.trial import unittest +from pkg_resources import resource_filename + +from leap.soledad.server._config import _load_config +from leap.soledad.server._config import CONFIG_DEFAULTS + + +class ConfigurationParsingTest(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_use_defaults_on_failure(self): + config = _load_config('this file will never exist') + expected = CONFIG_DEFAULTS + self.assertEquals(expected, config) + + def test_security_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = _load_config(config_path) + + # then + expected = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['user3', 'user4'], + 'admins_roles': ['role3', 'role3']} + self.assertDictEqual(expected, config['database-security']) + + def test_server_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = _load_config(config_path) + + # then + expected = {'couch_url': + 'http://soledad:passwd@localhost:5984', + 'create_cmd': + 'sudo -u soledad-admin /usr/bin/create-user-db', + 'admin_netrc': + '/etc/couchdb/couchdb-soledad-admin.netrc', + 'batching': False, + 'blobs': False, + 'blobs_path': '/srv/leap/soledad/blobs'} + self.assertDictEqual(expected, config['soledad-server']) diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py index 6710caaf..4a5ec43f 100644 --- a/testing/tests/server/test_server.py +++ b/testing/tests/server/test_server.py @@ -18,17 +18,13 @@ Tests for server-related functionality. """ import binascii -import mock import os import pytest -from hashlib import sha512 -from pkg_resources import resource_filename from urlparse import urljoin from uuid import uuid4 from twisted.internet import defer -from twisted.trial import unittest from leap.soledad.common.couch.state import CouchServerState from leap.soledad.common.couch import CouchDatabase @@ -38,253 +34,10 @@ from test_soledad.util import ( make_token_soledad_app, make_soledad_document_for_test, soledad_sync_target, - BaseSoledadTest, ) from leap.soledad.client import _crypto from leap.soledad.client import Soledad -from leap.soledad.server.config import load_configuration -from leap.soledad.server.config import CONFIG_DEFAULTS -from leap.soledad.server.auth import URLToAuthorization -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -class ServerAuthenticationMiddlewareTestCase(CouchDBTestCase): - - def setUp(self): - super(ServerAuthenticationMiddlewareTestCase, self).setUp() - app = mock.Mock() - self._state = CouchServerState(self.couch_url) - app.state = self._state - self.auth_middleware = SoledadTokenAuthMiddleware(app) - self._authorize('valid-uuid', 'valid-token') - - def _authorize(self, uuid, token): - token_doc = {} - token_doc['_id'] = sha512(token).hexdigest() - token_doc[self._state.TOKENS_USER_ID_KEY] = uuid - token_doc[self._state.TOKENS_TYPE_KEY] = \ - self._state.TOKENS_TYPE_DEF - dbname = self._state._tokens_dbname() - db = self.couch_server.create(dbname) - db.save(token_doc) - self.addCleanup(self.delete_db, db.name) - - def test_authorized_user(self): - is_authorized = self.auth_middleware._verify_authentication_data - self.assertTrue(is_authorized('valid-uuid', 'valid-token')) - self.assertFalse(is_authorized('valid-uuid', 'invalid-token')) - self.assertFalse(is_authorized('invalid-uuid', 'valid-token')) - self.assertFalse(is_authorized('eve', 'invalid-token')) - - -class ServerAuthorizationTestCase(BaseSoledadTest): - - """ - Tests related to Soledad server authorization. - """ - - def setUp(self): - pass - - def tearDown(self): - pass - - def _make_environ(self, path_info, request_method): - return { - 'PATH_INFO': path_info, - 'REQUEST_METHOD': request_method, - } - - def test_verify_action_with_correct_dbnames(self): - """ - Test encrypting and decrypting documents. - - The following table lists the authorized actions among all possible - u1db remote actions: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{id} | GET, PUT, DELETE - /shared-db/sync-from/{source} | - - /user-db | GET, PUT, DELETE - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid,) - dbname = authmap._user_db_name - # test global auth - self.assertTrue( - authmap.is_authorized(self._make_environ('/', 'GET'))) - # test shared-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'POST'))) - # test shared-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'POST'))) - # test shared-db doc resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'POST'))) - # test shared-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'POST'))) - # test user-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test user-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test user-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test user-db sync resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) - - def test_verify_action_with_wrong_dbnames(self): - """ - Test if authorization fails for a wrong dbname. - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid) - dbname = 'somedb' - # test wrong-db database resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test wrong-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test wrong-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test wrong-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) @pytest.mark.usefixtures("method_tmpdir") @@ -382,7 +135,7 @@ class EncryptedSyncTestCase( user=user, prefix='x', auth_token='auth-token', - secrets_path=sol1._secrets_path, + secrets_path=sol1.secrets_path, passphrase=passphrase) # ensure remote db exists before syncing @@ -474,45 +227,3 @@ class EncryptedSyncTestCase( Test if Soledad can sync many smallfiles. """ return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) - - -class ConfigurationParsingTest(unittest.TestCase): - - def setUp(self): - self.maxDiff = None - - def test_use_defaults_on_failure(self): - config = load_configuration('this file will never exist') - expected = CONFIG_DEFAULTS - self.assertEquals(expected, config) - - def test_security_values_configuration(self): - # given - config_path = resource_filename('test_soledad', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['user3', 'user4'], - 'admins_roles': ['role3', 'role3']} - self.assertDictEqual(expected, config['database-security']) - - def test_server_values_configuration(self): - # given - config_path = resource_filename('test_soledad', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'couch_url': - 'http://soledad:passwd@localhost:5984', - 'create_cmd': - 'sudo -u soledad-admin /usr/bin/create-user-db', - 'admin_netrc': - '/etc/couchdb/couchdb-soledad-admin.netrc', - 'batching': False} - self.assertDictEqual(expected, config['soledad-server']) diff --git a/testing/tests/server/test_session.py b/testing/tests/server/test_session.py new file mode 100644 index 00000000..ebb94476 --- /dev/null +++ b/testing/tests/server/test_session.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +# test_session.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for server session entrypoint. +""" +from twisted.trial import unittest + +from twisted.cred import portal +from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse +from twisted.cred.credentials import IUsernamePassword +from twisted.web.resource import getChildForRequest +from twisted.web.static import Data +from twisted.web.test.requesthelper import DummyRequest +from twisted.web.test.test_httpauth import b64encode +from twisted.web.test.test_httpauth import Realm +from twisted.web._auth.wrapper import UnauthorizedResource + +from leap.soledad.server.session import SoledadSession + + +class SoledadSessionTestCase(unittest.TestCase): + """ + Tests adapted from for + L{twisted.web.test.test_httpauth.HTTPAuthSessionWrapper}. + """ + + def makeRequest(self, *args, **kwargs): + request = DummyRequest(*args, **kwargs) + request.path = '/' + return request + + def setUp(self): + self.username = b'foo bar' + self.password = b'bar baz' + self.avatarContent = b"contents of the avatar resource itself" + self.childName = b"foo-child" + self.childContent = b"contents of the foo child of the avatar" + self.checker = InMemoryUsernamePasswordDatabaseDontUse() + self.checker.addUser(self.username, self.password) + self.avatar = Data(self.avatarContent, 'text/plain') + self.avatar.putChild( + self.childName, Data(self.childContent, 'text/plain')) + self.avatars = {self.username: self.avatar} + self.realm = Realm(self.avatars.get) + self.portal = portal.Portal(self.realm, [self.checker]) + self.wrapper = SoledadSession(self.portal) + + def _authorizedTokenLogin(self, request): + authorization = b64encode( + self.username + b':' + self.password) + request.requestHeaders.addRawHeader(b'authorization', + b'Token ' + authorization) + return getChildForRequest(self.wrapper, request) + + def test_getChildWithDefault(self): + request = self.makeRequest([self.childName]) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(result): + self.assertEqual(request.responseCode, 401) + + d.addCallback(cbFinished) + request.render(child) + return d + + def _invalidAuthorizationTest(self, response): + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', response) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(result): + self.assertEqual(request.responseCode, 401) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_getChildWithDefaultUnauthorizedUser(self): + return self._invalidAuthorizationTest( + b'Basic ' + b64encode(b'foo:bar')) + + def test_getChildWithDefaultUnauthorizedPassword(self): + return self._invalidAuthorizationTest( + b'Basic ' + b64encode(self.username + b':bar')) + + def test_getChildWithDefaultUnrecognizedScheme(self): + return self._invalidAuthorizationTest(b'Quux foo bar baz') + + def test_getChildWithDefaultAuthorized(self): + request = self.makeRequest([self.childName]) + child = self._authorizedTokenLogin(request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [self.childContent]) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_renderAuthorized(self): + # Request it exactly, not any of its children. + request = self.makeRequest([]) + child = self._authorizedTokenLogin(request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [self.avatarContent]) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_decodeRaises(self): + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', + b'Token decode should fail') + child = getChildForRequest(self.wrapper, request) + self.assertIsInstance(child, UnauthorizedResource) + + def test_parseResponse(self): + basicAuthorization = b'Basic abcdef123456' + self.assertEqual( + self.wrapper._parseHeader(basicAuthorization), + None) + tokenAuthorization = b'Token abcdef123456' + self.assertEqual( + self.wrapper._parseHeader(tokenAuthorization), + b'abcdef123456') + + def test_unexpectedDecodeError(self): + + class UnexpectedException(Exception): + pass + + class BadFactory(object): + scheme = b'bad' + + def getChallenge(self, client): + return {} + + def decode(self, response, request): + print "decode raised" + raise UnexpectedException() + + self.wrapper._credentialFactory = BadFactory() + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', b'Bad abc') + child = getChildForRequest(self.wrapper, request) + request.render(child) + self.assertEqual(request.responseCode, 500) + errors = self.flushLoggedErrors(UnexpectedException) + self.assertEqual(len(errors), 1) + + def test_unexpectedLoginError(self): + class UnexpectedException(Exception): + pass + + class BrokenChecker(object): + credentialInterfaces = (IUsernamePassword,) + + def requestAvatarId(self, credentials): + raise UnexpectedException() + + self.portal.registerChecker(BrokenChecker()) + request = self.makeRequest([self.childName]) + child = self._authorizedTokenLogin(request) + request.render(child) + self.assertEqual(request.responseCode, 500) + self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) diff --git a/testing/tests/server/test_url_mapper.py b/testing/tests/server/test_url_mapper.py new file mode 100644 index 00000000..fa99cae7 --- /dev/null +++ b/testing/tests/server/test_url_mapper.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# test_url_mapper.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for server-related functionality. +""" + +from twisted.trial import unittest +from uuid import uuid4 + +from leap.soledad.server.url_mapper import URLMapper + + +class URLMapperTestCase(unittest.TestCase): + """ + Test if the URLMapper behaves as expected. + + The following table lists the authorized actions among all possible + u1db remote actions: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{id} | - + /shared-db/sync-from/{source} | - + /user-db | - + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST + """ + + def setUp(self): + self._uuid = uuid4().hex + self._urlmap = URLMapper() + self._dbname = 'user-%s' % self._uuid + + def test_root_authorized(self): + match = self._urlmap.match('/', 'GET') + self.assertIsNotNone(match) + + def test_shared_authorized(self): + self.assertIsNotNone(self._urlmap.match('/shared', 'GET')) + + def test_shared_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared', 'POST')) + + def test_shared_docs_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/docs', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'POST')) + + def test_shared_doc_authorized(self): + match = self._urlmap.match('/shared/doc/x', 'GET') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = self._urlmap.match('/shared/doc/x', 'PUT') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = self._urlmap.match('/shared/doc/x', 'DELETE') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + def test_shared_doc_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/doc/x', 'POST')) + + def test_shared_sync_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'POST')) + + def test_user_db_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'POST')) + + def test_user_db_docs_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'POST')) + + def test_user_db_doc_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'POST')) + + def test_user_db_sync_authorized(self): + uuid = self._uuid + dbname = self._dbname + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'GET') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'PUT') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'POST') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + def test_user_db_sync_unauthorized(self): + dbname = self._dbname + self.assertIsNone( + self._urlmap.match('/%s/sync-from/x' % dbname, 'DELETE')) diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index 6ce9a5c5..25136ba1 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -63,7 +63,8 @@ class TestSoledadParseReceivedDocResponse(unittest.TestCase): """ def parse(self, stream): - parser = DocStreamReceiver(None, None, lambda *_: defer.succeed(42)) + parser = DocStreamReceiver(None, defer.Deferred(), + lambda *_: defer.succeed(42)) parser.dataReceived(stream) parser.finish() @@ -838,7 +839,7 @@ class TestSoledadDbSync( # already created on some setUp method. import binascii tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) + key = tohex(self._soledad.secrets.local_key) dbpath = self._soledad._local_db_path self.opts = SQLCipherOptions( |