[pkg] unify client and server into a single python package

We have been discussing about this merge for a while.
Its main goal is to simplify things: code navigation, but also
packaging.

The rationale is that the code is more cohesive in this way, and there's
only one source package to install.

Dependencies that are only for the server or the client will not be
installed by default, and they are expected to be provided by the
environment. There are setuptools extras defined for the client and the
server.

Debianization is still expected to split the single source package into
3 binaries.

Another avantage is that the documentation can now install a single
package with a single step, and therefore include the docstrings into
the generated docs.

- Resolves: #8896

4 files changed, 450 insertions, 0 deletions

diff --git a/src/leap/soledad/client/_secrets/__init__.py b/src/leap/soledad/client/_secrets/__init__.py
new file mode 100644
index 00000000..b6c81cda
--- /dev/null
+++ b/src/leap/soledad/client/_secrets/__init__.py
@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+# _secrets/__init__.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import scrypt
+
+from leap.soledad.common.log import getLogger
+
+from leap.soledad.client._secrets.storage import SecretsStorage
+from leap.soledad.client._secrets.crypto import SecretsCrypto
+from leap.soledad.client._secrets.util import emit, UserDataMixin
+
+
+logger = getLogger(__name__)
+
+
+class Secrets(UserDataMixin):
+
+    lengths = {
+        'remote_secret': 512,  # remote_secret is used to encrypt remote data.
+        'local_salt': 64,      # local_salt is used in conjunction with
+        'local_secret': 448,   # local_secret to derive a local_key for storage
+    }
+
+    def __init__(self, soledad):
+        self._soledad = soledad
+        self._secrets = {}
+        self.crypto = SecretsCrypto(soledad)
+        self.storage = SecretsStorage(soledad)
+        self._bootstrap()
+
+    #
+    # bootstrap
+    #
+
+    def _bootstrap(self):
+
+        # attempt to load secrets from local storage
+        encrypted = self.storage.load_local()
+        if encrypted:
+            self._secrets = self.crypto.decrypt(encrypted)
+            # maybe update the format of storage of local secret.
+            if encrypted['version'] < self.crypto.VERSION:
+                self.store_secrets()
+            return
+
+        # no secret was found in local storage, so this is a first run of
+        # soledad for this user in this device. It is mandatory that we check
+        # if there's a secret stored in server.
+        encrypted = self.storage.load_remote()
+        if encrypted:
+            self._secrets = self.crypto.decrypt(encrypted)
+            self.store_secrets()
+            return
+
+        # we have *not* found a secret neither in local nor in remote storage,
+        # so we have to generate a new one, and then store it.
+        self._secrets = self._generate()
+        self.store_secrets()
+
+    #
+    # generation
+    #
+
+    @emit('creating')
+    def _generate(self):
+        logger.info("generating new set of secrets...")
+        secrets = {}
+        for name, length in self.lengths.iteritems():
+            secret = os.urandom(length)
+            secrets[name] = secret
+        logger.info("new set of secrets successfully generated")
+        return secrets
+
+    #
+    # crypto
+    #
+
+    def store_secrets(self):
+        # TODO: we have to improve the logic here, as we want to make sure that
+        # whatever is stored locally should only be used after remote storage
+        # is successful. Otherwise, this soledad could start encrypting with a
+        # secret while another soledad in another device could start encrypting
+        # with another secret, which would lead to decryption failures during
+        # sync.
+        encrypted = self.crypto.encrypt(self._secrets)
+        self.storage.save_local(encrypted)
+        self.storage.save_remote(encrypted)
+
+    #
+    # secrets
+    #
+
+    @property
+    def remote_secret(self):
+        return self._secrets.get('remote_secret')
+
+    @property
+    def local_salt(self):
+        return self._secrets.get('local_salt')
+
+    @property
+    def local_secret(self):
+        return self._secrets.get('local_secret')
+
+    @property
+    def local_key(self):
+        # local storage key is scrypt-derived from `local_secret` and
+        # `local_salt` above
+        secret = scrypt.hash(
+            password=self.local_secret,
+            salt=self.local_salt,
+            buflen=32,  # we need a key with 256 bits (32 bytes)
+        )
+        return secret
diff --git a/src/leap/soledad/client/_secrets/crypto.py b/src/leap/soledad/client/_secrets/crypto.py
new file mode 100644
index 00000000..8148151d
--- /dev/null
+++ b/src/leap/soledad/client/_secrets/crypto.py
@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# _secrets/crypto.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import binascii
+import json
+import os
+import scrypt
+
+from leap.soledad.common import soledad_assert
+from leap.soledad.common.log import getLogger
+
+from leap.soledad.client._crypto import encrypt_sym, decrypt_sym, ENC_METHOD
+from leap.soledad.client._secrets.util import SecretsError
+
+
+logger = getLogger(__name__)
+
+
+class SecretsCrypto(object):
+
+    VERSION = 2
+
+    def __init__(self, soledad):
+        self._soledad = soledad
+
+    def _get_key(self, salt):
+        passphrase = self._soledad.passphrase.encode('utf8')
+        key = scrypt.hash(passphrase, salt, buflen=32)
+        return key
+
+    #
+    # encryption
+    #
+
+    def encrypt(self, secrets):
+        encoded = {}
+        for name, value in secrets.iteritems():
+            encoded[name] = binascii.b2a_base64(value)
+        plaintext = json.dumps(encoded)
+        salt = os.urandom(64)  # TODO: get salt length from somewhere else
+        key = self._get_key(salt)
+        iv, ciphertext = encrypt_sym(plaintext, key,
+                                     method=ENC_METHOD.aes_256_gcm)
+        encrypted = {
+            'version': self.VERSION,
+            'kdf': 'scrypt',
+            'kdf_salt': binascii.b2a_base64(salt),
+            'kdf_length': len(key),
+            'cipher': ENC_METHOD.aes_256_gcm,
+            'length': len(plaintext),
+            'iv': str(iv),
+            'secrets': binascii.b2a_base64(ciphertext),
+        }
+        return encrypted
+
+    #
+    # decryption
+    #
+
+    def decrypt(self, data):
+        version = data.setdefault('version', 1)
+        method = getattr(self, '_decrypt_v%d' % version)
+        try:
+            return method(data)
+        except Exception as e:
+            logger.error('error decrypting secrets: %r' % e)
+            raise SecretsError(e)
+
+    def _decrypt_v1(self, data):
+        # get encrypted secret from dictionary: the old format allowed for
+        # storage of more than one secret, but this feature was never used and
+        # soledad has been using only one secret so far. As there is a corner
+        # case where the old 'active_secret' key might not be set, we just
+        # ignore it and pop the only secret found in the 'storage_secrets' key.
+        secret_id = data['storage_secrets'].keys().pop()
+        encrypted = data['storage_secrets'][secret_id]
+
+        # assert that we know how to decrypt the secret
+        soledad_assert('cipher' in encrypted)
+        cipher = encrypted['cipher']
+        if cipher == 'aes256':
+            cipher = ENC_METHOD.aes_256_ctr
+        soledad_assert(cipher in ENC_METHOD)
+
+        # decrypt
+        salt = binascii.a2b_base64(encrypted['kdf_salt'])
+        key = self._get_key(salt)
+        separator = ':'
+        iv, ciphertext = encrypted['secret'].split(separator, 1)
+        ciphertext = binascii.a2b_base64(ciphertext)
+        plaintext = self._decrypt(key, iv, ciphertext, encrypted, cipher)
+
+        # create secrets dictionary
+        secrets = {
+            'remote_secret': plaintext[0:512],
+            'local_salt': plaintext[512:576],
+            'local_secret': plaintext[576:1024],
+        }
+        return secrets
+
+    def _decrypt_v2(self, encrypted):
+        cipher = encrypted['cipher']
+        soledad_assert(cipher in ENC_METHOD)
+
+        salt = binascii.a2b_base64(encrypted['kdf_salt'])
+        key = self._get_key(salt)
+        iv = encrypted['iv']
+        ciphertext = binascii.a2b_base64(encrypted['secrets'])
+        plaintext = self._decrypt(
+            key, iv, ciphertext, encrypted, cipher)
+        encoded = json.loads(plaintext)
+        secrets = {}
+        for name, value in encoded.iteritems():
+            secrets[name] = binascii.a2b_base64(value)
+        return secrets
+
+    def _decrypt(self, key, iv, ciphertext, encrypted, method):
+        # assert some properties of the stored secret
+        soledad_assert(encrypted['kdf'] == 'scrypt')
+        soledad_assert(encrypted['kdf_length'] == len(key))
+        # decrypt
+        plaintext = decrypt_sym(ciphertext, key, iv, method)
+        soledad_assert(encrypted['length'] == len(plaintext))
+        return plaintext
diff --git a/src/leap/soledad/client/_secrets/storage.py b/src/leap/soledad/client/_secrets/storage.py
new file mode 100644
index 00000000..85713a48
--- /dev/null
+++ b/src/leap/soledad/client/_secrets/storage.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# _secrets/storage.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import six.moves.urllib.parse as urlparse
+
+from hashlib import sha256
+
+from leap.soledad.common import SHARED_DB_NAME
+from leap.soledad.common.log import getLogger
+
+from leap.soledad.client.shared_db import SoledadSharedDatabase
+from leap.soledad.client._document import Document
+from leap.soledad.client._secrets.util import emit, UserDataMixin
+
+
+logger = getLogger(__name__)
+
+
+class SecretsStorage(UserDataMixin):
+
+    def __init__(self, soledad):
+        self._soledad = soledad
+        self._shared_db = self._soledad.shared_db or self._init_shared_db()
+        self.__remote_doc = None
+
+    @property
+    def _creds(self):
+        uuid = self._soledad.uuid
+        token = self._soledad.token
+        return {'token': {'uuid': uuid, 'token': token}}
+
+    #
+    # local storage
+    #
+
+    def load_local(self):
+        path = self._soledad.secrets_path
+        logger.info("trying to load secrets from disk: %s" % path)
+        try:
+            with open(path, 'r') as f:
+                encrypted = json.loads(f.read())
+            logger.info("secrets loaded successfully from disk")
+            return encrypted
+        except IOError:
+            logger.warn("secrets not found in disk")
+        return None
+
+    def save_local(self, encrypted):
+        path = self._soledad.secrets_path
+        json_data = json.dumps(encrypted)
+        with open(path, 'w') as f:
+            f.write(json_data)
+
+    #
+    # remote storage
+    #
+
+    def _init_shared_db(self):
+        url = urlparse.urljoin(self._soledad.server_url, SHARED_DB_NAME)
+        creds = self._creds
+        db = SoledadSharedDatabase.open_database(url, creds)
+        return db
+
+    def _remote_doc_id(self):
+        passphrase = self._soledad.passphrase.encode('utf8')
+        uuid = self._soledad.uuid
+        text = '%s%s' % (passphrase, uuid)
+        digest = sha256(text).hexdigest()
+        return digest
+
+    @property
+    def _remote_doc(self):
+        if not self.__remote_doc and self._shared_db:
+            doc = self._get_remote_doc()
+            self.__remote_doc = doc
+        return self.__remote_doc
+
+    @emit('downloading')
+    def _get_remote_doc(self):
+        logger.info('trying to load secrets from server...')
+        doc = self._shared_db.get_doc(self._remote_doc_id())
+        if doc:
+            logger.info('secrets loaded successfully from server')
+        else:
+            logger.warn('secrets not found in server')
+        return doc
+
+    def load_remote(self):
+        doc = self._remote_doc
+        if not doc:
+            return None
+        encrypted = doc.content
+        return encrypted
+
+    @emit('uploading')
+    def save_remote(self, encrypted):
+        doc = self._remote_doc
+        if not doc:
+            doc = Document(doc_id=self._remote_doc_id())
+        doc.content = encrypted
+        db = self._shared_db
+        if not db:
+            logger.warn('no shared db found')
+            return
+        db.put_doc(doc)
diff --git a/src/leap/soledad/client/_secrets/util.py b/src/leap/soledad/client/_secrets/util.py
new file mode 100644
index 00000000..6401889b
--- /dev/null
+++ b/src/leap/soledad/client/_secrets/util.py
@@ -0,0 +1,63 @@
+# -*- coding:utf-8 -*-
+# _secrets/util.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from leap.soledad.client import events
+
+
+class SecretsError(Exception):
+    pass
+
+
+class UserDataMixin(object):
+    """
+    When emitting an event, we have to pass a dictionary containing user data.
+    This class only defines a property so we don't have to define it in
+    multiple places.
+    """
+
+    @property
+    def _user_data(self):
+        uuid = self._soledad.uuid
+        userid = self._soledad.userid
+        # TODO: seems that uuid and userid hold the same value! We should check
+        # whether we should pass something different or if the events api
+        # really needs two different values.
+        return {'uuid': uuid, 'userid': userid}
+
+
+def emit(verb):
+    def _decorator(method):
+        def _decorated(self, *args, **kwargs):
+
+            # emit starting event
+            user_data = self._user_data
+            name = 'SOLEDAD_' + verb.upper() + '_KEYS'
+            event = getattr(events, name)
+            events.emit_async(event, user_data)
+
+            # run the method
+            result = method(self, *args, **kwargs)
+
+            # emit a finished event
+            name = 'SOLEDAD_DONE_' + verb.upper() + '_KEYS'
+            event = getattr(events, name)
+            events.emit_async(event, user_data)
+
+            return result
+        return _decorated
+    return _decorator