[refactor] add SoledadCrypto interface

4 files changed, 37 insertions, 408 deletions

diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index a2de0ae1..2a523144 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -29,6 +29,7 @@ import time
 
 from io import BytesIO
 from cStringIO import StringIO
+from collections import namedtuple
 
 import six
 
@@ -51,7 +52,6 @@ from cryptography.hazmat.backends.openssl.backend \
 from zope.interface import implements
 
 from leap.common.config import get_path_prefix
-from leap.soledad.client.secrets import SoledadSecrets
 
 
 log = Logger()
@@ -86,23 +86,49 @@ class SoledadCrypto(object):
         self.secret = secret
 
     def encrypt_doc(self, doc):
+
+        def put_raw(blob):
+            return '{"raw": "' + blob.getvalue() + '"}'
+
         content = BytesIO()
         content.write(str(doc.get_json()))
         info = docinfo(doc.doc_id, doc.rev)
         del doc
         encryptor = BlobEncryptor(info, content, secret=self.secret)
-        return encryptor.encrypt()
+        d = encryptor.encrypt()
+        d.addCallback(put_raw)
+        return d
 
     def decrypt_doc(self, doc):
         info = docinfo(doc.doc_id, doc.rev)
         ciphertext = BytesIO()
-        ciphertext.write(doc.get_json())
-        ciphertext.seek(0)
+        payload = doc.content['raw']
         del doc
+        ciphertext.write(str(payload))
+        ciphertext.seek(0)
         decryptor = BlobDecryptor(info, ciphertext, secret=self.secret)
         return decryptor.decrypt()
 
 
+def encrypt_sym(data, key):
+    iv = os.urandom(16)
+    encryptor = AESEncryptor(key, iv)
+    encryptor.write(data)
+    encryptor.end()
+    ciphertext = encryptor.fd.getvalue()
+    return base64.urlsafe_b64encode(iv), ciphertext
+
+
+def decrypt_sym(data, key, iv):
+    _iv = base64.urlsafe_b64decode(iv)
+    decryptor = AESDecryptor(key, _iv)
+    decryptor.write(data)
+    decryptor.end()
+    plaintext = decryptor.fd.getvalue()
+    return plaintext
+
+
+
 class BlobEncryptor(object):
 
     """
@@ -122,7 +148,7 @@ class BlobEncryptor(object):
         self.doc_id = doc_info.doc_id
         self.rev = doc_info.rev
 
-        self._producer = FileBodyProducer(content_fd, readSize=2**8)
+        self._producer = FileBodyProducer(content_fd, readSize=2**16)
 
         self._preamble = BytesIO()
         if result is None:
@@ -176,7 +202,7 @@ class BlobEncryptor(object):
         self._aes_fd.close()
         self._hmac.result.close()
         self.result.seek(0)
-        return defer.succeed('ok')
+        return defer.succeed(self.result)
 
 
 class BlobDecryptor(object):
@@ -274,7 +300,6 @@ class AESEncryptor(object):
 
     def write(self, data):
         encrypted = self.encryptor.update(data)
-        encode = binascii.b2a_hex
         self.fd.write(encrypted)
         return encrypted
 
@@ -318,7 +343,7 @@ class AESDecryptor(object):
 
     implements(interfaces.IConsumer)
 
-    def __init__(self, key, iv, fd):
+    def __init__(self, key, iv, fd=None):
         if iv is None:
             iv = os.urandom(16)
         if len(key) != 32:
@@ -329,6 +354,8 @@ class AESDecryptor(object):
         cipher = _get_aes_ctr_cipher(key, iv)
         self.decryptor = cipher.decryptor()
 
+        if fd is None:
+            fd = BytesIO()
         self.fd = fd
         self.done = False
         self.deferred = defer.Deferred()
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 74ebaddc..de44f526 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -225,7 +225,6 @@ class Soledad(object):
             # have to close any thread-related stuff we have already opened
             # here, otherwise there might be zombie threads that may clog the
             # reactor.
-            self._sync_db.close()
             if hasattr(self, '_dbpool'):
                 self._dbpool.close()
             raise
@@ -288,16 +287,12 @@ class Soledad(object):
         tohex = binascii.b2a_hex
         # sqlcipher only accepts the hex version
         key = tohex(self._secrets.get_local_storage_key())
-        sync_db_key = tohex(self._secrets.get_sync_db_key())
 
         opts = sqlcipher.SQLCipherOptions(
             self._local_db_path, key,
             is_raw_key=True, create=True,
-            defer_encryption=self._defer_encryption,
-            sync_db_key=sync_db_key,
-        )
+            defer_encryption=self._defer_encryption)
         self._sqlcipher_opts = opts
-
         self._dbpool = adbapi.getConnectionPool(opts)
 
     def _init_u1db_syncer(self):
@@ -332,10 +327,6 @@ class Soledad(object):
         self._dbpool.close()
         if getattr(self, '_dbsyncer', None):
             self._dbsyncer.close()
-        # close the sync database
-        if self._sync_db:
-            self._sync_db.close()
-        self._sync_db = None
 
     #
     # ILocalStorage
@@ -850,28 +841,6 @@ class Soledad(object):
 
     token = property(_get_token, _set_token, doc='The authentication Token.')
 
-    def _initialize_sync_db(self, opts):
-        """
-        Initialize the Symmetrically-Encrypted document to be synced database,
-        and the queue to communicate with subprocess workers.
-
-        :param opts:
-        :type opts: SQLCipherOptions
-        """
-        soledad_assert(opts.sync_db_key is not None)
-        sync_db_path = None
-        if opts.path != ":memory:":
-            sync_db_path = "%s-sync" % opts.path
-        else:
-            sync_db_path = ":memory:"
-
-        # we copy incoming options because the opts object might be used
-        # somewhere else
-        sync_opts = sqlcipher.SQLCipherOptions.copy(
-            opts, path=sync_db_path, create=True)
-        self._sync_db = sqlcipher.getConnectionPool(
-            sync_opts, extra_queries=self._sync_db_extra_init)
-
 
     #
     # ISecretsStorage
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 55c49d9c..ecc0a0cf 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -32,370 +32,3 @@ from leap.soledad.common import soledad_assert
 from leap.soledad.common import soledad_assert_type
 from leap.soledad.common import crypto
 from leap.soledad.common.log import getLogger
-
-
-logger = getLogger(__name__)
-
-
-MAC_KEY_LENGTH = 64
-
-crypto_backend = MultiBackend([OpenSSLBackend()])
-
-
-# TODO -- deprecate.
-# Secrets still using this.
-
-def encrypt_sym(data, key):
-    """
-    Encrypt data using AES-256 cipher in CTR mode.
-
-    :param data: The data to be encrypted.
-    :type data: str
-    :param key: The key used to encrypt data (must be 256 bits long).
-    :type key: str
-
-    :return: A tuple with the initialization vector and the encrypted data.
-    :rtype: (long, str)
-    """
-    soledad_assert_type(key, str)
-    soledad_assert(
-        len(key) == 32,  # 32 x 8 = 256 bits.
-        'Wrong key size: %s bits (must be 256 bits long).' %
-        (len(key) * 8))
-
-    iv = os.urandom(16)
-    cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
-    encryptor = cipher.encryptor()
-    ciphertext = encryptor.update(data) + encryptor.finalize()
-
-    return binascii.b2a_base64(iv), ciphertext
-
-
-# FIXME decryption of the secrets doc is still using b64
-# Deprecate that, move to hex.
-
-def decrypt_sym(data, key, iv, encoding='base64'):
-    """
-    Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
-
-    :param data: The data to be decrypted.
-    :type data: str
-    :param key: The symmetric key used to decrypt data (must be 256 bits
-                long).
-    :type key: str
-    :param iv: The initialization vector.
-    :type iv: str (it's b64 encoded by secrets, hex by deserializing from wire)
-
-    :return: The decrypted data.
-    :rtype: str
-    """
-    soledad_assert_type(key, str)
-    # assert params
-    soledad_assert(
-        len(key) == 32,  # 32 x 8 = 256 bits.
-        'Wrong key size: %s (must be 256 bits long).' % len(key))
-
-    if encoding == 'base64':
-        iv = binascii.a2b_base64(iv)
-    elif encoding == 'hex':
-        iv = binascii.a2b_hex(iv)
-
-    cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
-    decryptor = cipher.decryptor()
-    return decryptor.update(data) + decryptor.finalize()
-
-
-def doc_mac_key(doc_id, secret):
-    """
-    Generate a key for calculating a MAC for a document whose id is
-    C{doc_id}.
-
-    The key is derived using HMAC having sha256 as underlying hash
-    function. The key used for HMAC is the first MAC_KEY_LENGTH characters
-    of Soledad's storage secret. The HMAC message is C{doc_id}.
-
-    :param doc_id: The id of the document.
-    :type doc_id: str
-
-    :param secret: The Soledad storage secret
-    :type secret: str
-
-    :return: The key.
-    :rtype: str
-    """
-    soledad_assert(secret is not None)
-    return hmac.new(
-        secret[:MAC_KEY_LENGTH],
-        doc_id,
-        hashlib.sha256).digest()
-
-
-#
-# Crypto utilities for a SoledadDocument.
-#
-
-# TODO should be ported to streaming consumer
-
-def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
-            mac_method, secret):
-    """
-    Calculate a MAC for C{doc} using C{ciphertext}.
-
-    Current MAC method used is HMAC, with the following parameters:
-
-        * key: sha256(storage_secret, doc_id)
-        * msg: doc_id + doc_rev + ciphertext
-        * digestmod: sha256
-
-    :param doc_id: The id of the document.
-    :type doc_id: str
-    :param doc_rev: The revision of the document.
-    :type doc_rev: str
-    :param ciphertext: The content of the document.
-    :type ciphertext: str
-    :param enc_scheme: The encryption scheme.
-    :type enc_scheme: bytes
-    :param enc_method: The encryption method.
-    :type enc_method: str
-    :param enc_iv: The encryption initialization vector.
-    :type enc_iv: str
-    :param mac_method: The MAC method to use.
-    :type mac_method: str
-    :param secret: The Soledad storage secret
-    :type secret: str
-
-    :return: The calculated MAC.
-    :rtype: str
-
-    :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
-    """
-    try:
-        soledad_assert(mac_method == crypto.MacMethods.HMAC)
-    except AssertionError:
-        raise crypto.UnknownMacMethodError
-
-    template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}"
-    content = template.format(
-        doc_id=doc_id,
-        doc_rev=doc_rev,
-        ciphertext=ciphertext,
-        enc_scheme=enc_scheme,
-        enc_method=enc_method,
-        enc_iv=enc_iv)
-
-    digest = hmac.new(
-        doc_mac_key(doc_id, secret),
-        content,
-        hashlib.sha256).digest()
-    return digest
-
-
-#def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
-    #"""
-    #Encrypt C{doc}'s content.
-#
-    #Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
-    #string representing the following:
-#
-        #{
-            #crypto.ENC_JSON_KEY: '<encrypted doc JSON string>',
-            #crypto.ENC_SCHEME_KEY: 'symkey',
-            #crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR,
-            #crypto.ENC_IV_KEY: '<the initial value used to encrypt>',
-            #MAC_KEY: '<mac>'
-            #crypto.MAC_METHOD_KEY: 'hmac'
-        #}
-#
-    #:param docstr: A representation of the document to be encrypted.
-    #:type docstr: str or unicode.
-#
-    #:param doc_id: The document id.
-    #:type doc_id: str
-#
-    #:param doc_rev: The document revision.
-    #:type doc_rev: str
-#
-    #:param key: The key used to encrypt ``data`` (must be 256 bits long).
-    #:type key: str
-#
-    #:param secret: The Soledad storage secret (used for MAC auth).
-    #:type secret: str
-#
-    #:return: The JSON serialization of the dict representing the encrypted
-             #content.
-    #:rtype: str
-    #"""
-    #enc_scheme = crypto.EncryptionSchemes.SYMKEY
-    #enc_method = crypto.EncryptionMethods.AES_256_CTR
-    #mac_method = crypto.MacMethods.HMAC
-    #enc_iv, ciphertext = encrypt_sym(
-        #str(docstr),  # encryption/decryption routines expect str
-        #key)
-    #mac = binascii.b2a_hex(  # store the mac as hex.
-        #mac_doc(
-            #doc_id,
-            #doc_rev,
-            #ciphertext,
-            #enc_scheme,
-            #enc_method,
-            #enc_iv,
-            #mac_method,
-            #secret))
-    # Return a representation for the encrypted content. In the following, we
-    # convert binary data to hexadecimal representation so the JSON
-    # serialization does not complain about what it tries to serialize.
-    #hex_ciphertext = binascii.b2a_hex(ciphertext)
-    #log.debug("Encrypting doc: %s" % doc_id)
-    #return json.dumps({
-        #crypto.ENC_JSON_KEY: hex_ciphertext,
-        #crypto.ENC_SCHEME_KEY: enc_scheme,
-        #crypto.ENC_METHOD_KEY: enc_method,
-        #crypto.ENC_IV_KEY: enc_iv,
-        #crypto.MAC_KEY: mac,
-        #crypto.MAC_METHOD_KEY: mac_method,
-    #})
-#
-
-
-# TODO port to _crypto
-def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
-                    enc_iv, mac_method, secret, doc_mac):
-    """
-    Verify that C{doc_mac} is a correct MAC for the given document.
-
-    :param doc_id: The id of the document.
-    :type doc_id: str
-    :param doc_rev: The revision of the document.
-    :type doc_rev: str
-    :param ciphertext: The content of the document.
-    :type ciphertext: str
-    :param enc_scheme: The encryption scheme.
-    :type enc_scheme: str
-    :param enc_method: The encryption method.
-    :type enc_method: str
-    :param enc_iv: The encryption initialization vector.
-    :type enc_iv: str
-    :param mac_method: The MAC method to use.
-    :type mac_method: str
-    :param secret: The Soledad storage secret
-    :type secret: str
-    :param doc_mac: The MAC to be verified against.
-    :type doc_mac: str
-
-    :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
-    :raise crypto.WrongMacError: Raised when MAC could not be verified.
-    """
-    # TODO mac_doc should be ported to Streaming also
-    calculated_mac = mac_doc(
-        doc_id,
-        doc_rev,
-        ciphertext,
-        enc_scheme,
-        enc_method,
-        enc_iv,
-        mac_method,
-        secret)
-    
-    ok = hmac.compare_digest(
-        str(calculated_mac),
-        binascii.a2b_hex(doc_mac))
-
-    if not ok:
-        loggger.warn("wrong MAC while decrypting doc...")
-        loggger.info(u'given:      %s' % doc_mac)
-        loggger.info(u'calculated: %s' % binascii.b2a_hex(calculated_mac))
-        raise crypto.WrongMacError("Could not authenticate document's "
-                                   "contents.")
-
-
-def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
-    """
-    Decrypt a symmetrically encrypted C{doc}'s content.
-
-    Return the JSON string representation of the document's decrypted content.
-
-    The passed doc_dict argument should have the following structure:
-
-        {
-            crypto.ENC_JSON_KEY: '<enc_blob>',
-            crypto.ENC_SCHEME_KEY: '<enc_scheme>',
-            crypto.ENC_METHOD_KEY: '<enc_method>',
-            crypto.ENC_IV_KEY: '<initial value used to encrypt>',  # (optional)
-            MAC_KEY: '<mac>'
-            crypto.MAC_METHOD_KEY: 'hmac'
-        }
-
-    C{enc_blob} is the encryption of the JSON serialization of the document's
-    content. For now Soledad just deals with documents whose C{enc_scheme} is
-    crypto.EncryptionSchemes.SYMKEY and C{enc_method} is
-    crypto.EncryptionMethods.AES_256_CTR.
-
-    :param doc_dict: The content of the document to be decrypted.
-    :type doc_dict: dict
-
-    :param doc_id: The document id.
-    :type doc_id: str
-
-    :param doc_rev: The document revision.
-    :type doc_rev: str
-
-    :param key: The key used to encrypt ``data`` (must be 256 bits long).
-    :type key: str
-
-    :param secret: The Soledad storage secret.
-    :type secret: str
-
-    :return: The JSON serialization of the decrypted content.
-    :rtype: str
-
-    :raise UnknownEncryptionMethodError: Raised when trying to decrypt from an
-        unknown encryption method.
-    """
-    # assert document dictionary structure
-    expected_keys = set([
-        crypto.ENC_JSON_KEY,
-        crypto.ENC_SCHEME_KEY,
-        crypto.ENC_METHOD_KEY,
-        crypto.ENC_IV_KEY,
-        crypto.MAC_KEY,
-        crypto.MAC_METHOD_KEY,
-    ])
-    soledad_assert(expected_keys.issubset(set(doc_dict.keys())))
-
-    d = doc_dict
-    decode = binascii.a2b_hex
-
-    enc_scheme = d[crypto.ENC_SCHEME_KEY]
-    enc_method = d[crypto.ENC_METHOD_KEY]
-    doc_mac = d[crypto.MAC_KEY]
-    mac_method = d[crypto.MAC_METHOD_KEY]
-    enc_iv = d[crypto.ENC_IV_KEY]
-
-    ciphertext_hex = d[crypto.ENC_JSON_KEY]
-    ciphertext = decode(ciphertext_hex)
-
-    soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY)
-
-    _verify_doc_mac(
-        doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
-        enc_iv, mac_method, secret, doc_mac)
-
-    decr = decrypt_sym(ciphertext, key, enc_iv, encoding='hex')
-    return decr
-
-
-# TODO deprecate
-def is_symmetrically_encrypted(doc):
-    """
-    Return True if the document was symmetrically encrypted.
-
-    :param doc: The document to check.
-    :type doc: SoledadDocument
-
-    :rtype: bool
-    """
-    if doc.content and crypto.ENC_SCHEME_KEY in doc.content:
-        if doc.content[crypto.ENC_SCHEME_KEY] \
-                == crypto.EncryptionSchemes.SYMKEY:
-            return True
-    return False
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index ad1db2b8..8543df01 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -34,7 +34,7 @@ from leap.soledad.common import soledad_assert_type
 from leap.soledad.common import document
 from leap.soledad.common.log import getLogger
 from leap.soledad.client import events
-from leap.soledad.client.crypto import encrypt_sym, decrypt_sym
+from leap.soledad.client._crypto import encrypt_sym, decrypt_sym
 
 
 logger = getLogger(__name__)