[feature] blob encryptor / decryptor

3 files changed, 334 insertions, 116 deletions

diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index ed861fdd..61a190c7 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -24,9 +24,14 @@ import base64
 import hashlib
 import hmac
 import os
+import struct
+import time
 
+from io import BytesIO
 from cStringIO import StringIO
 
+import six
+
 from twisted.internet import defer
 from twisted.internet import interfaces
 from twisted.internet import reactor
@@ -35,6 +40,9 @@ from twisted.persisted import dirdbm
 from twisted.web import client
 from twisted.web.client import FileBodyProducer
 
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.hmac import HMAC
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.backends.multibackend import MultiBackend
 from cryptography.hazmat.backends.openssl.backend \
@@ -53,63 +61,223 @@ MAC_KEY_LENGTH = 64
 crypto_backend = MultiBackend([OpenSSLBackend()])
 
 
-class EncryptionError(Exception):
+class ENC_SCHEME:
+    symkey = 1
+
+
+class ENC_METHOD:
+    aes_256_ctr = 1
+
+
+class EncryptionDecryptionError(Exception):
     pass
 
 
-class AESWriter(object):
+class InvalidBlob(Exception):
+    pass
 
-    implements(interfaces.IConsumer)
 
-    def __init__(self, key, fd, iv=None):
+
+class BlobEncryptor(object):
+
+    """
+    Encrypts a payload associated with a given Document.
+    """
+
+    def __init__(self, doc_info, content_fd, result=None, secret=None, iv=None):
+
         if iv is None:
             iv = os.urandom(16)
+        else:
+            log.warn('Using a fixed IV. Use only for testing!')
+        self.iv = iv
+        if not secret:
+            raise EncryptionDecryptionError('no secret given')
+
+        self.doc_id = doc_info.doc_id
+        self.rev = doc_info.rev
+
+        self._producer = FileBodyProducer(content_fd, readSize=2**8)
+
+        self._preamble = BytesIO()
+        if result is None:
+            result = BytesIO()
+        self.result = result
+
+        sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
+        mac_key = _get_mac_key_for_doc(doc_info.doc_id, secret)
+
+        self._aes_fd = BytesIO()
+        self._aes = AESEncryptor(sym_key, self.iv, self._aes_fd)
+        self._hmac = HMACWriter(mac_key)
+        self._write_preamble()
+
+        self._crypter = VerifiedEncrypter(self._aes, self._hmac)
+
+    def encrypt(self):
+        d = self._producer.startProducing(self._crypter)
+        d.addCallback(self._end_crypto_stream)
+        return d
+
+    def _write_preamble(self):
+
+        def write(data):
+            self._preamble.write(data)
+            self._hmac.write(data)
+        
+        current_time = int(time.time())
+
+        write(b'\x80')
+        write(struct.pack(
+            'Qbb', 
+            current_time,
+            ENC_SCHEME.symkey,
+            ENC_METHOD.aes_256_ctr))
+        write(self.iv)
+        write(self.doc_id)
+        write(self.rev)
+
+    def _end_crypto_stream(self, ignored):
+        self._aes.end()
+        self._hmac.end()
+
+        preamble = self._preamble.getvalue()
+        encrypted = self._aes_fd.getvalue()
+        hmac = self._hmac.result.getvalue()
+
+        self.result.write(
+            base64.urlsafe_b64encode(preamble + encrypted + hmac))
+        self._preamble.close()
+        self._aes_fd.close()
+        self._hmac.result.close()
+        self.result.seek(0)
+        return defer.succeed('ok')
+
+
+class BlobDecryptor(object):
+    """
+    Decrypts an encrypted blob associated with a given Document.
+
+    Will raise an exception if the blob doesn't have the expected structure, or
+    if the HMAC doesn't verify.
+    """
+
+    def __init__(self, doc_info, ciphertext_fd, result=None,
+                 secret=None):
+        self.doc_id = doc_info.doc_id
+        self.rev = doc_info.rev
+
+        self.ciphertext = ciphertext_fd
+
+        self.sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
+        self.mac_key = _get_mac_key_for_doc(doc_info.doc_id, secret)
+
+        if result is None:
+            result = BytesIO()
+        self.result = result
+
+    def decrypt(self):
+
+        try:
+            data = base64.urlsafe_b64decode(self.ciphertext.getvalue())
+        except (TypeError, binascii.Error):
+            raise InvalidBlob
+        self.ciphertext.close()
+
+        current_time = int(time.time())
+        if not data or six.indexbytes(data, 0) != 0x80:
+            raise InvalidBlob
+        try:
+            ts, sch, meth = struct.unpack("Qbb", data[1:11])
+        except struct.error:
+            raise InvalidBlob
+
+        # TODO check timestamp
+        if sch != ENC_SCHEME.symkey:
+            raise InvalidBlob('invalid scheme')
+        # TODO should adapt the assymetric-gpg too, rigth?
+        if meth != ENC_METHOD.aes_256_ctr:
+            raise InvalidBlob('invalid encryption scheme')
+
+        iv = data[11:27]
+        docidlen = len(self.doc_id)
+        ciph_idx = 26 + docidlen
+        doc_id = data[26:ciph_idx]
+        revlen = len(self.rev)
+        rev_idx = ciph_idx + 1 + revlen
+        rev = data[ciph_idx + 1:rev_idx]
+
+        if rev != self.rev:
+            raise InvalidBlob('invalid revision')
+
+        ciphertext = data[rev_idx:-64]
+        hmac = data[-64:]
+
+        h = HMAC(self.mac_key, hashes.SHA512(), backend=crypto_backend)
+        h.update(data[:-64])
+        try:
+            h.verify(hmac)
+        except InvalidSignature:
+            raise InvalidBlob('HMAC could not be verifed')
+
+        decryptor = _get_aes_ctr_cipher(self.sym_key, iv).decryptor()
+
+        # TODO pass chunks, streaming, instead
+        # Use AESDecryptor below
+        self.result.write(decryptor.update(ciphertext))
+        self.result.write(decryptor.finalize())
+        return self.result
+
+
+class AESEncryptor(object):
+
+    implements(interfaces.IConsumer)
+
+    def __init__(self, key, iv, fd=None):
         if len(key) != 32:
-            raise EncryptionError('key is not 256 bits')
+            raise EncryptionDecryptionError('key is not 256 bits')
         if len(iv) != 16:
-            raise EncryptionError('iv is not 128 bits')
+            raise EncryptionDecryptionError('iv is not 128 bits')
 
         cipher = _get_aes_ctr_cipher(key, iv)
         self.encryptor = cipher.encryptor()
-
+        
+        if fd is None:
+            fd = BytesIO()
         self.fd = fd
+
         self.done = False
-        self.deferred = defer.Deferred()
 
     def write(self, data):
         encrypted = self.encryptor.update(data)
+        encode = binascii.b2a_hex
         self.fd.write(encrypted)
         return encrypted
 
     def end(self):
         if not self.done:
-            self.encryptor.finalize()
-            self.deferred.callback(self.fd)
+            final = self.encryptor.finalize()
         self.done = True
 
 
 class HMACWriter(object):
 
     implements(interfaces.IConsumer)
+    hashtype = 'sha512'
 
     def __init__(self, key):
-        self.done = False
-        self.deferred = defer.Deferred()
-
-        self.digest = ''
-        self._hmac = hmac.new(key, '', hashlib.sha256)
+        self._hmac = hmac.new(key, '', getattr(hashlib, self.hashtype))
+        self.result = BytesIO('')
 
     def write(self, data):
         self._hmac.update(data)
 
     def end(self):
-        if not self.done:
-            self.digest = self._hmac.digest()
-            self.deferred.callback(self.digest)
-        self.done = True
+        self.result.write(self._hmac.digest())
+
 
 
-class EncryptAndHMAC(object):
+class VerifiedEncrypter(object):
 
     implements(interfaces.IConsumer)
 
@@ -122,99 +290,39 @@ class EncryptAndHMAC(object):
         self.hmac.write(enc_chunk)
         
 
+class AESDecryptor(object):
 
-class DocEncrypter(object):
-
-    staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging')
-    staged_template = """{"_enc_scheme": "symkey", "_enc_method":
-        "aes-256-ctr", "_mac_method": "hmac", "_mac_hash": "sha256",
-        "_encoding": "ENCODING", "_enc_json": "CIPHERTEXT", "_enc_iv": "IV", "_mac": "MAC"}"""
-
-
-    def __init__(self, content_fd, doc_id, rev, secret=None):
-        self._content_fd  = content_fd
-        self._contentFileProducer = FileBodyProducer(
-            content_fd, readSize=2**8)
-        self.doc_id = doc_id
-        self.rev = rev
-        self._encrypted_fd = StringIO()
-
-        self.iv = os.urandom(16)
-
-        sym_key = _get_sym_key_for_doc(doc_id, secret)
-        mac_key = _get_mac_key_for_doc(doc_id, secret)
+    implements(interfaces.IConsumer)
 
-        crypter = AESWriter(sym_key, self._encrypted_fd, self.iv)
-        hmac = HMACWriter(mac_key)
+    def __init__(self, key, iv, fd):
+        if iv is None:
+            iv = os.urandom(16)
+        if len(key) != 32:
+            raise EncryptionhDecryptionError('key is not 256 bits')
+        if len(iv) != 16:
+            raise EncryptionDecryptionError('iv is not 128 bits')
 
-        self.crypter_consumer = crypter
-        self.hmac_consumer = hmac
+        cipher = _get_aes_ctr_cipher(key, iv)
+        self.decryptor = cipher.decryptor()
 
-        self._prime_hmac()
-        self.encrypt_and_mac_consumer = EncryptAndHMAC(crypter, hmac)
+        self.fd = fd
+        self.done = False
+        self.deferred = defer.Deferred()
 
-    def encrypt_stream(self):
-        d = self._contentFileProducer.startProducing(
-            self.encrypt_and_mac_consumer)
-        d.addCallback(self.end_crypto_stream)
-        d.addCallback(self.persist_encrypted_doc)
-        return d
 
-    def end_crypto_stream(self, ignored):
-        self.crypter_consumer.end()
-        self._post_hmac()
-        self.hmac_consumer.end()
-        return defer.succeed('ok')
+    def write(self, data):
+        decrypted = self.decryptor.update(data)
+        self.fd.write(decrypted)
+        return decrypted
 
-    # TODO make this pluggable:
-    # pass another class (CryptoSerializer) to which we pass
-    # the doc info, the encrypted_fd and the mac_digest
+    def end(self):
+        if not self.done:
+            self.decryptor.finalize()
+            self.deferred.callback(self.fd)
+        self.done = True
 
-    def persist_encrypted_doc(self, ignored, encoding='hex'):
-        # TODO -- transition to b64: needs migration FIXME
-        if encoding == 'b64':
-            encode = binascii.b2a_base64
-        elif encoding == 'hex':
-            encode = binascii.b2a_hex
-        else:
-            raise RuntimeError('Unknown encoding: %s' % encoding)
-
-        # TODO to avoid blocking on io, this can use a
-        # version of dbm that chunks the writes to the 
-        # disk fd by using the same FileBodyProducer strategy
-        # that we're using here, long live to the Cooperator.
-
-
-        db = dirdbm.DirDBM(self.staging_path)
-        key = '{doc_id}@{rev}'.format(
-            doc_id=self.doc_id, rev=self.rev)
-        ciphertext = encode(self._encrypted_fd.getvalue())
-        value = self.staged_template.replace(
-            'ENCODING', encoding).replace(
-            'CIPHERTEXT', ciphertext).replace(
-            'IV', encode(self.iv)).replace(
-            'MAC', encode(self.hmac_consumer.digest)).replace(
-            '\n', '')
-        self._encrypted_fd.seek(0)
-
-        log.debug('persisting %s' % key)
-        db[key] = value
-
-        self._content_fd.close()
-        self._encrypted_fd.close()
-
-    def _prime_hmac(self):
-        pre = '{doc_id}{rev}'.format(
-            doc_id=self.doc_id, rev=self.rev)
-        self.hmac_consumer.write(pre)
-
-    def _post_hmac(self):
-        post = '{enc_scheme}{enc_method}{enc_iv}'.format(
-            enc_scheme='symkey',
-            enc_method='aes-256-ctr',
-            enc_iv=binascii.b2a_hex(self.iv))
-        self.hmac_consumer.write(post)
 
+# utils
 
 
 def _hmac_sha256(key, data):
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 6b257669..98613df2 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -61,7 +61,7 @@ from leap.soledad.client.secrets import SoledadSecrets
 from leap.soledad.client.shared_db import SoledadSharedDatabase
 from leap.soledad.client import sqlcipher
 from leap.soledad.client import encdecpool
-from leap.soledad.client._crypto import DocEncrypter
+#from leap.soledad.client._crypto import DocEncrypter
 
 logger = getLogger(__name__)
 
@@ -360,8 +360,10 @@ class Soledad(object):
         contentfd.seek(0)
 
         sikret = self._secrets.remote_storage_secret
-        crypter = DocEncrypter(
-            contentfd, doc.doc_id, doc.rev, secret=sikret)
+
+        # TODO use BlobEncrypter
+        #crypter = DocEncrypter(
+            #contentfd, doc.doc_id, doc.rev, secret=sikret)
         d = crypter.encrypt_stream()
         d.addCallback(lambda _: result)
         return d
diff --git a/testing/tests/client/test_crypto2.py b/testing/tests/client/test_crypto2.py
index ae280020..f0f6c4af 100644
--- a/testing/tests/client/test_crypto2.py
+++ b/testing/tests/client/test_crypto2.py
@@ -19,16 +19,28 @@
 Tests for the _crypto module
 """
 
+import base64
+import binascii
+import time
+import struct
 import StringIO
 
-
 import leap.soledad.client
 from leap.soledad.client import _crypto
 
-
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.backends import default_backend
 
+from twisted.trial import unittest
+
+
+snowden1 = (
+    "You can't come up against "
+    "the world's most powerful intelligence "
+    "agencies and not accept the risk. "
+    "If they want to get you, over time "
+    "they will.")
+
 
 def _aes_encrypt(key, iv, data):
     backend = default_backend()
@@ -36,20 +48,21 @@ def _aes_encrypt(key, iv, data):
     encryptor = cipher.encryptor()
     return encryptor.update(data) + encryptor.finalize()
 
+def _aes_decrypt(key, iv, data):
+    backend = default_backend()
+    cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
+    decryptor = cipher.decryptor()
+    return decryptor.update(data) + decryptor.finalize()
+
 
 def test_chunked_encryption():
     key = 'A' * 32
     iv = 'A' * 16
-    data = (
-        "You can't come up against "
-        "the world's most powerful intelligence "
-        "agencies and not accept the risk. "
-        "If they want to get you, over time "
-        "they will.")
 
     fd = StringIO.StringIO()
-    aes = _crypto.AESWriter(key, fd, iv)
+    aes = _crypto.AESEncryptor(key, iv, fd)
 
+    data = snowden1
     block = 16
 
     for i in range(len(data)/block):
@@ -61,3 +74,98 @@ def test_chunked_encryption():
     ciphertext = _aes_encrypt(key, iv, data)
 
     assert ciphertext_chunked == ciphertext
+
+
+def test_decrypt():
+    key = 'A' * 32
+    iv = 'A' * 16
+
+    data = snowden1
+    block = 16
+
+    ciphertext = _aes_encrypt(key, iv, data)
+
+    fd = StringIO.StringIO()
+    aes = _crypto.AESDecryptor(key, iv, fd)
+
+    for i in range(len(ciphertext)/block):
+        chunk = ciphertext[i * block:(i+1)*block]
+        aes.write(chunk)
+    aes.end()
+
+    cleartext_chunked = fd.getvalue()
+    assert cleartext_chunked == data
+
+
+
+class BlobTestCase(unittest.TestCase):
+
+    class doc_info:
+        doc_id = 'D-deadbeef'
+        rev = '397932e0c77f45fcb7c3732930e7e9b2:1'
+
+    def test_blob_encryptor(self):
+
+        inf = StringIO.StringIO()
+        inf.write(snowden1)
+        inf.seek(0)
+        outf = StringIO.StringIO()
+
+        blob = _crypto.BlobEncryptor(
+            self.doc_info, inf, result=outf,
+            secret='A' * 96, iv='B'*16)
+
+        d = blob.encrypt()
+        d.addCallback(self._test_blob_encryptor_cb, outf)
+        return d
+
+    def _test_blob_encryptor_cb(self, _, outf):
+        encrypted = outf.getvalue()
+        data = base64.urlsafe_b64decode(encrypted)
+
+        assert data[0] == '\x80'
+        ts, sch, meth  = struct.unpack(
+            'Qbb', data[1:11])
+        assert sch == 1
+        assert meth == 1
+        iv = data[11:27]
+        assert iv == 'B' * 16
+        doc_id = data[27:37]
+        assert doc_id == 'D-deadbeef'
+
+        rev = data[37:71]
+        assert rev == self.doc_info.rev
+
+        ciphertext = data[71:-64]
+        aes_key = _crypto._get_sym_key_for_doc(
+            self.doc_info.doc_id, 'A'*96)
+        assert ciphertext == _aes_encrypt(aes_key, 'B'*16, snowden1)
+
+        decrypted = _aes_decrypt(aes_key, 'B'*16, ciphertext)
+        assert str(decrypted) == snowden1
+
+    def test_blob_decryptor(self):
+
+        inf = StringIO.StringIO()
+        inf.write(snowden1)
+        inf.seek(0)
+        outf = StringIO.StringIO()
+
+        blob = _crypto.BlobEncryptor(
+            self.doc_info, inf, result=outf,
+            secret='A' * 96, iv='B' * 16)
+
+        def do_decrypt(_, outf):
+            decryptor = _crypto.BlobDecryptor(
+                self.doc_info, outf,
+                secret='A' * 96)
+            d = decryptor.decrypt()
+            return d
+
+        d = blob.encrypt()
+        d.addCallback(do_decrypt, outf)
+        d.addCallback(self._test_blob_decryptor_cb)
+        return d
+
+    def _test_blob_decryptor_cb(self, decrypted):
+        assert decrypted.getvalue() == snowden1