[feature] make _crypto stream on decryption

We are already doing this on encryption, now we can stream also from
decryption. This unblocks the reactor and will be valuable for blobs-io.

3 files changed, 64 insertions, 25 deletions

diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index a235e246..163c9e4e 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -39,9 +39,6 @@ from twisted.internet import interfaces
 from twisted.logger import Logger
 from twisted.web.client import FileBodyProducer
 
-from cryptography.exceptions import InvalidSignature
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives.hmac import HMAC
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.backends.multibackend import MultiBackend
 from cryptography.hazmat.backends.openssl.backend \
@@ -133,8 +130,7 @@ class SoledadCrypto(object):
         del doc
         ciphertext.write(str(payload))
         decryptor = BlobDecryptor(info, ciphertext, secret=self.secret)
-        buf = decryptor.decrypt()
-        return buf.getvalue()
+        return decryptor.decrypt()
 
 
 def encrypt_sym(data, key):
@@ -291,29 +287,45 @@ class BlobDecryptor(object):
 
     def __init__(self, doc_info, ciphertext_fd, result=None,
                  secret=None):
+        if not secret:
+            raise EncryptionDecryptionError('no secret given')
+        ciphertext_fd.seek(0)
+
         self.doc_id = doc_info.doc_id
         self.rev = doc_info.rev
 
-        self.ciphertext = ciphertext_fd
-
         self.sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
         self.mac_key = _get_mac_key_for_doc(doc_info.doc_id, secret)
 
+        self._read_preamble(ciphertext_fd)
+
+        self._producer = FileBodyProducer(self.ciphertext, readSize=2**16)
+        self._content_fd = self.ciphertext
+
         self.result = result or BytesIO()
 
-    def decrypt(self):
+        self._aes_fd = BytesIO()
+        self._aes = AESDecryptor(self.sym_key, self.iv, self.result)
+        self._hmac = HMACWriter(self.mac_key)
+        self._hmac.write(self.preamble)
+
+        self._decrypter = VerifiedDecrypter(self._aes, self._hmac)
+
+    def _read_preamble(self, ciphertext):
         try:
-            preamble, ciphertext = _split(self.ciphertext.getvalue())
-            hmac, ciphertext = ciphertext[-64:], ciphertext[:-64]
+            self.preamble, ciphertext = _split(ciphertext.getvalue())
+            self.doc_hmac, self.ciphertext = ciphertext[-64:], ciphertext[:-64]
         except (TypeError, binascii.Error):
             raise InvalidBlob
-        self.ciphertext.close()
-        if len(preamble) != PACMAN.size:
+        self.ciphertext = BytesIO(self.ciphertext)
+
+        if len(self.preamble) != PACMAN.size:
             raise InvalidBlob
 
         try:
-            unpacked_data = PACMAN.unpack(preamble)
+            unpacked_data = PACMAN.unpack(self.preamble)
             pad, ts, sch, meth, iv, doc_id, rev = unpacked_data
+            self.iv = iv
         except struct.error:
             raise InvalidBlob
         if pad != '\x80':
@@ -329,18 +341,28 @@ class BlobDecryptor(object):
         if rev != self.rev:
             raise InvalidBlob('invalid revision')
 
-        h = HMAC(self.mac_key, hashes.SHA512(), backend=crypto_backend)
-        h.update(preamble)
-        h.update(ciphertext)
-        try:
-            h.verify(hmac)
-        except InvalidSignature:
+    def _check_hmac(self):
+        if self._hmac._hmac.digest() != self.doc_hmac:
             raise InvalidBlob('HMAC could not be verifed')
 
-        decryptor = _get_aes_ctr_cipher(self.sym_key, iv).decryptor()
+    def decrypt(self):
+        """
+        Starts producing encrypted data from the cleartext data.
+
+        :return: A deferred which will be fired when encryption ends and whose
+            callback will be invoked with the resulting ciphertext.
+        :rtype: twisted.internet.defer.Deferred
+        """
+        d = self._producer.startProducing(self._decrypter)
+        d.addCallback(lambda _: self._check_hmac())
+        d.addCallback(lambda _: self.result.getvalue())
+        return d
 
-        # TODO pass chunks, streaming, instead
-        # Use AESDecryptor below
+    def decrypt_whole(self):
+        ciphertext = self.ciphertext.getvalue()
+        self.hmac_obj.update(ciphertext)
+        self._check_hmac()
+        decryptor = _get_aes_ctr_cipher(self.sym_key, self.iv).decryptor()
 
         self.result.write(decryptor.update(ciphertext))
         self.result.write(decryptor.finalize())
@@ -412,6 +434,23 @@ class VerifiedEncrypter(object):
         self.hmac.write(enc_chunk)
 
 
+class VerifiedDecrypter(object):
+    """
+    A Twisted's Consumer implementation combining AESDecryptor and HMACWriter.
+    It directs the resulting ciphertext into HMAC-SHA512 processing, then
+    decrypt.
+    """
+    implements(interfaces.IConsumer)
+
+    def __init__(self, decrypter, hmac):
+        self.decrypter = decrypter
+        self.hmac = hmac
+
+    def write(self, enc_chunk):
+        self.hmac.write(enc_chunk)
+        self.decrypter.write(enc_chunk)
+
+
 class AESDecryptor(object):
     """
     A Twisted's Consumer implementation that consumes data encrypted with
diff --git a/testing/tests/benchmarks/test_crypto.py b/testing/tests/benchmarks/test_crypto.py
index 75ad9a30..8ee9b899 100644
--- a/testing/tests/benchmarks/test_crypto.py
+++ b/testing/tests/benchmarks/test_crypto.py
@@ -39,7 +39,7 @@ def create_doc_encryption(size):
 def create_doc_decryption(size):
     @pytest.inlineCallbacks
     @pytest.mark.benchmark(group="test_crypto_decrypt_doc")
-    def test_doc_decryption(soledad_client, benchmark, payload):
+    def test_doc_decryption(soledad_client, txbenchmark, payload):
         crypto = soledad_client()._crypto
 
         DOC_CONTENT = {'payload': payload(size)}
@@ -50,7 +50,7 @@ def create_doc_decryption(size):
         encrypted_doc = yield crypto.encrypt_doc(doc)
         doc.set_json(encrypted_doc)
 
-        benchmark(crypto.decrypt_doc, doc)
+        yield txbenchmark(crypto.decrypt_doc, doc)
     return test_doc_decryption
 
 
diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py
index 78da8d24..863873f7 100644
--- a/testing/tests/client/test_crypto.py
+++ b/testing/tests/client/test_crypto.py
@@ -139,7 +139,7 @@ class BlobTestCase(unittest.TestCase):
             self.doc_info, ciphertext,
             secret='A' * 96)
         decrypted = yield decryptor.decrypt()
-        assert decrypted.getvalue() == snowden1
+        assert decrypted == snowden1
 
     @defer.inlineCallbacks
     def test_encrypt_and_decrypt(self):