[refactor] remove encryption pool

13 files changed, 298 insertions, 339 deletions

diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 245a8971..3a114021 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -21,6 +21,7 @@ from leap.soledad.client.api import Soledad
 from leap.soledad.common import soledad_assert
 
 from ._version import get_versions
+
 __version__ = get_versions()['version']
 del get_versions
 
diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index e4093a9e..ed861fdd 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -1,14 +1,37 @@
+# -*- coding: utf-8 -*-
+# _crypto.py
+# Copyright (C) 2016 LEAP Encryption Access Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Cryptographic operations for the soledad client
+"""
+
 import binascii
+import base64
 import hashlib
 import hmac
 import os
 
 from cStringIO import StringIO
 
-from twisted.persisted import dirdbm
 from twisted.internet import defer
 from twisted.internet import interfaces
 from twisted.internet import reactor
+from twisted.logger import Logger
+from twisted.persisted import dirdbm
 from twisted.web import client
 from twisted.web.client import FileBodyProducer
 
@@ -23,11 +46,17 @@ from leap.common.config import get_path_prefix
 from leap.soledad.client.secrets import SoledadSecrets
 
 
+log = Logger()
+
 MAC_KEY_LENGTH = 64
 
 crypto_backend = MultiBackend([OpenSSLBackend()])
 
 
+class EncryptionError(Exception):
+    pass
+
+
 class AESWriter(object):
 
     implements(interfaces.IConsumer)
@@ -35,6 +64,10 @@ class AESWriter(object):
     def __init__(self, key, fd, iv=None):
         if iv is None:
             iv = os.urandom(16)
+        if len(key) != 32:
+            raise EncryptionError('key is not 256 bits')
+        if len(iv) != 16:
+            raise EncryptionError('iv is not 128 bits')
 
         cipher = _get_aes_ctr_cipher(key, iv)
         self.encryptor = cipher.encryptor()
@@ -51,7 +84,6 @@ class AESWriter(object):
     def end(self):
         if not self.done:
             self.encryptor.finalize()
-            self.fd.seek(0)
             self.deferred.callback(self.fd)
         self.done = True
 
@@ -91,18 +123,12 @@ class EncryptAndHMAC(object):
         
 
 
-class NewDocCryptoStreamer(object):
+class DocEncrypter(object):
 
     staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging')
-    staged_template = """
-     {"_enc_scheme": "symkey",
-      "_enc_method": "aes-256-ctr",
-      "_mac_method": "hmac",
-      "_mac_hash": "sha256",
-      "_encoding": "ENCODING",
-      "_enc_json": "ENC",
-      "_enc_iv": "IV",
-      "_mac": "MAC"}"""
+    staged_template = """{"_enc_scheme": "symkey", "_enc_method":
+        "aes-256-ctr", "_mac_method": "hmac", "_mac_hash": "sha256",
+        "_encoding": "ENCODING", "_enc_json": "CIPHERTEXT", "_enc_iv": "IV", "_mac": "MAC"}"""
 
 
     def __init__(self, content_fd, doc_id, rev, secret=None):
@@ -140,14 +166,12 @@ class NewDocCryptoStreamer(object):
         self.hmac_consumer.end()
         return defer.succeed('ok')
 
-    def persist_encrypted_doc(self, ignored, encoding='hex'):
-        # TODO to avoid blocking on io, this can use a
-        # version of dbm that chunks the writes to the 
-        # disk fd by using the same FileBodyProducer strategy
-        # that we're using here, long live to the Cooperator.
-        # this will benefit 
+    # TODO make this pluggable:
+    # pass another class (CryptoSerializer) to which we pass
+    # the doc info, the encrypted_fd and the mac_digest
 
-        # TODO -- transition to hex: needs migration FIXME
+    def persist_encrypted_doc(self, ignored, encoding='hex'):
+        # TODO -- transition to b64: needs migration FIXME
         if encoding == 'b64':
             encode = binascii.b2a_base64
         elif encoding == 'hex':
@@ -155,14 +179,25 @@ class NewDocCryptoStreamer(object):
         else:
             raise RuntimeError('Unknown encoding: %s' % encoding)
 
+        # TODO to avoid blocking on io, this can use a
+        # version of dbm that chunks the writes to the 
+        # disk fd by using the same FileBodyProducer strategy
+        # that we're using here, long live to the Cooperator.
+
+
         db = dirdbm.DirDBM(self.staging_path)
         key = '{doc_id}@{rev}'.format(
             doc_id=self.doc_id, rev=self.rev)
+        ciphertext = encode(self._encrypted_fd.getvalue())
         value = self.staged_template.replace(
             'ENCODING', encoding).replace(
-            'ENC', encode(self._encrypted_fd.read())).replace(
-            'IV', binascii.b2a_base64(self.iv)).replace(
-            'MAC', encode(self.hmac_consumer.digest))
+            'CIPHERTEXT', ciphertext).replace(
+            'IV', encode(self.iv)).replace(
+            'MAC', encode(self.hmac_consumer.digest)).replace(
+            '\n', '')
+        self._encrypted_fd.seek(0)
+
+        log.debug('persisting %s' % key)
         db[key] = value
 
         self._content_fd.close()
@@ -174,14 +209,14 @@ class NewDocCryptoStreamer(object):
         self.hmac_consumer.write(pre)
 
     def _post_hmac(self):
-        # FIXME -- original impl passed b64 encoded iv
         post = '{enc_scheme}{enc_method}{enc_iv}'.format(
             enc_scheme='symkey',
             enc_method='aes-256-ctr',
-            enc_iv=binascii.b2a_base64(self.iv))
+            enc_iv=binascii.b2a_hex(self.iv))
         self.hmac_consumer.write(post)
 
 
+
 def _hmac_sha256(key, data):
     return hmac.new(key, data, hashlib.sha256).digest()
 
diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py
index ce9bec05..a5328d2b 100644
--- a/client/src/leap/soledad/client/adbapi.py
+++ b/client/src/leap/soledad/client/adbapi.py
@@ -50,8 +50,7 @@ How many times a SQLCipher query should be retried in case of timeout.
 SQLCIPHER_MAX_RETRIES = 10
 
 
-def getConnectionPool(opts, openfun=None, driver="pysqlcipher",
-                      sync_enc_pool=None):
+def getConnectionPool(opts, openfun=None, driver="pysqlcipher"):
     """
     Return a connection pool.
 
@@ -72,7 +71,7 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher",
     if openfun is None and driver == "pysqlcipher":
         openfun = partial(set_init_pragmas, opts=opts)
     return U1DBConnectionPool(
-        opts, sync_enc_pool,
+        opts,
         # the following params are relayed "as is" to twisted's
         # ConnectionPool.
         "%s.dbapi2" % driver, opts.path, timeout=SQLCIPHER_CONNECTION_TIMEOUT,
@@ -89,7 +88,7 @@ class U1DBConnection(adbapi.Connection):
     The U1DB wrapper to use.
     """
 
-    def __init__(self, pool, sync_enc_pool, init_u1db=False):
+    def __init__(self, pool, init_u1db=False):
         """
         :param pool: The pool of connections to that owns this connection.
         :type pool: adbapi.ConnectionPool
@@ -97,7 +96,6 @@ class U1DBConnection(adbapi.Connection):
         :type init_u1db: bool
         """
         self.init_u1db = init_u1db
-        self._sync_enc_pool = sync_enc_pool
         try:
             adbapi.Connection.__init__(self, pool)
         except dbapi2.DatabaseError as e:
@@ -116,8 +114,7 @@ class U1DBConnection(adbapi.Connection):
         if self.init_u1db:
             self._u1db = self.u1db_wrapper(
                 self._connection,
-                self._pool.opts,
-                self._sync_enc_pool)
+                self._pool.opts)
 
     def __getattr__(self, name):
         """
@@ -162,12 +159,11 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
     connectionFactory = U1DBConnection
     transactionFactory = U1DBTransaction
 
-    def __init__(self, opts, sync_enc_pool, *args, **kwargs):
+    def __init__(self, opts, *args, **kwargs):
         """
         Initialize the connection pool.
         """
         self.opts = opts
-        self._sync_enc_pool = sync_enc_pool
         try:
             adbapi.ConnectionPool.__init__(self, *args, **kwargs)
         except dbapi2.DatabaseError as e:
@@ -182,7 +178,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
 
         try:
             conn = self.connectionFactory(
-                self, self._sync_enc_pool, init_u1db=True)
+                self, init_u1db=True)
             replica_uid = conn._u1db._real_replica_uid
             setProxiedObject(self.replica_uid, replica_uid)
         except DatabaseAccessError as e:
@@ -257,7 +253,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
         tid = self.threadID()
         u1db = self._u1dbconnections.get(tid)
         conn = self.connectionFactory(
-            self, self._sync_enc_pool, init_u1db=not bool(u1db))
+            self, init_u1db=not bool(u1db))
 
         if self.replica_uid is None:
             replica_uid = conn._u1db._real_replica_uid
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index cbcae4f7..6b257669 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -61,6 +61,7 @@ from leap.soledad.client.secrets import SoledadSecrets
 from leap.soledad.client.shared_db import SoledadSharedDatabase
 from leap.soledad.client import sqlcipher
 from leap.soledad.client import encdecpool
+from leap.soledad.client._crypto import DocEncrypter
 
 logger = getLogger(__name__)
 
@@ -190,7 +191,6 @@ class Soledad(object):
         self._server_url = server_url
         self._defer_encryption = defer_encryption
         self._secrets_path = None
-        self._sync_enc_pool = None
         self._dbsyncer = None
 
         self.shared_db = shared_db
@@ -299,12 +299,7 @@ class Soledad(object):
         )
         self._sqlcipher_opts = opts
 
-        # the sync_db is used both for deferred encryption, so
-        # we want to initialize it anyway to allow for all combinations of
-        # deferred encryption configurations.
-        self._initialize_sync_db(opts)
-        self._dbpool = adbapi.getConnectionPool(
-            opts, sync_enc_pool=self._sync_enc_pool)
+        self._dbpool = adbapi.getConnectionPool(opts)
 
     def _init_u1db_syncer(self):
         """
@@ -314,9 +309,7 @@ class Soledad(object):
         self._dbsyncer = sqlcipher.SQLCipherU1DBSync(
             self._sqlcipher_opts, self._crypto, replica_uid,
             SOLEDAD_CERT,
-            defer_encryption=self._defer_encryption,
-            sync_db=self._sync_db,
-            sync_enc_pool=self._sync_enc_pool)
+            sync_db=self._sync_db)
 
     def sync_stats(self):
         sync_phase = 0
@@ -345,8 +338,6 @@ class Soledad(object):
         if self._sync_db:
             self._sync_db.close()
         self._sync_db = None
-        if self._defer_encryption:
-            self._sync_enc_pool.stop()
 
     #
     # ILocalStorage
@@ -363,6 +354,19 @@ class Soledad(object):
         """
         return self._dbpool.runU1DBQuery(meth, *args, **kw)
 
+    def stream_encryption(self, result, doc):
+        contentfd = StringIO()
+        contentfd.write(doc.get_json())
+        contentfd.seek(0)
+
+        sikret = self._secrets.remote_storage_secret
+        crypter = DocEncrypter(
+            contentfd, doc.doc_id, doc.rev, secret=sikret)
+        d = crypter.encrypt_stream()
+        d.addCallback(lambda _: result)
+        return d
+
+
     def put_doc(self, doc):
         """
         Update a document.
@@ -385,7 +389,9 @@ class Soledad(object):
             also be updated.
         :rtype: twisted.internet.defer.Deferred
         """
-        return self._defer("put_doc", doc)
+        d = self._defer("put_doc", doc)
+        d.addCallback(self.stream_encryption, doc)
+        return d
 
     def delete_doc(self, doc):
         """
@@ -479,7 +485,9 @@ class Soledad(object):
         # create_doc (and probably to put_doc too). There are cases (mail
         # payloads for example) in which we already have the encoding in the
         # headers, so we don't need to guess it.
-        return self._defer("create_doc", content, doc_id=doc_id)
+        d = self._defer("create_doc", content, doc_id=doc_id)
+        d.addCallback(lambda doc: self.stream_encryption('', doc))
+        return d
 
     def create_doc_from_json(self, json, doc_id=None):
         """
@@ -846,11 +854,6 @@ class Soledad(object):
             opts, path=sync_db_path, create=True)
         self._sync_db = sqlcipher.getConnectionPool(
             sync_opts, extra_queries=self._sync_db_extra_init)
-        if self._defer_encryption:
-            # initialize syncing queue encryption pool
-            self._sync_enc_pool = encdecpool.SyncEncrypterPool(
-                self._crypto, self._sync_db)
-            self._sync_enc_pool.start()
 
     @property
     def _sync_db_extra_init(self):
@@ -860,11 +863,6 @@ class Soledad(object):
 
         :rtype: tuple of strings
         """
-        maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)"
-        encr = encdecpool.SyncEncrypterPool
-        sql_encr_table_query = (maybe_create % (
-            encr.TABLE_NAME, encr.FIELD_NAMES))
-        return (sql_encr_table_query,)
 
     #
     # ISecretsStorage
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index d81c883b..da067237 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -42,6 +42,9 @@ MAC_KEY_LENGTH = 64
 crypto_backend = MultiBackend([OpenSSLBackend()])
 
 
+# TODO -- deprecate.
+# Secrets still using this.
+
 def encrypt_sym(data, key):
     """
     Encrypt data using AES-256 cipher in CTR mode.
@@ -68,7 +71,10 @@ def encrypt_sym(data, key):
     return binascii.b2a_base64(iv), ciphertext
 
 
-def decrypt_sym(data, key, iv):
+# FIXME decryption of the secrets doc is still using b64
+# Deprecate that, move to hex.
+
+def decrypt_sym(data, key, iv, encoding='base64'):
     """
     Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
 
@@ -78,7 +84,7 @@ def decrypt_sym(data, key, iv):
                 long).
     :type key: str
     :param iv: The initialization vector.
-    :type iv: long
+    :type iv: str (it's b64 encoded by secrets, hex by deserializing from wire)
 
     :return: The decrypted data.
     :rtype: str
@@ -88,7 +94,12 @@ def decrypt_sym(data, key, iv):
     soledad_assert(
         len(key) == 32,  # 32 x 8 = 256 bits.
         'Wrong key size: %s (must be 256 bits long).' % len(key))
-    iv = binascii.a2b_base64(iv)
+
+    if encoding == 'base64':
+        iv = binascii.a2b_base64(iv)
+    elif encoding == 'hex':
+        iv = binascii.a2b_hex(iv)
+
     cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
     decryptor = cipher.decryptor()
     return decryptor.update(data) + decryptor.finalize()
@@ -159,17 +170,17 @@ class SoledadCrypto(object):
             doc_id,
             hashlib.sha256).digest()
 
-    def encrypt_doc(self, doc):
-        """
-        Wrapper around encrypt_docstr that accepts the document as argument.
-
-        :param doc: the document.
-        :type doc: SoledadDocument
-        """
-        key = self.doc_passphrase(doc.doc_id)
-
-        return encrypt_docstr(
-            doc.get_json(), doc.doc_id, doc.rev, key, self._secret)
+    #def encrypt_doc(self, doc):
+        #"""
+        #Wrapper around encrypt_docstr that accepts the document as argument.
+#
+        #:param doc: the document.
+        #:type doc: SoledadDocument
+        #"""
+        #key = self.doc_passphrase(doc.doc_id)
+#
+        #return encrypt_docstr(
+            #doc.get_json(), doc.doc_id, doc.rev, key, self._secret)
 
     def decrypt_doc(self, doc):
         """
@@ -194,6 +205,8 @@ class SoledadCrypto(object):
 # Crypto utilities for a SoledadDocument.
 #
 
+# TODO should be ported to streaming consumer
+
 def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
             mac_method, secret):
     """
@@ -212,7 +225,7 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
     :param ciphertext: The content of the document.
     :type ciphertext: str
     :param enc_scheme: The encryption scheme.
-    :type enc_scheme: str
+    :type enc_scheme: bytes
     :param enc_method: The encryption method.
     :type enc_method: str
     :param enc_iv: The encryption initialization vector.
@@ -231,6 +244,7 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
         soledad_assert(mac_method == crypto.MacMethods.HMAC)
     except AssertionError:
         raise crypto.UnknownMacMethodError
+
     template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}"
     content = template.format(
         doc_id=doc_id,
@@ -239,78 +253,82 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
         enc_scheme=enc_scheme,
         enc_method=enc_method,
         enc_iv=enc_iv)
-    return hmac.new(
+
+    digest = hmac.new(
         doc_mac_key(doc_id, secret),
         content,
         hashlib.sha256).digest()
+    return digest
 
 
-def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
-    """
-    Encrypt C{doc}'s content.
-
-    Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
-    string representing the following:
-
-        {
-            crypto.ENC_JSON_KEY: '<encrypted doc JSON string>',
-            crypto.ENC_SCHEME_KEY: 'symkey',
-            crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR,
-            crypto.ENC_IV_KEY: '<the initial value used to encrypt>',
-            MAC_KEY: '<mac>'
-            crypto.MAC_METHOD_KEY: 'hmac'
-        }
-
-    :param docstr: A representation of the document to be encrypted.
-    :type docstr: str or unicode.
-
-    :param doc_id: The document id.
-    :type doc_id: str
-
-    :param doc_rev: The document revision.
-    :type doc_rev: str
-
-    :param key: The key used to encrypt ``data`` (must be 256 bits long).
-    :type key: str
-
-    :param secret: The Soledad storage secret (used for MAC auth).
-    :type secret: str
-
-    :return: The JSON serialization of the dict representing the encrypted
-             content.
-    :rtype: str
-    """
-    enc_scheme = crypto.EncryptionSchemes.SYMKEY
-    enc_method = crypto.EncryptionMethods.AES_256_CTR
-    mac_method = crypto.MacMethods.HMAC
-    enc_iv, ciphertext = encrypt_sym(
-        str(docstr),  # encryption/decryption routines expect str
-        key)
-    mac = binascii.b2a_hex(  # store the mac as hex.
-        mac_doc(
-            doc_id,
-            doc_rev,
-            ciphertext,
-            enc_scheme,
-            enc_method,
-            enc_iv,
-            mac_method,
-            secret))
+#def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
+    #"""
+    #Encrypt C{doc}'s content.
+#
+    #Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
+    #string representing the following:
+#
+        #{
+            #crypto.ENC_JSON_KEY: '<encrypted doc JSON string>',
+            #crypto.ENC_SCHEME_KEY: 'symkey',
+            #crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR,
+            #crypto.ENC_IV_KEY: '<the initial value used to encrypt>',
+            #MAC_KEY: '<mac>'
+            #crypto.MAC_METHOD_KEY: 'hmac'
+        #}
+#
+    #:param docstr: A representation of the document to be encrypted.
+    #:type docstr: str or unicode.
+#
+    #:param doc_id: The document id.
+    #:type doc_id: str
+#
+    #:param doc_rev: The document revision.
+    #:type doc_rev: str
+#
+    #:param key: The key used to encrypt ``data`` (must be 256 bits long).
+    #:type key: str
+#
+    #:param secret: The Soledad storage secret (used for MAC auth).
+    #:type secret: str
+#
+    #:return: The JSON serialization of the dict representing the encrypted
+             #content.
+    #:rtype: str
+    #"""
+    #enc_scheme = crypto.EncryptionSchemes.SYMKEY
+    #enc_method = crypto.EncryptionMethods.AES_256_CTR
+    #mac_method = crypto.MacMethods.HMAC
+    #enc_iv, ciphertext = encrypt_sym(
+        #str(docstr),  # encryption/decryption routines expect str
+        #key)
+    #mac = binascii.b2a_hex(  # store the mac as hex.
+        #mac_doc(
+            #doc_id,
+            #doc_rev,
+            #ciphertext,
+            #enc_scheme,
+            #enc_method,
+            #enc_iv,
+            #mac_method,
+            #secret))
     # Return a representation for the encrypted content. In the following, we
     # convert binary data to hexadecimal representation so the JSON
     # serialization does not complain about what it tries to serialize.
-    hex_ciphertext = binascii.b2a_hex(ciphertext)
-    logger.debug("encrypting doc: %s" % doc_id)
-    return json.dumps({
-        crypto.ENC_JSON_KEY: hex_ciphertext,
-        crypto.ENC_SCHEME_KEY: enc_scheme,
-        crypto.ENC_METHOD_KEY: enc_method,
-        crypto.ENC_IV_KEY: enc_iv,
-        crypto.MAC_KEY: mac,
-        crypto.MAC_METHOD_KEY: mac_method,
-    })
+    #hex_ciphertext = binascii.b2a_hex(ciphertext)
+    #log.debug("Encrypting doc: %s" % doc_id)
+    #return json.dumps({
+        #crypto.ENC_JSON_KEY: hex_ciphertext,
+        #crypto.ENC_SCHEME_KEY: enc_scheme,
+        #crypto.ENC_METHOD_KEY: enc_method,
+        #crypto.ENC_IV_KEY: enc_iv,
+        #crypto.MAC_KEY: mac,
+        #crypto.MAC_METHOD_KEY: mac_method,
+    #})
+#
 
 
+# TODO port to _crypto
 def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
                     enc_iv, mac_method, secret, doc_mac):
     """
@@ -338,6 +356,7 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
     :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
     :raise crypto.WrongMacError: Raised when MAC could not be verified.
     """
+    # TODO mac_doc should be ported to Streaming also
     calculated_mac = mac_doc(
         doc_id,
         doc_rev,
@@ -347,16 +366,15 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
         enc_iv,
         mac_method,
         secret)
-    # we compare mac's hashes to avoid possible timing attacks that might
-    # exploit python's builtin comparison operator behaviour, which fails
-    # immediatelly when non-matching bytes are found.
-    doc_mac_hash = hashlib.sha256(
-        binascii.a2b_hex(  # the mac is stored as hex
-            doc_mac)).digest()
-    calculated_mac_hash = hashlib.sha256(calculated_mac).digest()
-
-    if doc_mac_hash != calculated_mac_hash:
-        logger.warn("wrong MAC while decrypting doc...")
+    
+    ok = hmac.compare_digest(
+        str(calculated_mac),
+        binascii.a2b_hex(doc_mac))
+
+    if not ok:
+        loggger.warn("wrong MAC while decrypting doc...")
+        loggger.info(u'given:      %s' % doc_mac)
+        loggger.info(u'calculated: %s' % binascii.b2a_hex(calculated_mac))
         raise crypto.WrongMacError("Could not authenticate document's "
                                    "contents.")
 
@@ -415,12 +433,17 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
     ])
     soledad_assert(expected_keys.issubset(set(doc_dict.keys())))
 
-    ciphertext = binascii.a2b_hex(doc_dict[crypto.ENC_JSON_KEY])
-    enc_scheme = doc_dict[crypto.ENC_SCHEME_KEY]
-    enc_method = doc_dict[crypto.ENC_METHOD_KEY]
-    enc_iv = doc_dict[crypto.ENC_IV_KEY]
-    doc_mac = doc_dict[crypto.MAC_KEY]
-    mac_method = doc_dict[crypto.MAC_METHOD_KEY]
+    d = doc_dict
+    decode = binascii.a2b_hex
+
+    enc_scheme = d[crypto.ENC_SCHEME_KEY]
+    enc_method = d[crypto.ENC_METHOD_KEY]
+    doc_mac = d[crypto.MAC_KEY]
+    mac_method = d[crypto.MAC_METHOD_KEY]
+    enc_iv = d[crypto.ENC_IV_KEY]
+
+    ciphertext_hex = d[crypto.ENC_JSON_KEY]
+    ciphertext = decode(ciphertext_hex)
 
     soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY)
 
@@ -428,7 +451,8 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
         doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
         enc_iv, mac_method, secret, doc_mac)
 
-    return decrypt_sym(ciphertext, key, enc_iv)
+    decr = decrypt_sym(ciphertext, key, enc_iv, encoding='hex')
+    return decr
 
 
 def is_symmetrically_encrypted(doc):
diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 8eaefa77..b5cfb3ca 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -28,7 +28,6 @@ from twisted.internet import defer
 from leap.soledad.common import soledad_assert
 from leap.soledad.common.log import getLogger
 
-from leap.soledad.client.crypto import encrypt_docstr
 from leap.soledad.client.crypto import decrypt_doc_dict
 
 
@@ -104,150 +103,6 @@ class SyncEncryptDecryptPool(object):
         return self._sync_db.runQuery(query, *args)
 
 
-def encrypt_doc_task(doc_id, doc_rev, content, key, secret):
-    """
-    Encrypt the content of the given document.
-
-    :param doc_id: The document id.
-    :type doc_id: str
-    :param doc_rev: The document revision.
-    :type doc_rev: str
-    :param content: The serialized content of the document.
-    :type content: str
-    :param key: The encryption key.
-    :type key: str
-    :param secret: The Soledad storage secret (used for MAC auth).
-    :type secret: str
-
-    :return: A tuple containing the doc id, revision and encrypted content.
-    :rtype: tuple(str, str, str)
-    """
-    encrypted_content = encrypt_docstr(
-        content, doc_id, doc_rev, key, secret)
-    return doc_id, doc_rev, encrypted_content
-
-
-class SyncEncrypterPool(SyncEncryptDecryptPool):
-    """
-    Pool of workers that spawn subprocesses to execute the symmetric encryption
-    of documents to be synced.
-    """
-    TABLE_NAME = "docs_tosync"
-    FIELD_NAMES = "doc_id PRIMARY KEY, rev, content"
-
-    ENCRYPT_LOOP_PERIOD = 2
-
-    def __init__(self, *args, **kwargs):
-        """
-        Initialize the sync encrypter pool.
-        """
-        SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
-        # TODO delete already synced files from database
-
-    def start(self):
-        """
-        Start the encrypter pool.
-        """
-        SyncEncryptDecryptPool.start(self)
-        logger.debug("starting the encryption loop...")
-
-    def stop(self):
-        """
-        Stop the encrypter pool.
-        """
-
-        SyncEncryptDecryptPool.stop(self)
-
-    def encrypt_doc(self, doc):
-        """
-        Encrypt document asynchronously then insert it on
-        local staging database.
-
-        :param doc: The document to be encrypted.
-        :type doc: SoledadDocument
-        """
-        soledad_assert(self._crypto is not None, "need a crypto object")
-        docstr = doc.get_json()
-        key = self._crypto.doc_passphrase(doc.doc_id)
-        secret = self._crypto.secret
-        args = doc.doc_id, doc.rev, docstr, key, secret
-        # encrypt asynchronously
-        # TODO use dedicated threadpool / move to ampoule
-        d = threads.deferToThread(
-            encrypt_doc_task, *args)
-        d.addCallback(self._encrypt_doc_cb)
-        return d
-
-    def _encrypt_doc_cb(self, result):
-        """
-        Insert results of encryption routine into the local sync database.
-
-        :param result: A tuple containing the doc id, revision and encrypted
-                       content.
-        :type result: tuple(str, str, str)
-        """
-        doc_id, doc_rev, content = result
-        return self._insert_encrypted_local_doc(doc_id, doc_rev, content)
-
-    def _insert_encrypted_local_doc(self, doc_id, doc_rev, content):
-        """
-        Insert the contents of the encrypted doc into the local sync
-        database.
-
-        :param doc_id: The document id.
-        :type doc_id: str
-        :param doc_rev: The document revision.
-        :type doc_rev: str
-        :param content: The serialized content of the document.
-        :type content: str
-        """
-        query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?)" \
-                % (self.TABLE_NAME,)
-        return self._runOperation(query, (doc_id, doc_rev, content))
-
-    @defer.inlineCallbacks
-    def get_encrypted_doc(self, doc_id, doc_rev):
-        """
-        Get an encrypted document from the sync db.
-
-        :param doc_id: The id of the document.
-        :type doc_id: str
-        :param doc_rev: The revision of the document.
-        :type doc_rev: str
-
-        :return: A deferred that will fire with the encrypted content of the
-                 document or None if the document was not found in the sync
-                 db.
-        :rtype: twisted.internet.defer.Deferred
-        """
-        query = "SELECT content FROM %s WHERE doc_id=? and rev=?" \
-                % self.TABLE_NAME
-        result = yield self._runQuery(query, (doc_id, doc_rev))
-        if result:
-            logger.debug("found doc on sync db: %s" % doc_id)
-            val = result.pop()
-            defer.returnValue(val[0])
-        logger.debug("did not find doc on sync db: %s" % doc_id)
-        defer.returnValue(None)
-
-    def delete_encrypted_doc(self, doc_id, doc_rev):
-        """
-        Delete an encrypted document from the sync db.
-
-        :param doc_id: The id of the document.
-        :type doc_id: str
-        :param doc_rev: The revision of the document.
-        :type doc_rev: str
-
-        :return: A deferred that will fire when the operation in the database
-                 has finished.
-        :rtype: twisted.internet.defer.Deferred
-        """
-        query = "DELETE FROM %s WHERE doc_id=? and rev=?" \
-                % self.TABLE_NAME
-        self._runOperation(query, (doc_id, doc_rev))
-
-
 def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret,
                      idx):
     """
diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py
index 94de2feb..5dc87fcb 100644
--- a/client/src/leap/soledad/client/http_target/__init__.py
+++ b/client/src/leap/soledad/client/http_target/__init__.py
@@ -54,7 +54,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
     written to the main database.
     """
     def __init__(self, url, source_replica_uid, creds, crypto, cert_file,
-                 sync_db=None, sync_enc_pool=None):
+                 sync_db=None):
         """
         Initialize the sync target.
 
@@ -78,10 +78,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
                         instead of retreiving it from the dedicated
                         database.
         :type sync_db: Sqlite handler
-        :param sync_enc_pool: The encryption pool to use to defer encryption.
-                              If None is passed the encryption will not be
-                              deferred.
-        :type sync_enc_pool: leap.soledad.client.encdecpool.SyncEncrypterPool
         """
         if url.endswith("/"):
             url = url[:-1]
@@ -92,7 +88,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
         self.set_creds(creds)
         self._crypto = crypto
         self._sync_db = sync_db
-        self._sync_enc_pool = sync_enc_pool
         self._insert_doc_cb = None
         # asynchronous encryption/decryption attributes
         self._decryption_callback = None
diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py
index 00b943e1..2d51d94f 100644
--- a/client/src/leap/soledad/client/http_target/api.py
+++ b/client/src/leap/soledad/client/http_target/api.py
@@ -42,8 +42,6 @@ class SyncTargetAPI(SyncTarget):
 
     @defer.inlineCallbacks
     def close(self):
-        if self._sync_enc_pool:
-            self._sync_enc_pool.stop()
         yield self._http.close()
 
     @property
@@ -68,10 +66,6 @@ class SyncTargetAPI(SyncTarget):
     def _base_header(self):
         return self._auth_header.copy() if self._auth_header else {}
 
-    @property
-    def _defer_encryption(self):
-        return self._sync_enc_pool is not None
-
     def _http_request(self, url, method='GET', body=None, headers=None,
                       content_type=None, body_reader=readBody,
                       body_producer=None):
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index 50e89a2a..541ec1d2 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -146,7 +146,20 @@ class HTTPDocFetcher(object):
         # make sure we have replica_uid from fresh new dbs
         if self._ensure_callback and 'replica_uid' in metadata:
             self._ensure_callback(metadata['replica_uid'])
-        return number_of_changes, new_generation, new_transaction_id
+        # parse incoming document info
+        entries = []
+        for index in xrange(1, len(data[1:]), 2):
+            try:
+                line, comma = utils.check_and_strip_comma(data[index])
+                content, _ = utils.check_and_strip_comma(data[index + 1])
+                entry = json.loads(line)
+                entries.append((entry['id'], entry['rev'], content,
+                                entry['gen'], entry['trans_id']))
+            except (IndexError, KeyError):
+                raise errors.BrokenSyncStream
+        return new_generation, new_transaction_id, number_of_changes, \
+            entries
+
 
 
 def _emit_receive_status(user_data, received_docs, total):
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index fcda9bd7..86744ec2 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -15,10 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 import json
+import os
 
 from twisted.internet import defer
+from twisted.persisted import dirdbm
 
 from leap.soledad.common.log import getLogger
+from leap.common.config import get_path_prefix
 from leap.soledad.client.events import emit_async
 from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS
 from leap.soledad.client.http_target.support import RequestBody
@@ -39,6 +42,8 @@ class HTTPDocSender(object):
     # Any class inheriting from this one should provide a meaningful attribute
     # if the sync status event is meant to be used somewhere else.
 
+    staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging')
+
     uuid = 'undefined'
     userid = 'undefined'
 
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index 1eb6f31d..ad1db2b8 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -266,7 +266,11 @@ class SoledadSecrets(object):
         # read storage secrets from file
         content = None
         with open(self._secrets_path, 'r') as f:
-            content = json.loads(f.read())
+            raw = f.read()
+            raw = raw.replace('\n', '')
+            content = json.loads(raw)
+
+        print "LOADING", content
         _, active_secret, version = self._import_recovery_document(content)
 
         self._maybe_set_active_secret(active_secret)
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index ba341bbf..8cbc3aea 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -266,26 +266,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
             'ALTER TABLE document '
             'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
 
-    #
-    # Document operations
-    #
-
-    def put_doc(self, doc):
-        """
-        Overwrite the put_doc method, to enqueue the modified document for
-        encryption before sync.
-
-        :param doc: The document to be put.
-        :type doc: u1db.Document
-
-        :return: The new document revision.
-        :rtype: str
-        """
-        doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc)
-        if self.defer_encryption:
-            # TODO move to api?
-            self._sync_enc_pool.encrypt_doc(doc)
-        return doc_rev
 
     #
     # SQLCipher API methods
@@ -426,14 +406,13 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
     ENCRYPT_LOOP_PERIOD = 1
 
     def __init__(self, opts, soledad_crypto, replica_uid, cert_file,
-                 defer_encryption=False, sync_db=None, sync_enc_pool=None):
+                 sync_db=None):
 
         self._opts = opts
         self._path = opts.path
         self._crypto = soledad_crypto
         self.__replica_uid = replica_uid
         self._cert_file = cert_file
-        self._sync_enc_pool = sync_enc_pool
 
         self._sync_db = sync_db
 
@@ -538,8 +517,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
                     creds=creds,
                     crypto=self._crypto,
                     cert_file=self._cert_file,
-                    sync_db=self._sync_db,
-                    sync_enc_pool=self._sync_enc_pool))
+                    sync_db=self._sync_db))
             self._syncers[url] = (h, syncer)
         # in order to reuse the same synchronizer multiple times we have to
         # reset its state (i.e. the number of documents received from target
@@ -597,14 +575,12 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase):
     It can be used from adbapi to initialize a soledad database after
     getting a regular connection to a sqlcipher database.
     """
-    def __init__(self, conn, opts, sync_enc_pool):
+    def __init__(self, conn, opts):
         self._db_handle = conn
         self._real_replica_uid = None
         self._ensure_schema()
         self.set_document_factory(soledad_doc_factory)
         self._prime_replica_uid()
-        self.defer_encryption = opts.defer_encryption
-        self._sync_enc_pool = sync_enc_pool
 
 
 def _assert_db_is_encrypted(opts):
diff --git a/testing/tests/client/test_crypto2.py b/testing/tests/client/test_crypto2.py
new file mode 100644
index 00000000..ae280020
--- /dev/null
+++ b/testing/tests/client/test_crypto2.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# test_crypto2.py
+# Copyright (C) 2016 LEAP Encryption Access Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Tests for the _crypto module
+"""
+
+import StringIO
+
+
+import leap.soledad.client
+from leap.soledad.client import _crypto
+
+
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.backends import default_backend
+
+
+def _aes_encrypt(key, iv, data):
+    backend = default_backend()
+    cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
+    encryptor = cipher.encryptor()
+    return encryptor.update(data) + encryptor.finalize()
+
+
+def test_chunked_encryption():
+    key = 'A' * 32
+    iv = 'A' * 16
+    data = (
+        "You can't come up against "
+        "the world's most powerful intelligence "
+        "agencies and not accept the risk. "
+        "If they want to get you, over time "
+        "they will.")
+
+    fd = StringIO.StringIO()
+    aes = _crypto.AESWriter(key, fd, iv)
+
+    block = 16
+
+    for i in range(len(data)/block):
+        chunk = data[i * block:(i+1)*block]
+        aes.write(chunk)
+    aes.end()
+
+    ciphertext_chunked = fd.getvalue()
+    ciphertext = _aes_encrypt(key, iv, data)
+
+    assert ciphertext_chunked == ciphertext