[refactor] adapt fetcher to decryptor

6 files changed, 110 insertions, 135 deletions

diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index 61a190c7..a2de0ae1 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -77,6 +77,31 @@ class InvalidBlob(Exception):
     pass
 
 
+docinfo = namedtuple('docinfo', 'doc_id rev')
+
+
+class SoledadCrypto(object):
+
+    def __init__(self, secret):
+        self.secret = secret
+
+    def encrypt_doc(self, doc):
+        content = BytesIO()
+        content.write(str(doc.get_json()))
+        info = docinfo(doc.doc_id, doc.rev)
+        del doc
+        encryptor = BlobEncryptor(info, content, secret=self.secret)
+        return encryptor.encrypt()
+
+    def decrypt_doc(self, doc):
+        info = docinfo(doc.doc_id, doc.rev)
+        ciphertext = BytesIO()
+        ciphertext.write(doc.get_json())
+        ciphertext.seek(0)
+        del doc
+        decryptor = BlobDecryptor(info, ciphertext, secret=self.secret)
+        return decryptor.decrypt()
+
 
 class BlobEncryptor(object):
 
@@ -134,8 +159,8 @@ class BlobEncryptor(object):
             ENC_SCHEME.symkey,
             ENC_METHOD.aes_256_ctr))
         write(self.iv)
-        write(self.doc_id)
-        write(self.rev)
+        write(str(self.doc_id))
+        write(str(self.rev))
 
     def _end_crypto_stream(self, ignored):
         self._aes.end()
@@ -177,7 +202,6 @@ class BlobDecryptor(object):
         self.result = result
 
     def decrypt(self):
-
         try:
             data = base64.urlsafe_b64decode(self.ciphertext.getvalue())
         except (TypeError, binascii.Error):
@@ -341,3 +365,9 @@ def _get_sym_key_for_doc(doc_id, secret):
 
 def _get_aes_ctr_cipher(key, iv):
     return Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
+
+
+def is_symmetrically_encrypted(payload):
+    header = base64.urlsafe_b64decode(enc[:15] + '===')
+    ts, sch, meth = struct.unpack('Qbb', header[1:11])
+    return sch == ENC_SCHEME.symkey
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 98613df2..74ebaddc 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -56,12 +56,11 @@ from leap.soledad.common.errors import DatabaseAccessError
 from leap.soledad.client import adbapi
 from leap.soledad.client import events as soledad_events
 from leap.soledad.client import interfaces as soledad_interfaces
-from leap.soledad.client.crypto import SoledadCrypto
+from leap.soledad.client import sqlcipher
 from leap.soledad.client.secrets import SoledadSecrets
 from leap.soledad.client.shared_db import SoledadSharedDatabase
-from leap.soledad.client import sqlcipher
-from leap.soledad.client import encdecpool
-#from leap.soledad.client._crypto import DocEncrypter
+from leap.soledad.client._crypto import SoledadCrypto
+from leap.soledad.client._crypto import BlobEncryptor
 
 logger = getLogger(__name__)
 
@@ -308,8 +307,7 @@ class Soledad(object):
         replica_uid = self._dbpool.replica_uid
         self._dbsyncer = sqlcipher.SQLCipherU1DBSync(
             self._sqlcipher_opts, self._crypto, replica_uid,
-            SOLEDAD_CERT,
-            sync_db=self._sync_db)
+            SOLEDAD_CERT)
 
     def sync_stats(self):
         sync_phase = 0
@@ -354,19 +352,38 @@ class Soledad(object):
         """
         return self._dbpool.runU1DBQuery(meth, *args, **kw)
 
-    def stream_encryption(self, result, doc):
-        contentfd = StringIO()
-        contentfd.write(doc.get_json())
-        contentfd.seek(0)
-
-        sikret = self._secrets.remote_storage_secret
-
-        # TODO use BlobEncrypter
-        #crypter = DocEncrypter(
-            #contentfd, doc.doc_id, doc.rev, secret=sikret)
-        d = crypter.encrypt_stream()
-        d.addCallback(lambda _: result)
-        return d
+    #def stream_encryption(self, result, doc):
+        #print 'streaming encryption'
+        #contentfd = StringIO()
+        #contentfd.write(str(doc.get_json()))
+        #contentfd.seek(0)
+#
+        #sikret = self._secrets.remote_storage_secret
+        #docinfo = DocInfo(doc.doc_id, doc.rev)
+#
+        # -------------------------------------------------------
+        # TODO need to pass a fd to stage this!!!
+        # in the long run, we could connect this to the uploader
+        # but in the meantime, I thikn it's easy if we just
+        # serialize this to disk.
+        # 
+        # To do this:
+        # 1. open a file, with a known name:
+        #     soledad/staging/docid@rev.bin
+        # 2. pass that fd to BlobEncrypter as result (it's a fd)
+        # 3. On the upload part of the sync, just open again a read-only fd 
+        #    to this staging path and read it.
+        #    that's the encrypted blob, ready to upload!
+        # -------------------------------------------------------
+#
+        #crypter = BlobEncryptor(
+            #docinfo, contentfd, secret=sikret)
+        #del doc
+#
+#
+        #d = crypter.encrypt()
+        #d.addCallback(lambda _: result)
+        #return d
 
 
     def put_doc(self, doc):
@@ -392,7 +409,6 @@ class Soledad(object):
         :rtype: twisted.internet.defer.Deferred
         """
         d = self._defer("put_doc", doc)
-        d.addCallback(self.stream_encryption, doc)
         return d
 
     def delete_doc(self, doc):
@@ -488,7 +504,6 @@ class Soledad(object):
         # payloads for example) in which we already have the encoding in the
         # headers, so we don't need to guess it.
         d = self._defer("create_doc", content, doc_id=doc_id)
-        d.addCallback(lambda doc: self.stream_encryption('', doc))
         return d
 
     def create_doc_from_json(self, json, doc_id=None):
@@ -857,14 +872,6 @@ class Soledad(object):
         self._sync_db = sqlcipher.getConnectionPool(
             sync_opts, extra_queries=self._sync_db_extra_init)
 
-    @property
-    def _sync_db_extra_init(self):
-        """
-        Queries for creating tables for the local sync documents db if needed.
-        They are passed as extra initialization to initialize_sqlciphjer_db
-
-        :rtype: tuple of strings
-        """
 
     #
     # ISecretsStorage
@@ -1034,5 +1041,13 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
         match_hostname(self.sock.getpeercert(), self.host)
 
 
+# TODO move this to a common module
+
+class DocInfo:
+    def __init__(self, doc_id, rev):
+        self.doc_id = doc_id
+        self.rev = rev
+
+
 old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection
 http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index da067237..55c49d9c 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -130,77 +130,6 @@ def doc_mac_key(doc_id, secret):
         hashlib.sha256).digest()
 
 
-class SoledadCrypto(object):
-    """
-    General cryptographic functionality encapsulated in a
-    object that can be passed along.
-    """
-    def __init__(self, secret):
-        """
-        Initialize the crypto object.
-
-        :param secret: The Soledad remote storage secret.
-        :type secret: str
-        """
-        self._secret = secret
-
-    def doc_mac_key(self, doc_id):
-        return doc_mac_key(doc_id, self._secret)
-
-    def doc_passphrase(self, doc_id):
-        """
-        Generate a passphrase for symmetric encryption of document's contents.
-
-        The password is derived using HMAC having sha256 as underlying hash
-        function. The key used for HMAC are the first
-        C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage
-        secret stripped from the first MAC_KEY_LENGTH characters. The HMAC
-        message is C{doc_id}.
-
-        :param doc_id: The id of the document that will be encrypted using
-            this passphrase.
-        :type doc_id: str
-
-        :return: The passphrase.
-        :rtype: str
-        """
-        soledad_assert(self._secret is not None)
-        return hmac.new(
-            self._secret[MAC_KEY_LENGTH:],
-            doc_id,
-            hashlib.sha256).digest()
-
-    #def encrypt_doc(self, doc):
-        #"""
-        #Wrapper around encrypt_docstr that accepts the document as argument.
-#
-        #:param doc: the document.
-        #:type doc: SoledadDocument
-        #"""
-        #key = self.doc_passphrase(doc.doc_id)
-#
-        #return encrypt_docstr(
-            #doc.get_json(), doc.doc_id, doc.rev, key, self._secret)
-
-    def decrypt_doc(self, doc):
-        """
-        Wrapper around decrypt_doc_dict that accepts the document as argument.
-
-        :param doc: the document.
-        :type doc: SoledadDocument
-
-        :return: json string with the decrypted document
-        :rtype: str
-        """
-        key = self.doc_passphrase(doc.doc_id)
-        return decrypt_doc_dict(
-            doc.content, doc.doc_id, doc.rev, key, self._secret)
-
-    @property
-    def secret(self):
-        return self._secret
-
-
 #
 # Crypto utilities for a SoledadDocument.
 #
@@ -455,6 +384,7 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
     return decr
 
 
+# TODO deprecate
 def is_symmetrically_encrypted(doc):
     """
     Return True if the document was symmetrically encrypted.
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index 541ec1d2..2e54ca70 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -18,9 +18,9 @@ from twisted.internet import defer
 
 from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS
 from leap.soledad.client.events import emit_async
-from leap.soledad.client.crypto import is_symmetrically_encrypted
 from leap.soledad.client.http_target.support import RequestBody
 from leap.soledad.common.log import getLogger
+from leap.soledad.client._crypto import is_symmetrically_encrypted
 from leap.soledad.common.document import SoledadDocument
 from leap.soledad.common.l2db import errors
 
@@ -50,6 +50,8 @@ class HTTPDocFetcher(object):
     def _receive_docs(self, last_known_generation, last_known_trans_id,
                       ensure_callback, sync_id):
 
+        print 'receiving.....', sync_id
+
         new_generation = last_known_generation
         new_transaction_id = last_known_trans_id
 
@@ -90,6 +92,7 @@ class HTTPDocFetcher(object):
             content_type='application/x-soledad-sync-get',
             body_reader=body_reader)
 
+    @defer.inlineCallbacks
     def _doc_parser(self, doc_info, content):
         """
         Insert a received document into the local replica.
@@ -102,13 +105,19 @@ class HTTPDocFetcher(object):
         :type total: int
         """
         # decrypt incoming document and insert into local database
-        # ---------------------------------------------------------
-        # symmetric decryption of document's contents
-        # ---------------------------------------------------------
         # If arriving content was symmetrically encrypted, we decrypt
+
         doc = SoledadDocument(doc_info['id'], doc_info['rev'], content)
-        if is_symmetrically_encrypted(doc):
-            doc.set_json(self._crypto.decrypt_doc(doc))
+
+        print "GOT.....", doc
+
+        payload = doc['raw']
+        if is_symmetrically_encrypted(payload):
+            print "SHOULD DECRYPT!!!!", content
+            decrypted = yield self._crypto.decrypt_doc(doc)
+            doc.set_json(decrypted)
+
+        # TODO insert blobs here on the blob backend
         self._insert_doc_cb(doc, doc_info['gen'], doc_info['trans_id'])
         self._received_docs += 1
         user_data = {'uuid': self.uuid, 'userid': self.userid}
@@ -125,17 +134,6 @@ class HTTPDocFetcher(object):
                  content, gen, trans_id)
         :rtype: tuple
         """
-        # decode incoming stream
-        # parts = response.splitlines()
-        # if not parts or parts[0] != '[' or parts[-1] != ']':
-        #    raise errors.BrokenSyncStream
-        # data = parts[1:-1]
-        # decode metadata
-        # try:
-        #    line, comma = utils.check_and_strip_comma(data[0])
-        #    metadata = None
-        # except (IndexError):
-        #    raise errors.BrokenSyncStream
         try:
             # metadata = json.loads(line)
             new_generation = metadata['new_generation']
@@ -146,20 +144,7 @@ class HTTPDocFetcher(object):
         # make sure we have replica_uid from fresh new dbs
         if self._ensure_callback and 'replica_uid' in metadata:
             self._ensure_callback(metadata['replica_uid'])
-        # parse incoming document info
-        entries = []
-        for index in xrange(1, len(data[1:]), 2):
-            try:
-                line, comma = utils.check_and_strip_comma(data[index])
-                content, _ = utils.check_and_strip_comma(data[index + 1])
-                entry = json.loads(line)
-                entries.append((entry['id'], entry['rev'], content,
-                                entry['gen'], entry['trans_id']))
-            except (IndexError, KeyError):
-                raise errors.BrokenSyncStream
-        return new_generation, new_transaction_id, number_of_changes, \
-            entries
-
+        return number_of_changes, new_generation, new_transaction_id
 
 
 def _emit_receive_status(user_data, received_docs, total):
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index 86744ec2..6f5893b1 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -109,7 +109,10 @@ class HTTPDocSender(object):
         if doc.is_tombstone():
             defer.returnValue((doc, None))
         else:
-            defer.returnValue((doc, self._crypto.encrypt_doc(doc)))
+            # TODO -- for blobs, should stream the doc raw content
+            # TODO -- get rid of this json encoding
+            content = yield self._crypto.encrypt_doc(doc)
+            defer.returnValue((doc, content.getvalue()))
 
 
 def _emit_send_status(user_data, idx, total):
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index e12ebf8a..533ce778 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -17,6 +17,7 @@
 """
 Server side synchronization infrastructure.
 """
+<<<<<<< a64e0fad3a8b1a07887c567d99fd32e3dcf54b23
 import time
 from leap.soledad.common.l2db import sync
 from leap.soledad.common.l2db.remote import http_app
@@ -24,6 +25,15 @@ from leap.soledad.server.caching import get_cache_for
 from leap.soledad.server.state import ServerSyncState
 from leap.soledad.common.document import ServerDocument
 from itertools import izip
+=======
+from itertools import izip
+import cjson
+
+from leap.soledad.common.l2db import sync, Document
+from leap.soledad.common.l2db.remote import http_app
+from leap.soledad.server.caching import get_cache_for
+from leap.soledad.server.state import ServerSyncState
+>>>>>>> wip: adapt crypto to streaming flow
 
 
 MAX_REQUEST_SIZE = 6000  # in Mb
@@ -199,6 +209,7 @@ class SyncResource(http_app.SyncResource):
                        not already exist.
         :type ensure: bool
         """
+        print "POST ARGS"
         # create or open the database
         cache = get_cache_for('db-' + sync_id + self.dbname, expire=120)
         if ensure:
@@ -271,6 +282,7 @@ class SyncResource(http_app.SyncResource):
                          client on the current sync session.
         :type received: int
         """
+        print 'IN POST GET'
 
         def send_doc(doc, gen, trans_id):
             entry = dict(id=doc.doc_id, rev=doc.rev,