summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2016-09-16 21:43:35 -0400
committerdrebs <drebs@leap.se>2016-12-12 09:11:59 -0200
commitd7740272be029db6229ec5372f277d2934815e98 (patch)
tree65be67cae50f9cb86c5411b7227bb38ce334080e
parent510c0ba3a0c0ade334090a1c36dab9ccae0ba1b4 (diff)
[refactor] adapt fetcher to decryptor
-rw-r--r--client/src/leap/soledad/client/_crypto.py36
-rw-r--r--client/src/leap/soledad/client/api.py73
-rw-r--r--client/src/leap/soledad/client/crypto.py72
-rw-r--r--client/src/leap/soledad/client/http_target/fetch.py47
-rw-r--r--client/src/leap/soledad/client/http_target/send.py5
-rw-r--r--server/src/leap/soledad/server/sync.py12
6 files changed, 110 insertions, 135 deletions
diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index 61a190c7..a2de0ae1 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -77,6 +77,31 @@ class InvalidBlob(Exception):
pass
+docinfo = namedtuple('docinfo', 'doc_id rev')
+
+
+class SoledadCrypto(object):
+
+ def __init__(self, secret):
+ self.secret = secret
+
+ def encrypt_doc(self, doc):
+ content = BytesIO()
+ content.write(str(doc.get_json()))
+ info = docinfo(doc.doc_id, doc.rev)
+ del doc
+ encryptor = BlobEncryptor(info, content, secret=self.secret)
+ return encryptor.encrypt()
+
+ def decrypt_doc(self, doc):
+ info = docinfo(doc.doc_id, doc.rev)
+ ciphertext = BytesIO()
+ ciphertext.write(doc.get_json())
+ ciphertext.seek(0)
+ del doc
+ decryptor = BlobDecryptor(info, ciphertext, secret=self.secret)
+ return decryptor.decrypt()
+
class BlobEncryptor(object):
@@ -134,8 +159,8 @@ class BlobEncryptor(object):
ENC_SCHEME.symkey,
ENC_METHOD.aes_256_ctr))
write(self.iv)
- write(self.doc_id)
- write(self.rev)
+ write(str(self.doc_id))
+ write(str(self.rev))
def _end_crypto_stream(self, ignored):
self._aes.end()
@@ -177,7 +202,6 @@ class BlobDecryptor(object):
self.result = result
def decrypt(self):
-
try:
data = base64.urlsafe_b64decode(self.ciphertext.getvalue())
except (TypeError, binascii.Error):
@@ -341,3 +365,9 @@ def _get_sym_key_for_doc(doc_id, secret):
def _get_aes_ctr_cipher(key, iv):
return Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
+
+
+def is_symmetrically_encrypted(payload):
+ header = base64.urlsafe_b64decode(enc[:15] + '===')
+ ts, sch, meth = struct.unpack('Qbb', header[1:11])
+ return sch == ENC_SCHEME.symkey
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 98613df2..74ebaddc 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -56,12 +56,11 @@ from leap.soledad.common.errors import DatabaseAccessError
from leap.soledad.client import adbapi
from leap.soledad.client import events as soledad_events
from leap.soledad.client import interfaces as soledad_interfaces
-from leap.soledad.client.crypto import SoledadCrypto
+from leap.soledad.client import sqlcipher
from leap.soledad.client.secrets import SoledadSecrets
from leap.soledad.client.shared_db import SoledadSharedDatabase
-from leap.soledad.client import sqlcipher
-from leap.soledad.client import encdecpool
-#from leap.soledad.client._crypto import DocEncrypter
+from leap.soledad.client._crypto import SoledadCrypto
+from leap.soledad.client._crypto import BlobEncryptor
logger = getLogger(__name__)
@@ -308,8 +307,7 @@ class Soledad(object):
replica_uid = self._dbpool.replica_uid
self._dbsyncer = sqlcipher.SQLCipherU1DBSync(
self._sqlcipher_opts, self._crypto, replica_uid,
- SOLEDAD_CERT,
- sync_db=self._sync_db)
+ SOLEDAD_CERT)
def sync_stats(self):
sync_phase = 0
@@ -354,19 +352,38 @@ class Soledad(object):
"""
return self._dbpool.runU1DBQuery(meth, *args, **kw)
- def stream_encryption(self, result, doc):
- contentfd = StringIO()
- contentfd.write(doc.get_json())
- contentfd.seek(0)
-
- sikret = self._secrets.remote_storage_secret
-
- # TODO use BlobEncrypter
- #crypter = DocEncrypter(
- #contentfd, doc.doc_id, doc.rev, secret=sikret)
- d = crypter.encrypt_stream()
- d.addCallback(lambda _: result)
- return d
+ #def stream_encryption(self, result, doc):
+ #print 'streaming encryption'
+ #contentfd = StringIO()
+ #contentfd.write(str(doc.get_json()))
+ #contentfd.seek(0)
+#
+ #sikret = self._secrets.remote_storage_secret
+ #docinfo = DocInfo(doc.doc_id, doc.rev)
+#
+ # -------------------------------------------------------
+ # TODO need to pass a fd to stage this!!!
+ # in the long run, we could connect this to the uploader
+ # but in the meantime, I thikn it's easy if we just
+ # serialize this to disk.
+ #
+ # To do this:
+ # 1. open a file, with a known name:
+ # soledad/staging/docid@rev.bin
+ # 2. pass that fd to BlobEncrypter as result (it's a fd)
+ # 3. On the upload part of the sync, just open again a read-only fd
+ # to this staging path and read it.
+ # that's the encrypted blob, ready to upload!
+ # -------------------------------------------------------
+#
+ #crypter = BlobEncryptor(
+ #docinfo, contentfd, secret=sikret)
+ #del doc
+#
+#
+ #d = crypter.encrypt()
+ #d.addCallback(lambda _: result)
+ #return d
def put_doc(self, doc):
@@ -392,7 +409,6 @@ class Soledad(object):
:rtype: twisted.internet.defer.Deferred
"""
d = self._defer("put_doc", doc)
- d.addCallback(self.stream_encryption, doc)
return d
def delete_doc(self, doc):
@@ -488,7 +504,6 @@ class Soledad(object):
# payloads for example) in which we already have the encoding in the
# headers, so we don't need to guess it.
d = self._defer("create_doc", content, doc_id=doc_id)
- d.addCallback(lambda doc: self.stream_encryption('', doc))
return d
def create_doc_from_json(self, json, doc_id=None):
@@ -857,14 +872,6 @@ class Soledad(object):
self._sync_db = sqlcipher.getConnectionPool(
sync_opts, extra_queries=self._sync_db_extra_init)
- @property
- def _sync_db_extra_init(self):
- """
- Queries for creating tables for the local sync documents db if needed.
- They are passed as extra initialization to initialize_sqlciphjer_db
-
- :rtype: tuple of strings
- """
#
# ISecretsStorage
@@ -1034,5 +1041,13 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
match_hostname(self.sock.getpeercert(), self.host)
+# TODO move this to a common module
+
+class DocInfo:
+ def __init__(self, doc_id, rev):
+ self.doc_id = doc_id
+ self.rev = rev
+
+
old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection
http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index da067237..55c49d9c 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -130,77 +130,6 @@ def doc_mac_key(doc_id, secret):
hashlib.sha256).digest()
-class SoledadCrypto(object):
- """
- General cryptographic functionality encapsulated in a
- object that can be passed along.
- """
- def __init__(self, secret):
- """
- Initialize the crypto object.
-
- :param secret: The Soledad remote storage secret.
- :type secret: str
- """
- self._secret = secret
-
- def doc_mac_key(self, doc_id):
- return doc_mac_key(doc_id, self._secret)
-
- def doc_passphrase(self, doc_id):
- """
- Generate a passphrase for symmetric encryption of document's contents.
-
- The password is derived using HMAC having sha256 as underlying hash
- function. The key used for HMAC are the first
- C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage
- secret stripped from the first MAC_KEY_LENGTH characters. The HMAC
- message is C{doc_id}.
-
- :param doc_id: The id of the document that will be encrypted using
- this passphrase.
- :type doc_id: str
-
- :return: The passphrase.
- :rtype: str
- """
- soledad_assert(self._secret is not None)
- return hmac.new(
- self._secret[MAC_KEY_LENGTH:],
- doc_id,
- hashlib.sha256).digest()
-
- #def encrypt_doc(self, doc):
- #"""
- #Wrapper around encrypt_docstr that accepts the document as argument.
-#
- #:param doc: the document.
- #:type doc: SoledadDocument
- #"""
- #key = self.doc_passphrase(doc.doc_id)
-#
- #return encrypt_docstr(
- #doc.get_json(), doc.doc_id, doc.rev, key, self._secret)
-
- def decrypt_doc(self, doc):
- """
- Wrapper around decrypt_doc_dict that accepts the document as argument.
-
- :param doc: the document.
- :type doc: SoledadDocument
-
- :return: json string with the decrypted document
- :rtype: str
- """
- key = self.doc_passphrase(doc.doc_id)
- return decrypt_doc_dict(
- doc.content, doc.doc_id, doc.rev, key, self._secret)
-
- @property
- def secret(self):
- return self._secret
-
-
#
# Crypto utilities for a SoledadDocument.
#
@@ -455,6 +384,7 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
return decr
+# TODO deprecate
def is_symmetrically_encrypted(doc):
"""
Return True if the document was symmetrically encrypted.
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index 541ec1d2..2e54ca70 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -18,9 +18,9 @@ from twisted.internet import defer
from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS
from leap.soledad.client.events import emit_async
-from leap.soledad.client.crypto import is_symmetrically_encrypted
from leap.soledad.client.http_target.support import RequestBody
from leap.soledad.common.log import getLogger
+from leap.soledad.client._crypto import is_symmetrically_encrypted
from leap.soledad.common.document import SoledadDocument
from leap.soledad.common.l2db import errors
@@ -50,6 +50,8 @@ class HTTPDocFetcher(object):
def _receive_docs(self, last_known_generation, last_known_trans_id,
ensure_callback, sync_id):
+ print 'receiving.....', sync_id
+
new_generation = last_known_generation
new_transaction_id = last_known_trans_id
@@ -90,6 +92,7 @@ class HTTPDocFetcher(object):
content_type='application/x-soledad-sync-get',
body_reader=body_reader)
+ @defer.inlineCallbacks
def _doc_parser(self, doc_info, content):
"""
Insert a received document into the local replica.
@@ -102,13 +105,19 @@ class HTTPDocFetcher(object):
:type total: int
"""
# decrypt incoming document and insert into local database
- # ---------------------------------------------------------
- # symmetric decryption of document's contents
- # ---------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt
+
doc = SoledadDocument(doc_info['id'], doc_info['rev'], content)
- if is_symmetrically_encrypted(doc):
- doc.set_json(self._crypto.decrypt_doc(doc))
+
+ print "GOT.....", doc
+
+ payload = doc['raw']
+ if is_symmetrically_encrypted(payload):
+ print "SHOULD DECRYPT!!!!", content
+ decrypted = yield self._crypto.decrypt_doc(doc)
+ doc.set_json(decrypted)
+
+ # TODO insert blobs here on the blob backend
self._insert_doc_cb(doc, doc_info['gen'], doc_info['trans_id'])
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
@@ -125,17 +134,6 @@ class HTTPDocFetcher(object):
content, gen, trans_id)
:rtype: tuple
"""
- # decode incoming stream
- # parts = response.splitlines()
- # if not parts or parts[0] != '[' or parts[-1] != ']':
- # raise errors.BrokenSyncStream
- # data = parts[1:-1]
- # decode metadata
- # try:
- # line, comma = utils.check_and_strip_comma(data[0])
- # metadata = None
- # except (IndexError):
- # raise errors.BrokenSyncStream
try:
# metadata = json.loads(line)
new_generation = metadata['new_generation']
@@ -146,20 +144,7 @@ class HTTPDocFetcher(object):
# make sure we have replica_uid from fresh new dbs
if self._ensure_callback and 'replica_uid' in metadata:
self._ensure_callback(metadata['replica_uid'])
- # parse incoming document info
- entries = []
- for index in xrange(1, len(data[1:]), 2):
- try:
- line, comma = utils.check_and_strip_comma(data[index])
- content, _ = utils.check_and_strip_comma(data[index + 1])
- entry = json.loads(line)
- entries.append((entry['id'], entry['rev'], content,
- entry['gen'], entry['trans_id']))
- except (IndexError, KeyError):
- raise errors.BrokenSyncStream
- return new_generation, new_transaction_id, number_of_changes, \
- entries
-
+ return number_of_changes, new_generation, new_transaction_id
def _emit_receive_status(user_data, received_docs, total):
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index 86744ec2..6f5893b1 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -109,7 +109,10 @@ class HTTPDocSender(object):
if doc.is_tombstone():
defer.returnValue((doc, None))
else:
- defer.returnValue((doc, self._crypto.encrypt_doc(doc)))
+ # TODO -- for blobs, should stream the doc raw content
+ # TODO -- get rid of this json encoding
+ content = yield self._crypto.encrypt_doc(doc)
+ defer.returnValue((doc, content.getvalue()))
def _emit_send_status(user_data, idx, total):
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index e12ebf8a..533ce778 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -17,6 +17,7 @@
"""
Server side synchronization infrastructure.
"""
+<<<<<<< a64e0fad3a8b1a07887c567d99fd32e3dcf54b23
import time
from leap.soledad.common.l2db import sync
from leap.soledad.common.l2db.remote import http_app
@@ -24,6 +25,15 @@ from leap.soledad.server.caching import get_cache_for
from leap.soledad.server.state import ServerSyncState
from leap.soledad.common.document import ServerDocument
from itertools import izip
+=======
+from itertools import izip
+import cjson
+
+from leap.soledad.common.l2db import sync, Document
+from leap.soledad.common.l2db.remote import http_app
+from leap.soledad.server.caching import get_cache_for
+from leap.soledad.server.state import ServerSyncState
+>>>>>>> wip: adapt crypto to streaming flow
MAX_REQUEST_SIZE = 6000 # in Mb
@@ -199,6 +209,7 @@ class SyncResource(http_app.SyncResource):
not already exist.
:type ensure: bool
"""
+ print "POST ARGS"
# create or open the database
cache = get_cache_for('db-' + sync_id + self.dbname, expire=120)
if ensure:
@@ -271,6 +282,7 @@ class SyncResource(http_app.SyncResource):
client on the current sync session.
:type received: int
"""
+ print 'IN POST GET'
def send_doc(doc, gen, trans_id):
entry = dict(id=doc.doc_id, rev=doc.rev,