diff options
| -rw-r--r-- | client/src/leap/soledad/client/_crypto.py | 36 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/api.py | 73 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/crypto.py | 72 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/http_target/fetch.py | 47 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/http_target/send.py | 5 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/sync.py | 12 |
6 files changed, 110 insertions, 135 deletions
diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index 61a190c7..a2de0ae1 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -77,6 +77,31 @@ class InvalidBlob(Exception): pass +docinfo = namedtuple('docinfo', 'doc_id rev') + + +class SoledadCrypto(object): + + def __init__(self, secret): + self.secret = secret + + def encrypt_doc(self, doc): + content = BytesIO() + content.write(str(doc.get_json())) + info = docinfo(doc.doc_id, doc.rev) + del doc + encryptor = BlobEncryptor(info, content, secret=self.secret) + return encryptor.encrypt() + + def decrypt_doc(self, doc): + info = docinfo(doc.doc_id, doc.rev) + ciphertext = BytesIO() + ciphertext.write(doc.get_json()) + ciphertext.seek(0) + del doc + decryptor = BlobDecryptor(info, ciphertext, secret=self.secret) + return decryptor.decrypt() + class BlobEncryptor(object): @@ -134,8 +159,8 @@ class BlobEncryptor(object): ENC_SCHEME.symkey, ENC_METHOD.aes_256_ctr)) write(self.iv) - write(self.doc_id) - write(self.rev) + write(str(self.doc_id)) + write(str(self.rev)) def _end_crypto_stream(self, ignored): self._aes.end() @@ -177,7 +202,6 @@ class BlobDecryptor(object): self.result = result def decrypt(self): - try: data = base64.urlsafe_b64decode(self.ciphertext.getvalue()) except (TypeError, binascii.Error): @@ -341,3 +365,9 @@ def _get_sym_key_for_doc(doc_id, secret): def _get_aes_ctr_cipher(key, iv): return Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend) + + +def is_symmetrically_encrypted(payload): + header = base64.urlsafe_b64decode(enc[:15] + '===') + ts, sch, meth = struct.unpack('Qbb', header[1:11]) + return sch == ENC_SCHEME.symkey diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 98613df2..74ebaddc 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -56,12 +56,11 @@ from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events from leap.soledad.client import interfaces as soledad_interfaces -from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client import sqlcipher from leap.soledad.client.secrets import SoledadSecrets from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client import sqlcipher -from leap.soledad.client import encdecpool -#from leap.soledad.client._crypto import DocEncrypter +from leap.soledad.client._crypto import SoledadCrypto +from leap.soledad.client._crypto import BlobEncryptor logger = getLogger(__name__) @@ -308,8 +307,7 @@ class Soledad(object): replica_uid = self._dbpool.replica_uid self._dbsyncer = sqlcipher.SQLCipherU1DBSync( self._sqlcipher_opts, self._crypto, replica_uid, - SOLEDAD_CERT, - sync_db=self._sync_db) + SOLEDAD_CERT) def sync_stats(self): sync_phase = 0 @@ -354,19 +352,38 @@ class Soledad(object): """ return self._dbpool.runU1DBQuery(meth, *args, **kw) - def stream_encryption(self, result, doc): - contentfd = StringIO() - contentfd.write(doc.get_json()) - contentfd.seek(0) - - sikret = self._secrets.remote_storage_secret - - # TODO use BlobEncrypter - #crypter = DocEncrypter( - #contentfd, doc.doc_id, doc.rev, secret=sikret) - d = crypter.encrypt_stream() - d.addCallback(lambda _: result) - return d + #def stream_encryption(self, result, doc): + #print 'streaming encryption' + #contentfd = StringIO() + #contentfd.write(str(doc.get_json())) + #contentfd.seek(0) +# + #sikret = self._secrets.remote_storage_secret + #docinfo = DocInfo(doc.doc_id, doc.rev) +# + # ------------------------------------------------------- + # TODO need to pass a fd to stage this!!! + # in the long run, we could connect this to the uploader + # but in the meantime, I thikn it's easy if we just + # serialize this to disk. + # + # To do this: + # 1. open a file, with a known name: + # soledad/staging/docid@rev.bin + # 2. pass that fd to BlobEncrypter as result (it's a fd) + # 3. On the upload part of the sync, just open again a read-only fd + # to this staging path and read it. + # that's the encrypted blob, ready to upload! + # ------------------------------------------------------- +# + #crypter = BlobEncryptor( + #docinfo, contentfd, secret=sikret) + #del doc +# +# + #d = crypter.encrypt() + #d.addCallback(lambda _: result) + #return d def put_doc(self, doc): @@ -392,7 +409,6 @@ class Soledad(object): :rtype: twisted.internet.defer.Deferred """ d = self._defer("put_doc", doc) - d.addCallback(self.stream_encryption, doc) return d def delete_doc(self, doc): @@ -488,7 +504,6 @@ class Soledad(object): # payloads for example) in which we already have the encoding in the # headers, so we don't need to guess it. d = self._defer("create_doc", content, doc_id=doc_id) - d.addCallback(lambda doc: self.stream_encryption('', doc)) return d def create_doc_from_json(self, json, doc_id=None): @@ -857,14 +872,6 @@ class Soledad(object): self._sync_db = sqlcipher.getConnectionPool( sync_opts, extra_queries=self._sync_db_extra_init) - @property - def _sync_db_extra_init(self): - """ - Queries for creating tables for the local sync documents db if needed. - They are passed as extra initialization to initialize_sqlciphjer_db - - :rtype: tuple of strings - """ # # ISecretsStorage @@ -1034,5 +1041,13 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): match_hostname(self.sock.getpeercert(), self.host) +# TODO move this to a common module + +class DocInfo: + def __init__(self, doc_id, rev): + self.doc_id = doc_id + self.rev = rev + + old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index da067237..55c49d9c 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -130,77 +130,6 @@ def doc_mac_key(doc_id, secret): hashlib.sha256).digest() -class SoledadCrypto(object): - """ - General cryptographic functionality encapsulated in a - object that can be passed along. - """ - def __init__(self, secret): - """ - Initialize the crypto object. - - :param secret: The Soledad remote storage secret. - :type secret: str - """ - self._secret = secret - - def doc_mac_key(self, doc_id): - return doc_mac_key(doc_id, self._secret) - - def doc_passphrase(self, doc_id): - """ - Generate a passphrase for symmetric encryption of document's contents. - - The password is derived using HMAC having sha256 as underlying hash - function. The key used for HMAC are the first - C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage - secret stripped from the first MAC_KEY_LENGTH characters. The HMAC - message is C{doc_id}. - - :param doc_id: The id of the document that will be encrypted using - this passphrase. - :type doc_id: str - - :return: The passphrase. - :rtype: str - """ - soledad_assert(self._secret is not None) - return hmac.new( - self._secret[MAC_KEY_LENGTH:], - doc_id, - hashlib.sha256).digest() - - #def encrypt_doc(self, doc): - #""" - #Wrapper around encrypt_docstr that accepts the document as argument. -# - #:param doc: the document. - #:type doc: SoledadDocument - #""" - #key = self.doc_passphrase(doc.doc_id) -# - #return encrypt_docstr( - #doc.get_json(), doc.doc_id, doc.rev, key, self._secret) - - def decrypt_doc(self, doc): - """ - Wrapper around decrypt_doc_dict that accepts the document as argument. - - :param doc: the document. - :type doc: SoledadDocument - - :return: json string with the decrypted document - :rtype: str - """ - key = self.doc_passphrase(doc.doc_id) - return decrypt_doc_dict( - doc.content, doc.doc_id, doc.rev, key, self._secret) - - @property - def secret(self): - return self._secret - - # # Crypto utilities for a SoledadDocument. # @@ -455,6 +384,7 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): return decr +# TODO deprecate def is_symmetrically_encrypted(doc): """ Return True if the document was symmetrically encrypted. diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 541ec1d2..2e54ca70 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -18,9 +18,9 @@ from twisted.internet import defer from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS from leap.soledad.client.events import emit_async -from leap.soledad.client.crypto import is_symmetrically_encrypted from leap.soledad.client.http_target.support import RequestBody from leap.soledad.common.log import getLogger +from leap.soledad.client._crypto import is_symmetrically_encrypted from leap.soledad.common.document import SoledadDocument from leap.soledad.common.l2db import errors @@ -50,6 +50,8 @@ class HTTPDocFetcher(object): def _receive_docs(self, last_known_generation, last_known_trans_id, ensure_callback, sync_id): + print 'receiving.....', sync_id + new_generation = last_known_generation new_transaction_id = last_known_trans_id @@ -90,6 +92,7 @@ class HTTPDocFetcher(object): content_type='application/x-soledad-sync-get', body_reader=body_reader) + @defer.inlineCallbacks def _doc_parser(self, doc_info, content): """ Insert a received document into the local replica. @@ -102,13 +105,19 @@ class HTTPDocFetcher(object): :type total: int """ # decrypt incoming document and insert into local database - # --------------------------------------------------------- - # symmetric decryption of document's contents - # --------------------------------------------------------- # If arriving content was symmetrically encrypted, we decrypt + doc = SoledadDocument(doc_info['id'], doc_info['rev'], content) - if is_symmetrically_encrypted(doc): - doc.set_json(self._crypto.decrypt_doc(doc)) + + print "GOT.....", doc + + payload = doc['raw'] + if is_symmetrically_encrypted(payload): + print "SHOULD DECRYPT!!!!", content + decrypted = yield self._crypto.decrypt_doc(doc) + doc.set_json(decrypted) + + # TODO insert blobs here on the blob backend self._insert_doc_cb(doc, doc_info['gen'], doc_info['trans_id']) self._received_docs += 1 user_data = {'uuid': self.uuid, 'userid': self.userid} @@ -125,17 +134,6 @@ class HTTPDocFetcher(object): content, gen, trans_id) :rtype: tuple """ - # decode incoming stream - # parts = response.splitlines() - # if not parts or parts[0] != '[' or parts[-1] != ']': - # raise errors.BrokenSyncStream - # data = parts[1:-1] - # decode metadata - # try: - # line, comma = utils.check_and_strip_comma(data[0]) - # metadata = None - # except (IndexError): - # raise errors.BrokenSyncStream try: # metadata = json.loads(line) new_generation = metadata['new_generation'] @@ -146,20 +144,7 @@ class HTTPDocFetcher(object): # make sure we have replica_uid from fresh new dbs if self._ensure_callback and 'replica_uid' in metadata: self._ensure_callback(metadata['replica_uid']) - # parse incoming document info - entries = [] - for index in xrange(1, len(data[1:]), 2): - try: - line, comma = utils.check_and_strip_comma(data[index]) - content, _ = utils.check_and_strip_comma(data[index + 1]) - entry = json.loads(line) - entries.append((entry['id'], entry['rev'], content, - entry['gen'], entry['trans_id'])) - except (IndexError, KeyError): - raise errors.BrokenSyncStream - return new_generation, new_transaction_id, number_of_changes, \ - entries - + return number_of_changes, new_generation, new_transaction_id def _emit_receive_status(user_data, received_docs, total): diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index 86744ec2..6f5893b1 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -109,7 +109,10 @@ class HTTPDocSender(object): if doc.is_tombstone(): defer.returnValue((doc, None)) else: - defer.returnValue((doc, self._crypto.encrypt_doc(doc))) + # TODO -- for blobs, should stream the doc raw content + # TODO -- get rid of this json encoding + content = yield self._crypto.encrypt_doc(doc) + defer.returnValue((doc, content.getvalue())) def _emit_send_status(user_data, idx, total): diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index e12ebf8a..533ce778 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -17,6 +17,7 @@ """ Server side synchronization infrastructure. """ +<<<<<<< a64e0fad3a8b1a07887c567d99fd32e3dcf54b23 import time from leap.soledad.common.l2db import sync from leap.soledad.common.l2db.remote import http_app @@ -24,6 +25,15 @@ from leap.soledad.server.caching import get_cache_for from leap.soledad.server.state import ServerSyncState from leap.soledad.common.document import ServerDocument from itertools import izip +======= +from itertools import izip +import cjson + +from leap.soledad.common.l2db import sync, Document +from leap.soledad.common.l2db.remote import http_app +from leap.soledad.server.caching import get_cache_for +from leap.soledad.server.state import ServerSyncState +>>>>>>> wip: adapt crypto to streaming flow MAX_REQUEST_SIZE = 6000 # in Mb @@ -199,6 +209,7 @@ class SyncResource(http_app.SyncResource): not already exist. :type ensure: bool """ + print "POST ARGS" # create or open the database cache = get_cache_for('db-' + sync_id + self.dbname, expire=120) if ensure: @@ -271,6 +282,7 @@ class SyncResource(http_app.SyncResource): client on the current sync session. :type received: int """ + print 'IN POST GET' def send_doc(doc, gen, trans_id): entry = dict(id=doc.doc_id, rev=doc.rev, |