diff options
Diffstat (limited to 'client/src')
| -rw-r--r-- | client/src/leap/soledad/client/__init__.py | 1 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/_crypto.py | 83 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/adbapi.py | 18 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/api.py | 46 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/crypto.py | 212 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/encdecpool.py | 145 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/http_target/__init__.py | 7 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/http_target/api.py | 6 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/http_target/fetch.py | 15 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/http_target/send.py | 5 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/secrets.py | 6 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/sqlcipher.py | 30 |
12 files changed, 235 insertions, 339 deletions
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 245a8971..3a114021 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -21,6 +21,7 @@ from leap.soledad.client.api import Soledad from leap.soledad.common import soledad_assert from ._version import get_versions + __version__ = get_versions()['version'] del get_versions diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index e4093a9e..ed861fdd 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -1,14 +1,37 @@ +# -*- coding: utf-8 -*- +# _crypto.py +# Copyright (C) 2016 LEAP Encryption Access Project +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Cryptographic operations for the soledad client +""" + import binascii +import base64 import hashlib import hmac import os from cStringIO import StringIO -from twisted.persisted import dirdbm from twisted.internet import defer from twisted.internet import interfaces from twisted.internet import reactor +from twisted.logger import Logger +from twisted.persisted import dirdbm from twisted.web import client from twisted.web.client import FileBodyProducer @@ -23,11 +46,17 @@ from leap.common.config import get_path_prefix from leap.soledad.client.secrets import SoledadSecrets +log = Logger() + MAC_KEY_LENGTH = 64 crypto_backend = MultiBackend([OpenSSLBackend()]) +class EncryptionError(Exception): + pass + + class AESWriter(object): implements(interfaces.IConsumer) @@ -35,6 +64,10 @@ class AESWriter(object): def __init__(self, key, fd, iv=None): if iv is None: iv = os.urandom(16) + if len(key) != 32: + raise EncryptionError('key is not 256 bits') + if len(iv) != 16: + raise EncryptionError('iv is not 128 bits') cipher = _get_aes_ctr_cipher(key, iv) self.encryptor = cipher.encryptor() @@ -51,7 +84,6 @@ class AESWriter(object): def end(self): if not self.done: self.encryptor.finalize() - self.fd.seek(0) self.deferred.callback(self.fd) self.done = True @@ -91,18 +123,12 @@ class EncryptAndHMAC(object): -class NewDocCryptoStreamer(object): +class DocEncrypter(object): staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging') - staged_template = """ - {"_enc_scheme": "symkey", - "_enc_method": "aes-256-ctr", - "_mac_method": "hmac", - "_mac_hash": "sha256", - "_encoding": "ENCODING", - "_enc_json": "ENC", - "_enc_iv": "IV", - "_mac": "MAC"}""" + staged_template = """{"_enc_scheme": "symkey", "_enc_method": + "aes-256-ctr", "_mac_method": "hmac", "_mac_hash": "sha256", + "_encoding": "ENCODING", "_enc_json": "CIPHERTEXT", "_enc_iv": "IV", "_mac": "MAC"}""" def __init__(self, content_fd, doc_id, rev, secret=None): @@ -140,14 +166,12 @@ class NewDocCryptoStreamer(object): self.hmac_consumer.end() return defer.succeed('ok') - def persist_encrypted_doc(self, ignored, encoding='hex'): - # TODO to avoid blocking on io, this can use a - # version of dbm that chunks the writes to the - # disk fd by using the same FileBodyProducer strategy - # that we're using here, long live to the Cooperator. - # this will benefit + # TODO make this pluggable: + # pass another class (CryptoSerializer) to which we pass + # the doc info, the encrypted_fd and the mac_digest - # TODO -- transition to hex: needs migration FIXME + def persist_encrypted_doc(self, ignored, encoding='hex'): + # TODO -- transition to b64: needs migration FIXME if encoding == 'b64': encode = binascii.b2a_base64 elif encoding == 'hex': @@ -155,14 +179,25 @@ class NewDocCryptoStreamer(object): else: raise RuntimeError('Unknown encoding: %s' % encoding) + # TODO to avoid blocking on io, this can use a + # version of dbm that chunks the writes to the + # disk fd by using the same FileBodyProducer strategy + # that we're using here, long live to the Cooperator. + + db = dirdbm.DirDBM(self.staging_path) key = '{doc_id}@{rev}'.format( doc_id=self.doc_id, rev=self.rev) + ciphertext = encode(self._encrypted_fd.getvalue()) value = self.staged_template.replace( 'ENCODING', encoding).replace( - 'ENC', encode(self._encrypted_fd.read())).replace( - 'IV', binascii.b2a_base64(self.iv)).replace( - 'MAC', encode(self.hmac_consumer.digest)) + 'CIPHERTEXT', ciphertext).replace( + 'IV', encode(self.iv)).replace( + 'MAC', encode(self.hmac_consumer.digest)).replace( + '\n', '') + self._encrypted_fd.seek(0) + + log.debug('persisting %s' % key) db[key] = value self._content_fd.close() @@ -174,14 +209,14 @@ class NewDocCryptoStreamer(object): self.hmac_consumer.write(pre) def _post_hmac(self): - # FIXME -- original impl passed b64 encoded iv post = '{enc_scheme}{enc_method}{enc_iv}'.format( enc_scheme='symkey', enc_method='aes-256-ctr', - enc_iv=binascii.b2a_base64(self.iv)) + enc_iv=binascii.b2a_hex(self.iv)) self.hmac_consumer.write(post) + def _hmac_sha256(key, data): return hmac.new(key, data, hashlib.sha256).digest() diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index ce9bec05..a5328d2b 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -50,8 +50,7 @@ How many times a SQLCipher query should be retried in case of timeout. SQLCIPHER_MAX_RETRIES = 10 -def getConnectionPool(opts, openfun=None, driver="pysqlcipher", - sync_enc_pool=None): +def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): """ Return a connection pool. @@ -72,7 +71,7 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher", if openfun is None and driver == "pysqlcipher": openfun = partial(set_init_pragmas, opts=opts) return U1DBConnectionPool( - opts, sync_enc_pool, + opts, # the following params are relayed "as is" to twisted's # ConnectionPool. "%s.dbapi2" % driver, opts.path, timeout=SQLCIPHER_CONNECTION_TIMEOUT, @@ -89,7 +88,7 @@ class U1DBConnection(adbapi.Connection): The U1DB wrapper to use. """ - def __init__(self, pool, sync_enc_pool, init_u1db=False): + def __init__(self, pool, init_u1db=False): """ :param pool: The pool of connections to that owns this connection. :type pool: adbapi.ConnectionPool @@ -97,7 +96,6 @@ class U1DBConnection(adbapi.Connection): :type init_u1db: bool """ self.init_u1db = init_u1db - self._sync_enc_pool = sync_enc_pool try: adbapi.Connection.__init__(self, pool) except dbapi2.DatabaseError as e: @@ -116,8 +114,7 @@ class U1DBConnection(adbapi.Connection): if self.init_u1db: self._u1db = self.u1db_wrapper( self._connection, - self._pool.opts, - self._sync_enc_pool) + self._pool.opts) def __getattr__(self, name): """ @@ -162,12 +159,11 @@ class U1DBConnectionPool(adbapi.ConnectionPool): connectionFactory = U1DBConnection transactionFactory = U1DBTransaction - def __init__(self, opts, sync_enc_pool, *args, **kwargs): + def __init__(self, opts, *args, **kwargs): """ Initialize the connection pool. """ self.opts = opts - self._sync_enc_pool = sync_enc_pool try: adbapi.ConnectionPool.__init__(self, *args, **kwargs) except dbapi2.DatabaseError as e: @@ -182,7 +178,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): try: conn = self.connectionFactory( - self, self._sync_enc_pool, init_u1db=True) + self, init_u1db=True) replica_uid = conn._u1db._real_replica_uid setProxiedObject(self.replica_uid, replica_uid) except DatabaseAccessError as e: @@ -257,7 +253,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): tid = self.threadID() u1db = self._u1dbconnections.get(tid) conn = self.connectionFactory( - self, self._sync_enc_pool, init_u1db=not bool(u1db)) + self, init_u1db=not bool(u1db)) if self.replica_uid is None: replica_uid = conn._u1db._real_replica_uid diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index cbcae4f7..6b257669 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -61,6 +61,7 @@ from leap.soledad.client.secrets import SoledadSecrets from leap.soledad.client.shared_db import SoledadSharedDatabase from leap.soledad.client import sqlcipher from leap.soledad.client import encdecpool +from leap.soledad.client._crypto import DocEncrypter logger = getLogger(__name__) @@ -190,7 +191,6 @@ class Soledad(object): self._server_url = server_url self._defer_encryption = defer_encryption self._secrets_path = None - self._sync_enc_pool = None self._dbsyncer = None self.shared_db = shared_db @@ -299,12 +299,7 @@ class Soledad(object): ) self._sqlcipher_opts = opts - # the sync_db is used both for deferred encryption, so - # we want to initialize it anyway to allow for all combinations of - # deferred encryption configurations. - self._initialize_sync_db(opts) - self._dbpool = adbapi.getConnectionPool( - opts, sync_enc_pool=self._sync_enc_pool) + self._dbpool = adbapi.getConnectionPool(opts) def _init_u1db_syncer(self): """ @@ -314,9 +309,7 @@ class Soledad(object): self._dbsyncer = sqlcipher.SQLCipherU1DBSync( self._sqlcipher_opts, self._crypto, replica_uid, SOLEDAD_CERT, - defer_encryption=self._defer_encryption, - sync_db=self._sync_db, - sync_enc_pool=self._sync_enc_pool) + sync_db=self._sync_db) def sync_stats(self): sync_phase = 0 @@ -345,8 +338,6 @@ class Soledad(object): if self._sync_db: self._sync_db.close() self._sync_db = None - if self._defer_encryption: - self._sync_enc_pool.stop() # # ILocalStorage @@ -363,6 +354,19 @@ class Soledad(object): """ return self._dbpool.runU1DBQuery(meth, *args, **kw) + def stream_encryption(self, result, doc): + contentfd = StringIO() + contentfd.write(doc.get_json()) + contentfd.seek(0) + + sikret = self._secrets.remote_storage_secret + crypter = DocEncrypter( + contentfd, doc.doc_id, doc.rev, secret=sikret) + d = crypter.encrypt_stream() + d.addCallback(lambda _: result) + return d + + def put_doc(self, doc): """ Update a document. @@ -385,7 +389,9 @@ class Soledad(object): also be updated. :rtype: twisted.internet.defer.Deferred """ - return self._defer("put_doc", doc) + d = self._defer("put_doc", doc) + d.addCallback(self.stream_encryption, doc) + return d def delete_doc(self, doc): """ @@ -479,7 +485,9 @@ class Soledad(object): # create_doc (and probably to put_doc too). There are cases (mail # payloads for example) in which we already have the encoding in the # headers, so we don't need to guess it. - return self._defer("create_doc", content, doc_id=doc_id) + d = self._defer("create_doc", content, doc_id=doc_id) + d.addCallback(lambda doc: self.stream_encryption('', doc)) + return d def create_doc_from_json(self, json, doc_id=None): """ @@ -846,11 +854,6 @@ class Soledad(object): opts, path=sync_db_path, create=True) self._sync_db = sqlcipher.getConnectionPool( sync_opts, extra_queries=self._sync_db_extra_init) - if self._defer_encryption: - # initialize syncing queue encryption pool - self._sync_enc_pool = encdecpool.SyncEncrypterPool( - self._crypto, self._sync_db) - self._sync_enc_pool.start() @property def _sync_db_extra_init(self): @@ -860,11 +863,6 @@ class Soledad(object): :rtype: tuple of strings """ - maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)" - encr = encdecpool.SyncEncrypterPool - sql_encr_table_query = (maybe_create % ( - encr.TABLE_NAME, encr.FIELD_NAMES)) - return (sql_encr_table_query,) # # ISecretsStorage diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index d81c883b..da067237 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -42,6 +42,9 @@ MAC_KEY_LENGTH = 64 crypto_backend = MultiBackend([OpenSSLBackend()]) +# TODO -- deprecate. +# Secrets still using this. + def encrypt_sym(data, key): """ Encrypt data using AES-256 cipher in CTR mode. @@ -68,7 +71,10 @@ def encrypt_sym(data, key): return binascii.b2a_base64(iv), ciphertext -def decrypt_sym(data, key, iv): +# FIXME decryption of the secrets doc is still using b64 +# Deprecate that, move to hex. + +def decrypt_sym(data, key, iv, encoding='base64'): """ Decrypt some data previously encrypted using AES-256 cipher in CTR mode. @@ -78,7 +84,7 @@ def decrypt_sym(data, key, iv): long). :type key: str :param iv: The initialization vector. - :type iv: long + :type iv: str (it's b64 encoded by secrets, hex by deserializing from wire) :return: The decrypted data. :rtype: str @@ -88,7 +94,12 @@ def decrypt_sym(data, key, iv): soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s (must be 256 bits long).' % len(key)) - iv = binascii.a2b_base64(iv) + + if encoding == 'base64': + iv = binascii.a2b_base64(iv) + elif encoding == 'hex': + iv = binascii.a2b_hex(iv) + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend) decryptor = cipher.decryptor() return decryptor.update(data) + decryptor.finalize() @@ -159,17 +170,17 @@ class SoledadCrypto(object): doc_id, hashlib.sha256).digest() - def encrypt_doc(self, doc): - """ - Wrapper around encrypt_docstr that accepts the document as argument. - - :param doc: the document. - :type doc: SoledadDocument - """ - key = self.doc_passphrase(doc.doc_id) - - return encrypt_docstr( - doc.get_json(), doc.doc_id, doc.rev, key, self._secret) + #def encrypt_doc(self, doc): + #""" + #Wrapper around encrypt_docstr that accepts the document as argument. +# + #:param doc: the document. + #:type doc: SoledadDocument + #""" + #key = self.doc_passphrase(doc.doc_id) +# + #return encrypt_docstr( + #doc.get_json(), doc.doc_id, doc.rev, key, self._secret) def decrypt_doc(self, doc): """ @@ -194,6 +205,8 @@ class SoledadCrypto(object): # Crypto utilities for a SoledadDocument. # +# TODO should be ported to streaming consumer + def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, mac_method, secret): """ @@ -212,7 +225,7 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, :param ciphertext: The content of the document. :type ciphertext: str :param enc_scheme: The encryption scheme. - :type enc_scheme: str + :type enc_scheme: bytes :param enc_method: The encryption method. :type enc_method: str :param enc_iv: The encryption initialization vector. @@ -231,6 +244,7 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, soledad_assert(mac_method == crypto.MacMethods.HMAC) except AssertionError: raise crypto.UnknownMacMethodError + template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}" content = template.format( doc_id=doc_id, @@ -239,78 +253,82 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, enc_scheme=enc_scheme, enc_method=enc_method, enc_iv=enc_iv) - return hmac.new( + + digest = hmac.new( doc_mac_key(doc_id, secret), content, hashlib.sha256).digest() + return digest -def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): - """ - Encrypt C{doc}'s content. - - Encrypt doc's contents using AES-256 CTR mode and return a valid JSON - string representing the following: - - { - crypto.ENC_JSON_KEY: '<encrypted doc JSON string>', - crypto.ENC_SCHEME_KEY: 'symkey', - crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR, - crypto.ENC_IV_KEY: '<the initial value used to encrypt>', - MAC_KEY: '<mac>' - crypto.MAC_METHOD_KEY: 'hmac' - } - - :param docstr: A representation of the document to be encrypted. - :type docstr: str or unicode. - - :param doc_id: The document id. - :type doc_id: str - - :param doc_rev: The document revision. - :type doc_rev: str - - :param key: The key used to encrypt ``data`` (must be 256 bits long). - :type key: str - - :param secret: The Soledad storage secret (used for MAC auth). - :type secret: str - - :return: The JSON serialization of the dict representing the encrypted - content. - :rtype: str - """ - enc_scheme = crypto.EncryptionSchemes.SYMKEY - enc_method = crypto.EncryptionMethods.AES_256_CTR - mac_method = crypto.MacMethods.HMAC - enc_iv, ciphertext = encrypt_sym( - str(docstr), # encryption/decryption routines expect str - key) - mac = binascii.b2a_hex( # store the mac as hex. - mac_doc( - doc_id, - doc_rev, - ciphertext, - enc_scheme, - enc_method, - enc_iv, - mac_method, - secret)) +#def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): + #""" + #Encrypt C{doc}'s content. +# + #Encrypt doc's contents using AES-256 CTR mode and return a valid JSON + #string representing the following: +# + #{ + #crypto.ENC_JSON_KEY: '<encrypted doc JSON string>', + #crypto.ENC_SCHEME_KEY: 'symkey', + #crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR, + #crypto.ENC_IV_KEY: '<the initial value used to encrypt>', + #MAC_KEY: '<mac>' + #crypto.MAC_METHOD_KEY: 'hmac' + #} +# + #:param docstr: A representation of the document to be encrypted. + #:type docstr: str or unicode. +# + #:param doc_id: The document id. + #:type doc_id: str +# + #:param doc_rev: The document revision. + #:type doc_rev: str +# + #:param key: The key used to encrypt ``data`` (must be 256 bits long). + #:type key: str +# + #:param secret: The Soledad storage secret (used for MAC auth). + #:type secret: str +# + #:return: The JSON serialization of the dict representing the encrypted + #content. + #:rtype: str + #""" + #enc_scheme = crypto.EncryptionSchemes.SYMKEY + #enc_method = crypto.EncryptionMethods.AES_256_CTR + #mac_method = crypto.MacMethods.HMAC + #enc_iv, ciphertext = encrypt_sym( + #str(docstr), # encryption/decryption routines expect str + #key) + #mac = binascii.b2a_hex( # store the mac as hex. + #mac_doc( + #doc_id, + #doc_rev, + #ciphertext, + #enc_scheme, + #enc_method, + #enc_iv, + #mac_method, + #secret)) # Return a representation for the encrypted content. In the following, we # convert binary data to hexadecimal representation so the JSON # serialization does not complain about what it tries to serialize. - hex_ciphertext = binascii.b2a_hex(ciphertext) - logger.debug("encrypting doc: %s" % doc_id) - return json.dumps({ - crypto.ENC_JSON_KEY: hex_ciphertext, - crypto.ENC_SCHEME_KEY: enc_scheme, - crypto.ENC_METHOD_KEY: enc_method, - crypto.ENC_IV_KEY: enc_iv, - crypto.MAC_KEY: mac, - crypto.MAC_METHOD_KEY: mac_method, - }) + #hex_ciphertext = binascii.b2a_hex(ciphertext) + #log.debug("Encrypting doc: %s" % doc_id) + #return json.dumps({ + #crypto.ENC_JSON_KEY: hex_ciphertext, + #crypto.ENC_SCHEME_KEY: enc_scheme, + #crypto.ENC_METHOD_KEY: enc_method, + #crypto.ENC_IV_KEY: enc_iv, + #crypto.MAC_KEY: mac, + #crypto.MAC_METHOD_KEY: mac_method, + #}) +# +# TODO port to _crypto def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, mac_method, secret, doc_mac): """ @@ -338,6 +356,7 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown. :raise crypto.WrongMacError: Raised when MAC could not be verified. """ + # TODO mac_doc should be ported to Streaming also calculated_mac = mac_doc( doc_id, doc_rev, @@ -347,16 +366,15 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, mac_method, secret) - # we compare mac's hashes to avoid possible timing attacks that might - # exploit python's builtin comparison operator behaviour, which fails - # immediatelly when non-matching bytes are found. - doc_mac_hash = hashlib.sha256( - binascii.a2b_hex( # the mac is stored as hex - doc_mac)).digest() - calculated_mac_hash = hashlib.sha256(calculated_mac).digest() - - if doc_mac_hash != calculated_mac_hash: - logger.warn("wrong MAC while decrypting doc...") + + ok = hmac.compare_digest( + str(calculated_mac), + binascii.a2b_hex(doc_mac)) + + if not ok: + loggger.warn("wrong MAC while decrypting doc...") + loggger.info(u'given: %s' % doc_mac) + loggger.info(u'calculated: %s' % binascii.b2a_hex(calculated_mac)) raise crypto.WrongMacError("Could not authenticate document's " "contents.") @@ -415,12 +433,17 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): ]) soledad_assert(expected_keys.issubset(set(doc_dict.keys()))) - ciphertext = binascii.a2b_hex(doc_dict[crypto.ENC_JSON_KEY]) - enc_scheme = doc_dict[crypto.ENC_SCHEME_KEY] - enc_method = doc_dict[crypto.ENC_METHOD_KEY] - enc_iv = doc_dict[crypto.ENC_IV_KEY] - doc_mac = doc_dict[crypto.MAC_KEY] - mac_method = doc_dict[crypto.MAC_METHOD_KEY] + d = doc_dict + decode = binascii.a2b_hex + + enc_scheme = d[crypto.ENC_SCHEME_KEY] + enc_method = d[crypto.ENC_METHOD_KEY] + doc_mac = d[crypto.MAC_KEY] + mac_method = d[crypto.MAC_METHOD_KEY] + enc_iv = d[crypto.ENC_IV_KEY] + + ciphertext_hex = d[crypto.ENC_JSON_KEY] + ciphertext = decode(ciphertext_hex) soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY) @@ -428,7 +451,8 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, mac_method, secret, doc_mac) - return decrypt_sym(ciphertext, key, enc_iv) + decr = decrypt_sym(ciphertext, key, enc_iv, encoding='hex') + return decr def is_symmetrically_encrypted(doc): diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 8eaefa77..b5cfb3ca 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -28,7 +28,6 @@ from twisted.internet import defer from leap.soledad.common import soledad_assert from leap.soledad.common.log import getLogger -from leap.soledad.client.crypto import encrypt_docstr from leap.soledad.client.crypto import decrypt_doc_dict @@ -104,150 +103,6 @@ class SyncEncryptDecryptPool(object): return self._sync_db.runQuery(query, *args) -def encrypt_doc_task(doc_id, doc_rev, content, key, secret): - """ - Encrypt the content of the given document. - - :param doc_id: The document id. - :type doc_id: str - :param doc_rev: The document revision. - :type doc_rev: str - :param content: The serialized content of the document. - :type content: str - :param key: The encryption key. - :type key: str - :param secret: The Soledad storage secret (used for MAC auth). - :type secret: str - - :return: A tuple containing the doc id, revision and encrypted content. - :rtype: tuple(str, str, str) - """ - encrypted_content = encrypt_docstr( - content, doc_id, doc_rev, key, secret) - return doc_id, doc_rev, encrypted_content - - -class SyncEncrypterPool(SyncEncryptDecryptPool): - """ - Pool of workers that spawn subprocesses to execute the symmetric encryption - of documents to be synced. - """ - TABLE_NAME = "docs_tosync" - FIELD_NAMES = "doc_id PRIMARY KEY, rev, content" - - ENCRYPT_LOOP_PERIOD = 2 - - def __init__(self, *args, **kwargs): - """ - Initialize the sync encrypter pool. - """ - SyncEncryptDecryptPool.__init__(self, *args, **kwargs) - # TODO delete already synced files from database - - def start(self): - """ - Start the encrypter pool. - """ - SyncEncryptDecryptPool.start(self) - logger.debug("starting the encryption loop...") - - def stop(self): - """ - Stop the encrypter pool. - """ - - SyncEncryptDecryptPool.stop(self) - - def encrypt_doc(self, doc): - """ - Encrypt document asynchronously then insert it on - local staging database. - - :param doc: The document to be encrypted. - :type doc: SoledadDocument - """ - soledad_assert(self._crypto is not None, "need a crypto object") - docstr = doc.get_json() - key = self._crypto.doc_passphrase(doc.doc_id) - secret = self._crypto.secret - args = doc.doc_id, doc.rev, docstr, key, secret - # encrypt asynchronously - # TODO use dedicated threadpool / move to ampoule - d = threads.deferToThread( - encrypt_doc_task, *args) - d.addCallback(self._encrypt_doc_cb) - return d - - def _encrypt_doc_cb(self, result): - """ - Insert results of encryption routine into the local sync database. - - :param result: A tuple containing the doc id, revision and encrypted - content. - :type result: tuple(str, str, str) - """ - doc_id, doc_rev, content = result - return self._insert_encrypted_local_doc(doc_id, doc_rev, content) - - def _insert_encrypted_local_doc(self, doc_id, doc_rev, content): - """ - Insert the contents of the encrypted doc into the local sync - database. - - :param doc_id: The document id. - :type doc_id: str - :param doc_rev: The document revision. - :type doc_rev: str - :param content: The serialized content of the document. - :type content: str - """ - query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?)" \ - % (self.TABLE_NAME,) - return self._runOperation(query, (doc_id, doc_rev, content)) - - @defer.inlineCallbacks - def get_encrypted_doc(self, doc_id, doc_rev): - """ - Get an encrypted document from the sync db. - - :param doc_id: The id of the document. - :type doc_id: str - :param doc_rev: The revision of the document. - :type doc_rev: str - - :return: A deferred that will fire with the encrypted content of the - document or None if the document was not found in the sync - db. - :rtype: twisted.internet.defer.Deferred - """ - query = "SELECT content FROM %s WHERE doc_id=? and rev=?" \ - % self.TABLE_NAME - result = yield self._runQuery(query, (doc_id, doc_rev)) - if result: - logger.debug("found doc on sync db: %s" % doc_id) - val = result.pop() - defer.returnValue(val[0]) - logger.debug("did not find doc on sync db: %s" % doc_id) - defer.returnValue(None) - - def delete_encrypted_doc(self, doc_id, doc_rev): - """ - Delete an encrypted document from the sync db. - - :param doc_id: The id of the document. - :type doc_id: str - :param doc_rev: The revision of the document. - :type doc_rev: str - - :return: A deferred that will fire when the operation in the database - has finished. - :rtype: twisted.internet.defer.Deferred - """ - query = "DELETE FROM %s WHERE doc_id=? and rev=?" \ - % self.TABLE_NAME - self._runOperation(query, (doc_id, doc_rev)) - - def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret, idx): """ diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 94de2feb..5dc87fcb 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -54,7 +54,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): written to the main database. """ def __init__(self, url, source_replica_uid, creds, crypto, cert_file, - sync_db=None, sync_enc_pool=None): + sync_db=None): """ Initialize the sync target. @@ -78,10 +78,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): instead of retreiving it from the dedicated database. :type sync_db: Sqlite handler - :param sync_enc_pool: The encryption pool to use to defer encryption. - If None is passed the encryption will not be - deferred. - :type sync_enc_pool: leap.soledad.client.encdecpool.SyncEncrypterPool """ if url.endswith("/"): url = url[:-1] @@ -92,7 +88,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): self.set_creds(creds) self._crypto = crypto self._sync_db = sync_db - self._sync_enc_pool = sync_enc_pool self._insert_doc_cb = None # asynchronous encryption/decryption attributes self._decryption_callback = None diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py index 00b943e1..2d51d94f 100644 --- a/client/src/leap/soledad/client/http_target/api.py +++ b/client/src/leap/soledad/client/http_target/api.py @@ -42,8 +42,6 @@ class SyncTargetAPI(SyncTarget): @defer.inlineCallbacks def close(self): - if self._sync_enc_pool: - self._sync_enc_pool.stop() yield self._http.close() @property @@ -68,10 +66,6 @@ class SyncTargetAPI(SyncTarget): def _base_header(self): return self._auth_header.copy() if self._auth_header else {} - @property - def _defer_encryption(self): - return self._sync_enc_pool is not None - def _http_request(self, url, method='GET', body=None, headers=None, content_type=None, body_reader=readBody, body_producer=None): diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 50e89a2a..541ec1d2 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -146,7 +146,20 @@ class HTTPDocFetcher(object): # make sure we have replica_uid from fresh new dbs if self._ensure_callback and 'replica_uid' in metadata: self._ensure_callback(metadata['replica_uid']) - return number_of_changes, new_generation, new_transaction_id + # parse incoming document info + entries = [] + for index in xrange(1, len(data[1:]), 2): + try: + line, comma = utils.check_and_strip_comma(data[index]) + content, _ = utils.check_and_strip_comma(data[index + 1]) + entry = json.loads(line) + entries.append((entry['id'], entry['rev'], content, + entry['gen'], entry['trans_id'])) + except (IndexError, KeyError): + raise errors.BrokenSyncStream + return new_generation, new_transaction_id, number_of_changes, \ + entries + def _emit_receive_status(user_data, received_docs, total): diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index fcda9bd7..86744ec2 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import json +import os from twisted.internet import defer +from twisted.persisted import dirdbm from leap.soledad.common.log import getLogger +from leap.common.config import get_path_prefix from leap.soledad.client.events import emit_async from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS from leap.soledad.client.http_target.support import RequestBody @@ -39,6 +42,8 @@ class HTTPDocSender(object): # Any class inheriting from this one should provide a meaningful attribute # if the sync status event is meant to be used somewhere else. + staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging') + uuid = 'undefined' userid = 'undefined' diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 1eb6f31d..ad1db2b8 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -266,7 +266,11 @@ class SoledadSecrets(object): # read storage secrets from file content = None with open(self._secrets_path, 'r') as f: - content = json.loads(f.read()) + raw = f.read() + raw = raw.replace('\n', '') + content = json.loads(raw) + + print "LOADING", content _, active_secret, version = self._import_recovery_document(content) self._maybe_set_active_secret(active_secret) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index ba341bbf..8cbc3aea 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -266,26 +266,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') - # - # Document operations - # - - def put_doc(self, doc): - """ - Overwrite the put_doc method, to enqueue the modified document for - encryption before sync. - - :param doc: The document to be put. - :type doc: u1db.Document - - :return: The new document revision. - :rtype: str - """ - doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) - if self.defer_encryption: - # TODO move to api? - self._sync_enc_pool.encrypt_doc(doc) - return doc_rev # # SQLCipher API methods @@ -426,14 +406,13 @@ class SQLCipherU1DBSync(SQLCipherDatabase): ENCRYPT_LOOP_PERIOD = 1 def __init__(self, opts, soledad_crypto, replica_uid, cert_file, - defer_encryption=False, sync_db=None, sync_enc_pool=None): + sync_db=None): self._opts = opts self._path = opts.path self._crypto = soledad_crypto self.__replica_uid = replica_uid self._cert_file = cert_file - self._sync_enc_pool = sync_enc_pool self._sync_db = sync_db @@ -538,8 +517,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): creds=creds, crypto=self._crypto, cert_file=self._cert_file, - sync_db=self._sync_db, - sync_enc_pool=self._sync_enc_pool)) + sync_db=self._sync_db)) self._syncers[url] = (h, syncer) # in order to reuse the same synchronizer multiple times we have to # reset its state (i.e. the number of documents received from target @@ -597,14 +575,12 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase): It can be used from adbapi to initialize a soledad database after getting a regular connection to a sqlcipher database. """ - def __init__(self, conn, opts, sync_enc_pool): + def __init__(self, conn, opts): self._db_handle = conn self._real_replica_uid = None self._ensure_schema() self.set_document_factory(soledad_doc_factory) self._prime_replica_uid() - self.defer_encryption = opts.defer_encryption - self._sync_enc_pool = sync_enc_pool def _assert_db_is_encrypted(opts): |