summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2016-09-14 01:52:36 -0400
committerVictor Shyba <victor1984@riseup.net>2016-11-18 15:55:52 -0300
commitd3b5d85d701f8d898b49e84dd5426cf25eeb76c4 (patch)
tree98ce5f0b7e95035c77661879a622e61bc0d8746c
parentf95dd49dd67dd752ba52825b27cbea7c4f44ef24 (diff)
[refactor] remove encryption pool
-rw-r--r--client/src/leap/soledad/client/__init__.py1
-rw-r--r--client/src/leap/soledad/client/_crypto.py83
-rw-r--r--client/src/leap/soledad/client/adbapi.py18
-rw-r--r--client/src/leap/soledad/client/api.py46
-rw-r--r--client/src/leap/soledad/client/crypto.py212
-rw-r--r--client/src/leap/soledad/client/encdecpool.py145
-rw-r--r--client/src/leap/soledad/client/http_target/__init__.py7
-rw-r--r--client/src/leap/soledad/client/http_target/api.py6
-rw-r--r--client/src/leap/soledad/client/http_target/fetch.py15
-rw-r--r--client/src/leap/soledad/client/http_target/send.py5
-rw-r--r--client/src/leap/soledad/client/secrets.py6
-rw-r--r--client/src/leap/soledad/client/sqlcipher.py30
-rw-r--r--testing/tests/client/test_crypto2.py63
13 files changed, 298 insertions, 339 deletions
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 245a8971..3a114021 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -21,6 +21,7 @@ from leap.soledad.client.api import Soledad
from leap.soledad.common import soledad_assert
from ._version import get_versions
+
__version__ = get_versions()['version']
del get_versions
diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index e4093a9e..ed861fdd 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -1,14 +1,37 @@
+# -*- coding: utf-8 -*-
+# _crypto.py
+# Copyright (C) 2016 LEAP Encryption Access Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Cryptographic operations for the soledad client
+"""
+
import binascii
+import base64
import hashlib
import hmac
import os
from cStringIO import StringIO
-from twisted.persisted import dirdbm
from twisted.internet import defer
from twisted.internet import interfaces
from twisted.internet import reactor
+from twisted.logger import Logger
+from twisted.persisted import dirdbm
from twisted.web import client
from twisted.web.client import FileBodyProducer
@@ -23,11 +46,17 @@ from leap.common.config import get_path_prefix
from leap.soledad.client.secrets import SoledadSecrets
+log = Logger()
+
MAC_KEY_LENGTH = 64
crypto_backend = MultiBackend([OpenSSLBackend()])
+class EncryptionError(Exception):
+ pass
+
+
class AESWriter(object):
implements(interfaces.IConsumer)
@@ -35,6 +64,10 @@ class AESWriter(object):
def __init__(self, key, fd, iv=None):
if iv is None:
iv = os.urandom(16)
+ if len(key) != 32:
+ raise EncryptionError('key is not 256 bits')
+ if len(iv) != 16:
+ raise EncryptionError('iv is not 128 bits')
cipher = _get_aes_ctr_cipher(key, iv)
self.encryptor = cipher.encryptor()
@@ -51,7 +84,6 @@ class AESWriter(object):
def end(self):
if not self.done:
self.encryptor.finalize()
- self.fd.seek(0)
self.deferred.callback(self.fd)
self.done = True
@@ -91,18 +123,12 @@ class EncryptAndHMAC(object):
-class NewDocCryptoStreamer(object):
+class DocEncrypter(object):
staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging')
- staged_template = """
- {"_enc_scheme": "symkey",
- "_enc_method": "aes-256-ctr",
- "_mac_method": "hmac",
- "_mac_hash": "sha256",
- "_encoding": "ENCODING",
- "_enc_json": "ENC",
- "_enc_iv": "IV",
- "_mac": "MAC"}"""
+ staged_template = """{"_enc_scheme": "symkey", "_enc_method":
+ "aes-256-ctr", "_mac_method": "hmac", "_mac_hash": "sha256",
+ "_encoding": "ENCODING", "_enc_json": "CIPHERTEXT", "_enc_iv": "IV", "_mac": "MAC"}"""
def __init__(self, content_fd, doc_id, rev, secret=None):
@@ -140,14 +166,12 @@ class NewDocCryptoStreamer(object):
self.hmac_consumer.end()
return defer.succeed('ok')
- def persist_encrypted_doc(self, ignored, encoding='hex'):
- # TODO to avoid blocking on io, this can use a
- # version of dbm that chunks the writes to the
- # disk fd by using the same FileBodyProducer strategy
- # that we're using here, long live to the Cooperator.
- # this will benefit
+ # TODO make this pluggable:
+ # pass another class (CryptoSerializer) to which we pass
+ # the doc info, the encrypted_fd and the mac_digest
- # TODO -- transition to hex: needs migration FIXME
+ def persist_encrypted_doc(self, ignored, encoding='hex'):
+ # TODO -- transition to b64: needs migration FIXME
if encoding == 'b64':
encode = binascii.b2a_base64
elif encoding == 'hex':
@@ -155,14 +179,25 @@ class NewDocCryptoStreamer(object):
else:
raise RuntimeError('Unknown encoding: %s' % encoding)
+ # TODO to avoid blocking on io, this can use a
+ # version of dbm that chunks the writes to the
+ # disk fd by using the same FileBodyProducer strategy
+ # that we're using here, long live to the Cooperator.
+
+
db = dirdbm.DirDBM(self.staging_path)
key = '{doc_id}@{rev}'.format(
doc_id=self.doc_id, rev=self.rev)
+ ciphertext = encode(self._encrypted_fd.getvalue())
value = self.staged_template.replace(
'ENCODING', encoding).replace(
- 'ENC', encode(self._encrypted_fd.read())).replace(
- 'IV', binascii.b2a_base64(self.iv)).replace(
- 'MAC', encode(self.hmac_consumer.digest))
+ 'CIPHERTEXT', ciphertext).replace(
+ 'IV', encode(self.iv)).replace(
+ 'MAC', encode(self.hmac_consumer.digest)).replace(
+ '\n', '')
+ self._encrypted_fd.seek(0)
+
+ log.debug('persisting %s' % key)
db[key] = value
self._content_fd.close()
@@ -174,14 +209,14 @@ class NewDocCryptoStreamer(object):
self.hmac_consumer.write(pre)
def _post_hmac(self):
- # FIXME -- original impl passed b64 encoded iv
post = '{enc_scheme}{enc_method}{enc_iv}'.format(
enc_scheme='symkey',
enc_method='aes-256-ctr',
- enc_iv=binascii.b2a_base64(self.iv))
+ enc_iv=binascii.b2a_hex(self.iv))
self.hmac_consumer.write(post)
+
def _hmac_sha256(key, data):
return hmac.new(key, data, hashlib.sha256).digest()
diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py
index ce9bec05..a5328d2b 100644
--- a/client/src/leap/soledad/client/adbapi.py
+++ b/client/src/leap/soledad/client/adbapi.py
@@ -50,8 +50,7 @@ How many times a SQLCipher query should be retried in case of timeout.
SQLCIPHER_MAX_RETRIES = 10
-def getConnectionPool(opts, openfun=None, driver="pysqlcipher",
- sync_enc_pool=None):
+def getConnectionPool(opts, openfun=None, driver="pysqlcipher"):
"""
Return a connection pool.
@@ -72,7 +71,7 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher",
if openfun is None and driver == "pysqlcipher":
openfun = partial(set_init_pragmas, opts=opts)
return U1DBConnectionPool(
- opts, sync_enc_pool,
+ opts,
# the following params are relayed "as is" to twisted's
# ConnectionPool.
"%s.dbapi2" % driver, opts.path, timeout=SQLCIPHER_CONNECTION_TIMEOUT,
@@ -89,7 +88,7 @@ class U1DBConnection(adbapi.Connection):
The U1DB wrapper to use.
"""
- def __init__(self, pool, sync_enc_pool, init_u1db=False):
+ def __init__(self, pool, init_u1db=False):
"""
:param pool: The pool of connections to that owns this connection.
:type pool: adbapi.ConnectionPool
@@ -97,7 +96,6 @@ class U1DBConnection(adbapi.Connection):
:type init_u1db: bool
"""
self.init_u1db = init_u1db
- self._sync_enc_pool = sync_enc_pool
try:
adbapi.Connection.__init__(self, pool)
except dbapi2.DatabaseError as e:
@@ -116,8 +114,7 @@ class U1DBConnection(adbapi.Connection):
if self.init_u1db:
self._u1db = self.u1db_wrapper(
self._connection,
- self._pool.opts,
- self._sync_enc_pool)
+ self._pool.opts)
def __getattr__(self, name):
"""
@@ -162,12 +159,11 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
connectionFactory = U1DBConnection
transactionFactory = U1DBTransaction
- def __init__(self, opts, sync_enc_pool, *args, **kwargs):
+ def __init__(self, opts, *args, **kwargs):
"""
Initialize the connection pool.
"""
self.opts = opts
- self._sync_enc_pool = sync_enc_pool
try:
adbapi.ConnectionPool.__init__(self, *args, **kwargs)
except dbapi2.DatabaseError as e:
@@ -182,7 +178,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
try:
conn = self.connectionFactory(
- self, self._sync_enc_pool, init_u1db=True)
+ self, init_u1db=True)
replica_uid = conn._u1db._real_replica_uid
setProxiedObject(self.replica_uid, replica_uid)
except DatabaseAccessError as e:
@@ -257,7 +253,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool):
tid = self.threadID()
u1db = self._u1dbconnections.get(tid)
conn = self.connectionFactory(
- self, self._sync_enc_pool, init_u1db=not bool(u1db))
+ self, init_u1db=not bool(u1db))
if self.replica_uid is None:
replica_uid = conn._u1db._real_replica_uid
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index cbcae4f7..6b257669 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -61,6 +61,7 @@ from leap.soledad.client.secrets import SoledadSecrets
from leap.soledad.client.shared_db import SoledadSharedDatabase
from leap.soledad.client import sqlcipher
from leap.soledad.client import encdecpool
+from leap.soledad.client._crypto import DocEncrypter
logger = getLogger(__name__)
@@ -190,7 +191,6 @@ class Soledad(object):
self._server_url = server_url
self._defer_encryption = defer_encryption
self._secrets_path = None
- self._sync_enc_pool = None
self._dbsyncer = None
self.shared_db = shared_db
@@ -299,12 +299,7 @@ class Soledad(object):
)
self._sqlcipher_opts = opts
- # the sync_db is used both for deferred encryption, so
- # we want to initialize it anyway to allow for all combinations of
- # deferred encryption configurations.
- self._initialize_sync_db(opts)
- self._dbpool = adbapi.getConnectionPool(
- opts, sync_enc_pool=self._sync_enc_pool)
+ self._dbpool = adbapi.getConnectionPool(opts)
def _init_u1db_syncer(self):
"""
@@ -314,9 +309,7 @@ class Soledad(object):
self._dbsyncer = sqlcipher.SQLCipherU1DBSync(
self._sqlcipher_opts, self._crypto, replica_uid,
SOLEDAD_CERT,
- defer_encryption=self._defer_encryption,
- sync_db=self._sync_db,
- sync_enc_pool=self._sync_enc_pool)
+ sync_db=self._sync_db)
def sync_stats(self):
sync_phase = 0
@@ -345,8 +338,6 @@ class Soledad(object):
if self._sync_db:
self._sync_db.close()
self._sync_db = None
- if self._defer_encryption:
- self._sync_enc_pool.stop()
#
# ILocalStorage
@@ -363,6 +354,19 @@ class Soledad(object):
"""
return self._dbpool.runU1DBQuery(meth, *args, **kw)
+ def stream_encryption(self, result, doc):
+ contentfd = StringIO()
+ contentfd.write(doc.get_json())
+ contentfd.seek(0)
+
+ sikret = self._secrets.remote_storage_secret
+ crypter = DocEncrypter(
+ contentfd, doc.doc_id, doc.rev, secret=sikret)
+ d = crypter.encrypt_stream()
+ d.addCallback(lambda _: result)
+ return d
+
+
def put_doc(self, doc):
"""
Update a document.
@@ -385,7 +389,9 @@ class Soledad(object):
also be updated.
:rtype: twisted.internet.defer.Deferred
"""
- return self._defer("put_doc", doc)
+ d = self._defer("put_doc", doc)
+ d.addCallback(self.stream_encryption, doc)
+ return d
def delete_doc(self, doc):
"""
@@ -479,7 +485,9 @@ class Soledad(object):
# create_doc (and probably to put_doc too). There are cases (mail
# payloads for example) in which we already have the encoding in the
# headers, so we don't need to guess it.
- return self._defer("create_doc", content, doc_id=doc_id)
+ d = self._defer("create_doc", content, doc_id=doc_id)
+ d.addCallback(lambda doc: self.stream_encryption('', doc))
+ return d
def create_doc_from_json(self, json, doc_id=None):
"""
@@ -846,11 +854,6 @@ class Soledad(object):
opts, path=sync_db_path, create=True)
self._sync_db = sqlcipher.getConnectionPool(
sync_opts, extra_queries=self._sync_db_extra_init)
- if self._defer_encryption:
- # initialize syncing queue encryption pool
- self._sync_enc_pool = encdecpool.SyncEncrypterPool(
- self._crypto, self._sync_db)
- self._sync_enc_pool.start()
@property
def _sync_db_extra_init(self):
@@ -860,11 +863,6 @@ class Soledad(object):
:rtype: tuple of strings
"""
- maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)"
- encr = encdecpool.SyncEncrypterPool
- sql_encr_table_query = (maybe_create % (
- encr.TABLE_NAME, encr.FIELD_NAMES))
- return (sql_encr_table_query,)
#
# ISecretsStorage
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index d81c883b..da067237 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -42,6 +42,9 @@ MAC_KEY_LENGTH = 64
crypto_backend = MultiBackend([OpenSSLBackend()])
+# TODO -- deprecate.
+# Secrets still using this.
+
def encrypt_sym(data, key):
"""
Encrypt data using AES-256 cipher in CTR mode.
@@ -68,7 +71,10 @@ def encrypt_sym(data, key):
return binascii.b2a_base64(iv), ciphertext
-def decrypt_sym(data, key, iv):
+# FIXME decryption of the secrets doc is still using b64
+# Deprecate that, move to hex.
+
+def decrypt_sym(data, key, iv, encoding='base64'):
"""
Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
@@ -78,7 +84,7 @@ def decrypt_sym(data, key, iv):
long).
:type key: str
:param iv: The initialization vector.
- :type iv: long
+ :type iv: str (it's b64 encoded by secrets, hex by deserializing from wire)
:return: The decrypted data.
:rtype: str
@@ -88,7 +94,12 @@ def decrypt_sym(data, key, iv):
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
- iv = binascii.a2b_base64(iv)
+
+ if encoding == 'base64':
+ iv = binascii.a2b_base64(iv)
+ elif encoding == 'hex':
+ iv = binascii.a2b_hex(iv)
+
cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
decryptor = cipher.decryptor()
return decryptor.update(data) + decryptor.finalize()
@@ -159,17 +170,17 @@ class SoledadCrypto(object):
doc_id,
hashlib.sha256).digest()
- def encrypt_doc(self, doc):
- """
- Wrapper around encrypt_docstr that accepts the document as argument.
-
- :param doc: the document.
- :type doc: SoledadDocument
- """
- key = self.doc_passphrase(doc.doc_id)
-
- return encrypt_docstr(
- doc.get_json(), doc.doc_id, doc.rev, key, self._secret)
+ #def encrypt_doc(self, doc):
+ #"""
+ #Wrapper around encrypt_docstr that accepts the document as argument.
+#
+ #:param doc: the document.
+ #:type doc: SoledadDocument
+ #"""
+ #key = self.doc_passphrase(doc.doc_id)
+#
+ #return encrypt_docstr(
+ #doc.get_json(), doc.doc_id, doc.rev, key, self._secret)
def decrypt_doc(self, doc):
"""
@@ -194,6 +205,8 @@ class SoledadCrypto(object):
# Crypto utilities for a SoledadDocument.
#
+# TODO should be ported to streaming consumer
+
def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
mac_method, secret):
"""
@@ -212,7 +225,7 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
:param ciphertext: The content of the document.
:type ciphertext: str
:param enc_scheme: The encryption scheme.
- :type enc_scheme: str
+ :type enc_scheme: bytes
:param enc_method: The encryption method.
:type enc_method: str
:param enc_iv: The encryption initialization vector.
@@ -231,6 +244,7 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
soledad_assert(mac_method == crypto.MacMethods.HMAC)
except AssertionError:
raise crypto.UnknownMacMethodError
+
template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}"
content = template.format(
doc_id=doc_id,
@@ -239,78 +253,82 @@ def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
enc_scheme=enc_scheme,
enc_method=enc_method,
enc_iv=enc_iv)
- return hmac.new(
+
+ digest = hmac.new(
doc_mac_key(doc_id, secret),
content,
hashlib.sha256).digest()
+ return digest
-def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
- """
- Encrypt C{doc}'s content.
-
- Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
- string representing the following:
-
- {
- crypto.ENC_JSON_KEY: '<encrypted doc JSON string>',
- crypto.ENC_SCHEME_KEY: 'symkey',
- crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR,
- crypto.ENC_IV_KEY: '<the initial value used to encrypt>',
- MAC_KEY: '<mac>'
- crypto.MAC_METHOD_KEY: 'hmac'
- }
-
- :param docstr: A representation of the document to be encrypted.
- :type docstr: str or unicode.
-
- :param doc_id: The document id.
- :type doc_id: str
-
- :param doc_rev: The document revision.
- :type doc_rev: str
-
- :param key: The key used to encrypt ``data`` (must be 256 bits long).
- :type key: str
-
- :param secret: The Soledad storage secret (used for MAC auth).
- :type secret: str
-
- :return: The JSON serialization of the dict representing the encrypted
- content.
- :rtype: str
- """
- enc_scheme = crypto.EncryptionSchemes.SYMKEY
- enc_method = crypto.EncryptionMethods.AES_256_CTR
- mac_method = crypto.MacMethods.HMAC
- enc_iv, ciphertext = encrypt_sym(
- str(docstr), # encryption/decryption routines expect str
- key)
- mac = binascii.b2a_hex( # store the mac as hex.
- mac_doc(
- doc_id,
- doc_rev,
- ciphertext,
- enc_scheme,
- enc_method,
- enc_iv,
- mac_method,
- secret))
+#def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
+ #"""
+ #Encrypt C{doc}'s content.
+#
+ #Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
+ #string representing the following:
+#
+ #{
+ #crypto.ENC_JSON_KEY: '<encrypted doc JSON string>',
+ #crypto.ENC_SCHEME_KEY: 'symkey',
+ #crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR,
+ #crypto.ENC_IV_KEY: '<the initial value used to encrypt>',
+ #MAC_KEY: '<mac>'
+ #crypto.MAC_METHOD_KEY: 'hmac'
+ #}
+#
+ #:param docstr: A representation of the document to be encrypted.
+ #:type docstr: str or unicode.
+#
+ #:param doc_id: The document id.
+ #:type doc_id: str
+#
+ #:param doc_rev: The document revision.
+ #:type doc_rev: str
+#
+ #:param key: The key used to encrypt ``data`` (must be 256 bits long).
+ #:type key: str
+#
+ #:param secret: The Soledad storage secret (used for MAC auth).
+ #:type secret: str
+#
+ #:return: The JSON serialization of the dict representing the encrypted
+ #content.
+ #:rtype: str
+ #"""
+ #enc_scheme = crypto.EncryptionSchemes.SYMKEY
+ #enc_method = crypto.EncryptionMethods.AES_256_CTR
+ #mac_method = crypto.MacMethods.HMAC
+ #enc_iv, ciphertext = encrypt_sym(
+ #str(docstr), # encryption/decryption routines expect str
+ #key)
+ #mac = binascii.b2a_hex( # store the mac as hex.
+ #mac_doc(
+ #doc_id,
+ #doc_rev,
+ #ciphertext,
+ #enc_scheme,
+ #enc_method,
+ #enc_iv,
+ #mac_method,
+ #secret))
# Return a representation for the encrypted content. In the following, we
# convert binary data to hexadecimal representation so the JSON
# serialization does not complain about what it tries to serialize.
- hex_ciphertext = binascii.b2a_hex(ciphertext)
- logger.debug("encrypting doc: %s" % doc_id)
- return json.dumps({
- crypto.ENC_JSON_KEY: hex_ciphertext,
- crypto.ENC_SCHEME_KEY: enc_scheme,
- crypto.ENC_METHOD_KEY: enc_method,
- crypto.ENC_IV_KEY: enc_iv,
- crypto.MAC_KEY: mac,
- crypto.MAC_METHOD_KEY: mac_method,
- })
+ #hex_ciphertext = binascii.b2a_hex(ciphertext)
+ #log.debug("Encrypting doc: %s" % doc_id)
+ #return json.dumps({
+ #crypto.ENC_JSON_KEY: hex_ciphertext,
+ #crypto.ENC_SCHEME_KEY: enc_scheme,
+ #crypto.ENC_METHOD_KEY: enc_method,
+ #crypto.ENC_IV_KEY: enc_iv,
+ #crypto.MAC_KEY: mac,
+ #crypto.MAC_METHOD_KEY: mac_method,
+ #})
+#
+# TODO port to _crypto
def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
enc_iv, mac_method, secret, doc_mac):
"""
@@ -338,6 +356,7 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
:raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
:raise crypto.WrongMacError: Raised when MAC could not be verified.
"""
+ # TODO mac_doc should be ported to Streaming also
calculated_mac = mac_doc(
doc_id,
doc_rev,
@@ -347,16 +366,15 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
enc_iv,
mac_method,
secret)
- # we compare mac's hashes to avoid possible timing attacks that might
- # exploit python's builtin comparison operator behaviour, which fails
- # immediatelly when non-matching bytes are found.
- doc_mac_hash = hashlib.sha256(
- binascii.a2b_hex( # the mac is stored as hex
- doc_mac)).digest()
- calculated_mac_hash = hashlib.sha256(calculated_mac).digest()
-
- if doc_mac_hash != calculated_mac_hash:
- logger.warn("wrong MAC while decrypting doc...")
+
+ ok = hmac.compare_digest(
+ str(calculated_mac),
+ binascii.a2b_hex(doc_mac))
+
+ if not ok:
+ loggger.warn("wrong MAC while decrypting doc...")
+ loggger.info(u'given: %s' % doc_mac)
+ loggger.info(u'calculated: %s' % binascii.b2a_hex(calculated_mac))
raise crypto.WrongMacError("Could not authenticate document's "
"contents.")
@@ -415,12 +433,17 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
])
soledad_assert(expected_keys.issubset(set(doc_dict.keys())))
- ciphertext = binascii.a2b_hex(doc_dict[crypto.ENC_JSON_KEY])
- enc_scheme = doc_dict[crypto.ENC_SCHEME_KEY]
- enc_method = doc_dict[crypto.ENC_METHOD_KEY]
- enc_iv = doc_dict[crypto.ENC_IV_KEY]
- doc_mac = doc_dict[crypto.MAC_KEY]
- mac_method = doc_dict[crypto.MAC_METHOD_KEY]
+ d = doc_dict
+ decode = binascii.a2b_hex
+
+ enc_scheme = d[crypto.ENC_SCHEME_KEY]
+ enc_method = d[crypto.ENC_METHOD_KEY]
+ doc_mac = d[crypto.MAC_KEY]
+ mac_method = d[crypto.MAC_METHOD_KEY]
+ enc_iv = d[crypto.ENC_IV_KEY]
+
+ ciphertext_hex = d[crypto.ENC_JSON_KEY]
+ ciphertext = decode(ciphertext_hex)
soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY)
@@ -428,7 +451,8 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
enc_iv, mac_method, secret, doc_mac)
- return decrypt_sym(ciphertext, key, enc_iv)
+ decr = decrypt_sym(ciphertext, key, enc_iv, encoding='hex')
+ return decr
def is_symmetrically_encrypted(doc):
diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 8eaefa77..b5cfb3ca 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -28,7 +28,6 @@ from twisted.internet import defer
from leap.soledad.common import soledad_assert
from leap.soledad.common.log import getLogger
-from leap.soledad.client.crypto import encrypt_docstr
from leap.soledad.client.crypto import decrypt_doc_dict
@@ -104,150 +103,6 @@ class SyncEncryptDecryptPool(object):
return self._sync_db.runQuery(query, *args)
-def encrypt_doc_task(doc_id, doc_rev, content, key, secret):
- """
- Encrypt the content of the given document.
-
- :param doc_id: The document id.
- :type doc_id: str
- :param doc_rev: The document revision.
- :type doc_rev: str
- :param content: The serialized content of the document.
- :type content: str
- :param key: The encryption key.
- :type key: str
- :param secret: The Soledad storage secret (used for MAC auth).
- :type secret: str
-
- :return: A tuple containing the doc id, revision and encrypted content.
- :rtype: tuple(str, str, str)
- """
- encrypted_content = encrypt_docstr(
- content, doc_id, doc_rev, key, secret)
- return doc_id, doc_rev, encrypted_content
-
-
-class SyncEncrypterPool(SyncEncryptDecryptPool):
- """
- Pool of workers that spawn subprocesses to execute the symmetric encryption
- of documents to be synced.
- """
- TABLE_NAME = "docs_tosync"
- FIELD_NAMES = "doc_id PRIMARY KEY, rev, content"
-
- ENCRYPT_LOOP_PERIOD = 2
-
- def __init__(self, *args, **kwargs):
- """
- Initialize the sync encrypter pool.
- """
- SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
- # TODO delete already synced files from database
-
- def start(self):
- """
- Start the encrypter pool.
- """
- SyncEncryptDecryptPool.start(self)
- logger.debug("starting the encryption loop...")
-
- def stop(self):
- """
- Stop the encrypter pool.
- """
-
- SyncEncryptDecryptPool.stop(self)
-
- def encrypt_doc(self, doc):
- """
- Encrypt document asynchronously then insert it on
- local staging database.
-
- :param doc: The document to be encrypted.
- :type doc: SoledadDocument
- """
- soledad_assert(self._crypto is not None, "need a crypto object")
- docstr = doc.get_json()
- key = self._crypto.doc_passphrase(doc.doc_id)
- secret = self._crypto.secret
- args = doc.doc_id, doc.rev, docstr, key, secret
- # encrypt asynchronously
- # TODO use dedicated threadpool / move to ampoule
- d = threads.deferToThread(
- encrypt_doc_task, *args)
- d.addCallback(self._encrypt_doc_cb)
- return d
-
- def _encrypt_doc_cb(self, result):
- """
- Insert results of encryption routine into the local sync database.
-
- :param result: A tuple containing the doc id, revision and encrypted
- content.
- :type result: tuple(str, str, str)
- """
- doc_id, doc_rev, content = result
- return self._insert_encrypted_local_doc(doc_id, doc_rev, content)
-
- def _insert_encrypted_local_doc(self, doc_id, doc_rev, content):
- """
- Insert the contents of the encrypted doc into the local sync
- database.
-
- :param doc_id: The document id.
- :type doc_id: str
- :param doc_rev: The document revision.
- :type doc_rev: str
- :param content: The serialized content of the document.
- :type content: str
- """
- query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?)" \
- % (self.TABLE_NAME,)
- return self._runOperation(query, (doc_id, doc_rev, content))
-
- @defer.inlineCallbacks
- def get_encrypted_doc(self, doc_id, doc_rev):
- """
- Get an encrypted document from the sync db.
-
- :param doc_id: The id of the document.
- :type doc_id: str
- :param doc_rev: The revision of the document.
- :type doc_rev: str
-
- :return: A deferred that will fire with the encrypted content of the
- document or None if the document was not found in the sync
- db.
- :rtype: twisted.internet.defer.Deferred
- """
- query = "SELECT content FROM %s WHERE doc_id=? and rev=?" \
- % self.TABLE_NAME
- result = yield self._runQuery(query, (doc_id, doc_rev))
- if result:
- logger.debug("found doc on sync db: %s" % doc_id)
- val = result.pop()
- defer.returnValue(val[0])
- logger.debug("did not find doc on sync db: %s" % doc_id)
- defer.returnValue(None)
-
- def delete_encrypted_doc(self, doc_id, doc_rev):
- """
- Delete an encrypted document from the sync db.
-
- :param doc_id: The id of the document.
- :type doc_id: str
- :param doc_rev: The revision of the document.
- :type doc_rev: str
-
- :return: A deferred that will fire when the operation in the database
- has finished.
- :rtype: twisted.internet.defer.Deferred
- """
- query = "DELETE FROM %s WHERE doc_id=? and rev=?" \
- % self.TABLE_NAME
- self._runOperation(query, (doc_id, doc_rev))
-
-
def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret,
idx):
"""
diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py
index 94de2feb..5dc87fcb 100644
--- a/client/src/leap/soledad/client/http_target/__init__.py
+++ b/client/src/leap/soledad/client/http_target/__init__.py
@@ -54,7 +54,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
written to the main database.
"""
def __init__(self, url, source_replica_uid, creds, crypto, cert_file,
- sync_db=None, sync_enc_pool=None):
+ sync_db=None):
"""
Initialize the sync target.
@@ -78,10 +78,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
instead of retreiving it from the dedicated
database.
:type sync_db: Sqlite handler
- :param sync_enc_pool: The encryption pool to use to defer encryption.
- If None is passed the encryption will not be
- deferred.
- :type sync_enc_pool: leap.soledad.client.encdecpool.SyncEncrypterPool
"""
if url.endswith("/"):
url = url[:-1]
@@ -92,7 +88,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
self.set_creds(creds)
self._crypto = crypto
self._sync_db = sync_db
- self._sync_enc_pool = sync_enc_pool
self._insert_doc_cb = None
# asynchronous encryption/decryption attributes
self._decryption_callback = None
diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py
index 00b943e1..2d51d94f 100644
--- a/client/src/leap/soledad/client/http_target/api.py
+++ b/client/src/leap/soledad/client/http_target/api.py
@@ -42,8 +42,6 @@ class SyncTargetAPI(SyncTarget):
@defer.inlineCallbacks
def close(self):
- if self._sync_enc_pool:
- self._sync_enc_pool.stop()
yield self._http.close()
@property
@@ -68,10 +66,6 @@ class SyncTargetAPI(SyncTarget):
def _base_header(self):
return self._auth_header.copy() if self._auth_header else {}
- @property
- def _defer_encryption(self):
- return self._sync_enc_pool is not None
-
def _http_request(self, url, method='GET', body=None, headers=None,
content_type=None, body_reader=readBody,
body_producer=None):
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index 50e89a2a..541ec1d2 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -146,7 +146,20 @@ class HTTPDocFetcher(object):
# make sure we have replica_uid from fresh new dbs
if self._ensure_callback and 'replica_uid' in metadata:
self._ensure_callback(metadata['replica_uid'])
- return number_of_changes, new_generation, new_transaction_id
+ # parse incoming document info
+ entries = []
+ for index in xrange(1, len(data[1:]), 2):
+ try:
+ line, comma = utils.check_and_strip_comma(data[index])
+ content, _ = utils.check_and_strip_comma(data[index + 1])
+ entry = json.loads(line)
+ entries.append((entry['id'], entry['rev'], content,
+ entry['gen'], entry['trans_id']))
+ except (IndexError, KeyError):
+ raise errors.BrokenSyncStream
+ return new_generation, new_transaction_id, number_of_changes, \
+ entries
+
def _emit_receive_status(user_data, received_docs, total):
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index fcda9bd7..86744ec2 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -15,10 +15,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
+import os
from twisted.internet import defer
+from twisted.persisted import dirdbm
from leap.soledad.common.log import getLogger
+from leap.common.config import get_path_prefix
from leap.soledad.client.events import emit_async
from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS
from leap.soledad.client.http_target.support import RequestBody
@@ -39,6 +42,8 @@ class HTTPDocSender(object):
# Any class inheriting from this one should provide a meaningful attribute
# if the sync status event is meant to be used somewhere else.
+ staging_path = os.path.join(get_path_prefix(), 'leap', 'soledad', 'staging')
+
uuid = 'undefined'
userid = 'undefined'
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index 1eb6f31d..ad1db2b8 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -266,7 +266,11 @@ class SoledadSecrets(object):
# read storage secrets from file
content = None
with open(self._secrets_path, 'r') as f:
- content = json.loads(f.read())
+ raw = f.read()
+ raw = raw.replace('\n', '')
+ content = json.loads(raw)
+
+ print "LOADING", content
_, active_secret, version = self._import_recovery_document(content)
self._maybe_set_active_secret(active_secret)
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index ba341bbf..8cbc3aea 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -266,26 +266,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
'ALTER TABLE document '
'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
- #
- # Document operations
- #
-
- def put_doc(self, doc):
- """
- Overwrite the put_doc method, to enqueue the modified document for
- encryption before sync.
-
- :param doc: The document to be put.
- :type doc: u1db.Document
-
- :return: The new document revision.
- :rtype: str
- """
- doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc)
- if self.defer_encryption:
- # TODO move to api?
- self._sync_enc_pool.encrypt_doc(doc)
- return doc_rev
#
# SQLCipher API methods
@@ -426,14 +406,13 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
ENCRYPT_LOOP_PERIOD = 1
def __init__(self, opts, soledad_crypto, replica_uid, cert_file,
- defer_encryption=False, sync_db=None, sync_enc_pool=None):
+ sync_db=None):
self._opts = opts
self._path = opts.path
self._crypto = soledad_crypto
self.__replica_uid = replica_uid
self._cert_file = cert_file
- self._sync_enc_pool = sync_enc_pool
self._sync_db = sync_db
@@ -538,8 +517,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
creds=creds,
crypto=self._crypto,
cert_file=self._cert_file,
- sync_db=self._sync_db,
- sync_enc_pool=self._sync_enc_pool))
+ sync_db=self._sync_db))
self._syncers[url] = (h, syncer)
# in order to reuse the same synchronizer multiple times we have to
# reset its state (i.e. the number of documents received from target
@@ -597,14 +575,12 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase):
It can be used from adbapi to initialize a soledad database after
getting a regular connection to a sqlcipher database.
"""
- def __init__(self, conn, opts, sync_enc_pool):
+ def __init__(self, conn, opts):
self._db_handle = conn
self._real_replica_uid = None
self._ensure_schema()
self.set_document_factory(soledad_doc_factory)
self._prime_replica_uid()
- self.defer_encryption = opts.defer_encryption
- self._sync_enc_pool = sync_enc_pool
def _assert_db_is_encrypted(opts):
diff --git a/testing/tests/client/test_crypto2.py b/testing/tests/client/test_crypto2.py
new file mode 100644
index 00000000..ae280020
--- /dev/null
+++ b/testing/tests/client/test_crypto2.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# test_crypto2.py
+# Copyright (C) 2016 LEAP Encryption Access Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Tests for the _crypto module
+"""
+
+import StringIO
+
+
+import leap.soledad.client
+from leap.soledad.client import _crypto
+
+
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.backends import default_backend
+
+
+def _aes_encrypt(key, iv, data):
+ backend = default_backend()
+ cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
+ encryptor = cipher.encryptor()
+ return encryptor.update(data) + encryptor.finalize()
+
+
+def test_chunked_encryption():
+ key = 'A' * 32
+ iv = 'A' * 16
+ data = (
+ "You can't come up against "
+ "the world's most powerful intelligence "
+ "agencies and not accept the risk. "
+ "If they want to get you, over time "
+ "they will.")
+
+ fd = StringIO.StringIO()
+ aes = _crypto.AESWriter(key, fd, iv)
+
+ block = 16
+
+ for i in range(len(data)/block):
+ chunk = data[i * block:(i+1)*block]
+ aes.write(chunk)
+ aes.end()
+
+ ciphertext_chunked = fd.getvalue()
+ ciphertext = _aes_encrypt(key, iv, data)
+
+ assert ciphertext_chunked == ciphertext