Merge remote-tracking branch 'drebs/feature/2529-use-pycrypto-for-symmetric-encryption' into develop

5 files changed, 63 insertions, 29 deletions

diff --git a/changes/feature_change-symmetric-encryption-method-to-aes-256-ctr b/changes/feature_change-symmetric-encryption-method-to-aes-256-ctr
new file mode 100644
index 00000000..8c44436a
--- /dev/null
+++ b/changes/feature_change-symmetric-encryption-method-to-aes-256-ctr
@@ -0,0 +1 @@
+  o Change symmetric encryption method to AES-256 CTR mode.
diff --git a/setup.py b/setup.py
index d06a6640..340ebcd2 100644
--- a/setup.py
+++ b/setup.py
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+
 import os
 from setuptools import (
     setup,
@@ -52,7 +53,8 @@ install_requirements = [
 dependency_links = [
     #'git+git://git.futeisha.org/pysqlcipher.git@develop#egg=pysqlcipher',
     #'git+ssh://code.leap.se/leap_pycommon.git@develop#egg=leap.common',
-    'http://twistedmatrix.com/Releases/Twisted/13.0/Twisted-13.0.0.tar.bz2#egg=twisted-13.0.0'
+    'http://twistedmatrix.com/Releases/Twisted/13.0/Twisted-13.0.0.tar.bz2'
+    '#egg=twisted-13.0.0'  # break the string to adhere to pep8 style.
 ]
 
 
@@ -62,12 +64,14 @@ tests_requirements = [
     'testscenarios',
 ]
 
+
 if os.environ.get('VIRTUAL_ENV', None):
     data_files = None
 else:
     # XXX this should go only for linux/mac
     data_files = [("/etc/init.d/", ["pkg/soledad"])]
 
+
 setup(
     name='leap.soledad',
     # TODO: change version according to decisions regarding soledad versus
@@ -84,11 +88,21 @@ setup(
         "LEAP client, an API for data storage and sync."
     ),
     namespace_packages=["leap"],
-    packages=find_packages('src', exclude=['leap.soledad.tests']),
+    # For now, we do not exclude tests because of the circular dependency
+    # between leap.common and leap.soledad.
+    #packages=find_packages('src', exclude=['leap.soledad.tests']),
+    packages=find_packages('src'),
     package_dir={'': 'src'},
     test_suite='leap.soledad.tests',
     install_requires=install_requirements,
     tests_require=tests_requirements,
     dependency_links=dependency_links,
-    data_files = data_files
+    data_files=data_files,
+    # the following files are only used for testing, and might be removed if
+    # we manage or decide to not install tests in the future.
+    package_data={
+        'leap.soledad.tests.u1db_tests.testing-certs': [
+            '*.pem', '*.cert', '*.key'
+        ]
+    }
 )
diff --git a/src/leap/soledad/backends/leap_backend.py b/src/leap/soledad/backends/leap_backend.py
index 87f63432..8fa662e9 100644
--- a/src/leap/soledad/backends/leap_backend.py
+++ b/src/leap/soledad/backends/leap_backend.py
@@ -27,6 +27,7 @@ except ImportError:
     import json  # noqa
 import hashlib
 import hmac
+import binascii
 
 
 from u1db import Document
@@ -35,6 +36,11 @@ from u1db.errors import BrokenSyncStream
 from u1db.remote.http_target import HTTPSyncTarget
 
 
+from leap.common.crypto import (
+    EncryptionMethods,
+    encrypt_sym,
+    decrypt_sym,
+)
 from leap.common.keymanager import KeyManager
 from leap.common.check import leap_assert
 from leap.soledad.auth import TokenBasedAuth
@@ -100,6 +106,8 @@ class MacMethods(object):
 
 ENC_JSON_KEY = '_enc_json'
 ENC_SCHEME_KEY = '_enc_scheme'
+ENC_METHOD_KEY = '_enc_method'
+ENC_IV_KEY = '_enc_iv'
 MAC_KEY = '_mac'
 MAC_METHOD_KEY = '_mac_method'
 
@@ -132,7 +140,7 @@ def mac_doc(crypto, doc_id, doc_rev, ciphertext, mac_method):
         return hmac.new(
             crypto.doc_mac_key(doc_id),
             str(doc_id) + str(doc_rev) + ciphertext,
-            hashlib.sha256).hexdigest()
+            hashlib.sha256).digest()
     # raise if we do not know how to handle this MAC method
     raise UnknownMacMethod('Unknown MAC method: %s.' % mac_method)
 
@@ -141,11 +149,14 @@ def encrypt_doc(crypto, doc):
     """
     Encrypt C{doc}'s content.
 
-    Return a valid JSON string representing the following:
+    Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
+    string representing the following:
 
         {
             ENC_JSON_KEY: '<encrypted doc JSON string>',
             ENC_SCHEME_KEY: 'symkey',
+            ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
+            ENC_IV_KEY: '<the initial value used to encrypt>',
             MAC_KEY: '<mac>'
             MAC_METHOD_KEY: 'hmac'
         }
@@ -160,19 +171,24 @@ def encrypt_doc(crypto, doc):
     @rtype: str
     """
     leap_assert(doc.is_tombstone() is False)
-    # encrypt content
-    ciphertext = crypto.encrypt_sym(
+    # encrypt content using AES-256 CTR mode
+    iv, ciphertext = encrypt_sym(
         doc.get_json(),
-        crypto.doc_passphrase(doc.doc_id))
-    # verify it is indeed encrypted
-    if not crypto.is_encrypted_sym(ciphertext):
-        raise DocumentNotEncrypted('Failed encrypting document.')
-    # update doc's content with encrypted version
+        crypto.doc_passphrase(doc.doc_id),
+        method=EncryptionMethods.AES_256_CTR)
+    # Return a representation for the encrypted content. In the following, we
+    # convert binary data to hexadecimal representation so the JSON
+    # serialization does not complain about what it tries to serialize.
+    hex_ciphertext = binascii.b2a_hex(ciphertext)
     return json.dumps({
-        ENC_JSON_KEY: ciphertext,
+        ENC_JSON_KEY: hex_ciphertext,
         ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY,
-        MAC_KEY: mac_doc(
-            crypto, doc.doc_id, doc.rev, ciphertext, MacMethods.HMAC),
+        ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
+        ENC_IV_KEY: iv,
+        MAC_KEY: binascii.b2a_hex(mac_doc(  # store the mac as hex.
+            crypto, doc.doc_id, doc.rev,
+            ciphertext,
+            MacMethods.HMAC)),
         MAC_METHOD_KEY: MacMethods.HMAC,
     })
 
@@ -188,13 +204,16 @@ def decrypt_doc(crypto, doc):
         {
             ENC_JSON_KEY: '<enc_blob>',
             ENC_SCHEME_KEY: '<enc_scheme>',
+            ENC_METHOD_KEY: '<enc_method>',
+            ENC_IV_KEY: '<initial value used to encrypt>',  # (optional)
             MAC_KEY: '<mac>'
             MAC_METHOD_KEY: 'hmac'
         }
 
     C{enc_blob} is the encryption of the JSON serialization of the document's
     content. For now Soledad just deals with documents whose C{enc_scheme} is
-    EncryptionSchemes.SYMKEY.
+    EncryptionSchemes.SYMKEY and C{enc_method} is
+    EncryptionMethods.AES_256_CTR.
 
     @param crypto: A SoledadCryto instance to perform the encryption.
     @type crypto: leap.soledad.crypto.SoledadCrypto
@@ -207,28 +226,28 @@ def decrypt_doc(crypto, doc):
     leap_assert(doc.is_tombstone() is False)
     leap_assert(ENC_JSON_KEY in doc.content)
     leap_assert(ENC_SCHEME_KEY in doc.content)
+    leap_assert(ENC_METHOD_KEY in doc.content)
     leap_assert(MAC_KEY in doc.content)
     leap_assert(MAC_METHOD_KEY in doc.content)
     # verify MAC
+    ciphertext = binascii.a2b_hex(  # content is stored as hex.
+        doc.content[ENC_JSON_KEY])
     mac = mac_doc(
         crypto, doc.doc_id, doc.rev,
-        doc.content[ENC_JSON_KEY],
+        ciphertext,
         doc.content[MAC_METHOD_KEY])
-    if doc.content[MAC_KEY] != mac:
+    if binascii.a2b_hex(doc.content[MAC_KEY]) != mac:  # mac is stored as hex.
         raise WrongMac('Could not authenticate document\'s contents.')
     # decrypt doc's content
-    ciphertext = doc.content[ENC_JSON_KEY]
     enc_scheme = doc.content[ENC_SCHEME_KEY]
     plainjson = None
     if enc_scheme == EncryptionSchemes.SYMKEY:
-        if not crypto.is_encrypted_sym(ciphertext):
-            raise DocumentNotEncrypted(
-                'Unable to identify document encryption for incoming '
-                'document, although it is marked as being encrypted with a '
-                'symmetric key.')
-        plainjson = crypto.decrypt_sym(
+        leap_assert(ENC_IV_KEY in doc.content)
+        plainjson = decrypt_sym(
             ciphertext,
-            crypto.doc_passphrase(doc.doc_id))
+            crypto.doc_passphrase(doc.doc_id),
+            method=doc.content[ENC_METHOD_KEY],
+            iv=doc.content[ENC_IV_KEY])
     else:
         raise UnknownEncryptionScheme(enc_scheme)
     return plainjson
diff --git a/src/leap/soledad/crypto.py b/src/leap/soledad/crypto.py
index 6140ef31..d0e2c720 100644
--- a/src/leap/soledad/crypto.py
+++ b/src/leap/soledad/crypto.py
@@ -123,7 +123,7 @@ class SoledadCrypto(object):
         return hmac.new(
             self.secret[self.MAC_KEY_LENGTH:],
             doc_id,
-            hashlib.sha256).hexdigest()
+            hashlib.sha256).digest()
 
     def doc_mac_key(self, doc_id):
         """
@@ -147,7 +147,7 @@ class SoledadCrypto(object):
         return hmac.new(
             self.secret[:self.MAC_KEY_LENGTH],
             doc_id,
-            hashlib.sha256).hexdigest()
+            hashlib.sha256).digest()
 
     #
     # secret setters/getters
diff --git a/src/leap/soledad/tests/__init__.py b/src/leap/soledad/tests/__init__.py
index 07038910..79ee69c4 100644
--- a/src/leap/soledad/tests/__init__.py
+++ b/src/leap/soledad/tests/__init__.py
@@ -23,7 +23,7 @@ from leap.common.testing.basetest import BaseLeapTest
 
 class BaseSoledadTest(BaseLeapTest):
     """
-    Instantiates GPG and Soledad for usage in tests.
+    Instantiates Soledad for usage in tests.
     """
 
     def setUp(self):