Merge branch 'release-0.4.4'0.4.4

17 files changed, 267 insertions, 113 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 660d4160..83bb883e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,13 @@
+0.4.4 Dec 6:
+Client:
+  o Add MAC verirication to the recovery document and
+    soledad.json. Closes #4348.
+Common:
+  o Add unicode conversion to put_doc(). Closes #4095.
+  o Remove tests dependency on nose2. Closes #4258.
+
 0.4.3 Nov 15:
+Client:
   o Defaults detected encoding to utf-8 to avoid bug if detected
     encoding is None. Closes: #4417
   o Open db in autocommit mode, to avoid nested transactions problems.
diff --git a/README.rst b/README.rst
index 61a15ee4..fb909120 100644
--- a/README.rst
+++ b/README.rst
@@ -38,14 +38,8 @@ Tests
 Client and server tests are both included in leap.soledad.common. If you want
 to run tests in development mode you must do the following::
 
-  cd common
-  python setup.py develop
-  cd ../client
-  python setup.py develop
-  cd ../server
-  python setup.py develop
-  cd ../common
-  python setup.py test
+  scripts/develop_mode.sh
+  ./run_tests.sh
 
 Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your
 system.
diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/client/changes/VERSION_COMPAT
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index a159d773..62f93b3d 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -31,6 +31,7 @@ import os
 import socket
 import ssl
 import urlparse
+import hmac
 
 from hashlib import sha256
 
@@ -52,6 +53,13 @@ from leap.soledad.common.errors import (
     NotLockedError,
     AlreadyLockedError,
 )
+from leap.soledad.common.crypto import (
+    MacMethods,
+    UnknownMacMethod,
+    WrongMac,
+    MAC_KEY,
+    MAC_METHOD_KEY,
+)
 
 #
 # Signaling function
@@ -357,7 +365,12 @@ class Soledad(object):
                 logger.info(
                     'Found cryptographic secrets in shared recovery '
                     'database.')
-                self.import_recovery_document(doc.content)
+                _, mac = self.import_recovery_document(doc.content)
+                if mac is False:
+                    self.put_secrets_in_shared_db()
+                self._store_secrets()  # save new secrets in local file
+                if self._secret_id is None:
+                    self._set_secret_id(self._secrets.items()[0][0])
             else:
                 # STAGE 3 - there are no secrets in server also, so
                 # generate a secret and store it in remote db.
@@ -516,21 +529,6 @@ class Soledad(object):
     def _load_secrets(self):
         """
         Load storage secrets from local file.
-
-        The content of the file has the following format:
-
-            {
-                "storage_secrets": {
-                    "<secret_id>": {
-                        'kdf': 'scrypt',
-                        'kdf_salt': '<b64 repr of salt>'
-                        'kdf_length': <key length>
-                        "cipher": "aes256",
-                        "length": <secret length>,
-                        "secret": "<encrypted storage_secret 1>",
-                    }
-                }
-            }
         """
         # does the file exist in disk?
         if not os.path.isfile(self._secrets_path):
@@ -539,7 +537,10 @@ class Soledad(object):
         content = None
         with open(self._secrets_path, 'r') as f:
             content = json.loads(f.read())
-        self._secrets = content[self.STORAGE_SECRETS_KEY]
+        _, mac = self.import_recovery_document(content)
+        if mac is False:
+            self._store_secrets()
+            self._put_secrets_in_shared_db()
         # choose first secret if no secret_id was given
         if self._secret_id is None:
             self._set_secret_id(self._secrets.items()[0][0])
@@ -614,28 +615,12 @@ class Soledad(object):
 
     def _store_secrets(self):
         """
-        Store a secret in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
-
-        The contents of the stored file have the following format:
-
-            {
-                'storage_secrets': {
-                    '<secret_id>': {
-                        'kdf': 'scrypt',
-                        'kdf_salt': '<salt>'
-                        'kdf_length': <len>
-                        'cipher': 'aes256',
-                        'length': 1024,
-                        'secret': '<encrypted storage_secret 1>',
-                    }
-                }
-            }
+        Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
         """
-        data = {
-            self.STORAGE_SECRETS_KEY: self._secrets,
-        }
         with open(self._secrets_path, 'w') as f:
-            f.write(json.dumps(data))
+            f.write(
+                json.dumps(
+                    self.export_recovery_document()))
 
     def change_passphrase(self, new_passphrase):
         """
@@ -662,6 +647,7 @@ class Soledad(object):
         # get a 256-bit key
         key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32)
         iv, ciphertext = self._crypto.encrypt_sym(secret, key)
+        # XXX update all secrets in the dict
         self._secrets[self._secret_id] = {
             # leap.soledad.crypto submodule uses AES256 for symmetric
             # encryption.
@@ -673,9 +659,9 @@ class Soledad(object):
             self.SECRET_KEY: '%s%s%s' % (
                 str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
         }
-
-        self._store_secrets()
         self._passphrase = new_passphrase
+        self._store_secrets()
+        self._put_secrets_in_shared_db()
 
     #
     # General crypto utility methods.
@@ -743,7 +729,7 @@ class Soledad(object):
             doc = SoledadDocument(
                 doc_id=self._shared_db_doc_id())
         # fill doc with encrypted secrets
-        doc.content = self.export_recovery_document(include_uuid=False)
+        doc.content = self.export_recovery_document()
         # upload secrets to server
         signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
         db = self._shared_db
@@ -761,12 +747,20 @@ class Soledad(object):
         """
         Update a document in the local encrypted database.
 
+        ============================== WARNING ==============================
+        This method converts the document's contents to unicode in-place. This
+        meanse that after calling C{put_doc(doc)}, the contents of the
+        document, i.e. C{doc.content}, might be different from before the
+        call.
+        ============================== WARNING ==============================
+
         :param doc: the document to update
         :type doc: SoledadDocument
 
         :return: the new revision identifier for the document
         :rtype: str
         """
+        doc.content = self._convert_to_unicode(doc.content)
         return self._db.put_doc(doc)
 
     def delete_doc(self, doc):
@@ -1100,26 +1094,51 @@ class Soledad(object):
     # Recovery document export and import methods
     #
 
-    def export_recovery_document(self, include_uuid=True):
+    def export_recovery_document(self):
         """
-        Export the storage secrets and (optionally) the uuid.
+        Export the storage secrets.
 
         A recovery document has the following structure:
 
             {
-                self.STORAGE_SECRET_KEY: <secrets dict>,
-                self.UUID_KEY: '<uuid>',  # (optional)
+                'storage_secrets': {
+                    '<storage_secret id>': {
+                        'kdf': 'scrypt',
+                        'kdf_salt': '<b64 repr of salt>'
+                        'kdf_length': <key length>
+                        'cipher': 'aes256',
+                        'length': <secret length>,
+                        'secret': '<encrypted storage_secret>',
+                    },
+                },
+                'kdf': 'scrypt',
+                'kdf_salt': '<b64 repr of salt>',
+                'kdf_length: <key length>,
+                '_mac_method': 'hmac',
+                '_mac': '<mac>'
             }
 
-        :param include_uuid: Should the uuid be included?
-        :type include_uuid: bool
+        Note that multiple storage secrets might be stored in one recovery
+        document. This method will also calculate a MAC of a string
+        representation of the secrets dictionary.
 
         :return: The recovery document.
         :rtype: dict
         """
-        data = {self.STORAGE_SECRETS_KEY: self._secrets}
-        if include_uuid:
-            data[self.UUID_KEY] = self._uuid
+        # create salt and key for calculating MAC
+        salt = os.urandom(self.SALT_LENGTH)
+        key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+        data = {
+            self.STORAGE_SECRETS_KEY: self._secrets,
+            self.KDF_KEY: self.KDF_SCRYPT,
+            self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+            self.KDF_LENGTH_KEY: len(key),
+            MAC_METHOD_KEY: MacMethods.HMAC,
+            MAC_KEY: hmac.new(
+                key,
+                json.dumps(self._secrets),
+                sha256).hexdigest(),
+        }
         return data
 
     def import_recovery_document(self, data):
@@ -1127,27 +1146,49 @@ class Soledad(object):
         Import storage secrets for symmetric encryption and uuid (if present)
         from a recovery document.
 
-        A recovery document has the following structure:
-
-            {
-                self.STORAGE_SECRET_KEY: <secrets dict>,
-                self.UUID_KEY: '<uuid>',  # (optional)
-            }
+        Note that this method does not store the imported data on disk. For
+        that, use C{self._store_secrets()}.
 
         :param data: The recovery document.
         :type data: dict
-        """
-        # include new secrets in our secret pool.
+
+        :return: A tuple containing the number of imported secrets and whether
+                 there was MAC informationa available for authenticating.
+        :rtype: (int, bool)
+        """
+        soledad_assert(self.STORAGE_SECRETS_KEY in data)
+        # check mac of the recovery document
+        mac_auth = False
+        mac = None
+        if MAC_KEY in data:
+            soledad_assert(data[MAC_KEY] is not None)
+            soledad_assert(MAC_METHOD_KEY in data)
+            soledad_assert(self.KDF_KEY in data)
+            soledad_assert(self.KDF_SALT_KEY in data)
+            soledad_assert(self.KDF_LENGTH_KEY in data)
+            if data[MAC_METHOD_KEY] == MacMethods.HMAC:
+                key = scrypt.hash(
+                    self._passphrase_as_string(),
+                    binascii.a2b_base64(data[self.KDF_SALT_KEY]),
+                    buflen=32)
+                mac = hmac.new(
+                    key,
+                    json.dumps(data[self.STORAGE_SECRETS_KEY]),
+                    sha256).hexdigest()
+            else:
+                raise UnknownMacMethod('Unknown MAC method: %s.' %
+                                       data[MAC_METHOD_KEY])
+            if mac != data[MAC_KEY]:
+                raise WrongMac('Could not authenticate recovery document\'s '
+                               'contents.')
+            mac_auth = True
+        # include secrets in the secret pool.
+        secrets = 0
         for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items():
             if secret_id not in self._secrets:
+                secrets += 1
                 self._secrets[secret_id] = secret_data
-        self._store_secrets()  # save new secrets in local file
-        # set uuid if present
-        if self.UUID_KEY in data:
-            self._uuid = data[self.UUID_KEY]
-        # choose first secret to use is none is assigned
-        if self._secret_id is None:
-            self._set_secret_id(data[self.STORAGE_SECRETS_KEY].items()[0][0])
+        return secrets, mac
 
     #
     # Setters/getters
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 65639887..d8899a97 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -35,7 +35,10 @@ from u1db.remote.http_target import HTTPSyncTarget
 from leap.soledad.common import soledad_assert
 from leap.soledad.common.crypto import (
     EncryptionSchemes,
+    UnknownEncryptionScheme,
     MacMethods,
+    UnknownMacMethod,
+    WrongMac,
     ENC_JSON_KEY,
     ENC_SCHEME_KEY,
     ENC_METHOD_KEY,
@@ -62,27 +65,6 @@ class DocumentNotEncrypted(Exception):
     pass
 
 
-class UnknownEncryptionScheme(Exception):
-    """
-    Raised when trying to decrypt from unknown encryption schemes.
-    """
-    pass
-
-
-class UnknownMacMethod(Exception):
-    """
-    Raised when trying to authenticate document's content with unknown MAC
-    mehtod.
-    """
-    pass
-
-
-class WrongMac(Exception):
-    """
-    Raised when failing to authenticate document's contents based on MAC.
-    """
-
-
 #
 # Crypto utilities for a SoledadDocument.
 #
diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/common/changes/VERSION_COMPAT
diff --git a/common/pkg/requirements-testing.pip b/common/pkg/requirements-testing.pip
index 6ff52ff9..9302450c 100644
--- a/common/pkg/requirements-testing.pip
+++ b/common/pkg/requirements-testing.pip
@@ -1,10 +1,5 @@
 mock
-nose2
 testscenarios
 leap.common
 leap.soledad.server
 leap.soledad.client
-
-# Under quarantine...
-# I've been able to run all tests with six==1.3 -- kali
-# six==1.1.0  # some tests are incompatible with newer versions of six.
diff --git a/common/pkg/tools/with_venvwrapper.sh b/common/pkg/tools/with_venvwrapper.sh
new file mode 100755
index 00000000..b62bc10f
--- /dev/null
+++ b/common/pkg/tools/with_venvwrapper.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#Wraps a command in a virtualenwrapper passed as first argument.
+#Example:
+#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh
+
+wd=`pwd`
+alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"'
+
+source `which virtualenvwrapper.sh`
+echo "Activating virtualenv " $1
+echo "------------------------------------"
+workon $1
+cd $wd
+echo "running version: "
+echo `pyver leap.bitmask`
+echo `pyver leap.soledad.common`
+echo `pyver leap.keymanager`
+$2 $3 $4 $5
diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py
index 2c6bd7a3..56bb608a 100644
--- a/common/src/leap/soledad/common/crypto.py
+++ b/common/src/leap/soledad/common/crypto.py
@@ -35,6 +35,13 @@ class EncryptionSchemes(object):
     PUBKEY = 'pubkey'
 
 
+class UnknownEncryptionScheme(Exception):
+    """
+    Raised when trying to decrypt from unknown encryption schemes.
+    """
+    pass
+
+
 class MacMethods(object):
     """
     Representation of MAC methods used to authenticate document's contents.
@@ -43,6 +50,20 @@ class MacMethods(object):
     HMAC = 'hmac'
 
 
+class UnknownMacMethod(Exception):
+    """
+    Raised when trying to authenticate document's content with unknown MAC
+    mehtod.
+    """
+    pass
+
+
+class WrongMac(Exception):
+    """
+    Raised when failing to authenticate document's contents based on MAC.
+    """
+
+
 #
 # Crypto utilities for a SoledadDocument.
 #
diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py
index db217bb3..af11bc76 100644
--- a/common/src/leap/soledad/common/tests/test_crypto.py
+++ b/common/src/leap/soledad/common/tests/test_crypto.py
@@ -40,6 +40,7 @@ from leap.soledad.common.tests import (
     KEY_FINGERPRINT,
     PRIVATE_KEY,
 )
+from leap.soledad.common.crypto import WrongMac, UnknownMacMethod
 from leap.soledad.common.tests.u1db_tests import (
     simple_doc,
     nested_doc,
@@ -88,11 +89,9 @@ class RecoveryDocumentTestCase(BaseSoledadTest):
 
     def test_import_recovery_document(self):
         rd = self._soledad.export_recovery_document()
-        s = self._soledad_instance(user='anotheruser@leap.se')
+        s = self._soledad_instance()
         s.import_recovery_document(rd)
         s._set_secret_id(self._soledad._secret_id)
-        self.assertEqual(self._soledad._uuid,
-                         s._uuid, 'Failed setting user uuid.')
         self.assertEqual(self._soledad._get_storage_secret(),
                          s._get_storage_secret(),
                          'Failed settinng secret for symmetric encryption.')
@@ -164,7 +163,7 @@ class MacAuthTestCase(BaseSoledadTest):
         doc.content[target.MAC_KEY] = '1234567890ABCDEF'
         # try to decrypt doc
         self.assertRaises(
-            target.WrongMac,
+            WrongMac,
             target.decrypt_doc, self._soledad._crypto, doc)
 
     def test_decrypt_with_unknown_mac_method_raises(self):
@@ -182,7 +181,7 @@ class MacAuthTestCase(BaseSoledadTest):
         doc.content[target.MAC_METHOD_KEY] = 'mymac'
         # try to decrypt doc
         self.assertRaises(
-            target.UnknownMacMethod,
+            UnknownMacMethod,
             target.decrypt_doc, self._soledad._crypto, doc)
 
 
diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py
index 8970a437..035c5ac5 100644
--- a/common/src/leap/soledad/common/tests/test_soledad.py
+++ b/common/src/leap/soledad/common/tests/test_soledad.py
@@ -33,6 +33,7 @@ from leap.soledad.common.tests import (
 )
 from leap import soledad
 from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.crypto import WrongMac
 from leap.soledad.client import Soledad, PassphraseTooShort
 from leap.soledad.client.crypto import SoledadCrypto
 from leap.soledad.client.shared_db import SoledadSharedDatabase
@@ -119,7 +120,7 @@ class AuxMethodsTestCase(BaseSoledadTest):
         sol.change_passphrase(u'654321')
 
         self.assertRaises(
-            DatabaseError,
+            WrongMac,
             self._soledad_instance, 'leap@leap.se',
             passphrase=u'123',
             prefix=self.rand_prefix)
@@ -292,7 +293,7 @@ class SoledadSignalingTestCase(BaseSoledadTest):
         sol = self._soledad_instance()
         # create a document with secrets
         doc = SoledadDocument(doc_id=sol._shared_db_doc_id())
-        doc.content = sol.export_recovery_document(include_uuid=False)
+        doc.content = sol.export_recovery_document()
 
         class Stage2MockSharedDB(object):
 
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/README b/common/src/leap/soledad/common/tests/u1db_tests/README
index 605f01fa..d543f250 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/README
+++ b/common/src/leap/soledad/common/tests/u1db_tests/README
@@ -12,7 +12,6 @@ Dependencies
 
 u1db tests depend on the following python packages:
 
-  nose2
   unittest2
   mercurial
   hgtools
@@ -25,10 +24,3 @@ u1db tests depend on the following python packages:
   routes
   simplejson
   cython
-
-Running tests
--------------
-
-Use nose2 to run tests:
-
-  nose2 leap.soledad.tests.u1db_tests
diff --git a/run_tests.sh b/run_tests.sh
new file mode 100755
index 00000000..e36466f8
--- /dev/null
+++ b/run_tests.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+cd common
+python setup.py test
diff --git a/scripts/README.rst b/scripts/README.rst
new file mode 100644
index 00000000..fdd1d642
--- /dev/null
+++ b/scripts/README.rst
@@ -0,0 +1,17 @@
+Soledad Scripts
+===============
+
+The scripts in this directory are meant to be used for development purposes.
+
+Currently, the scripts are:
+
+  * server-side-db.py: Gives access to server-side soledad user database,
+    based on the configuration in /etc/leap/soledad-server.conf. One should
+    use it as:
+
+      python -i server-side-db.py <uuid>
+
+  * client-side-db.py: Gives access to client-side soledad user database,
+    based on data stored in ~/.config/leap/soledad. One should use it as:
+
+      python -i client-side-db.py <uuid> <passphrase>
diff --git a/scripts/client-side-db.py b/scripts/client-side-db.py
new file mode 100644
index 00000000..0c3df7a4
--- /dev/null
+++ b/scripts/client-side-db.py
@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+# This script gives client-side access to one Soledad user database by using
+# the data stored in ~/.config/leap/soledad/
+
+import sys
+import os
+
+from leap.common.config import get_path_prefix
+from leap.soledad.client import Soledad
+
+if len(sys.argv) != 3:
+    print 'Usage: %s <uuid> <passphrase>' % sys.argv[0]
+    exit(1)
+
+uuid = sys.argv[1]
+passphrase = unicode(sys.argv[2])
+
+secrets_path = os.path.join(get_path_prefix(), 'leap', 'soledad',
+                            '%s.secret' % uuid)
+local_db_path = os.path.join(get_path_prefix(), 'leap', 'soledad',
+                             '%s.db' % uuid)
+server_url = 'http://dummy-url'
+cert_file = 'cert'
+
+sol = Soledad(uuid, passphrase, secrets_path, local_db_path, server_url,
+             cert_file)
+db = sol._db
+
+# get replica info
+replica_uid = db._replica_uid
+gen, docs = db.get_all_docs()
+print "replica_uid: %s" % replica_uid
+print "generation:  %d" % gen
+gen, trans_id = db._get_generation_info()
+print "transaction_id: %s" % trans_id
diff --git a/scripts/develop_mode.sh b/scripts/develop_mode.sh
new file mode 100755
index 00000000..8d2ebaa8
--- /dev/null
+++ b/scripts/develop_mode.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+cd common
+python setup.py develop
+cd ../client
+python setup.py develop
+cd ../server
+python setup.py develop
diff --git a/scripts/server-side-db.py b/scripts/server-side-db.py
new file mode 100644
index 00000000..01a9aaac
--- /dev/null
+++ b/scripts/server-side-db.py
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+
+# This script gives server-side access to one Soledad user database by using
+# the configuration stored in /etc/leap/soledad-server.conf.
+
+import sys
+from ConfigParser import ConfigParser
+
+from leap.soledad.common.couch import CouchDatabase
+
+if len(sys.argv) != 2:
+    print 'Usage: %s <uuid>' % sys.argv[0]
+    exit(1)
+
+uuid = sys.argv[1]
+
+# get couch url
+cp = ConfigParser()
+cp.read('/etc/leap/soledad-server.conf')
+url = cp.get('soledad-server', 'couch_url')
+
+# access user db
+dbname = 'user-%s' % uuid
+db = CouchDatabase(url, dbname)
+
+# get replica info
+replica_uid = db._replica_uid
+gen, docs = db.get_all_docs()
+print "dbname:      %s" % dbname
+print "replica_uid: %s" % replica_uid
+print "generation:  %d" % gen
+
+# get relevant docs
+schemes = map(lambda d: d.content['_enc_scheme'], docs)
+pubenc = filter(lambda d: d.content['_enc_scheme'] == 'pubkey', docs)
+
+print "total number of docs:  %d" % len(docs)
+print "pubkey encrypted docs: %d" % len(pubenc)