diff options
| -rw-r--r-- | CHANGELOG | 9 | ||||
| -rw-r--r-- | README.rst | 10 | ||||
| -rw-r--r-- | client/changes/VERSION_COMPAT | 0 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/__init__.py | 169 | ||||
| -rw-r--r-- | client/src/leap/soledad/client/target.py | 24 | ||||
| -rw-r--r-- | common/changes/VERSION_COMPAT | 0 | ||||
| -rw-r--r-- | common/pkg/requirements-testing.pip | 5 | ||||
| -rwxr-xr-x | common/pkg/tools/with_venvwrapper.sh | 19 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/crypto.py | 21 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_crypto.py | 9 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/test_soledad.py | 5 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/README | 8 | ||||
| -rwxr-xr-x | run_tests.sh | 3 | ||||
| -rw-r--r-- | scripts/README.rst | 17 | ||||
| -rw-r--r-- | scripts/client-side-db.py | 36 | ||||
| -rwxr-xr-x | scripts/develop_mode.sh | 7 | ||||
| -rw-r--r-- | scripts/server-side-db.py | 38 |
17 files changed, 267 insertions, 113 deletions
@@ -1,4 +1,13 @@ +0.4.4 Dec 6: +Client: + o Add MAC verirication to the recovery document and + soledad.json. Closes #4348. +Common: + o Add unicode conversion to put_doc(). Closes #4095. + o Remove tests dependency on nose2. Closes #4258. + 0.4.3 Nov 15: +Client: o Defaults detected encoding to utf-8 to avoid bug if detected encoding is None. Closes: #4417 o Open db in autocommit mode, to avoid nested transactions problems. @@ -38,14 +38,8 @@ Tests Client and server tests are both included in leap.soledad.common. If you want to run tests in development mode you must do the following:: - cd common - python setup.py develop - cd ../client - python setup.py develop - cd ../server - python setup.py develop - cd ../common - python setup.py test + scripts/develop_mode.sh + ./run_tests.sh Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your system. diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/client/changes/VERSION_COMPAT diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a159d773..62f93b3d 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -31,6 +31,7 @@ import os import socket import ssl import urlparse +import hmac from hashlib import sha256 @@ -52,6 +53,13 @@ from leap.soledad.common.errors import ( NotLockedError, AlreadyLockedError, ) +from leap.soledad.common.crypto import ( + MacMethods, + UnknownMacMethod, + WrongMac, + MAC_KEY, + MAC_METHOD_KEY, +) # # Signaling function @@ -357,7 +365,12 @@ class Soledad(object): logger.info( 'Found cryptographic secrets in shared recovery ' 'database.') - self.import_recovery_document(doc.content) + _, mac = self.import_recovery_document(doc.content) + if mac is False: + self.put_secrets_in_shared_db() + self._store_secrets() # save new secrets in local file + if self._secret_id is None: + self._set_secret_id(self._secrets.items()[0][0]) else: # STAGE 3 - there are no secrets in server also, so # generate a secret and store it in remote db. @@ -516,21 +529,6 @@ class Soledad(object): def _load_secrets(self): """ Load storage secrets from local file. - - The content of the file has the following format: - - { - "storage_secrets": { - "<secret_id>": { - 'kdf': 'scrypt', - 'kdf_salt': '<b64 repr of salt>' - 'kdf_length': <key length> - "cipher": "aes256", - "length": <secret length>, - "secret": "<encrypted storage_secret 1>", - } - } - } """ # does the file exist in disk? if not os.path.isfile(self._secrets_path): @@ -539,7 +537,10 @@ class Soledad(object): content = None with open(self._secrets_path, 'r') as f: content = json.loads(f.read()) - self._secrets = content[self.STORAGE_SECRETS_KEY] + _, mac = self.import_recovery_document(content) + if mac is False: + self._store_secrets() + self._put_secrets_in_shared_db() # choose first secret if no secret_id was given if self._secret_id is None: self._set_secret_id(self._secrets.items()[0][0]) @@ -614,28 +615,12 @@ class Soledad(object): def _store_secrets(self): """ - Store a secret in C{Soledad.STORAGE_SECRETS_FILE_PATH}. - - The contents of the stored file have the following format: - - { - 'storage_secrets': { - '<secret_id>': { - 'kdf': 'scrypt', - 'kdf_salt': '<salt>' - 'kdf_length': <len> - 'cipher': 'aes256', - 'length': 1024, - 'secret': '<encrypted storage_secret 1>', - } - } - } + Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. """ - data = { - self.STORAGE_SECRETS_KEY: self._secrets, - } with open(self._secrets_path, 'w') as f: - f.write(json.dumps(data)) + f.write( + json.dumps( + self.export_recovery_document())) def change_passphrase(self, new_passphrase): """ @@ -662,6 +647,7 @@ class Soledad(object): # get a 256-bit key key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) iv, ciphertext = self._crypto.encrypt_sym(secret, key) + # XXX update all secrets in the dict self._secrets[self._secret_id] = { # leap.soledad.crypto submodule uses AES256 for symmetric # encryption. @@ -673,9 +659,9 @@ class Soledad(object): self.SECRET_KEY: '%s%s%s' % ( str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), } - - self._store_secrets() self._passphrase = new_passphrase + self._store_secrets() + self._put_secrets_in_shared_db() # # General crypto utility methods. @@ -743,7 +729,7 @@ class Soledad(object): doc = SoledadDocument( doc_id=self._shared_db_doc_id()) # fill doc with encrypted secrets - doc.content = self.export_recovery_document(include_uuid=False) + doc.content = self.export_recovery_document() # upload secrets to server signal(SOLEDAD_UPLOADING_KEYS, self._uuid) db = self._shared_db @@ -761,12 +747,20 @@ class Soledad(object): """ Update a document in the local encrypted database. + ============================== WARNING ============================== + This method converts the document's contents to unicode in-place. This + meanse that after calling C{put_doc(doc)}, the contents of the + document, i.e. C{doc.content}, might be different from before the + call. + ============================== WARNING ============================== + :param doc: the document to update :type doc: SoledadDocument :return: the new revision identifier for the document :rtype: str """ + doc.content = self._convert_to_unicode(doc.content) return self._db.put_doc(doc) def delete_doc(self, doc): @@ -1100,26 +1094,51 @@ class Soledad(object): # Recovery document export and import methods # - def export_recovery_document(self, include_uuid=True): + def export_recovery_document(self): """ - Export the storage secrets and (optionally) the uuid. + Export the storage secrets. A recovery document has the following structure: { - self.STORAGE_SECRET_KEY: <secrets dict>, - self.UUID_KEY: '<uuid>', # (optional) + 'storage_secrets': { + '<storage_secret id>': { + 'kdf': 'scrypt', + 'kdf_salt': '<b64 repr of salt>' + 'kdf_length': <key length> + 'cipher': 'aes256', + 'length': <secret length>, + 'secret': '<encrypted storage_secret>', + }, + }, + 'kdf': 'scrypt', + 'kdf_salt': '<b64 repr of salt>', + 'kdf_length: <key length>, + '_mac_method': 'hmac', + '_mac': '<mac>' } - :param include_uuid: Should the uuid be included? - :type include_uuid: bool + Note that multiple storage secrets might be stored in one recovery + document. This method will also calculate a MAC of a string + representation of the secrets dictionary. :return: The recovery document. :rtype: dict """ - data = {self.STORAGE_SECRETS_KEY: self._secrets} - if include_uuid: - data[self.UUID_KEY] = self._uuid + # create salt and key for calculating MAC + salt = os.urandom(self.SALT_LENGTH) + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + data = { + self.STORAGE_SECRETS_KEY: self._secrets, + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + MAC_METHOD_KEY: MacMethods.HMAC, + MAC_KEY: hmac.new( + key, + json.dumps(self._secrets), + sha256).hexdigest(), + } return data def import_recovery_document(self, data): @@ -1127,27 +1146,49 @@ class Soledad(object): Import storage secrets for symmetric encryption and uuid (if present) from a recovery document. - A recovery document has the following structure: - - { - self.STORAGE_SECRET_KEY: <secrets dict>, - self.UUID_KEY: '<uuid>', # (optional) - } + Note that this method does not store the imported data on disk. For + that, use C{self._store_secrets()}. :param data: The recovery document. :type data: dict - """ - # include new secrets in our secret pool. + + :return: A tuple containing the number of imported secrets and whether + there was MAC informationa available for authenticating. + :rtype: (int, bool) + """ + soledad_assert(self.STORAGE_SECRETS_KEY in data) + # check mac of the recovery document + mac_auth = False + mac = None + if MAC_KEY in data: + soledad_assert(data[MAC_KEY] is not None) + soledad_assert(MAC_METHOD_KEY in data) + soledad_assert(self.KDF_KEY in data) + soledad_assert(self.KDF_SALT_KEY in data) + soledad_assert(self.KDF_LENGTH_KEY in data) + if data[MAC_METHOD_KEY] == MacMethods.HMAC: + key = scrypt.hash( + self._passphrase_as_string(), + binascii.a2b_base64(data[self.KDF_SALT_KEY]), + buflen=32) + mac = hmac.new( + key, + json.dumps(data[self.STORAGE_SECRETS_KEY]), + sha256).hexdigest() + else: + raise UnknownMacMethod('Unknown MAC method: %s.' % + data[MAC_METHOD_KEY]) + if mac != data[MAC_KEY]: + raise WrongMac('Could not authenticate recovery document\'s ' + 'contents.') + mac_auth = True + # include secrets in the secret pool. + secrets = 0 for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): if secret_id not in self._secrets: + secrets += 1 self._secrets[secret_id] = secret_data - self._store_secrets() # save new secrets in local file - # set uuid if present - if self.UUID_KEY in data: - self._uuid = data[self.UUID_KEY] - # choose first secret to use is none is assigned - if self._secret_id is None: - self._set_secret_id(data[self.STORAGE_SECRETS_KEY].items()[0][0]) + return secrets, mac # # Setters/getters diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 65639887..d8899a97 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -35,7 +35,10 @@ from u1db.remote.http_target import HTTPSyncTarget from leap.soledad.common import soledad_assert from leap.soledad.common.crypto import ( EncryptionSchemes, + UnknownEncryptionScheme, MacMethods, + UnknownMacMethod, + WrongMac, ENC_JSON_KEY, ENC_SCHEME_KEY, ENC_METHOD_KEY, @@ -62,27 +65,6 @@ class DocumentNotEncrypted(Exception): pass -class UnknownEncryptionScheme(Exception): - """ - Raised when trying to decrypt from unknown encryption schemes. - """ - pass - - -class UnknownMacMethod(Exception): - """ - Raised when trying to authenticate document's content with unknown MAC - mehtod. - """ - pass - - -class WrongMac(Exception): - """ - Raised when failing to authenticate document's contents based on MAC. - """ - - # # Crypto utilities for a SoledadDocument. # diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/common/changes/VERSION_COMPAT diff --git a/common/pkg/requirements-testing.pip b/common/pkg/requirements-testing.pip index 6ff52ff9..9302450c 100644 --- a/common/pkg/requirements-testing.pip +++ b/common/pkg/requirements-testing.pip @@ -1,10 +1,5 @@ mock -nose2 testscenarios leap.common leap.soledad.server leap.soledad.client - -# Under quarantine... -# I've been able to run all tests with six==1.3 -- kali -# six==1.1.0 # some tests are incompatible with newer versions of six. diff --git a/common/pkg/tools/with_venvwrapper.sh b/common/pkg/tools/with_venvwrapper.sh new file mode 100755 index 00000000..b62bc10f --- /dev/null +++ b/common/pkg/tools/with_venvwrapper.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +#Wraps a command in a virtualenwrapper passed as first argument. +#Example: +#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh + +wd=`pwd` +alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"' + +source `which virtualenvwrapper.sh` +echo "Activating virtualenv " $1 +echo "------------------------------------" +workon $1 +cd $wd +echo "running version: " +echo `pyver leap.bitmask` +echo `pyver leap.soledad.common` +echo `pyver leap.keymanager` +$2 $3 $4 $5 diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py index 2c6bd7a3..56bb608a 100644 --- a/common/src/leap/soledad/common/crypto.py +++ b/common/src/leap/soledad/common/crypto.py @@ -35,6 +35,13 @@ class EncryptionSchemes(object): PUBKEY = 'pubkey' +class UnknownEncryptionScheme(Exception): + """ + Raised when trying to decrypt from unknown encryption schemes. + """ + pass + + class MacMethods(object): """ Representation of MAC methods used to authenticate document's contents. @@ -43,6 +50,20 @@ class MacMethods(object): HMAC = 'hmac' +class UnknownMacMethod(Exception): + """ + Raised when trying to authenticate document's content with unknown MAC + mehtod. + """ + pass + + +class WrongMac(Exception): + """ + Raised when failing to authenticate document's contents based on MAC. + """ + + # # Crypto utilities for a SoledadDocument. # diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index db217bb3..af11bc76 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -40,6 +40,7 @@ from leap.soledad.common.tests import ( KEY_FINGERPRINT, PRIVATE_KEY, ) +from leap.soledad.common.crypto import WrongMac, UnknownMacMethod from leap.soledad.common.tests.u1db_tests import ( simple_doc, nested_doc, @@ -88,11 +89,9 @@ class RecoveryDocumentTestCase(BaseSoledadTest): def test_import_recovery_document(self): rd = self._soledad.export_recovery_document() - s = self._soledad_instance(user='anotheruser@leap.se') + s = self._soledad_instance() s.import_recovery_document(rd) s._set_secret_id(self._soledad._secret_id) - self.assertEqual(self._soledad._uuid, - s._uuid, 'Failed setting user uuid.') self.assertEqual(self._soledad._get_storage_secret(), s._get_storage_secret(), 'Failed settinng secret for symmetric encryption.') @@ -164,7 +163,7 @@ class MacAuthTestCase(BaseSoledadTest): doc.content[target.MAC_KEY] = '1234567890ABCDEF' # try to decrypt doc self.assertRaises( - target.WrongMac, + WrongMac, target.decrypt_doc, self._soledad._crypto, doc) def test_decrypt_with_unknown_mac_method_raises(self): @@ -182,7 +181,7 @@ class MacAuthTestCase(BaseSoledadTest): doc.content[target.MAC_METHOD_KEY] = 'mymac' # try to decrypt doc self.assertRaises( - target.UnknownMacMethod, + UnknownMacMethod, target.decrypt_doc, self._soledad._crypto, doc) diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 8970a437..035c5ac5 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -33,6 +33,7 @@ from leap.soledad.common.tests import ( ) from leap import soledad from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import WrongMac from leap.soledad.client import Soledad, PassphraseTooShort from leap.soledad.client.crypto import SoledadCrypto from leap.soledad.client.shared_db import SoledadSharedDatabase @@ -119,7 +120,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol.change_passphrase(u'654321') self.assertRaises( - DatabaseError, + WrongMac, self._soledad_instance, 'leap@leap.se', passphrase=u'123', prefix=self.rand_prefix) @@ -292,7 +293,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): sol = self._soledad_instance() # create a document with secrets doc = SoledadDocument(doc_id=sol._shared_db_doc_id()) - doc.content = sol.export_recovery_document(include_uuid=False) + doc.content = sol.export_recovery_document() class Stage2MockSharedDB(object): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/README b/common/src/leap/soledad/common/tests/u1db_tests/README index 605f01fa..d543f250 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/README +++ b/common/src/leap/soledad/common/tests/u1db_tests/README @@ -12,7 +12,6 @@ Dependencies u1db tests depend on the following python packages: - nose2 unittest2 mercurial hgtools @@ -25,10 +24,3 @@ u1db tests depend on the following python packages: routes simplejson cython - -Running tests -------------- - -Use nose2 to run tests: - - nose2 leap.soledad.tests.u1db_tests diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 00000000..e36466f8 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd common +python setup.py test diff --git a/scripts/README.rst b/scripts/README.rst new file mode 100644 index 00000000..fdd1d642 --- /dev/null +++ b/scripts/README.rst @@ -0,0 +1,17 @@ +Soledad Scripts +=============== + +The scripts in this directory are meant to be used for development purposes. + +Currently, the scripts are: + + * server-side-db.py: Gives access to server-side soledad user database, + based on the configuration in /etc/leap/soledad-server.conf. One should + use it as: + + python -i server-side-db.py <uuid> + + * client-side-db.py: Gives access to client-side soledad user database, + based on data stored in ~/.config/leap/soledad. One should use it as: + + python -i client-side-db.py <uuid> <passphrase> diff --git a/scripts/client-side-db.py b/scripts/client-side-db.py new file mode 100644 index 00000000..0c3df7a4 --- /dev/null +++ b/scripts/client-side-db.py @@ -0,0 +1,36 @@ +#!/usr/bin/python + +# This script gives client-side access to one Soledad user database by using +# the data stored in ~/.config/leap/soledad/ + +import sys +import os + +from leap.common.config import get_path_prefix +from leap.soledad.client import Soledad + +if len(sys.argv) != 3: + print 'Usage: %s <uuid> <passphrase>' % sys.argv[0] + exit(1) + +uuid = sys.argv[1] +passphrase = unicode(sys.argv[2]) + +secrets_path = os.path.join(get_path_prefix(), 'leap', 'soledad', + '%s.secret' % uuid) +local_db_path = os.path.join(get_path_prefix(), 'leap', 'soledad', + '%s.db' % uuid) +server_url = 'http://dummy-url' +cert_file = 'cert' + +sol = Soledad(uuid, passphrase, secrets_path, local_db_path, server_url, + cert_file) +db = sol._db + +# get replica info +replica_uid = db._replica_uid +gen, docs = db.get_all_docs() +print "replica_uid: %s" % replica_uid +print "generation: %d" % gen +gen, trans_id = db._get_generation_info() +print "transaction_id: %s" % trans_id diff --git a/scripts/develop_mode.sh b/scripts/develop_mode.sh new file mode 100755 index 00000000..8d2ebaa8 --- /dev/null +++ b/scripts/develop_mode.sh @@ -0,0 +1,7 @@ +#!/bin/sh +cd common +python setup.py develop +cd ../client +python setup.py develop +cd ../server +python setup.py develop diff --git a/scripts/server-side-db.py b/scripts/server-side-db.py new file mode 100644 index 00000000..01a9aaac --- /dev/null +++ b/scripts/server-side-db.py @@ -0,0 +1,38 @@ +#!/usr/bin/python + +# This script gives server-side access to one Soledad user database by using +# the configuration stored in /etc/leap/soledad-server.conf. + +import sys +from ConfigParser import ConfigParser + +from leap.soledad.common.couch import CouchDatabase + +if len(sys.argv) != 2: + print 'Usage: %s <uuid>' % sys.argv[0] + exit(1) + +uuid = sys.argv[1] + +# get couch url +cp = ConfigParser() +cp.read('/etc/leap/soledad-server.conf') +url = cp.get('soledad-server', 'couch_url') + +# access user db +dbname = 'user-%s' % uuid +db = CouchDatabase(url, dbname) + +# get replica info +replica_uid = db._replica_uid +gen, docs = db.get_all_docs() +print "dbname: %s" % dbname +print "replica_uid: %s" % replica_uid +print "generation: %d" % gen + +# get relevant docs +schemes = map(lambda d: d.content['_enc_scheme'], docs) +pubenc = filter(lambda d: d.content['_enc_scheme'] == 'pubkey', docs) + +print "total number of docs: %d" % len(docs) +print "pubkey encrypted docs: %d" % len(pubenc) |