From d5d47dc216fd285ec60df1ccaab1e0740e29e3c5 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 18 Nov 2013 11:06:37 -0200 Subject: Add MAC verification to recovery document. Closes #4348. --- ...ature_4348_add-mac-verification-to-recovery-doc | 2 + client/src/leap/soledad/client/__init__.py | 161 +++++++++++++-------- client/src/leap/soledad/client/target.py | 24 +-- 3 files changed, 102 insertions(+), 85 deletions(-) create mode 100644 client/changes/feature_4348_add-mac-verification-to-recovery-doc (limited to 'client') diff --git a/client/changes/feature_4348_add-mac-verification-to-recovery-doc b/client/changes/feature_4348_add-mac-verification-to-recovery-doc new file mode 100644 index 00000000..692403f9 --- /dev/null +++ b/client/changes/feature_4348_add-mac-verification-to-recovery-doc @@ -0,0 +1,2 @@ + o Add MAC verirication to the recovery document and soledad.json. Closes + #4348. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a159d773..d50dde42 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -31,6 +31,7 @@ import os import socket import ssl import urlparse +import hmac from hashlib import sha256 @@ -52,6 +53,13 @@ from leap.soledad.common.errors import ( NotLockedError, AlreadyLockedError, ) +from leap.soledad.common.crypto import ( + MacMethods, + UnknownMacMethod, + WrongMac, + MAC_KEY, + MAC_METHOD_KEY, +) # # Signaling function @@ -357,7 +365,12 @@ class Soledad(object): logger.info( 'Found cryptographic secrets in shared recovery ' 'database.') - self.import_recovery_document(doc.content) + _, mac = self.import_recovery_document(doc.content) + if mac is False: + self.put_secrets_in_shared_db() + self._store_secrets() # save new secrets in local file + if self._secret_id is None: + self._set_secret_id(self._secrets.items()[0][0]) else: # STAGE 3 - there are no secrets in server also, so # generate a secret and store it in remote db. @@ -516,21 +529,6 @@ class Soledad(object): def _load_secrets(self): """ Load storage secrets from local file. - - The content of the file has the following format: - - { - "storage_secrets": { - "": { - 'kdf': 'scrypt', - 'kdf_salt': '' - 'kdf_length': - "cipher": "aes256", - "length": , - "secret": "", - } - } - } """ # does the file exist in disk? if not os.path.isfile(self._secrets_path): @@ -539,7 +537,10 @@ class Soledad(object): content = None with open(self._secrets_path, 'r') as f: content = json.loads(f.read()) - self._secrets = content[self.STORAGE_SECRETS_KEY] + _, mac = self.import_recovery_document(content) + if mac is False: + self._store_secrets() + self._put_secrets_in_shared_db() # choose first secret if no secret_id was given if self._secret_id is None: self._set_secret_id(self._secrets.items()[0][0]) @@ -614,28 +615,12 @@ class Soledad(object): def _store_secrets(self): """ - Store a secret in C{Soledad.STORAGE_SECRETS_FILE_PATH}. - - The contents of the stored file have the following format: - - { - 'storage_secrets': { - '': { - 'kdf': 'scrypt', - 'kdf_salt': '' - 'kdf_length': - 'cipher': 'aes256', - 'length': 1024, - 'secret': '', - } - } - } + Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. """ - data = { - self.STORAGE_SECRETS_KEY: self._secrets, - } with open(self._secrets_path, 'w') as f: - f.write(json.dumps(data)) + f.write( + json.dumps( + self.export_recovery_document())) def change_passphrase(self, new_passphrase): """ @@ -662,6 +647,7 @@ class Soledad(object): # get a 256-bit key key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) iv, ciphertext = self._crypto.encrypt_sym(secret, key) + # XXX update all secrets in the dict self._secrets[self._secret_id] = { # leap.soledad.crypto submodule uses AES256 for symmetric # encryption. @@ -673,9 +659,9 @@ class Soledad(object): self.SECRET_KEY: '%s%s%s' % ( str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), } - - self._store_secrets() self._passphrase = new_passphrase + self._store_secrets() + self._put_secrets_in_shared_db() # # General crypto utility methods. @@ -743,7 +729,7 @@ class Soledad(object): doc = SoledadDocument( doc_id=self._shared_db_doc_id()) # fill doc with encrypted secrets - doc.content = self.export_recovery_document(include_uuid=False) + doc.content = self.export_recovery_document() # upload secrets to server signal(SOLEDAD_UPLOADING_KEYS, self._uuid) db = self._shared_db @@ -1100,26 +1086,51 @@ class Soledad(object): # Recovery document export and import methods # - def export_recovery_document(self, include_uuid=True): + def export_recovery_document(self): """ - Export the storage secrets and (optionally) the uuid. + Export the storage secrets. A recovery document has the following structure: { - self.STORAGE_SECRET_KEY: , - self.UUID_KEY: '', # (optional) + 'storage_secrets': { + '': { + 'kdf': 'scrypt', + 'kdf_salt': '' + 'kdf_length': + 'cipher': 'aes256', + 'length': , + 'secret': '', + }, + }, + 'kdf': 'scrypt', + 'kdf_salt': '', + 'kdf_length: , + '_mac_method': 'hmac', + '_mac': '' } - :param include_uuid: Should the uuid be included? - :type include_uuid: bool + Note that multiple storage secrets might be stored in one recovery + document. This method will also calculate a MAC of a string + representation of the secrets dictionary. :return: The recovery document. :rtype: dict """ - data = {self.STORAGE_SECRETS_KEY: self._secrets} - if include_uuid: - data[self.UUID_KEY] = self._uuid + # create salt and key for calculating MAC + salt = os.urandom(self.SALT_LENGTH) + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + data = { + self.STORAGE_SECRETS_KEY: self._secrets, + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + MAC_METHOD_KEY: MacMethods.HMAC, + MAC_KEY: hmac.new( + key, + json.dumps(self._secrets), + sha256).hexdigest(), + } return data def import_recovery_document(self, data): @@ -1127,27 +1138,49 @@ class Soledad(object): Import storage secrets for symmetric encryption and uuid (if present) from a recovery document. - A recovery document has the following structure: - - { - self.STORAGE_SECRET_KEY: , - self.UUID_KEY: '', # (optional) - } + Note that this method does not store the imported data on disk. For + that, use C{self._store_secrets()}. :param data: The recovery document. :type data: dict - """ - # include new secrets in our secret pool. + + :return: A tuple containing the number of imported secrets and whether + there was MAC informationa available for authenticating. + :rtype: (int, bool) + """ + soledad_assert(self.STORAGE_SECRETS_KEY in data) + # check mac of the recovery document + mac_auth = False + mac = None + if MAC_KEY in data: + soledad_assert(data[MAC_KEY] is not None) + soledad_assert(MAC_METHOD_KEY in data) + soledad_assert(self.KDF_KEY in data) + soledad_assert(self.KDF_SALT_KEY in data) + soledad_assert(self.KDF_LENGTH_KEY in data) + if data[MAC_METHOD_KEY] == MacMethods.HMAC: + key = scrypt.hash( + self._passphrase_as_string(), + binascii.a2b_base64(data[self.KDF_SALT_KEY]), + buflen=32) + mac = hmac.new( + key, + json.dumps(data[self.STORAGE_SECRETS_KEY]), + sha256).hexdigest() + else: + raise UnknownMacMethod('Unknown MAC method: %s.' % + data[MAC_METHOD_KEY]) + if mac != data[MAC_KEY]: + raise WrongMac('Could not authenticate recovery document\'s ' + 'contents.') + mac_auth = True + # include secrets in the secret pool. + secrets = 0 for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): if secret_id not in self._secrets: + secrets += 1 self._secrets[secret_id] = secret_data - self._store_secrets() # save new secrets in local file - # set uuid if present - if self.UUID_KEY in data: - self._uuid = data[self.UUID_KEY] - # choose first secret to use is none is assigned - if self._secret_id is None: - self._set_secret_id(data[self.STORAGE_SECRETS_KEY].items()[0][0]) + return secrets, mac # # Setters/getters diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 65639887..d8899a97 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -35,7 +35,10 @@ from u1db.remote.http_target import HTTPSyncTarget from leap.soledad.common import soledad_assert from leap.soledad.common.crypto import ( EncryptionSchemes, + UnknownEncryptionScheme, MacMethods, + UnknownMacMethod, + WrongMac, ENC_JSON_KEY, ENC_SCHEME_KEY, ENC_METHOD_KEY, @@ -62,27 +65,6 @@ class DocumentNotEncrypted(Exception): pass -class UnknownEncryptionScheme(Exception): - """ - Raised when trying to decrypt from unknown encryption schemes. - """ - pass - - -class UnknownMacMethod(Exception): - """ - Raised when trying to authenticate document's content with unknown MAC - mehtod. - """ - pass - - -class WrongMac(Exception): - """ - Raised when failing to authenticate document's contents based on MAC. - """ - - # # Crypto utilities for a SoledadDocument. # -- cgit v1.2.3 From 75bc8c2220aa636a2c63fa0d151f271c22920040 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 21 Nov 2013 09:49:05 -0200 Subject: Add unicode conversion to put_doc. Closes #4095. --- client/src/leap/soledad/client/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index d50dde42..08cae892 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -753,6 +753,7 @@ class Soledad(object): :return: the new revision identifier for the document :rtype: str """ + doc.content = self._convert_to_unicode(doc.content) return self._db.put_doc(doc) def delete_doc(self, doc): -- cgit v1.2.3 From 69fd3a11a3821468e737b8fb0e3e77abef0a4b8a Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 21 Nov 2013 10:00:35 -0200 Subject: Add scripts for debugging client and server side databases. --- client/src/leap/soledad/client/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 08cae892..62f93b3d 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -747,6 +747,13 @@ class Soledad(object): """ Update a document in the local encrypted database. + ============================== WARNING ============================== + This method converts the document's contents to unicode in-place. This + meanse that after calling C{put_doc(doc)}, the contents of the + document, i.e. C{doc.content}, might be different from before the + call. + ============================== WARNING ============================== + :param doc: the document to update :type doc: SoledadDocument -- cgit v1.2.3 From 6eeed5c116d38ebae7d9309dd2f11660f6cff37e Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 29 Nov 2013 14:39:04 -0200 Subject: Stop creating per-user databases. Closes #4626. --- client/changes/feature_4626_stop-creating-per-user-databases | 2 ++ client/src/leap/soledad/client/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 client/changes/feature_4626_stop-creating-per-user-databases (limited to 'client') diff --git a/client/changes/feature_4626_stop-creating-per-user-databases b/client/changes/feature_4626_stop-creating-per-user-databases new file mode 100644 index 00000000..b967abd3 --- /dev/null +++ b/client/changes/feature_4626_stop-creating-per-user-databases @@ -0,0 +1,2 @@ + o Stop creating per-user databases, as now there's another service that does + this automatically (tapicero). Closes #4626. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 62f93b3d..c168bfdc 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1038,7 +1038,7 @@ class Soledad(object): if self._db: local_gen = self._db.sync( urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=True) + creds=self._creds, autocreate=False) signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) return local_gen -- cgit v1.2.3 From 5739d2c3a653639887f7af82795bcb5e8b4b9748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Wed, 4 Dec 2013 16:47:21 -0300 Subject: Revert "Stop creating per-user databases. Closes #4626." This reverts commit 6eeed5c116d38ebae7d9309dd2f11660f6cff37e. --- client/changes/feature_4626_stop-creating-per-user-databases | 2 -- client/src/leap/soledad/client/__init__.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 client/changes/feature_4626_stop-creating-per-user-databases (limited to 'client') diff --git a/client/changes/feature_4626_stop-creating-per-user-databases b/client/changes/feature_4626_stop-creating-per-user-databases deleted file mode 100644 index b967abd3..00000000 --- a/client/changes/feature_4626_stop-creating-per-user-databases +++ /dev/null @@ -1,2 +0,0 @@ - o Stop creating per-user databases, as now there's another service that does - this automatically (tapicero). Closes #4626. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index c168bfdc..62f93b3d 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1038,7 +1038,7 @@ class Soledad(object): if self._db: local_gen = self._db.sync( urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=False) + creds=self._creds, autocreate=True) signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) return local_gen -- cgit v1.2.3 From 5f2a29120ba2d22d7d2406628ce2f6735b60d09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 6 Dec 2013 15:42:59 -0300 Subject: Fold in changes --- client/changes/VERSION_COMPAT | 0 client/changes/feature_4348_add-mac-verification-to-recovery-doc | 2 -- 2 files changed, 2 deletions(-) create mode 100644 client/changes/VERSION_COMPAT delete mode 100644 client/changes/feature_4348_add-mac-verification-to-recovery-doc (limited to 'client') diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT new file mode 100644 index 00000000..e69de29b diff --git a/client/changes/feature_4348_add-mac-verification-to-recovery-doc b/client/changes/feature_4348_add-mac-verification-to-recovery-doc deleted file mode 100644 index 692403f9..00000000 --- a/client/changes/feature_4348_add-mac-verification-to-recovery-doc +++ /dev/null @@ -1,2 +0,0 @@ - o Add MAC verirication to the recovery document and soledad.json. Closes - #4348. -- cgit v1.2.3