Encrypt storage secret using scrypt KDF.

* Change format of storage.
* Refactor and modify export/import recovery documents logic.
* Change storage secret length to 512.
* Encrypt the storage document with derived function.
* Refactor property name inside crypto submodule.
* Add docstrings for new methods.
* Fix wrong method call import_recovery_document.
* Fix base64 encoding and secret/kdf length info.
* Add missing header.
* Add missing fields to comment.

Closes 2475 and 2423.

3 files changed, 47 insertions, 51 deletions

diff --git a/src/leap/soledad/tests/__init__.py b/src/leap/soledad/tests/__init__.py
index d947f5b3..5cd23e45 100644
--- a/src/leap/soledad/tests/__init__.py
+++ b/src/leap/soledad/tests/__init__.py
@@ -40,9 +40,9 @@ class BaseSoledadTest(BaseLeapTest):
         self._soledad._init_dirs()
         #self._soledad._gpg.import_keys(PUBLIC_KEY)
         self._soledad._crypto = SoledadCrypto(self._soledad)
-        if not self._soledad._has_symkey():
-            self._soledad._gen_symkey()
-        self._soledad._load_symkey()
+        if not self._soledad._has_secret():
+            self._soledad._gen_secret()
+        self._soledad._load_secret()
         self._soledad._init_db()
 
     def tearDown(self):
@@ -51,12 +51,13 @@ class BaseSoledadTest(BaseLeapTest):
         self._soledad.close()
 
     def _soledad_instance(self, user='leap@leap.se', prefix='',
-                          bootstrap=False, secret_path='/secret.gpg',
+                          bootstrap=False,
+                          secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
                           local_db_path='/soledad.u1db'):
         return Soledad(
             user,
             '123',
-            secret_path=self.tempdir+prefix+secret_path,
+            secrets_path=self.tempdir+prefix+secrets_path,
             local_db_path=self.tempdir+prefix+local_db_path,
             server_url='',  # Soledad will fail if not given an url.
             cert_file=None,
diff --git a/src/leap/soledad/tests/test_crypto.py b/src/leap/soledad/tests/test_crypto.py
index abe32661..8f9145c8 100644
--- a/src/leap/soledad/tests/test_crypto.py
+++ b/src/leap/soledad/tests/test_crypto.py
@@ -127,13 +127,13 @@ class EncryptedSyncTestCase(BaseSoledadTest):
 #
 #    def _soledad_instance(self, user='leap@leap.se', prefix='',
 #                          bootstrap=False, gnupg_home='/gnupg',
-#                          secret_path='/secret.gpg',
+#                          secrets_path='/secret.gpg',
 #                          local_db_path='/soledad.u1db'):
 #        return Soledad(
 #            user,
 #            '123',
 #            gnupg_home=self.tempdir+prefix+gnupg_home,
-#            secret_path=self.tempdir+prefix+secret_path,
+#            secrets_path=self.tempdir+prefix+secrets_path,
 #            local_db_path=self.tempdir+prefix+local_db_path,
 #            bootstrap=bootstrap)
 #
@@ -145,9 +145,9 @@ class EncryptedSyncTestCase(BaseSoledadTest):
 #        self._soledad = self._soledad_instance('leap@leap.se')
 #        self._soledad._init_dirs()
 #        self._soledad._crypto = SoledadCrypto(self._soledad)
-#        if not self._soledad._has_symkey():
-#            self._soledad._gen_symkey()
-#        self._soledad._load_symkey()
+#        if not self._soledad._has_get_storage_secret()():
+#            self._soledad._gen_get_storage_secret()()
+#        self._soledad._load_get_storage_secret()()
 #        self._soledad._init_db()
 #
 #    def tearDown(self):
@@ -186,15 +186,15 @@ class EncryptedSyncTestCase(BaseSoledadTest):
 class RecoveryDocumentTestCase(BaseSoledadTest):
 
     def test_export_recovery_document_raw(self):
-        rd = self._soledad.export_recovery_document(None)
-        self.assertEqual(
-            {
-                self._soledad.UUID_KEY: self._soledad._uuid,
-                self._soledad.SYMKEY_KEY: self._soledad._symkey
-            },
-            json.loads(rd),
-            "Could not export raw recovery document."
-        )
+        rd = json.loads(self._soledad.export_recovery_document(None))
+        secret_id = rd[self._soledad.STORAGE_SECRETS_KEY].items()[0][0]
+        secret = rd[self._soledad.STORAGE_SECRETS_KEY][secret_id]
+        self.assertEqual(secret_id, self._soledad._secret_id)
+        self.assertEqual(secret, self._soledad._secrets[secret_id])
+        self.assertTrue(self._soledad.CIPHER_KEY in secret)
+        self.assertTrue(secret[self._soledad.CIPHER_KEY] == 'aes256')
+        self.assertTrue(self._soledad.LENGTH_KEY in secret)
+        self.assertTrue(self._soledad.SECRET_KEY in secret)
 
     def test_export_recovery_document_crypt(self):
         rd = self._soledad.export_recovery_document('123456')
@@ -202,7 +202,7 @@ class RecoveryDocumentTestCase(BaseSoledadTest):
                          self._soledad._crypto.is_encrypted_sym(rd))
         data = {
             self._soledad.UUID_KEY: self._soledad._uuid,
-            self._soledad.SYMKEY_KEY: self._soledad._symkey,
+            self._soledad.STORAGE_SECRETS_KEY: self._soledad._secrets,
         }
         raw_data = json.loads(self._soledad._crypto.decrypt_sym(
             rd,
@@ -213,11 +213,6 @@ class RecoveryDocumentTestCase(BaseSoledadTest):
             "Could not export raw recovery document."
         )
 
-    def test_import_recovery_document_raises_exception(self):
-        rd = self._soledad.export_recovery_document(None)
-        self.assertRaises(KeyAlreadyExists,
-                          self._soledad.import_recovery_document, rd, None)
-
     def test_import_recovery_document_raw(self):
         rd = self._soledad.export_recovery_document(None)
         s = self._soledad_instance(user='anotheruser@leap.se', prefix='/2')
@@ -226,8 +221,8 @@ class RecoveryDocumentTestCase(BaseSoledadTest):
         s.import_recovery_document(rd, None)
         self.assertEqual(self._soledad._uuid,
                          s._uuid, 'Failed setting user uuid.')
-        self.assertEqual(self._soledad._symkey,
-                         s._symkey,
+        self.assertEqual(self._soledad._get_storage_secret(),
+                         s._get_storage_secret(),
                          'Failed settinng secret for symmetric encryption.')
 
     def test_import_recovery_document_crypt(self):
@@ -238,26 +233,26 @@ class RecoveryDocumentTestCase(BaseSoledadTest):
         s.import_recovery_document(rd, '123456')
         self.assertEqual(self._soledad._uuid,
                          s._uuid, 'Failed setting user uuid.')
-        self.assertEqual(self._soledad._symkey,
-                         s._symkey,
+        self.assertEqual(self._soledad._get_storage_secret(),
+                         s._get_storage_secret(),
                          'Failed settinng secret for symmetric encryption.')
 
 
 class CryptoMethodsTestCase(BaseSoledadTest):
 
-    def test__gen_symkey(self):
+    def test__gen_secret(self):
         sol = self._soledad_instance(user='user@leap.se', prefix='/3')
         sol._init_dirs()
         sol._crypto = SoledadCrypto(sol)
-        self.assertFalse(sol._has_symkey(), "Should not have a symkey at "
+        self.assertFalse(sol._has_secret(), "Should not have a secret at "
                                             "this point")
-        sol._gen_symkey()
-        self.assertTrue(sol._has_symkey(), "Could not generate symkey.")
+        sol._gen_secret()
+        self.assertTrue(sol._has_secret(), "Could not generate secret.")
 
-    def test__has_keys(self):
+    def test__has_secret(self):
         sol = self._soledad_instance(user='leap@leap.se', prefix='/5')
         sol._init_dirs()
         sol._crypto = SoledadCrypto(sol)
-        self.assertFalse(sol._has_keys())
-        sol._gen_symkey()
-        self.assertTrue(sol._has_keys())
+        self.assertFalse(sol._has_secret())
+        sol._gen_secret()
+        self.assertTrue(sol._has_secret())
diff --git a/src/leap/soledad/tests/test_soledad.py b/src/leap/soledad/tests/test_soledad.py
index 6031c704..018ce8d7 100644
--- a/src/leap/soledad/tests/test_soledad.py
+++ b/src/leap/soledad/tests/test_soledad.py
@@ -45,18 +45,18 @@ class AuxMethodsTestCase(BaseSoledadTest):
         sol = self._soledad_instance(prefix='/_init_dirs')
         sol._init_dirs()
         local_db_dir = os.path.dirname(sol.local_db_path)
-        secret_path = os.path.dirname(sol.secret_path)
+        secrets_path = os.path.dirname(sol.secrets_path)
         self.assertTrue(os.path.isdir(local_db_dir))
-        self.assertTrue(os.path.isdir(secret_path))
+        self.assertTrue(os.path.isdir(secrets_path))
 
     def test__init_db(self):
         sol = self._soledad_instance()
         sol._init_dirs()
         sol._crypto = SoledadCrypto(sol)
         #self._soledad._gpg.import_keys(PUBLIC_KEY)
-        if not sol._has_symkey():
-            sol._gen_symkey()
-        sol._load_symkey()
+        if not sol._has_secret():
+            sol._gen_secret()
+        sol._load_secret()
         sol._init_db()
         from leap.soledad.backends.sqlcipher import SQLCipherDatabase
         self.assertIsInstance(sol._db, SQLCipherDatabase)
@@ -66,11 +66,11 @@ class AuxMethodsTestCase(BaseSoledadTest):
         Test if configuration defaults point to the correct place.
         """
         sol = Soledad('leap@leap.se', passphrase='123', bootstrap=False,
-                      secret_path=None, local_db_path=None,
+                      secrets_path=None, local_db_path=None,
                       server_url='', cert_file=None)  # otherwise Soledad will fail.
         self.assertEquals(
-            os.path.join(sol.DEFAULT_PREFIX, 'secret.gpg'),
-            sol.secret_path)
+            os.path.join(sol.DEFAULT_PREFIX, Soledad.STORAGE_SECRETS_FILE_NAME),
+            sol.secrets_path)
         self.assertEquals(
             os.path.join(sol.DEFAULT_PREFIX, 'soledad.u1db'),
             sol.local_db_path)
@@ -83,11 +83,11 @@ class AuxMethodsTestCase(BaseSoledadTest):
             'leap@leap.se',
             passphrase='123',
             bootstrap=False,
-            secret_path='value_3',
+            secrets_path='value_3',
             local_db_path='value_2',
             server_url='value_1',
             cert_file=None)
-        self.assertEqual('value_3', sol.secret_path)
+        self.assertEqual('value_3', sol.secrets_path)
         self.assertEqual('value_2', sol.local_db_path)
         self.assertEqual('value_1', sol.server_url)
 
@@ -102,7 +102,7 @@ class SoledadSharedDBTestCase(BaseSoledadTest):
         self._shared_db = SoledadSharedDatabase(
             'https://provider/', LeapDocument, None)
 
-    def test__fetch_keys_from_shared_db(self):
+    def test__get_secrets_from_shared_db(self):
         """
         Ensure the shared db is queried with the correct doc_id.
         """
@@ -116,13 +116,13 @@ class SoledadSharedDBTestCase(BaseSoledadTest):
 
         self._soledad._shared_db = MockSharedDB()
         doc_id = self._soledad._uuid_hash()
-        self._soledad._fetch_keys_from_shared_db()
+        self._soledad._get_secrets_from_shared_db()
         self.assertTrue(
             self._soledad._shared_db().get_doc.assert_called_once_with(
                 doc_id) is None,
             'Wrong doc_id when fetching recovery document.')
 
-    def test__assert_keys_in_shared_db(self):
+    def test__put_secrets_in_shared_db(self):
         """
         Ensure recovery document is put into shared recover db.
         """
@@ -140,7 +140,7 @@ class SoledadSharedDBTestCase(BaseSoledadTest):
 
         self._soledad._shared_db = MockSharedDB()
         doc_id = self._soledad._uuid_hash()
-        self._soledad._assert_keys_in_shared_db()
+        self._soledad._put_secrets_in_shared_db()
         self.assertTrue(
             self._soledad._shared_db().get_doc.assert_called_once_with(
                 doc_id) is None,