Merge remote-tracking branch 'drebs/feature/2359-avoid-encoding-problems-when-storing-u1db-data' into develop

4 files changed, 36 insertions, 32 deletions

diff --git a/changes/feature_encode-all-u1db-data-in-couch-backend b/changes/feature_encode-all-u1db-data-in-couch-backend
new file mode 100644
index 00000000..03660557
--- /dev/null
+++ b/changes/feature_encode-all-u1db-data-in-couch-backend
@@ -0,0 +1 @@
+  o b64 encode all U1DB data in couch backend to avoid utf8 encoding problems.
diff --git a/setup.py b/setup.py
index 857ec6cb..b1c0b9db 100644
--- a/setup.py
+++ b/setup.py
@@ -82,21 +82,11 @@ setup(
         "LEAP client, an API for data storage and sync."
     ),
     namespace_packages=["leap"],
-    # For now, we do not exclude tests because of the circular dependency
-    # between leap.common and leap.soledad.
-    #packages=find_packages('src', exclude=['leap.soledad.tests']),
-    packages=find_packages('src'),
+    packages=find_packages('src', exclude=['leap.soledad.tests']),
     package_dir={'': 'src'},
     test_suite='leap.soledad.tests',
     install_requires=install_requirements,
     tests_require=tests_requirements,
     data_files=data_files,
     classifiers=trove_classifiers,
-    # the following files are only used for testing, and might be removed if
-    # we manage or decide to not install tests in the future.
-    package_data={
-        'leap.soledad.tests.u1db_tests.testing-certs': [
-            '*.pem', '*.cert', '*.key'
-        ]
-    }
 )
diff --git a/src/leap/soledad/backends/couch.py b/src/leap/soledad/backends/couch.py
index 0b9bff01..57885012 100644
--- a/src/leap/soledad/backends/couch.py
+++ b/src/leap/soledad/backends/couch.py
@@ -312,11 +312,13 @@ class CouchDatabase(ObjectStoreDatabase):
         # TODO: prevent user from overwriting a document with the same doc_id
         # as this one.
         doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
-        doc.content = {self.U1DB_TRANSACTION_LOG_KEY: [],
-                       self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
-                       self.U1DB_OTHER_GENERATIONS_KEY: {},
-                       self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
-                       self.U1DB_REPLICA_UID_KEY: self._replica_uid}
+        doc.content = {
+            self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
+            self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
+            self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
+            self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
+            self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
+        }
         self._put_doc(doc)
 
     def _fetch_u1db_data(self):
@@ -331,13 +333,15 @@ class CouchDatabase(ObjectStoreDatabase):
             cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
         content = json.loads(jsonstr)
         # set u1db database info
-        self._transaction_log = content[self.U1DB_TRANSACTION_LOG_KEY]
+        self._transaction_log = json.loads(
+            b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
         self._conflicts = json.loads(
             b64decode(content[self.U1DB_CONFLICTS_KEY]))
-        self._other_generations = content[self.U1DB_OTHER_GENERATIONS_KEY]
+        self._other_generations = json.loads(
+            b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
         self._indexes = self._load_indexes_from_json(
             b64decode(content[self.U1DB_INDEXES_KEY]))
-        self._replica_uid = content[self.U1DB_REPLICA_UID_KEY]
+        self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
         # save couch _rev
         self._couch_rev = cdoc[self.COUCH_REV_KEY]
 
@@ -349,14 +353,16 @@ class CouchDatabase(ObjectStoreDatabase):
         """
         doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
         doc.content = {
-            self.U1DB_TRANSACTION_LOG_KEY: self._transaction_log,
             # Here, the b64 encode ensures that document content
             # does not cause strange behaviour in couchdb because
             # of encoding.
+            self.U1DB_TRANSACTION_LOG_KEY:
+                b64encode(json.dumps(self._transaction_log)),
             self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
-            self.U1DB_OTHER_GENERATIONS_KEY: self._other_generations,
+            self.U1DB_OTHER_GENERATIONS_KEY:
+                b64encode(json.dumps(self._other_generations)),
             self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
-            self.U1DB_REPLICA_UID_KEY: self._replica_uid,
+            self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
             self.COUCH_REV_KEY: self._couch_rev}
         self._put_doc(doc)
 
diff --git a/src/leap/soledad/tests/test_couch.py b/src/leap/soledad/tests/test_couch.py
index b3cbc1bc..60a61b71 100644
--- a/src/leap/soledad/tests/test_couch.py
+++ b/src/leap/soledad/tests/test_couch.py
@@ -360,7 +360,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         content = self._fetch_u1db_data(db)
         self.assertEqual(
             self._listify(db._transaction_log),
-            self._listify(content['transaction_log']))
+            self._listify(
+                json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY]))))
 
     def test_conflict_log_storage_after_put_if_newer(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -372,7 +373,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         content = self._fetch_u1db_data(db)
         self.assertEqual(
             self._listify(db._conflicts),
-            self._listify(json.loads(b64decode(content['conflicts']))))
+            self._listify(
+                json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY]))))
 
     def test_other_gens_storage_after_set(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -382,7 +384,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
         content = self._fetch_u1db_data(db)
         self.assertEqual(
             self._listify(db._other_generations),
-            self._listify(content['other_generations']))
+            self._listify(
+                json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY]))))
 
     def test_index_storage_after_create(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -398,9 +401,10 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
                 'values': myind._values,
             }
         }
-        self.assertEqual(self._listify(index),
-                         self._listify(
-                             json.loads(b64decode(content['indexes']))))
+        self.assertEqual(
+            self._listify(index),
+            self._listify(
+                json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
 
     def test_index_storage_after_delete(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -418,15 +422,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
                 'values': myind._values,
             }
         }
-        self.assertEqual(self._listify(index),
-                         self._listify(
-                             json.loads(b64decode(content['indexes']))))
+        self.assertEqual(
+            self._listify(index),
+            self._listify(
+                json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
 
     def test_replica_uid_storage_after_db_creation(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
                                  'u1db_tests')
         content = self._fetch_u1db_data(db)
-        self.assertEqual(db._replica_uid, content['replica_uid'])
+        self.assertEqual(
+            db._replica_uid,
+            b64decode(content[db.U1DB_REPLICA_UID_KEY]))
 
 
 load_tests = tests.load_with_scenarios