diff options
| -rw-r--r-- | changes/feature_encode-all-u1db-data-in-couch-backend | 1 | ||||
| -rw-r--r-- | setup.py | 12 | ||||
| -rw-r--r-- | src/leap/soledad/backends/couch.py | 28 | ||||
| -rw-r--r-- | src/leap/soledad/tests/test_couch.py | 27 |
4 files changed, 36 insertions, 32 deletions
diff --git a/changes/feature_encode-all-u1db-data-in-couch-backend b/changes/feature_encode-all-u1db-data-in-couch-backend new file mode 100644 index 00000000..03660557 --- /dev/null +++ b/changes/feature_encode-all-u1db-data-in-couch-backend @@ -0,0 +1 @@ + o b64 encode all U1DB data in couch backend to avoid utf8 encoding problems. @@ -82,21 +82,11 @@ setup( "LEAP client, an API for data storage and sync." ), namespace_packages=["leap"], - # For now, we do not exclude tests because of the circular dependency - # between leap.common and leap.soledad. - #packages=find_packages('src', exclude=['leap.soledad.tests']), - packages=find_packages('src'), + packages=find_packages('src', exclude=['leap.soledad.tests']), package_dir={'': 'src'}, test_suite='leap.soledad.tests', install_requires=install_requirements, tests_require=tests_requirements, data_files=data_files, classifiers=trove_classifiers, - # the following files are only used for testing, and might be removed if - # we manage or decide to not install tests in the future. - package_data={ - 'leap.soledad.tests.u1db_tests.testing-certs': [ - '*.pem', '*.cert', '*.key' - ] - } ) diff --git a/src/leap/soledad/backends/couch.py b/src/leap/soledad/backends/couch.py index 0b9bff01..57885012 100644 --- a/src/leap/soledad/backends/couch.py +++ b/src/leap/soledad/backends/couch.py @@ -312,11 +312,13 @@ class CouchDatabase(ObjectStoreDatabase): # TODO: prevent user from overwriting a document with the same doc_id # as this one. doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) - doc.content = {self.U1DB_TRANSACTION_LOG_KEY: [], - self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})), - self.U1DB_OTHER_GENERATIONS_KEY: {}, - self.U1DB_INDEXES_KEY: b64encode(json.dumps({})), - self.U1DB_REPLICA_UID_KEY: self._replica_uid} + doc.content = { + self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])), + self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})), + self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})), + self.U1DB_INDEXES_KEY: b64encode(json.dumps({})), + self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid), + } self._put_doc(doc) def _fetch_u1db_data(self): @@ -331,13 +333,15 @@ class CouchDatabase(ObjectStoreDatabase): cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read() content = json.loads(jsonstr) # set u1db database info - self._transaction_log = content[self.U1DB_TRANSACTION_LOG_KEY] + self._transaction_log = json.loads( + b64decode(content[self.U1DB_TRANSACTION_LOG_KEY])) self._conflicts = json.loads( b64decode(content[self.U1DB_CONFLICTS_KEY])) - self._other_generations = content[self.U1DB_OTHER_GENERATIONS_KEY] + self._other_generations = json.loads( + b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY])) self._indexes = self._load_indexes_from_json( b64decode(content[self.U1DB_INDEXES_KEY])) - self._replica_uid = content[self.U1DB_REPLICA_UID_KEY] + self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY]) # save couch _rev self._couch_rev = cdoc[self.COUCH_REV_KEY] @@ -349,14 +353,16 @@ class CouchDatabase(ObjectStoreDatabase): """ doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) doc.content = { - self.U1DB_TRANSACTION_LOG_KEY: self._transaction_log, # Here, the b64 encode ensures that document content # does not cause strange behaviour in couchdb because # of encoding. + self.U1DB_TRANSACTION_LOG_KEY: + b64encode(json.dumps(self._transaction_log)), self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)), - self.U1DB_OTHER_GENERATIONS_KEY: self._other_generations, + self.U1DB_OTHER_GENERATIONS_KEY: + b64encode(json.dumps(self._other_generations)), self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()), - self.U1DB_REPLICA_UID_KEY: self._replica_uid, + self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid), self.COUCH_REV_KEY: self._couch_rev} self._put_doc(doc) diff --git a/src/leap/soledad/tests/test_couch.py b/src/leap/soledad/tests/test_couch.py index b3cbc1bc..60a61b71 100644 --- a/src/leap/soledad/tests/test_couch.py +++ b/src/leap/soledad/tests/test_couch.py @@ -360,7 +360,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase): content = self._fetch_u1db_data(db) self.assertEqual( self._listify(db._transaction_log), - self._listify(content['transaction_log'])) + self._listify( + json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY])))) def test_conflict_log_storage_after_put_if_newer(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), @@ -372,7 +373,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase): content = self._fetch_u1db_data(db) self.assertEqual( self._listify(db._conflicts), - self._listify(json.loads(b64decode(content['conflicts'])))) + self._listify( + json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY])))) def test_other_gens_storage_after_set(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), @@ -382,7 +384,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase): content = self._fetch_u1db_data(db) self.assertEqual( self._listify(db._other_generations), - self._listify(content['other_generations'])) + self._listify( + json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY])))) def test_index_storage_after_create(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), @@ -398,9 +401,10 @@ class CouchDatabaseStorageTests(CouchDBTestCase): 'values': myind._values, } } - self.assertEqual(self._listify(index), - self._listify( - json.loads(b64decode(content['indexes'])))) + self.assertEqual( + self._listify(index), + self._listify( + json.loads(b64decode(content[db.U1DB_INDEXES_KEY])))) def test_index_storage_after_delete(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), @@ -418,15 +422,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase): 'values': myind._values, } } - self.assertEqual(self._listify(index), - self._listify( - json.loads(b64decode(content['indexes'])))) + self.assertEqual( + self._listify(index), + self._listify( + json.loads(b64decode(content[db.U1DB_INDEXES_KEY])))) def test_replica_uid_storage_after_db_creation(self): db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), 'u1db_tests') content = self._fetch_u1db_data(db) - self.assertEqual(db._replica_uid, content['replica_uid']) + self.assertEqual( + db._replica_uid, + b64decode(content[db.U1DB_REPLICA_UID_KEY])) load_tests = tests.load_with_scenarios |