summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2013-06-01 11:36:04 -0300
committerdrebs <drebs@leap.se>2013-06-01 11:43:02 -0300
commit917a00257b9a8b6f40457dba75463b886cb143db (patch)
treeb6cd4cc161a667d6701a5458263a872e9d2ce466
parent626c83924a81c436cf5374b6ee7a448031fdc6ae (diff)
Encode all u1db data before storing.
-rw-r--r--changes/feature_encode-all-u1db-data-in-couch-backend1
-rw-r--r--setup.py12
-rw-r--r--src/leap/soledad/backends/couch.py28
-rw-r--r--src/leap/soledad/tests/test_couch.py27
4 files changed, 36 insertions, 32 deletions
diff --git a/changes/feature_encode-all-u1db-data-in-couch-backend b/changes/feature_encode-all-u1db-data-in-couch-backend
new file mode 100644
index 00000000..03660557
--- /dev/null
+++ b/changes/feature_encode-all-u1db-data-in-couch-backend
@@ -0,0 +1 @@
+ o b64 encode all U1DB data in couch backend to avoid utf8 encoding problems.
diff --git a/setup.py b/setup.py
index 857ec6cb..b1c0b9db 100644
--- a/setup.py
+++ b/setup.py
@@ -82,21 +82,11 @@ setup(
"LEAP client, an API for data storage and sync."
),
namespace_packages=["leap"],
- # For now, we do not exclude tests because of the circular dependency
- # between leap.common and leap.soledad.
- #packages=find_packages('src', exclude=['leap.soledad.tests']),
- packages=find_packages('src'),
+ packages=find_packages('src', exclude=['leap.soledad.tests']),
package_dir={'': 'src'},
test_suite='leap.soledad.tests',
install_requires=install_requirements,
tests_require=tests_requirements,
data_files=data_files,
classifiers=trove_classifiers,
- # the following files are only used for testing, and might be removed if
- # we manage or decide to not install tests in the future.
- package_data={
- 'leap.soledad.tests.u1db_tests.testing-certs': [
- '*.pem', '*.cert', '*.key'
- ]
- }
)
diff --git a/src/leap/soledad/backends/couch.py b/src/leap/soledad/backends/couch.py
index 0b9bff01..57885012 100644
--- a/src/leap/soledad/backends/couch.py
+++ b/src/leap/soledad/backends/couch.py
@@ -312,11 +312,13 @@ class CouchDatabase(ObjectStoreDatabase):
# TODO: prevent user from overwriting a document with the same doc_id
# as this one.
doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {self.U1DB_TRANSACTION_LOG_KEY: [],
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
- self.U1DB_OTHER_GENERATIONS_KEY: {},
- self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
- self.U1DB_REPLICA_UID_KEY: self._replica_uid}
+ doc.content = {
+ self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
+ self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
+ self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
+ self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
+ self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
+ }
self._put_doc(doc)
def _fetch_u1db_data(self):
@@ -331,13 +333,15 @@ class CouchDatabase(ObjectStoreDatabase):
cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
content = json.loads(jsonstr)
# set u1db database info
- self._transaction_log = content[self.U1DB_TRANSACTION_LOG_KEY]
+ self._transaction_log = json.loads(
+ b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
self._conflicts = json.loads(
b64decode(content[self.U1DB_CONFLICTS_KEY]))
- self._other_generations = content[self.U1DB_OTHER_GENERATIONS_KEY]
+ self._other_generations = json.loads(
+ b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
self._indexes = self._load_indexes_from_json(
b64decode(content[self.U1DB_INDEXES_KEY]))
- self._replica_uid = content[self.U1DB_REPLICA_UID_KEY]
+ self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
# save couch _rev
self._couch_rev = cdoc[self.COUCH_REV_KEY]
@@ -349,14 +353,16 @@ class CouchDatabase(ObjectStoreDatabase):
"""
doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
doc.content = {
- self.U1DB_TRANSACTION_LOG_KEY: self._transaction_log,
# Here, the b64 encode ensures that document content
# does not cause strange behaviour in couchdb because
# of encoding.
+ self.U1DB_TRANSACTION_LOG_KEY:
+ b64encode(json.dumps(self._transaction_log)),
self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
- self.U1DB_OTHER_GENERATIONS_KEY: self._other_generations,
+ self.U1DB_OTHER_GENERATIONS_KEY:
+ b64encode(json.dumps(self._other_generations)),
self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
- self.U1DB_REPLICA_UID_KEY: self._replica_uid,
+ self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
self.COUCH_REV_KEY: self._couch_rev}
self._put_doc(doc)
diff --git a/src/leap/soledad/tests/test_couch.py b/src/leap/soledad/tests/test_couch.py
index b3cbc1bc..60a61b71 100644
--- a/src/leap/soledad/tests/test_couch.py
+++ b/src/leap/soledad/tests/test_couch.py
@@ -360,7 +360,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
content = self._fetch_u1db_data(db)
self.assertEqual(
self._listify(db._transaction_log),
- self._listify(content['transaction_log']))
+ self._listify(
+ json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY]))))
def test_conflict_log_storage_after_put_if_newer(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -372,7 +373,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
content = self._fetch_u1db_data(db)
self.assertEqual(
self._listify(db._conflicts),
- self._listify(json.loads(b64decode(content['conflicts']))))
+ self._listify(
+ json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY]))))
def test_other_gens_storage_after_set(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -382,7 +384,8 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
content = self._fetch_u1db_data(db)
self.assertEqual(
self._listify(db._other_generations),
- self._listify(content['other_generations']))
+ self._listify(
+ json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY]))))
def test_index_storage_after_create(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -398,9 +401,10 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
'values': myind._values,
}
}
- self.assertEqual(self._listify(index),
- self._listify(
- json.loads(b64decode(content['indexes']))))
+ self.assertEqual(
+ self._listify(index),
+ self._listify(
+ json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
def test_index_storage_after_delete(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -418,15 +422,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
'values': myind._values,
}
}
- self.assertEqual(self._listify(index),
- self._listify(
- json.loads(b64decode(content['indexes']))))
+ self.assertEqual(
+ self._listify(index),
+ self._listify(
+ json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
def test_replica_uid_storage_after_db_creation(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
content = self._fetch_u1db_data(db)
- self.assertEqual(db._replica_uid, content['replica_uid'])
+ self.assertEqual(
+ db._replica_uid,
+ b64decode(content[db.U1DB_REPLICA_UID_KEY]))
load_tests = tests.load_with_scenarios