summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2013-08-31 19:22:20 -0300
committerdrebs <drebs@leap.se>2013-09-18 11:24:55 -0300
commit4ac07012159bee27dcc57509dd53bfb464cdb1a9 (patch)
tree16fa4c685344d51b95d918a120cc6773719789e1
parent6de096709c54a010e45d71a7508272f74ab25bed (diff)
Improve u1db data storage in couch.
-rw-r--r--common/changes/bug_3647-improve-u1db-data-storage-in-couch1
-rw-r--r--common/src/leap/soledad/common/couch.py269
-rw-r--r--common/src/leap/soledad/common/objectstore.py189
-rw-r--r--common/src/leap/soledad/common/tests/test_couch.py45
4 files changed, 289 insertions, 215 deletions
diff --git a/common/changes/bug_3647-improve-u1db-data-storage-in-couch b/common/changes/bug_3647-improve-u1db-data-storage-in-couch
new file mode 100644
index 00000000..d46475cd
--- /dev/null
+++ b/common/changes/bug_3647-improve-u1db-data-storage-in-couch
@@ -0,0 +1 @@
+ o Improve u1db data storage in couch. Closes #3647.
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 973f8b49..187d3035 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -18,14 +18,12 @@
"""A U1DB backend that uses CouchDB as its persistence layer."""
-import uuid
import re
import simplejson as json
import socket
import logging
-from base64 import b64encode, b64decode
from u1db import errors
from u1db.sync import Synchronizer
from u1db.backends.inmemory import InMemoryIndex
@@ -50,22 +48,146 @@ class InvalidURLError(Exception):
"""
+def persistent_class(cls):
+ """
+ Decorator that modifies a class to ensure u1db metadata persists on
+ underlying storage.
+
+ @param cls: The class that will be modified.
+ @type cls: type
+ """
+
+ def _create_persistent_method(old_method_name, key, load_method_name,
+ dump_method_name, store):
+ """
+ Create a persistent method to replace C{old_method_name}.
+
+ The new method will load C{key} using C{load_method_name} and stores
+ it using C{dump_method_name} depending on the value of C{store}.
+ """
+ # get methods
+ old_method = getattr(cls, old_method_name)
+ load_method = getattr(cls, load_method_name) \
+ if load_method_name is not None \
+ else lambda self, data: setattr(self, key, data)
+ dump_method = getattr(cls, dump_method_name) \
+ if dump_method_name is not None \
+ else lambda self: getattr(self, key)
+
+ def _new_method(self, *args, **kwargs):
+ # get u1db data from couch db
+ doc = self._get_doc('%s%s' %
+ (self.U1DB_DATA_DOC_ID_PREFIX, key))
+ load_method(self, doc.content['content'])
+ # run old method
+ retval = old_method(self, *args, **kwargs)
+ # store u1db data on couch
+ if store:
+ doc.content = {'content': dump_method(self)}
+ self._put_doc(doc)
+ return retval
+
+ return _new_method
+
+ # ensure the class has a persistency map
+ if not hasattr(cls, 'PERSISTENCY_MAP'):
+ logger.error('Class %s has no PERSISTENCY_MAP attribute, skipping '
+ 'persistent methods substitution.' % cls)
+ return cls
+ # replace old methods with new persistent ones
+ for key, ((load_method_name, dump_method_name),
+ persistent_methods) in cls.PERSISTENCY_MAP.iteritems():
+ for (method_name, store) in persistent_methods:
+ setattr(cls, method_name,
+ _create_persistent_method(
+ method_name,
+ key,
+ load_method_name,
+ dump_method_name,
+ store))
+ return cls
+
+
+@persistent_class
class CouchDatabase(ObjectStoreDatabase):
"""
A U1DB backend that uses Couch as its persistence layer.
"""
- U1DB_TRANSACTION_LOG_KEY = 'transaction_log'
- U1DB_CONFLICTS_KEY = 'conflicts'
- U1DB_OTHER_GENERATIONS_KEY = 'other_generations'
- U1DB_INDEXES_KEY = 'indexes'
- U1DB_REPLICA_UID_KEY = 'replica_uid'
+ U1DB_TRANSACTION_LOG_KEY = '_transaction_log'
+ U1DB_CONFLICTS_KEY = '_conflicts'
+ U1DB_OTHER_GENERATIONS_KEY = '_other_generations'
+ U1DB_INDEXES_KEY = '_indexes'
+ U1DB_REPLICA_UID_KEY = '_replica_uid'
+
+ U1DB_DATA_KEYS = [
+ U1DB_TRANSACTION_LOG_KEY,
+ U1DB_CONFLICTS_KEY,
+ U1DB_OTHER_GENERATIONS_KEY,
+ U1DB_INDEXES_KEY,
+ U1DB_REPLICA_UID_KEY,
+ ]
COUCH_ID_KEY = '_id'
COUCH_REV_KEY = '_rev'
COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json'
COUCH_U1DB_REV_KEY = 'u1db_rev'
+ # the following map describes information about methods usage of
+ # properties that have to persist on the underlying database. The format
+ # of the map is assumed to be:
+ #
+ # {
+ # 'property_name': [
+ # ('property_load_method_name', 'property_dump_method_name'),
+ # [('method_1_name', bool),
+ # ...
+ # ('method_N_name', bool)]],
+ # ...
+ # }
+ #
+ # where the booleans indicate if the property should be stored after
+ # each method execution (i.e. if the method alters the property). Property
+ # load/dump methods will be run after/before properties are read/written
+ # to the underlying db.
+ PERSISTENCY_MAP = {
+ U1DB_TRANSACTION_LOG_KEY: [
+ ('_load_transaction_log_from_json', None),
+ [('_get_transaction_log', False),
+ ('_get_generation', False),
+ ('_get_generation_info', False),
+ ('_get_trans_id_for_gen', False),
+ ('whats_changed', False),
+ ('_put_and_update_indexes', True)]],
+ U1DB_CONFLICTS_KEY: [
+ (None, None),
+ [('_has_conflicts', False),
+ ('get_doc_conflicts', False),
+ ('_prune_conflicts', False),
+ ('resolve_doc', False),
+ ('_replace_conflicts', True),
+ ('_force_doc_sync_conflict', True)]],
+ U1DB_OTHER_GENERATIONS_KEY: [
+ ('_load_other_generations_from_json', None),
+ [('_get_replica_gen_and_trans_id', False),
+ ('_do_set_replica_gen_and_trans_id', True)]],
+ U1DB_INDEXES_KEY: [
+ ('_load_indexes_from_json', '_dump_indexes_as_json'),
+ [('list_indexes', False),
+ ('get_from_index', False),
+ ('get_range_from_index', False),
+ ('get_index_keys', False),
+ ('_put_and_update_indexes', True),
+ ('create_index', True),
+ ('delete_index', True)]],
+ U1DB_REPLICA_UID_KEY: [
+ (None, None),
+ [('_allocate_doc_rev', False),
+ ('_put_doc_if_newer', False),
+ ('_ensure_maximal_rev', False),
+ ('_prune_conflicts', False),
+ ('_set_replica_uid', True)]]}
+
@classmethod
def open_database(cls, url, create):
"""
@@ -109,9 +231,11 @@ class CouchDatabase(ObjectStoreDatabase):
@param session: an http.Session instance or None for a default session
@type session: http.Session
"""
+ # save params
self._url = url
self._full_commit = full_commit
self._session = session
+ # configure couch
self._server = Server(url=self._url,
full_commit=self._full_commit,
session=self._session)
@@ -179,7 +303,7 @@ class CouchDatabase(ObjectStoreDatabase):
generation = self._get_generation()
results = []
for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID:
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
continue
doc = self._get_doc(doc_id, check_for_conflicts=True)
if doc.content is None and not include_deleted:
@@ -246,14 +370,12 @@ class CouchDatabase(ObjectStoreDatabase):
raise errors.IndexNameTakenError
index = InMemoryIndex(index_name, list(index_expressions))
for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID: # skip special file
- continue
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ continue # skip special files
doc = self._get_doc(doc_id)
if doc.content is not None:
index.add_json(doc_id, doc.get_json())
self._indexes[index_name] = index
- # save data in object store
- self._store_u1db_data()
def close(self):
"""
@@ -295,76 +417,25 @@ class CouchDatabase(ObjectStoreDatabase):
def _init_u1db_data(self):
"""
- Initialize U1DB info data structure in the couch db.
+ Initialize u1db configuration data on backend storage.
A U1DB database needs to keep track of all database transactions,
document conflicts, the generation of other replicas it has seen,
indexes created by users and so on.
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- if self._replica_uid is None:
- self._replica_uid = uuid.uuid4().hex
- # TODO: prevent user from overwriting a document with the same doc_id
- # as this one.
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
- self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
- self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- }
- self._put_doc(doc)
-
- def _fetch_u1db_data(self):
- """
- Fetch U1DB info from the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- # retrieve u1db data from couch db
- cdoc = self._database.get(self.U1DB_DATA_DOC_ID)
- jsonstr = self._database.get_attachment(
- cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
- content = json.loads(jsonstr)
- # set u1db database info
- self._transaction_log = json.loads(
- b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
- self._conflicts = json.loads(
- b64decode(content[self.U1DB_CONFLICTS_KEY]))
- self._other_generations = json.loads(
- b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
- self._indexes = self._load_indexes_from_json(
- b64decode(content[self.U1DB_INDEXES_KEY]))
- self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
- # save couch _rev
- self._couch_rev = cdoc[self.COUCH_REV_KEY]
-
- def _store_u1db_data(self):
- """
- Store U1DB info in the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- # Here, the b64 encode ensures that document content
- # does not cause strange behaviour in couchdb because
- # of encoding.
- self.U1DB_TRANSACTION_LOG_KEY:
- b64encode(json.dumps(self._transaction_log)),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
- self.U1DB_OTHER_GENERATIONS_KEY:
- b64encode(json.dumps(self._other_generations)),
- self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- self.COUCH_REV_KEY: self._couch_rev}
- self._put_doc(doc)
+ In this implementation, all this information is stored in special
+ documents stored in the underlying with doc_id prefix equal to
+ U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved: put_doc(),
+ get_doc() and delete_doc() will not allow documents with a doc_id with
+ that prefix to be accessed or modified.
+ """
+ for key in self.U1DB_DATA_KEYS:
+ doc_id = '%s%s' % (self.U1DB_DATA_DOC_ID_PREFIX, key)
+ doc = self._get_doc(doc_id)
+ if doc is None:
+ doc = self._factory(doc_id)
+ doc.content = {'content': getattr(self, key)}
+ self._put_doc(doc)
#-------------------------------------------------------------------------
# Couch specific methods
@@ -382,7 +453,7 @@ class CouchDatabase(ObjectStoreDatabase):
def _dump_indexes_as_json(self):
"""
- Dump index definitions as JSON string.
+ Dump index definitions as JSON.
"""
indexes = {}
for name, idx in self._indexes.iteritems():
@@ -390,25 +461,45 @@ class CouchDatabase(ObjectStoreDatabase):
for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY,
self.INDEX_VALUES_KEY]:
indexes[name][attr] = getattr(idx, '_' + attr)
- return json.dumps(indexes)
+ return indexes
def _load_indexes_from_json(self, indexes):
"""
- Load index definitions from JSON string.
+ Load index definitions from stored JSON.
- @param indexes: A JSON serialization of a list of [('index-name',
- ['field', 'field2'])].
+ @param indexes: A JSON representation of indexes as
+ [('index-name', ['field', 'field2', ...]), ...].
@type indexes: str
-
- @return: A dictionary with the index definitions.
- @rtype: dict
"""
- dict = {}
- for name, idx_dict in json.loads(indexes).iteritems():
+ self._indexes = {}
+ for name, idx_dict in indexes.iteritems():
idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY])
idx._values = idx_dict[self.INDEX_VALUES_KEY]
- dict[name] = idx
- return dict
+ self._indexes[name] = idx
+
+ def _load_transaction_log_from_json(self, transaction_log):
+ """
+ Load transaction log from stored JSON.
+
+ @param transaction_log: A JSON representation of transaction_log as
+ [('generation', 'transaction_id'), ...].
+ @type transaction_log: list
+ """
+ self._transaction_log = []
+ for gen, trans_id in transaction_log:
+ self._transaction_log.append((gen, trans_id))
+
+ def _load_other_generations_from_json(self, other_generations):
+ """
+ Load other generations from stored JSON.
+
+ @param other_generations: A JSON representation of other_generations
+ as {'replica_uid': ('generation', 'transaction_id'), ...}.
+ @type other_generations: dict
+ """
+ self._other_generations = {}
+ for replica_uid, [gen, trans_id] in other_generations.iteritems():
+ self._other_generations[replica_uid] = (gen, trans_id)
class CouchSyncTarget(ObjectStoreSyncTarget):
diff --git a/common/src/leap/soledad/common/objectstore.py b/common/src/leap/soledad/common/objectstore.py
index 921cf075..7aff3e32 100644
--- a/common/src/leap/soledad/common/objectstore.py
+++ b/common/src/leap/soledad/common/objectstore.py
@@ -20,6 +20,10 @@
Abstract U1DB backend to handle storage using object stores (like CouchDB, for
example).
+This backend uses special documents to store all U1DB data (replica uid,
+indexes, transaction logs and info about other dbs). The id of these documents
+are reserved and have prefix equal to ObjectStore.U1DB_DATA_DOC_ID_PREFIX.
+
Right now, this is only used by CouchDatabase backend, but can also be
extended to implement OpenStack or Amazon S3 storage, for example.
@@ -27,6 +31,13 @@ See U1DB documentation for more information on how to use databases.
"""
+from base64 import b64encode, b64decode
+
+
+import uuid
+import simplejson as json
+
+
from u1db import errors
from u1db.backends.inmemory import (
InMemoryDatabase,
@@ -39,6 +50,8 @@ class ObjectStoreDatabase(InMemoryDatabase):
A backend for storing u1db data in an object store.
"""
+ U1DB_DATA_DOC_ID_PREFIX = 'u1db/'
+
@classmethod
def open_database(cls, url, create, document_factory=None):
"""
@@ -71,24 +84,52 @@ class ObjectStoreDatabase(InMemoryDatabase):
self,
replica_uid,
document_factory=document_factory)
- # sync data in memory with data in object store
- if not self._get_doc(self.U1DB_DATA_DOC_ID):
- self._init_u1db_data()
- self._fetch_u1db_data()
+ if self._replica_uid is None:
+ self._replica_uid = uuid.uuid4().hex
+ self._init_u1db_data()
+
+ def _init_u1db_data(self):
+ """
+ Initialize u1db configuration data on backend storage.
+
+ A U1DB database needs to keep track of all database transactions,
+ document conflicts, the generation of other replicas it has seen,
+ indexes created by users and so on.
+
+ In this implementation, all this information is stored in special
+ documents stored in the couch db with id prefix equal to
+ U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved:
+ put_doc(), get_doc() and delete_doc() will not allow documents with
+ a doc_id with that prefix to be accessed or modified.
+ """
+ raise NotImplementedError(self._init_u1db_data)
#-------------------------------------------------------------------------
# methods from Database
#-------------------------------------------------------------------------
- def _set_replica_uid(self, replica_uid):
+ def put_doc(self, doc):
"""
- Force the replica_uid to be set.
+ Update a document.
- @param replica_uid: The uid of the replica.
- @type replica_uid: str
+ If the document currently has conflicts, put will fail.
+ If the database specifies a maximum document size and the document
+ exceeds it, put will fail and raise a DocumentTooBig exception.
+
+ This method prevents from updating the document with doc_id equals to
+ self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
+ @param doc: A Document with new content.
+ @type doc: Document
+
+ @return: new_doc_rev - The new revision identifier for the document.
+ The Document object will also be updated.
+ @rtype: str
"""
- InMemoryDatabase._set_replica_uid(self, replica_uid)
- self._store_u1db_data()
+ if doc.doc_id is not None and \
+ doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ raise errors.InvalidDocId()
+ return InMemoryDatabase.put_doc(self, doc)
def _put_doc(self, doc):
"""
@@ -106,6 +147,27 @@ class ObjectStoreDatabase(InMemoryDatabase):
"""
raise NotImplementedError(self._put_doc)
+ def get_doc(self, doc_id, include_deleted=False):
+ """
+ Get the JSON string for the given document.
+
+ This method prevents from getting the document with doc_id equals to
+ self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
+ @param doc_id: The unique document identifier
+ @type doc_id: str
+ @param include_deleted: If set to True, deleted documents will be
+ returned with empty content. Otherwise asking for a deleted
+ document will return None.
+ @type include_deleted: bool
+
+ @return: a Document object.
+ @rtype: Document
+ """
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ raise errors.InvalidDocId()
+ return InMemoryDatabase.get_doc(self, doc_id, include_deleted)
+
def _get_doc(self, doc_id):
"""
Get just the document content, without fancy handling.
@@ -136,18 +198,32 @@ class ObjectStoreDatabase(InMemoryDatabase):
the documents in the database.
@rtype: tuple
"""
- raise NotImplementedError(self.get_all_docs)
+ generation = self._get_generation()
+ results = []
+ for doc_id in self._database:
+ if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ continue
+ doc = self._get_doc(doc_id, check_for_conflicts=True)
+ if doc.content is None and not include_deleted:
+ continue
+ results.append(doc)
+ return (generation, results)
def delete_doc(self, doc):
"""
Mark a document as deleted.
+ This method prevents from deleting the document with doc_id equals to
+ self.U1DB_DATA_DOC_ID, which contains U1DB data.
+
@param doc: The document to mark as deleted.
@type doc: u1db.Document
@return: The new revision id of the document.
@type: str
"""
+ if doc.doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
+ raise errors.InvalidDocId()
old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
if old_doc is None:
raise errors.DocumentDoesNotExist
@@ -177,58 +253,6 @@ class ObjectStoreDatabase(InMemoryDatabase):
"""
raise NotImplementedError(self.create_index)
- def delete_index(self, index_name):
- """
- Remove a named index.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param index_name: The name of the index we are removing.
- @type index_name: str
- """
- InMemoryDatabase.delete_index(self, index_name)
- self._store_u1db_data()
-
- def _replace_conflicts(self, doc, conflicts):
- """
- Set new conflicts for a document.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param doc: The document with a new set of conflicts.
- @param conflicts: The new set of conflicts.
- @type conflicts: list
- """
- InMemoryDatabase._replace_conflicts(self, doc, conflicts)
- self._store_u1db_data()
-
- def _do_set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation,
- other_transaction_id):
- """
- Set the last-known generation and transaction id for the other
- database replica.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param other_replica_uid: The U1DB identifier for the other replica.
- @type other_replica_uid: str
- @param other_generation: The generation number for the other replica.
- @type other_generation: int
- @param other_transaction_id: The transaction id associated with the
- generation.
- @type other_transaction_id: str
- """
- InMemoryDatabase._do_set_replica_gen_and_trans_id(
- self,
- other_replica_uid,
- other_generation,
- other_transaction_id)
- self._store_u1db_data()
-
#-------------------------------------------------------------------------
# implemented methods from CommonBackend
#-------------------------------------------------------------------------
@@ -250,45 +274,6 @@ class ObjectStoreDatabase(InMemoryDatabase):
trans_id = self._allocate_transaction_id()
self._put_doc(doc)
self._transaction_log.append((doc.doc_id, trans_id))
- self._store_u1db_data()
-
- #-------------------------------------------------------------------------
- # methods specific for object stores
- #-------------------------------------------------------------------------
-
- U1DB_DATA_DOC_ID = 'u1db_data'
-
- def _fetch_u1db_data(self):
- """
- Fetch u1db configuration data from backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._fetch_u1db_data)
-
- def _store_u1db_data(self):
- """
- Store u1db configuration data on backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._store_u1db_data)
-
- def _init_u1db_data(self):
- """
- Initialize u1db configuration data on backend storage.
-
- A U1DB database needs to keep track of all database transactions,
- document conflicts, the generation of other replicas it has seen,
- indexes created by users and so on.
-
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- NotImplementedError(self._init_u1db_data)
class ObjectStoreSyncTarget(InMemorySyncTarget):
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 3b49f39e..42edf9fe 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -178,7 +178,12 @@ def copy_couch_database_for_test(test, db):
new_db._conflicts = copy.deepcopy(db._conflicts)
new_db._other_generations = copy.deepcopy(db._other_generations)
new_db._indexes = copy.deepcopy(db._indexes)
- new_db._store_u1db_data()
+ # save u1db data on couch
+ for key in new_db.U1DB_DATA_KEYS:
+ doc_id = '%s%s' % (new_db.U1DB_DATA_DOC_ID_PREFIX, key)
+ doc = new_db._get_doc(doc_id)
+ doc.content = {'content': getattr(new_db, key)}
+ new_db._put_doc(doc)
return new_db
@@ -353,20 +358,18 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
return [self._listify(i) for i in l]
return l
- def _fetch_u1db_data(self, db):
- cdoc = db._database.get(db.U1DB_DATA_DOC_ID)
- jsonstr = db._database.get_attachment(cdoc, 'u1db_json').getvalue()
- return json.loads(jsonstr)
+ def _fetch_u1db_data(self, db, key):
+ doc = db._get_doc("%s%s" % (db.U1DB_DATA_DOC_ID_PREFIX, key))
+ return doc.content['content']
def test_transaction_log_storage_after_put(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
db.create_doc({'simple': 'doc'})
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_TRANSACTION_LOG_KEY)
self.assertEqual(
self._listify(db._transaction_log),
- self._listify(
- json.loads(b64decode(content[db.U1DB_TRANSACTION_LOG_KEY]))))
+ self._listify(content))
def test_conflict_log_storage_after_put_if_newer(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -375,29 +378,27 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
doc.set_json(nested_doc)
doc.rev = db._replica_uid + ':2'
db._force_doc_sync_conflict(doc)
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_CONFLICTS_KEY)
self.assertEqual(
self._listify(db._conflicts),
- self._listify(
- json.loads(b64decode(content[db.U1DB_CONFLICTS_KEY]))))
+ self._listify(content))
def test_other_gens_storage_after_set(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
doc = db.create_doc({'simple': 'doc'})
db._set_replica_gen_and_trans_id('a', 'b', 'c')
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_OTHER_GENERATIONS_KEY)
self.assertEqual(
self._listify(db._other_generations),
- self._listify(
- json.loads(b64decode(content[db.U1DB_OTHER_GENERATIONS_KEY]))))
+ self._listify(content))
def test_index_storage_after_create(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
doc = db.create_doc({'name': 'john'})
db.create_index('myindex', 'name')
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
myind = db._indexes['myindex']
index = {
'myindex': {
@@ -408,8 +409,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
}
self.assertEqual(
self._listify(index),
- self._listify(
- json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
+ self._listify(content))
def test_index_storage_after_delete(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
@@ -418,7 +418,7 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
db.create_index('myindex', 'name')
db.create_index('myindex2', 'name')
db.delete_index('myindex')
- content = self._fetch_u1db_data(db)
+ content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
myind = db._indexes['myindex2']
index = {
'myindex2': {
@@ -429,16 +429,13 @@ class CouchDatabaseStorageTests(CouchDBTestCase):
}
self.assertEqual(
self._listify(index),
- self._listify(
- json.loads(b64decode(content[db.U1DB_INDEXES_KEY]))))
+ self._listify(content))
def test_replica_uid_storage_after_db_creation(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
'u1db_tests')
- content = self._fetch_u1db_data(db)
- self.assertEqual(
- db._replica_uid,
- b64decode(content[db.U1DB_REPLICA_UID_KEY]))
+ content = self._fetch_u1db_data(db, db.U1DB_REPLICA_UID_KEY)
+ self.assertEqual(db._replica_uid, content)
load_tests = tests.load_with_scenarios