From f89f2e0fe490899ecc4baf3395f3441111da328f Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Dec 2012 11:00:10 -0200 Subject: Refactor to add ObjectStore class. --- src/leap/soledad/backends/objectstore.py | 153 +++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 src/leap/soledad/backends/objectstore.py (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py new file mode 100644 index 00000000..e36df72d --- /dev/null +++ b/src/leap/soledad/backends/objectstore.py @@ -0,0 +1,153 @@ +from u1db.backends import CommonBackend + + +class ObjectStore(CommonBackend): + + def __init__(self): + self._sync_log = SyncLog() + self._transaction_log = TransactionLog() + + #------------------------------------------------------------------------- + # implemented methods from Database + #------------------------------------------------------------------------- + + def set_document_factory(self, factory): + self._factory = factory + + def set_document_size_limit(self, limit): + raise NotImplementedError(self.set_document_size_limit) + + def whats_changed(self, old_generation=0): + self._get_u1db_data() + return self._transaction_log.whats_changed(old_generation) + + def get_doc(self, doc_id, include_deleted=False): + doc = self._get_doc(doc_id, check_for_conflicts=True) + if doc is None: + return None + if doc.is_tombstone() and not include_deleted: + return None + return doc + + def delete_doc(self, doc): + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc is None: + raise errors.DocumentDoesNotExist + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + if old_doc.is_tombstone(): + raise errors.DocumentAlreadyDeleted + if old_doc.has_conflicts: + raise errors.ConflictedDoc() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + doc.make_tombstone() + self._put_doc(olddoc) + return new_rev + + # start of index-related methods: these are not supported by this backend. + + def create_index(self, index_name, *index_expressions): + return False + + def delete_index(self, index_name): + return False + + def list_indexes(self): + return [] + + def get_from_index(self, index_name, *key_values): + return [] + + def get_range_from_index(self, index_name, start_value=None, + end_value=None): + return [] + + def get_index_keys(self, index_name): + return [] + + # end of index-related methods: these are not supported by this backend. + + def get_doc_conflicts(self, doc_id): + return [] + + def resolve_doc(self, doc, conflicted_doc_revs): + raise NotImplementedError(self.resolve_doc) + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + self._get_u1db_data() + return self._sync_log.get_replica_gen_and_trans_id(other_replica_uid) + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + self._get_u1db_data() + self._sync_log.set_replica_gen_and_trans_id(other_replica_uid, + other_generation, + other_transaction_id) + self._set_u1db_data() + + #------------------------------------------------------------------------- + # implemented methods from CommonBackend + #------------------------------------------------------------------------- + + def _get_generation(self): + self._get_u1db_data() + return self._transaction_log.get_generation() + + def _get_generation_info(self): + self._get_u1db_data() + return self._transaction_log.get_generation_info() + + def _has_conflicts(self, doc_id): + # Documents never have conflicts on server. + return False + + def _put_and_update_indexes(self, doc_id, old_doc, new_rev, content): + raise NotImplementedError(self._put_and_update_indexes) + + + def _get_trans_id_for_gen(self, generation): + self._get_u1db_data() + trans_id = self._transaction_log.get_trans_id_for_gen(generation) + if trans_id is None: + raise errors.InvalidGeneration + return trans_id + + def _ensure_u1db_data(self): + """ + Guarantee that u1db data exists in store. + """ + if not self._is_initialized(): + self._initialize() + u1db_data = self._get_doc('u1db_data') + self._sync_log.log = u1db_data.content['sync_log'] + self._transaction_log.log = u1db_data.content['transaction_log'] + + def _is_initialized(self): + """ + Verify if u1db data exists in store. + """ + if not self._get_doc('u1db_data'): + return False + return True + + def _initialize(self): + """ + Create u1db data object in store. + """ + content = { 'transaction_log' : [], + 'sync_log' : [] } + doc = self.create_doc('u1db_data', content) + + def _get_u1db_data(self): + data = self.get_doc('u1db_data').content + self._transaction_log = data['transaction_log'] + self._sync_log = data['sync_log'] + + def _set_u1db_data(self): + doc = self._factory('u1db_data') + doc.content = { 'transaction_log' : self._transaction_log, + 'sync_log' : self._sync_log } + self.put_doc(doc) + + -- cgit v1.2.3 From b3090f710e3777bad2a9f996444e5099883c9f03 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Dec 2012 12:05:31 -0200 Subject: Add CouchDB u1db backend. --- src/leap/soledad/backends/objectstore.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index e36df72d..456892b3 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -1,11 +1,17 @@ from u1db.backends import CommonBackend +from soledad import SyncLog, TransactionLog class ObjectStore(CommonBackend): def __init__(self): + # This initialization method should be called after the connection + # with the database is established, so it can ensure that u1db data is + # configured and up-to-date. + self.set_document_factory(LeapDocument) self._sync_log = SyncLog() self._transaction_log = TransactionLog() + self._ensure_u1db_data() #------------------------------------------------------------------------- # implemented methods from Database @@ -29,6 +35,26 @@ class ObjectStore(CommonBackend): return None return doc + def _put_doc(self, doc) + raise NotImplementedError(self._put_doc) + + def put_doc(self, doc) + # consistency check + if doc.doc_id is None: + raise errors.InvalidDocId() + self._check_doc_id(doc.doc_id) + self._check_doc_size(doc) + # put the document + new_rev = self._allocate_doc_rev(doc.rev) + self._put_doc(doc, new_rev) + doc.rev = new_rev + # update u1db generation and logs + new_gen = self._get_generation() + 1 + trans_id = self._allocate_transaction_id() + self._transaction_log.append((new_gen, doc.doc_id, trans_id)) + self._set_u1db_data() + return new_rev + def delete_doc(self, doc): old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) if old_doc is None: -- cgit v1.2.3 From 817d4a1dab5cfce6228593ad61951e1593777eeb Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Dec 2012 14:43:08 -0200 Subject: Fix lack of collons on some methods. --- src/leap/soledad/backends/objectstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index 456892b3..d9ab7cbd 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -35,10 +35,10 @@ class ObjectStore(CommonBackend): return None return doc - def _put_doc(self, doc) + def _put_doc(self, doc): raise NotImplementedError(self._put_doc) - def put_doc(self, doc) + def put_doc(self, doc): # consistency check if doc.doc_id is None: raise errors.InvalidDocId() -- cgit v1.2.3 From 002d2bfdbc4ca62733478524ec588cf0aa9f9383 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Dec 2012 18:39:56 -0200 Subject: CouchDB backend can put and get objects. --- src/leap/soledad/backends/objectstore.py | 43 ++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 19 deletions(-) (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index d9ab7cbd..5bd864c8 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -1,5 +1,7 @@ +import uuid from u1db.backends import CommonBackend from soledad import SyncLog, TransactionLog +from soledad.backends.leap import LeapDocument class ObjectStore(CommonBackend): @@ -45,15 +47,14 @@ class ObjectStore(CommonBackend): self._check_doc_id(doc.doc_id) self._check_doc_size(doc) # put the document - new_rev = self._allocate_doc_rev(doc.rev) - self._put_doc(doc, new_rev) - doc.rev = new_rev + doc.rev = self._allocate_doc_rev(doc.rev) + self._put_doc(doc) # update u1db generation and logs new_gen = self._get_generation() + 1 trans_id = self._allocate_transaction_id() self._transaction_log.append((new_gen, doc.doc_id, trans_id)) self._set_u1db_data() - return new_rev + return doc.rev def delete_doc(self, doc): old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) @@ -145,15 +146,16 @@ class ObjectStore(CommonBackend): """ if not self._is_initialized(): self._initialize() - u1db_data = self._get_doc('u1db_data') - self._sync_log.log = u1db_data.content['sync_log'] - self._transaction_log.log = u1db_data.content['transaction_log'] + self._get_u1db_data() + + U1DB_DATA_DOC_ID = 'u1db_data' def _is_initialized(self): """ Verify if u1db data exists in store. """ - if not self._get_doc('u1db_data'): + doc = self._get_doc(self.U1DB_DATA_DOC_ID) + if not self._get_doc(self.U1DB_DATA_DOC_ID): return False return True @@ -161,19 +163,22 @@ class ObjectStore(CommonBackend): """ Create u1db data object in store. """ - content = { 'transaction_log' : [], - 'sync_log' : [] } - doc = self.create_doc('u1db_data', content) + self._replica_uid = uuid.uuid4().hex + doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) + doc.content = { 'transaction_log' : [], + 'sync_log' : [], + 'replica_uid' : self._replica_uid } + self._put_doc(doc) - def _get_u1db_data(self): - data = self.get_doc('u1db_data').content - self._transaction_log = data['transaction_log'] - self._sync_log = data['sync_log'] + def _get_u1db_data(self, u1db_data_doc_id): + NotImplementedError(self._get_u1db_data) def _set_u1db_data(self): - doc = self._factory('u1db_data') - doc.content = { 'transaction_log' : self._transaction_log, - 'sync_log' : self._sync_log } - self.put_doc(doc) + doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) + doc.content = { 'transaction_log' : self._transaction_log.log, + 'sync_log' : self._sync_log.log, + 'replica_uid' : self._replica_uid, + '_rev' : self._couch_rev} + self._put_doc(doc) -- cgit v1.2.3 From d5816c05136c9c018b8984b5f8a104c164676e9f Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 11 Dec 2012 11:47:16 -0200 Subject: Fix ObjectStore's put_doc. --- src/leap/soledad/backends/objectstore.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index 5bd864c8..298bdda3 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -1,5 +1,6 @@ import uuid from u1db.backends import CommonBackend +from u1db import errors from soledad import SyncLog, TransactionLog from soledad.backends.leap import LeapDocument @@ -46,8 +47,21 @@ class ObjectStore(CommonBackend): raise errors.InvalidDocId() self._check_doc_id(doc.doc_id) self._check_doc_size(doc) - # put the document - doc.rev = self._allocate_doc_rev(doc.rev) + # check if document exists + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc and old_doc.has_conflicts: + raise errors.ConflictedDoc() + if old_doc and doc.rev is None and old_doc.is_tombstone(): + new_rev = self._allocate_doc_rev(old_doc.rev) + else: + if old_doc is not None: + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + else: + if doc.rev is not None: + raise errors.RevisionConflict() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev self._put_doc(doc) # update u1db generation and logs new_gen = self._get_generation() + 1 @@ -69,7 +83,7 @@ class ObjectStore(CommonBackend): new_rev = self._allocate_doc_rev(doc.rev) doc.rev = new_rev doc.make_tombstone() - self._put_doc(olddoc) + self._put_doc(doc) return new_rev # start of index-related methods: these are not supported by this backend. @@ -171,9 +185,15 @@ class ObjectStore(CommonBackend): self._put_doc(doc) def _get_u1db_data(self, u1db_data_doc_id): + """ + Fetch u1db configuration data from backend storage. + """ NotImplementedError(self._get_u1db_data) def _set_u1db_data(self): + """ + Save u1db configuration data on backend storage. + """ doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) doc.content = { 'transaction_log' : self._transaction_log.log, 'sync_log' : self._sync_log.log, -- cgit v1.2.3 From 7a67c36efd95d86dea04ab0741c68f5307a95c09 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 18 Dec 2012 18:51:01 -0200 Subject: Refactor and symmetric encryption --- src/leap/soledad/backends/objectstore.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index 298bdda3..a8e139f7 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -1,8 +1,7 @@ import uuid from u1db.backends import CommonBackend -from u1db import errors -from soledad import SyncLog, TransactionLog -from soledad.backends.leap import LeapDocument +from u1db import errors, Document +from soledad.util import SyncLog, TransactionLog class ObjectStore(CommonBackend): @@ -11,7 +10,7 @@ class ObjectStore(CommonBackend): # This initialization method should be called after the connection # with the database is established, so it can ensure that u1db data is # configured and up-to-date. - self.set_document_factory(LeapDocument) + self.set_document_factory(Document) self._sync_log = SyncLog() self._transaction_log = TransactionLog() self._ensure_u1db_data() -- cgit v1.2.3 From 277f17aa7b7bbcc48583149a3d72d8621f83c0ff Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 24 Dec 2012 10:13:12 -0200 Subject: Document ObjectStore --- src/leap/soledad/backends/objectstore.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/leap/soledad/backends/objectstore.py') diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index a8e139f7..61445a1f 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -5,6 +5,9 @@ from soledad.util import SyncLog, TransactionLog class ObjectStore(CommonBackend): + """ + A backend for storing u1db data in an object store. + """ def __init__(self): # This initialization method should be called after the connection @@ -153,9 +156,13 @@ class ObjectStore(CommonBackend): raise errors.InvalidGeneration return trans_id + #------------------------------------------------------------------------- + # methods specific for object stores + #------------------------------------------------------------------------- + def _ensure_u1db_data(self): """ - Guarantee that u1db data exists in store. + Guarantee that u1db data (logs and replica info) exists in store. """ if not self._is_initialized(): self._initialize() -- cgit v1.2.3