From 002d2bfdbc4ca62733478524ec588cf0aa9f9383 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Dec 2012 18:39:56 -0200 Subject: CouchDB backend can put and get objects. --- src/leap/soledad/backends/couch.py | 115 +++++++++++++++++++++++++++++++ src/leap/soledad/backends/couchdb.py | 97 -------------------------- src/leap/soledad/backends/leap.py | 1 + src/leap/soledad/backends/objectstore.py | 43 +++++++----- src/leap/soledad/backends/openstack.py | 2 +- 5 files changed, 141 insertions(+), 117 deletions(-) create mode 100644 src/leap/soledad/backends/couch.py delete mode 100644 src/leap/soledad/backends/couchdb.py (limited to 'src/leap/soledad/backends') diff --git a/src/leap/soledad/backends/couch.py b/src/leap/soledad/backends/couch.py new file mode 100644 index 00000000..5586ea9c --- /dev/null +++ b/src/leap/soledad/backends/couch.py @@ -0,0 +1,115 @@ +from u1db import errors +from u1db.remote.http_target import HTTPSyncTarget +from couchdb.client import Server, Document +from couchdb.http import ResourceNotFound +from soledad.backends.objectstore import ObjectStore +from soledad.backends.leap import LeapDocument + + +class CouchDatabase(ObjectStore): + """A U1DB implementation that uses Couch as its persistence layer.""" + + def __init__(self, url, database, full_commit=True, session=None): + """Create a new Couch data container.""" + self._url = url + self._full_commit = full_commit + self._session = session + self._server = Server(url=self._url, + full_commit=self._full_commit, + session=self._session) + # this will ensure that transaction and sync logs exist and are + # up-to-date. + self.set_document_factory(LeapDocument) + try: + self._database = self._server[database] + except ResourceNotFound: + self._server.create(database) + self._database = self._server[database] + super(CouchDatabase, self).__init__() + + #------------------------------------------------------------------------- + # implemented methods from Database + #------------------------------------------------------------------------- + + def _get_doc(self, doc_id, check_for_conflicts=False): + """Get just the document content, without fancy handling. + + Conflicts do not happen on server side, so there's no need to check + for them. + """ + cdoc = self._database.get(doc_id) + if cdoc is None: + return None + content = {} + for (key, value) in cdoc.items(): + if key not in ['_id', '_rev', 'u1db_rev']: + content[key] = value + doc = self._factory(doc_id=doc_id, rev=cdoc['u1db_rev']) + doc.content = content + return doc + + def get_all_docs(self, include_deleted=False): + """Get all documents from the database.""" + generation = self._get_generation() + results = [] + for doc_id in self._database: + doc = self._get_doc(doc_id) + if doc.content is None and not include_deleted: + continue + results.append(doc) + return (generation, results) + + def _put_doc(self, doc): + # map u1db metadata to couch + content = doc.content + cdoc = Document() + cdoc['_id'] = doc.doc_id + cdoc['u1db_rev'] = doc.rev + for (key, value) in content.items(): + cdoc[key] = value + self._database.save(cdoc) + + def get_sync_target(self): + return CouchSyncTarget(self) + + def close(self): + raise NotImplementedError(self.close) + + def sync(self, url, creds=None, autocreate=True): + from u1db.sync import Synchronizer + from u1db.remote.http_target import CouchSyncTarget + return Synchronizer(self, CouchSyncTarget(url, creds=creds)).sync( + autocreate=autocreate) + + def _get_u1db_data(self): + cdoc = self._database.get(self.U1DB_DATA_DOC_ID) + self._sync_log.log = cdoc['sync_log'] + self._transaction_log.log = cdoc['transaction_log'] + self._replica_uid = cdoc['replica_uid'] + self._couch_rev = cdoc['_rev'] + + #------------------------------------------------------------------------- + # Couch specific methods + #------------------------------------------------------------------------- + + # no specific methods so far. + +class CouchSyncTarget(HTTPSyncTarget): + + def get_sync_info(self, source_replica_uid): + source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( + source_replica_uid) + my_gen, my_trans_id = self._db._get_generation_info() + return ( + self._db._replica_uid, my_gen, my_trans_id, source_gen, + source_trans_id) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_replica_transaction_id): + if self._trace_hook: + self._trace_hook('record_sync_info') + self._db._set_replica_gen_and_trans_id( + source_replica_uid, source_replica_generation, + source_replica_transaction_id) + + diff --git a/src/leap/soledad/backends/couchdb.py b/src/leap/soledad/backends/couchdb.py deleted file mode 100644 index 89b713f9..00000000 --- a/src/leap/soledad/backends/couchdb.py +++ /dev/null @@ -1,97 +0,0 @@ -from u1db import errors -from u1db.remote.http_target import HTTPSyncTarget -from couchdb import * -from soledad.backends.objectstore import ObjectStore - - -class CouchDatabase(ObjectStore): - """A U1DB implementation that uses Couch as its persistence layer.""" - - def __init__(self, url, database, full_commit=True, session=None): - """Create a new Couch data container.""" - self._url = url - self._full_commit = full_commit - self._session = session - self._server = couchdb.Server(url=self._url, - full_commit=self._full_commit, - session=self._session) - # this will ensure that transaction and sync logs exist and are - # up-to-date. - super(CouchDatabase, self) - self._database = self._server[database] - - #------------------------------------------------------------------------- - # implemented methods from Database - #------------------------------------------------------------------------- - - def _get_doc(self, doc_id, check_for_conflicts=False): - """Get just the document content, without fancy handling. - - Conflicts do not happen on server side, so there's no need to check - for them. - """ - cdoc = self._database.get(doc_id) - if cdoc is not None: - content = {} - for key, value in content: - if not key in ['_id', '_rev', '_u1db_rev']: - content[key] = value - doc = self._factory(doc_id=doc_id, rev=cdoc['_u1db_rev']) - doc.content = content - return doc - - def get_all_docs(self, include_deleted=False): - """Get all documents from the database.""" - generation = self._get_generation() - results = [] - for doc_id in self._database: - doc = self._get_doc(doc_id) - if doc.content is None and not include_deleted: - continue - results.append(doc) - return (generation, results) - - def _put_doc(self, doc, new_rev): - # map u1db metadata to couch - content = doc.content - content['_id'] = doc.doc_id - content['_u1db_rev'] = new_rev - self._database.save(doc.content) - - def get_sync_target(self): - return CouchSyncTarget(self) - - def close(self): - raise NotImplementedError(self.close) - - def sync(self, url, creds=None, autocreate=True): - from u1db.sync import Synchronizer - from u1db.remote.http_target import CouchSyncTarget - return Synchronizer(self, CouchSyncTarget(url, creds=creds)).sync( - autocreate=autocreate) - - #------------------------------------------------------------------------- - # Couch specific methods - #------------------------------------------------------------------------- - - # no specific methods so far. - -class CouchSyncTarget(HTTPSyncTarget): - - def get_sync_info(self, source_replica_uid): - source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( - source_replica_uid) - my_gen, my_trans_id = self._db._get_generation_info() - return ( - self._db._replica_uid, my_gen, my_trans_id, source_gen, - source_trans_id) - - def record_sync_info(self, source_replica_uid, source_replica_generation, - source_replica_transaction_id): - if self._trace_hook: - self._trace_hook('record_sync_info') - self._db._set_replica_gen_and_trans_id( - source_replica_uid, source_replica_generation, - source_replica_transaction_id) - - diff --git a/src/leap/soledad/backends/leap.py b/src/leap/soledad/backends/leap.py index 2c815632..ce00c8f3 100644 --- a/src/leap/soledad/backends/leap.py +++ b/src/leap/soledad/backends/leap.py @@ -7,6 +7,7 @@ from u1db import Document from u1db.remote.http_target import HTTPSyncTarget from u1db.remote.http_database import HTTPDatabase import base64 +from soledad import GPGWrapper class NoDefaultKey(Exception): diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py index d9ab7cbd..5bd864c8 100644 --- a/src/leap/soledad/backends/objectstore.py +++ b/src/leap/soledad/backends/objectstore.py @@ -1,5 +1,7 @@ +import uuid from u1db.backends import CommonBackend from soledad import SyncLog, TransactionLog +from soledad.backends.leap import LeapDocument class ObjectStore(CommonBackend): @@ -45,15 +47,14 @@ class ObjectStore(CommonBackend): self._check_doc_id(doc.doc_id) self._check_doc_size(doc) # put the document - new_rev = self._allocate_doc_rev(doc.rev) - self._put_doc(doc, new_rev) - doc.rev = new_rev + doc.rev = self._allocate_doc_rev(doc.rev) + self._put_doc(doc) # update u1db generation and logs new_gen = self._get_generation() + 1 trans_id = self._allocate_transaction_id() self._transaction_log.append((new_gen, doc.doc_id, trans_id)) self._set_u1db_data() - return new_rev + return doc.rev def delete_doc(self, doc): old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) @@ -145,15 +146,16 @@ class ObjectStore(CommonBackend): """ if not self._is_initialized(): self._initialize() - u1db_data = self._get_doc('u1db_data') - self._sync_log.log = u1db_data.content['sync_log'] - self._transaction_log.log = u1db_data.content['transaction_log'] + self._get_u1db_data() + + U1DB_DATA_DOC_ID = 'u1db_data' def _is_initialized(self): """ Verify if u1db data exists in store. """ - if not self._get_doc('u1db_data'): + doc = self._get_doc(self.U1DB_DATA_DOC_ID) + if not self._get_doc(self.U1DB_DATA_DOC_ID): return False return True @@ -161,19 +163,22 @@ class ObjectStore(CommonBackend): """ Create u1db data object in store. """ - content = { 'transaction_log' : [], - 'sync_log' : [] } - doc = self.create_doc('u1db_data', content) + self._replica_uid = uuid.uuid4().hex + doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) + doc.content = { 'transaction_log' : [], + 'sync_log' : [], + 'replica_uid' : self._replica_uid } + self._put_doc(doc) - def _get_u1db_data(self): - data = self.get_doc('u1db_data').content - self._transaction_log = data['transaction_log'] - self._sync_log = data['sync_log'] + def _get_u1db_data(self, u1db_data_doc_id): + NotImplementedError(self._get_u1db_data) def _set_u1db_data(self): - doc = self._factory('u1db_data') - doc.content = { 'transaction_log' : self._transaction_log, - 'sync_log' : self._sync_log } - self.put_doc(doc) + doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID) + doc.content = { 'transaction_log' : self._transaction_log.log, + 'sync_log' : self._sync_log.log, + 'replica_uid' : self._replica_uid, + '_rev' : self._couch_rev} + self._put_doc(doc) diff --git a/src/leap/soledad/backends/openstack.py b/src/leap/soledad/backends/openstack.py index 5f2a2771..c027231c 100644 --- a/src/leap/soledad/backends/openstack.py +++ b/src/leap/soledad/backends/openstack.py @@ -17,7 +17,7 @@ class OpenStackDatabase(ObjectStore): self._auth_key) self._get_auth() # this will ensure transaction and sync logs exist and are up-to-date. - super(OpenStackDatabase, self) + super(OpenStackDatabase, self).__init__() #------------------------------------------------------------------------- # implemented methods from Database -- cgit v1.2.3