diff options
Diffstat (limited to 'common')
14 files changed, 9 insertions, 498 deletions
diff --git a/common/src/leap/soledad/common/.gitignore b/common/src/leap/soledad/common/.gitignore deleted file mode 100644 index 3378c78a..00000000 --- a/common/src/leap/soledad/common/.gitignore +++ /dev/null @@ -1 +0,0 @@ -ddocs.py diff --git a/common/src/leap/soledad/common/README.txt b/common/src/leap/soledad/common/README.txt index 38b9858e..0a252650 100644 --- a/common/src/leap/soledad/common/README.txt +++ b/common/src/leap/soledad/common/README.txt @@ -60,15 +60,6 @@ implemented in a way that all changes will be pushed with just one operation. * delete_index * create_index -Couch views and update functions are used in order to achieve atomicity on the -Couch backend. Transactions are stored in the `u1db_transactions` field of the -couch document. Document's content and conflicted versions are stored as couch -document attachments with names, respectivelly, `u1db_content` and -`u1db_conflicts`. - -A map of methods and couch query URI can be found on the `./ddocs/README.txt` -document. - Notes: * Currently, the couch backend does not implement indexing, so what is diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py index 6cad2b19..ca0c2855 100644 --- a/common/src/leap/soledad/common/couch/__init__.py +++ b/common/src/leap/soledad/common/couch/__init__.py @@ -37,7 +37,6 @@ from couchdb.client import Server, Database from couchdb.http import ( ResourceConflict, ResourceNotFound, - ServerError, Session, urljoin as couch_urljoin, Resource, @@ -50,9 +49,6 @@ from leap.soledad.common.l2db.errors import ( from leap.soledad.common.l2db.remote import http_app -from leap.soledad.common import ddocs -from .errors import raise_server_error -from .errors import raise_missing_design_doc_error from .support import MultipartWriter from leap.soledad.common.errors import InvalidURLError from leap.soledad.common.document import ServerDocument @@ -177,7 +173,6 @@ class CouchDatabase(object): self.batch_generation = None self.batch_docs = {} if ensure_ddocs: - self.ensure_ddocs_on_db() self.ensure_security_ddoc(database_security) def batch_start(self): @@ -212,22 +207,6 @@ class CouchDatabase(object): except ResourceNotFound: raise DatabaseDoesNotExist() - def ensure_ddocs_on_db(self): - """ - Ensure that the design documents used by the backend exist on the - couch database. - """ - for ddoc_name in ['docs', 'syncs']: - try: - self.json_from_resource(['_design'] + - ddoc_name.split('/') + ['_info'], - check_missing_ddoc=False) - except ResourceNotFound: - ddoc = json.loads( - binascii.a2b_base64( - getattr(ddocs, ddoc_name))) - self._database.save(ddoc) - def ensure_security_ddoc(self, security_config=None): """ Make sure that only soledad user is able to access this database as @@ -539,7 +518,6 @@ class CouchDatabase(object): try: response = self.json_from_resource([doc_id, 'u1db_conflicts'], - check_missing_ddoc=False, **params) return conflicts + self._build_conflicts( doc_id, json.loads(response.read())) @@ -652,48 +630,23 @@ class CouchDatabase(object): gen_doc = rows.pop()['doc'] return gen_doc['gen'], gen_doc['trans_id'] - def json_from_resource(self, ddoc_path, check_missing_ddoc=True, - **kwargs): + def json_from_resource(self, doc_path, **kwargs): """ Get a resource from it's path and gets a doc's JSON using provided - parameters, also checking for missing design docs by default. + parameters. - :param ddoc_path: The path to resource. - :type ddoc_path: [str] - :param check_missing_ddoc: Raises info on what design doc is missing. - :type check_missin_ddoc: bool + :param doc_path: The path to resource. + :type doc_path: [str] :return: The request's data parsed from JSON to a dict. :rtype: dict - - :raise MissingDesignDocError: Raised when tried to access a missing - design document. - :raise MissingDesignDocListFunctionError: Raised when trying to access - a missing list function on a - design document. - :raise MissingDesignDocNamedViewError: Raised when trying to access a - missing named view on a design - document. - :raise MissingDesignDocDeletedError: Raised when trying to access a - deleted design document. - :raise MissingDesignDocUnknownError: Raised when failed to access a - design document for an yet - unknown reason. - """ - if ddoc_path is not None: - resource = self._database.resource(*ddoc_path) + """ + if doc_path is not None: + resource = self._database.resource(*doc_path) else: resource = self._database.resource() - try: - _, _, data = resource.get_json(**kwargs) - return data - except ResourceNotFound as e: - if check_missing_ddoc: - raise_missing_design_doc_error(e, ddoc_path) - else: - raise e - except ServerError as e: - raise_server_error(e, ddoc_path) + _, _, data = resource.get_json(**kwargs) + return data def save_document(self, old_doc, doc, transaction_id): """ @@ -710,19 +663,6 @@ class CouchDatabase(object): :raise RevisionConflict: Raised when trying to update a document but couch revisions mismatch. - :raise MissingDesignDocError: Raised when tried to access a missing - design document. - :raise MissingDesignDocListFunctionError: Raised when trying to access - a missing list function on a - design document. - :raise MissingDesignDocNamedViewError: Raised when trying to access a - missing named view on a design - document. - :raise MissingDesignDocDeletedError: Raised when trying to access a - deleted design document. - :raise MissingDesignDocUnknownError: Raised when failed to access a - design document for an yet - unknown reason. """ attachments = {} # we save content and conflicts as attachments parts = [] # and we put it using couch's multipart PUT diff --git a/common/src/leap/soledad/common/couch/errors.py b/common/src/leap/soledad/common/couch/errors.py deleted file mode 100644 index 9b287c76..00000000 --- a/common/src/leap/soledad/common/couch/errors.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -# errors.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -from leap.soledad.common.errors import SoledadError, BackendNotReadyError -from leap.soledad.common.errors import register_exception - -""" -Specific errors that can be raised by CouchDatabase. -""" - - -@register_exception -class MissingDesignDocError(BackendNotReadyError): - - """ - Raised when trying to access a missing couch design document. - """ - - wire_description = "missing design document" - status = 500 - - -@register_exception -class MissingDesignDocNamedViewError(SoledadError): - - """ - Raised when trying to access a missing named view on a couch design - document. - """ - - wire_description = "missing design document named function" - status = 500 - - -@register_exception -class MissingDesignDocListFunctionError(SoledadError): - - """ - Raised when trying to access a missing list function on a couch design - document. - """ - - wire_description = "missing design document list function" - status = 500 - - -@register_exception -class MissingDesignDocDeletedError(SoledadError): - - """ - Raised when trying to access a deleted couch design document. - """ - - wire_description = "design document was deleted" - status = 500 - - -@register_exception -class DesignDocUnknownError(SoledadError): - - """ - Raised when trying to access a couch design document and getting an - unknown error. - """ - - wire_description = "missing design document unknown error" - status = 500 - - -def raise_missing_design_doc_error(exc, ddoc_path): - """ - Raise an appropriate exception when catching a ResourceNotFound when - accessing a design document. - - :param exc: The exception cought. - :type exc: ResourceNotFound - :param ddoc_path: A list representing the requested path. - :type ddoc_path: list - - :raise MissingDesignDocError: Raised when tried to access a missing design - document. - :raise MissingDesignDocListFunctionError: Raised when trying to access a - missing list function on a - design document. - :raise MissingDesignDocNamedViewError: Raised when trying to access a - missing named view on a design - document. - :raise MissingDesignDocDeletedError: Raised when trying to access a - deleted design document. - :raise MissingDesignDocUnknownError: Raised when failed to access a design - document for an yet unknown reason. - """ - path = "".join(ddoc_path) - if exc.message[1] == 'missing': - raise MissingDesignDocError(path) - elif exc.message[1] == 'missing function' or \ - exc.message[1].startswith('missing lists function'): - raise MissingDesignDocListFunctionError(path) - elif exc.message[1] == 'missing_named_view': - raise MissingDesignDocNamedViewError(path) - elif exc.message[1] == 'deleted': - raise MissingDesignDocDeletedError(path) - # other errors are unknown for now - raise DesignDocUnknownError("%s: %s" % (path, str(exc.message))) - - -def raise_server_error(exc, ddoc_path): - """ - Raise an appropriate exception when catching a ServerError when - accessing a design document. - - :param exc: The exception cought. - :type exc: ResourceNotFound - :param ddoc_path: A list representing the requested path. - :type ddoc_path: list - - :raise MissingDesignDocListFunctionError: Raised when trying to access a - missing list function on a - design document. - :raise MissingDesignDocUnknownError: Raised when failed to access a design - document for an yet unknown reason. - """ - path = "".join(ddoc_path) - msg = exc.message[1][0] - if msg == 'unnamed_error': - raise MissingDesignDocListFunctionError(path) - elif msg == 'TypeError': - if 'point is undefined' in exc.message[1][1]: - raise MissingDesignDocListFunctionError - # other errors are unknown for now - raise DesignDocUnknownError("%s: %s" % (path, str(exc.message))) diff --git a/common/src/leap/soledad/common/ddocs/README.txt b/common/src/leap/soledad/common/ddocs/README.txt deleted file mode 100644 index 5569d929..00000000 --- a/common/src/leap/soledad/common/ddocs/README.txt +++ /dev/null @@ -1,34 +0,0 @@ -This directory holds a folder structure containing javascript files that -represent the design documents needed by the CouchDB U1DB backend. These files -are compiled into the `../ddocs.py` file by setuptools when creating the -source distribution. - -The following table depicts the U1DB CouchDB backend method and the URI that -is queried to obtain/update data from/to the server. - - +----------------------------------+------------------------------------------------------------------+ - | u1db backend method | URI | - |----------------------------------+------------------------------------------------------------------| - | _get_generation | _design/transactions/_list/generation/log | - | _get_generation_info | _design/transactions/_list/generation/log | - | _get_trans_id_for_gen | _design/transactions/_list/trans_id_for_gen/log | - | _get_transaction_log | _design/transactions/_view/log | - | _get_doc (*) | _design/docs/_view/get?key=<doc_id> | - | _has_conflicts | _design/docs/_view/get?key=<doc_id> | - | get_all_docs | _design/docs/_view/get | - | _put_doc | _design/docs/_update/put/<doc_id> | - | _whats_changed | _design/transactions/_list/whats_changed/log?old_gen=<gen> | - | _get_conflicts (*) | _design/docs/_view/conflicts?key=<doc_id> | - | _get_replica_gen_and_trans_id | _design/syncs/_view/log?other_replica_uid=<uid> | - | _do_set_replica_gen_and_trans_id | _design/syncs/_update/put/u1db_sync_log | - | _add_conflict | _design/docs/_update/add_conflict/<doc_id> | - | _delete_conflicts | _design/docs/_update/delete_conflicts/<doc_id>?doc_rev=<doc_rev> | - | list_indexes | not implemented | - | _get_index_definition | not implemented | - | delete_index | not implemented | - | _get_indexed_fields | not implemented | - | _put_and_update_indexes | not implemented | - +----------------------------------+------------------------------------------------------------------+ - -(*) These methods also request CouchDB document attachments that store U1DB - document contents. diff --git a/common/src/leap/soledad/common/ddocs/docs/views/get/map.js b/common/src/leap/soledad/common/ddocs/docs/views/get/map.js deleted file mode 100644 index ae08d9e9..00000000 --- a/common/src/leap/soledad/common/ddocs/docs/views/get/map.js +++ /dev/null @@ -1,20 +0,0 @@ -function(doc) { - if (doc.u1db_rev) { - var is_tombstone = true; - var has_conflicts = false; - if (doc._attachments) { - if (doc._attachments.u1db_content) - is_tombstone = false; - if (doc._attachments.u1db_conflicts) - has_conflicts = true; - } - emit(doc._id, - { - "couch_rev": doc._rev, - "u1db_rev": doc.u1db_rev, - "is_tombstone": is_tombstone, - "has_conflicts": has_conflicts, - } - ); - } -} diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js deleted file mode 100644 index d62aeb40..00000000 --- a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js +++ /dev/null @@ -1,105 +0,0 @@ -/** - * This update handler stores information about ongoing synchronization - * attempts from distinct source replicas. - * - * Normally, u1db synchronization occurs during one POST request. In order to - * split that into many serial POST requests, we store the state of each sync - * in the server, using a document with id 'u1db_sync_state'. To identify - * each sync attempt, we use a sync_id sent by the client. If we ever receive - * a new sync_id, we trash current data for that source replica and start - * over. - * - * We expect the following in the document body: - * - * { - * 'source_replica_uid': '<source_replica_uid>', - * 'sync_id': '<sync_id>', - * 'seen_ids': [['<doc_id>', <at_gen>], ...], // optional - * 'changes_to_return': [ // optional - * 'gen': <gen>, - * 'trans_id': '<trans_id>', - * 'changes_to_return': [[<doc_id>', <gen>, '<trans_id>'], ...] - * ], - * } - * - * The format of the final document stored on server is: - * - * { - * '_id': '<str>', - * '_rev' '<str>', - * 'ongoing_syncs': { - * '<source_replica_uid>': { - * 'sync_id': '<sync_id>', - * 'seen_ids': [['<doc_id>', <at_gen>[, ...], - * 'changes_to_return': { - * 'gen': <gen>, - * 'trans_id': '<trans_id>', - * 'changes_to_return': [ - * ['<doc_id>', <gen>, '<trans_id>'], - * ..., - * ], - * }, - * }, - * ... // info about other source replicas here - * } - * } - */ -function(doc, req) { - - // prevent updates to alien documents - if (doc != null && doc['_id'] != 'u1db_sync_state') - return [null, 'invalid data']; - - // create the document if it doesn't exist - if (!doc) - doc = { - '_id': 'u1db_sync_state', - 'ongoing_syncs': {}, - }; - - // parse and validate incoming data - var body = JSON.parse(req.body); - if (body['source_replica_uid'] == null) - return [null, 'invalid data']; - var source_replica_uid = body['source_replica_uid']; - - if (body['sync_id'] == null) - return [null, 'invalid data']; - var sync_id = body['sync_id']; - - // trash outdated sync data for that replica if that exists - if (doc['ongoing_syncs'][source_replica_uid] != null && - doc['ongoing_syncs'][source_replica_uid]['sync_id'] != sync_id) - delete doc['ongoing_syncs'][source_replica_uid]; - - // create an entry for that source replica - if (doc['ongoing_syncs'][source_replica_uid] == null) - doc['ongoing_syncs'][source_replica_uid] = { - 'sync_id': sync_id, - 'seen_ids': {}, - 'changes_to_return': null, - }; - - // incoming meta-data values should be exclusive, so we count how many - // arrived and deny to accomplish the transaction if the count is high. - var incoming_values = 0; - var info = doc['ongoing_syncs'][source_replica_uid] - - // add incoming seen id - if ('seen_id' in body) { - info['seen_ids'][body['seen_id'][0]] = body['seen_id'][1]; - incoming_values += 1; - } - - // add incoming changes_to_return - if ('changes_to_return' in body) { - info['changes_to_return'] = body['changes_to_return']; - incoming_values += 1; - } - - if (incoming_values != 1) - return [null, 'invalid data']; - - return [doc, 'ok']; -} - diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js deleted file mode 100644 index 94b7e767..00000000 --- a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js +++ /dev/null @@ -1,20 +0,0 @@ -function(doc) { - if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) - for (var source_replica_uid in doc['ongoing_syncs']) { - var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return']; - var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id']; - if (changes == null) - emit([source_replica_uid, sync_id, 0], null); - else if (changes.length == 0) - emit([source_replica_uid, sync_id, 0], []); - else - for (var i = 0; i < changes['changes_to_return'].length; i++) - emit( - [source_replica_uid, sync_id, i], - { - 'gen': changes['gen'], - 'trans_id': changes['trans_id'], - 'next_change_to_return': changes['changes_to_return'][i], - }); - } -} diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js deleted file mode 100644 index 16118e88..00000000 --- a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js +++ /dev/null @@ -1,11 +0,0 @@ -function(doc) { - if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) - for (var source_replica_uid in doc['ongoing_syncs']) { - var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id']; - emit( - [source_replica_uid, sync_id], - { - 'seen_ids': doc['ongoing_syncs'][source_replica_uid]['seen_ids'], - }); - } -} diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js deleted file mode 100644 index e88c6ebb..00000000 --- a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js +++ /dev/null @@ -1,17 +0,0 @@ -function(doc) { - if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) - for (var source_replica_uid in doc['ongoing_syncs']) { - var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return']; - var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id']; - if (changes == null) - emit([source_replica_uid, sync_id], null); - else - emit( - [source_replica_uid, sync_id], - { - 'gen': changes['gen'], - 'trans_id': changes['trans_id'], - 'number_of_changes': changes['changes_to_return'].length - }); - } -} diff --git a/common/src/leap/soledad/common/ddocs/transactions/lists/generation.js b/common/src/leap/soledad/common/ddocs/transactions/lists/generation.js deleted file mode 100644 index dbdfff0d..00000000 --- a/common/src/leap/soledad/common/ddocs/transactions/lists/generation.js +++ /dev/null @@ -1,20 +0,0 @@ -function(head, req) { - var row; - var rows=[]; - // fetch all rows - while(row = getRow()) { - rows.push(row); - } - if (rows.length > 0) - send(JSON.stringify({ - "generation": rows.length, - "doc_id": rows[rows.length-1]['id'], - "transaction_id": rows[rows.length-1]['value'] - })); - else - send(JSON.stringify({ - "generation": 0, - "doc_id": "", - "transaction_id": "", - })); -} diff --git a/common/src/leap/soledad/common/ddocs/transactions/lists/trans_id_for_gen.js b/common/src/leap/soledad/common/ddocs/transactions/lists/trans_id_for_gen.js deleted file mode 100644 index 2ec91794..00000000 --- a/common/src/leap/soledad/common/ddocs/transactions/lists/trans_id_for_gen.js +++ /dev/null @@ -1,19 +0,0 @@ -function(head, req) { - var row; - var rows=[]; - var i = 1; - var gen = 1; - if (req.query.gen) - gen = parseInt(req.query['gen']); - // fetch all rows - while(row = getRow()) - rows.push(row); - if (gen <= rows.length) - send(JSON.stringify({ - "generation": gen, - "doc_id": rows[gen-1]['id'], - "transaction_id": rows[gen-1]['value'], - })); - else - send('{}'); -} diff --git a/common/src/leap/soledad/common/ddocs/transactions/lists/whats_changed.js b/common/src/leap/soledad/common/ddocs/transactions/lists/whats_changed.js deleted file mode 100644 index b35cdf51..00000000 --- a/common/src/leap/soledad/common/ddocs/transactions/lists/whats_changed.js +++ /dev/null @@ -1,22 +0,0 @@ -function(head, req) { - var row; - var gen = 1; - var old_gen = 0; - if (req.query.old_gen) - old_gen = parseInt(req.query['old_gen']); - send('{"transactions":[\n'); - // fetch all rows - while(row = getRow()) { - if (gen > old_gen) { - if (gen > old_gen+1) - send(',\n'); - send(JSON.stringify({ - "generation": gen, - "doc_id": row["id"], - "transaction_id": row["value"] - })); - } - gen++; - } - send('\n]}'); -} diff --git a/common/src/leap/soledad/common/ddocs/transactions/views/log/map.js b/common/src/leap/soledad/common/ddocs/transactions/views/log/map.js deleted file mode 100644 index 94ef63ca..00000000 --- a/common/src/leap/soledad/common/ddocs/transactions/views/log/map.js +++ /dev/null @@ -1,7 +0,0 @@ -function(doc) { - if (doc.u1db_transactions) - doc.u1db_transactions.forEach(function(t) { - emit(t[0], // use timestamp as key so the results are ordered - t[1]); // value is the transaction_id - }); -} |