From 8d504fa812da93df3a26c4b4b761a74685d40f25 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 24 Dec 2013 08:17:37 -0200 Subject: Avoid concurrent sync attempts from the same replica in the client (#4451). --- .../tests/test_couch_operations_atomicity.py | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'common') diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index a0c473b1..8b001859 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -337,3 +337,46 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): self.assertEqual( 1, len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_concurrent_syncs_do_not_fail(self): + """ + Assert that concurrent attempts to sync end up being executed + sequentially and do not fail. + """ + threads = [] + docs = [] + pool = threading.BoundedSemaphore(value=1) + self.startServer() + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + def _run_method(self): + # create a lot of documents + doc = self._params['sol'].create_doc({}) + # do the sync! + sol.sync() + pool.acquire() + docs.append(doc.doc_id) + pool.release() + + # launch threads to create documents in parallel + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread( + {'sol': sol, 'syncs': i}, + _run_method) + thread.start() + threads.append(thread) + + # wait for threads to finish + for thread in threads: + thread.join() + + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) -- cgit v1.2.3 From 89d3e4a1321ff9701ac67933f8e649cfecd1d95e Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jan 2014 10:29:43 -0200 Subject: Add proper error reporting to shared db lock. --- common/changes/bug_4435_add-lock-timeout-error | 1 + common/src/leap/soledad/common/errors.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 common/changes/bug_4435_add-lock-timeout-error (limited to 'common') diff --git a/common/changes/bug_4435_add-lock-timeout-error b/common/changes/bug_4435_add-lock-timeout-error new file mode 100644 index 00000000..8f220b1d --- /dev/null +++ b/common/changes/bug_4435_add-lock-timeout-error @@ -0,0 +1 @@ + o Add lock timeout HTTP error (#4435). diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 7c2d7296..62de19f8 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -57,14 +57,26 @@ class AlreadyLockedError(errors.U1DBError): wire_description = "lock is locked" status = 403 + +class LockTimedOutError(errors.U1DBError): + """ + Exception raised when timing out while trying to lock the shared database. + """ + + wire_description = "lock timed out" + status = 408 + + # update u1db "wire description to status" and "wire description to exception" # maps. -for e in [InvalidTokenError, NotLockedError, AlreadyLockedError]: +for e in [InvalidTokenError, NotLockedError, AlreadyLockedError, + LockTimedOutError]: http_errors.wire_description_to_status.update({ e.wire_description: e.status}) errors.wire_description_to_exc.update({ e.wire_description: e}) + # u1db error statuses also have to be updated http_errors.ERROR_STATUSES = set( http_errors.wire_description_to_status.values()) -- cgit v1.2.3 From 82628d8284c5c11452da75a6604f2f68b8dd8520 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 7 Jan 2014 11:05:39 -0200 Subject: Use temp dir for server side locks (#4918). --- common/src/leap/soledad/common/errors.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'common') diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 62de19f8..c41e0b0f 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -67,10 +67,19 @@ class LockTimedOutError(errors.U1DBError): status = 408 +class CouldNotObtainLockError(errors.U1DBError): + """ + Exception raised when timing out while trying to lock the shared database. + """ + + wire_description = "error obtaining lock" + status = 500 + + # update u1db "wire description to status" and "wire description to exception" # maps. for e in [InvalidTokenError, NotLockedError, AlreadyLockedError, - LockTimedOutError]: + LockTimedOutError, CouldNotObtainLockError]: http_errors.wire_description_to_status.update({ e.wire_description: e.status}) errors.wire_description_to_exc.update({ -- cgit v1.2.3 From 3587985a014daefbae75351911b06485c3c91134 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 16 Jan 2014 19:02:01 -0200 Subject: Improve error reporting on couch backend design documents access failure (#4994). --- .../feature_4994_add-missing-design-doc-exception | 1 + common/src/leap/soledad/common/couch.py | 415 ++++++++++++++++----- common/src/leap/soledad/common/errors.py | 74 +++- common/src/leap/soledad/common/tests/test_couch.py | 219 ++++++++++- 4 files changed, 597 insertions(+), 112 deletions(-) create mode 100644 common/changes/feature_4994_add-missing-design-doc-exception (limited to 'common') diff --git a/common/changes/feature_4994_add-missing-design-doc-exception b/common/changes/feature_4994_add-missing-design-doc-exception new file mode 100644 index 00000000..d5291cc1 --- /dev/null +++ b/common/changes/feature_4994_add-missing-design-doc-exception @@ -0,0 +1 @@ + o Add exceptions to deal with missing design documents. Fixes #4994. diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index d2414477..45ca4282 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -27,14 +27,23 @@ import socket from couchdb.client import Server -from couchdb.http import ResourceNotFound, Unauthorized -from u1db import errors, query_parser, vectorclock +from couchdb.http import ResourceNotFound, Unauthorized, ServerError +from u1db import query_parser, vectorclock +from u1db.errors import ( + DatabaseDoesNotExist, + InvalidGeneration, + RevisionConflict, + InvalidDocId, + ConflictedDoc, + DocumentDoesNotExist, + DocumentAlreadyDeleted, +) from u1db.backends import CommonBackend, CommonSyncTarget from u1db.remote import http_app from u1db.remote.server_state import ServerState -from leap.soledad.common import USER_DB_PREFIX, ddocs +from leap.soledad.common import USER_DB_PREFIX, ddocs, errors from leap.soledad.common.document import SoledadDocument @@ -153,6 +162,66 @@ class CouchDocument(SoledadDocument): http_app.Document = CouchDocument +def raise_missing_design_doc_error(exc, ddoc_path): + """ + Raise an appropriate exception when catching a ResourceNotFound when + accessing a design document. + + :param exc: The exception cought. + :type exc: ResourceNotFound + :param ddoc_path: A list representing the requested path. + :type ddoc_path: list + + :raise MissingDesignDocError: Raised when tried to access a missing design + document. + :raise MissingDesignDocListFunctionError: Raised when trying to access a + missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a design + document for an yet unknown reason. + """ + path = "".join(ddoc_path) + if exc.message[1] == 'missing': + raise errors.MissingDesignDocError(path) + elif exc.message[1] == 'missing function' or \ + exc.message[1].startswith('missing lists function'): + raise errors.MissingDesignDocListFunctionError(path) + elif exc.message[1] == 'missing_named_view': + raise errors.MissingDesignDocNamedViewError(path) + elif exc.message[1] == 'deleted': + raise errors.MissingDesignDocDeletedError(path) + # other errors are unknown for now + raise errors.DesignDocUnknownError(path) + + +def raise_server_error(exc, ddoc_path): + """ + Raise an appropriate exception when catching a ServerError when + accessing a design document. + + :param exc: The exception cought. + :type exc: ResourceNotFound + :param ddoc_path: A list representing the requested path. + :type ddoc_path: list + + :raise MissingDesignDocListFunctionError: Raised when trying to access a + missing list function on a + design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a design + document for an yet unknown reason. + """ + path = "".join(ddoc_path) + if exc.message[1][0] == 'unnamed_error': + raise errors.MissingDesignDocListFunctionError(path) + # other errors are unknown for now + raise errors.DesignDocUnknownError(path) + + class CouchDatabase(CommonBackend): """ A U1DB implementation that uses CouchDB as its persistence layer. @@ -182,7 +251,7 @@ class CouchDatabase(CommonBackend): server[dbname] except ResourceNotFound: if not create: - raise errors.DatabaseDoesNotExist() + raise DatabaseDoesNotExist() return cls(url, dbname) def __init__(self, url, dbname, replica_uid=None, full_commit=True, @@ -318,12 +387,31 @@ class CouchDatabase(CommonBackend): :return: The current generation. :rtype: int + + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ # query a couch list function - res = self._database.resource( - '_design', 'transactions', '_list', 'generation', 'log') - response = res.get_json() - return response[2]['generation'] + ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log'] + res = self._database.resource(*ddoc_path) + try: + response = res.get_json() + return response[2]['generation'] + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) + except ServerError as e: + raise_server_error(e, ddoc_path) def _get_generation_info(self): """ @@ -331,12 +419,31 @@ class CouchDatabase(CommonBackend): :return: A tuple containing the current generation and transaction id. :rtype: (int, str) + + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ # query a couch list function - res = self._database.resource( - '_design', 'transactions', '_list', 'generation', 'log') - response = res.get_json() - return (response[2]['generation'], response[2]['transaction_id']) + ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log'] + res = self._database.resource(*ddoc_path) + try: + response = res.get_json() + return (response[2]['generation'], response[2]['transaction_id']) + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) + except ServerError as e: + raise_server_error(e, ddoc_path) def _get_trans_id_for_gen(self, generation): """ @@ -349,16 +456,36 @@ class CouchDatabase(CommonBackend): :rtype: str :raise InvalidGeneration: Raised when the generation does not exist. + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ if generation == 0: return '' # query a couch list function - res = self._database.resource( - '_design', 'transactions', '_list', 'trans_id_for_gen', 'log') - response = res.get_json(gen=generation) - if response[2] == {}: - raise errors.InvalidGeneration - return response[2]['transaction_id'] + ddoc_path = [ + '_design', 'transactions', '_list', 'trans_id_for_gen', 'log' + ] + res = self._database.resource(*ddoc_path) + try: + response = res.get_json(gen=generation) + if response[2] == {}: + raise InvalidGeneration + return response[2]['transaction_id'] + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) + except ServerError as e: + raise_server_error(e, ddoc_path) def _get_transaction_log(self): """ @@ -366,12 +493,31 @@ class CouchDatabase(CommonBackend): :return: The complete transaction log. :rtype: [(str, str)] + + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ # query a couch view - res = self._database.resource( - '_design', 'transactions', '_view', 'log') - response = res.get_json() - return map(lambda row: (row['id'], row['value']), response[2]['rows']) + ddoc_path = ['_design', 'transactions', '_view', 'log'] + res = self._database.resource(*ddoc_path) + try: + response = res.get_json() + return map( + lambda row: (row['id'], row['value']), + response[2]['rows']) + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) def _get_doc(self, doc_id, check_for_conflicts=False): """ @@ -472,6 +618,19 @@ class CouchDatabase(CommonBackend): :raise RevisionConflict: Raised when trying to update a document but couch revisions mismatch. + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ trans_id = self._allocate_transaction_id() # encode content @@ -489,26 +648,29 @@ class CouchDatabase(CommonBackend): doc.get_conflicts())) )[:-1] # exclude \n # perform the request - resource = self._database.resource( - '_design', 'docs', '_update', 'put', doc.doc_id) - response = resource.put_json( - body={ - 'couch_rev': old_doc.couch_rev - if old_doc is not None else None, - 'u1db_rev': doc.rev, - 'content': content, - 'trans_id': trans_id, - 'conflicts': conflicts, - 'update_conflicts': update_conflicts, - }, - headers={'content-type': 'application/json'}) - # the document might have been updated in between, so we check for the - # return message - msg = response[2].read() - if msg == 'ok': - return - elif msg == 'revision conflict': - raise errors.RevisionConflict() + ddoc_path = ['_design', 'docs', '_update', 'put', doc.doc_id] + resource = self._database.resource(*ddoc_path) + try: + response = resource.put_json( + body={ + 'couch_rev': old_doc.couch_rev + if old_doc is not None else None, + 'u1db_rev': doc.rev, + 'content': content, + 'trans_id': trans_id, + 'conflicts': conflicts, + 'update_conflicts': update_conflicts, + }, + headers={'content-type': 'application/json'}) + # the document might have been updated in between, so we check for + # the return message + msg = response[2].read() + if msg == 'ok': + return + elif msg == 'revision conflict': + raise RevisionConflict() + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) def put_doc(self, doc): """ @@ -522,26 +684,26 @@ class CouchDatabase(CommonBackend): :return: new_doc_rev - The new revision identifier for the document. The Document object will also be updated. - :raise errors.InvalidDocId: Raised if the document's id is invalid. - :raise errors.DocumentTooBig: Raised if the document size is too big. - :raise errors.ConflictedDoc: Raised if the document has conflicts. + :raise InvalidDocId: Raised if the document's id is invalid. + :raise DocumentTooBig: Raised if the document size is too big. + :raise ConflictedDoc: Raised if the document has conflicts. """ if doc.doc_id is None: - raise errors.InvalidDocId() + raise InvalidDocId() self._check_doc_id(doc.doc_id) self._check_doc_size(doc) old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) if old_doc and old_doc.has_conflicts: - raise errors.ConflictedDoc() + raise ConflictedDoc() if old_doc and doc.rev is None and old_doc.is_tombstone(): new_rev = self._allocate_doc_rev(old_doc.rev) else: if old_doc is not None: if old_doc.rev != doc.rev: - raise errors.RevisionConflict() + raise RevisionConflict() else: if doc.rev is not None: - raise errors.RevisionConflict() + raise RevisionConflict() new_rev = self._allocate_doc_rev(doc.rev) doc.rev = new_rev self._put_doc(old_doc, doc) @@ -563,32 +725,53 @@ class CouchDatabase(CommonBackend): to the last intervening change and sorted by generation (old changes first) :rtype: (int, str, [(str, int, str)]) + + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ # query a couch list function - res = self._database.resource( - '_design', 'transactions', '_list', 'whats_changed', 'log') - response = res.get_json(old_gen=old_generation) - results = map( - lambda row: - (row['generation'], row['doc_id'], row['transaction_id']), - response[2]['transactions']) - results.reverse() - cur_gen = old_generation - seen = set() - changes = [] - newest_trans_id = '' - for generation, doc_id, trans_id in results: - if doc_id not in seen: - changes.append((doc_id, generation, trans_id)) - seen.add(doc_id) - if changes: - cur_gen = changes[0][1] # max generation - newest_trans_id = changes[0][2] - changes.reverse() - else: - cur_gen, newest_trans_id = self._get_generation_info() + ddoc_path = [ + '_design', 'transactions', '_list', 'whats_changed', 'log' + ] + res = self._database.resource(*ddoc_path) + try: + response = res.get_json(old_gen=old_generation) + results = map( + lambda row: + (row['generation'], row['doc_id'], row['transaction_id']), + response[2]['transactions']) + results.reverse() + cur_gen = old_generation + seen = set() + changes = [] + newest_trans_id = '' + for generation, doc_id, trans_id in results: + if doc_id not in seen: + changes.append((doc_id, generation, trans_id)) + seen.add(doc_id) + if changes: + cur_gen = changes[0][1] # max generation + newest_trans_id = changes[0][2] + changes.reverse() + else: + cur_gen, newest_trans_id = self._get_generation_info() - return cur_gen, newest_trans_id, changes + return cur_gen, newest_trans_id, changes + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) + except ServerError as e: + raise_server_error(e, ddoc_path) def delete_doc(self, doc): """ @@ -600,22 +783,22 @@ class CouchDatabase(CommonBackend): :param doc: The document to mark as deleted. :type doc: CouchDocument. - :raise errors.DocumentDoesNotExist: Raised if the document does not + :raise DocumentDoesNotExist: Raised if the document does not exist. - :raise errors.RevisionConflict: Raised if the revisions do not match. - :raise errors.DocumentAlreadyDeleted: Raised if the document is + :raise RevisionConflict: Raised if the revisions do not match. + :raise DocumentAlreadyDeleted: Raised if the document is already deleted. - :raise errors.ConflictedDoc: Raised if the doc has conflicts. + :raise ConflictedDoc: Raised if the doc has conflicts. """ old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) if old_doc is None: - raise errors.DocumentDoesNotExist + raise DocumentDoesNotExist if old_doc.rev != doc.rev: - raise errors.RevisionConflict() + raise RevisionConflict() if old_doc.is_tombstone(): - raise errors.DocumentAlreadyDeleted + raise DocumentAlreadyDeleted if old_doc.has_conflicts: - raise errors.ConflictedDoc() + raise ConflictedDoc() new_rev = self._allocate_doc_rev(doc.rev) doc.rev = new_rev doc.make_tombstone() @@ -737,17 +920,34 @@ class CouchDatabase(CommonBackend): :param other_transaction_id: The transaction id associated with the generation. :type other_transaction_id: str + + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ # query a couch update function - res = self._database.resource( - '_design', 'syncs', '_update', 'put', 'u1db_sync_log') - res.put_json( - body={ - 'other_replica_uid': other_replica_uid, - 'other_generation': other_generation, - 'other_transaction_id': other_transaction_id, - }, - headers={'content-type': 'application/json'}) + ddoc_path = ['_design', 'syncs', '_update', 'put', 'u1db_sync_log'] + res = self._database.resource(*ddoc_path) + try: + res.put_json( + body={ + 'other_replica_uid': other_replica_uid, + 'other_generation': other_generation, + 'other_transaction_id': other_transaction_id, + }, + headers={'content-type': 'application/json'}) + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) def _add_conflict(self, doc, my_doc_rev, my_content): """ @@ -774,7 +974,7 @@ class CouchDatabase(CommonBackend): """ Delete the conflicted revisions from the list of conflicts of C{doc}. - Note that thie method does not actually update the backed; rather, it + Note that this method does not actually update the backend; rather, it updates the CouchDocument object which will provide the conflict data when the atomic document update is made. @@ -842,6 +1042,20 @@ class CouchDatabase(CommonBackend): :param conflicted_doc_revs: A list of revisions that the new content supersedes. :type conflicted_doc_revs: [str] + + :raise MissingDesignDocError: Raised when tried to access a missing + design document. + :raise MissingDesignDocListFunctionError: Raised when trying to access + a missing list function on a + design document. + :raise MissingDesignDocNamedViewError: Raised when trying to access a + missing named view on a design + document. + :raise MissingDesignDocDeletedError: Raised when trying to access a + deleted design document. + :raise MissingDesignDocUnknownError: Raised when failed to access a + design document for an yet + unknown reason. """ cur_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) new_rev = self._ensure_maximal_rev(cur_doc.rev, @@ -855,8 +1069,10 @@ class CouchDatabase(CommonBackend): self._add_conflict(doc, new_rev, doc.get_json()) self._delete_conflicts(doc, superseded_revs) # perform request to resolve document in server - resource = self._database.resource( - '_design', 'docs', '_update', 'resolve_doc', doc.doc_id) + ddoc_path = [ + '_design', 'docs', '_update', 'resolve_doc', doc.doc_id + ] + resource = self._database.resource(*ddoc_path) conflicts = None if doc.has_conflicts: conflicts = binascii.b2a_base64( @@ -864,12 +1080,15 @@ class CouchDatabase(CommonBackend): map(lambda cdoc: (cdoc.rev, cdoc.content), doc.get_conflicts())) )[:-1] # exclude \n - response = resource.put_json( - body={ - 'couch_rev': cur_doc.couch_rev, - 'conflicts': conflicts, - }, - headers={'content-type': 'application/json'}) + try: + response = resource.put_json( + body={ + 'couch_rev': cur_doc.couch_rev, + 'conflicts': conflicts, + }, + headers={'content-type': 'application/json'}) + except ResourceNotFound as e: + raise_missing_design_doc_error(e, ddoc_path) def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, replica_trans_id=''): diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index c41e0b0f..f241ee06 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -25,11 +25,17 @@ from u1db import errors from u1db.remote import http_errors +class SoledadError(errors.U1DBError): + """ + Base Soledad HTTP errors. + """ + pass + # -# LockResource: a lock based on a document in the shared database. +# LockResource errors # -class InvalidTokenError(errors.U1DBError): +class InvalidTokenError(SoledadError): """ Exception raised when trying to unlock shared database with invalid token. """ @@ -38,7 +44,7 @@ class InvalidTokenError(errors.U1DBError): status = 401 -class NotLockedError(errors.U1DBError): +class NotLockedError(SoledadError): """ Exception raised when trying to unlock shared database when it is not locked. @@ -48,7 +54,7 @@ class NotLockedError(errors.U1DBError): status = 404 -class AlreadyLockedError(errors.U1DBError): +class AlreadyLockedError(SoledadError): """ Exception raised when trying to lock shared database but it is already locked. @@ -58,7 +64,7 @@ class AlreadyLockedError(errors.U1DBError): status = 403 -class LockTimedOutError(errors.U1DBError): +class LockTimedOutError(SoledadError): """ Exception raised when timing out while trying to lock the shared database. """ @@ -67,7 +73,7 @@ class LockTimedOutError(errors.U1DBError): status = 408 -class CouldNotObtainLockError(errors.U1DBError): +class CouldNotObtainLockError(SoledadError): """ Exception raised when timing out while trying to lock the shared database. """ @@ -76,10 +82,64 @@ class CouldNotObtainLockError(errors.U1DBError): status = 500 +# +# CouchDatabase errors +# + +class MissingDesignDocError(SoledadError): + """ + Raised when trying to access a missing couch design document. + """ + + wire_description = "missing design document" + status = 500 + + +class MissingDesignDocNamedViewError(SoledadError): + """ + Raised when trying to access a missing named view on a couch design + document. + """ + + wire_description = "missing design document named function" + status = 500 + + +class MissingDesignDocListFunctionError(SoledadError): + """ + Raised when trying to access a missing list function on a couch design + document. + """ + + wire_description = "missing design document list function" + status = 500 + + +class MissingDesignDocDeletedError(SoledadError): + """ + Raised when trying to access a deleted couch design document. + """ + + wire_description = "design document was deleted" + status = 500 + + +class DesignDocUnknownError(SoledadError): + """ + Raised when trying to access a couch design document and getting an + unknown error. + """ + + wire_description = "missing design document unknown error" + status = 500 + + # update u1db "wire description to status" and "wire description to exception" # maps. for e in [InvalidTokenError, NotLockedError, AlreadyLockedError, - LockTimedOutError, CouldNotObtainLockError]: + LockTimedOutError, CouldNotObtainLockError, MissingDesignDocError, + MissingDesignDocListFunctionError, MissingDesignDocNamedViewError, + MissingDesignDocDeletedError, DesignDocUnknownError]: http_errors.wire_description_to_status.update({ e.wire_description: e.status}) errors.wire_description_to_exc.update({ diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 72346333..0e07575d 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -24,16 +24,17 @@ import re import copy import shutil from base64 import b64decode +from mock import Mock from couchdb.client import Server -from u1db import errors +from u1db import errors as u1db_errors from leap.common.files import mkdir_p from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common import couch +from leap.soledad.common import couch, errors import simplejson as json @@ -356,7 +357,7 @@ class IndexedCouchDatabase(couch.CouchDatabase): def __init__(self, url, dbname, replica_uid=None, full_commit=True, session=None, ensure_ddocs=True): old_class.__init__(self, url, dbname, replica_uid, full_commit, - session, ensure_ddocs=True) + session, ensure_ddocs=ensure_ddocs) self._indexes = {} def _put_doc(self, old_doc, doc): @@ -372,7 +373,7 @@ class IndexedCouchDatabase(couch.CouchDatabase): if self._indexes[index_name]._definition == list( index_expressions): return - raise errors.IndexNameTakenError + raise u1db_errors.IndexNameTakenError index = InMemoryIndex(index_name, list(index_expressions)) _, all_docs = self.get_all_docs() for doc in all_docs: @@ -392,7 +393,7 @@ class IndexedCouchDatabase(couch.CouchDatabase): try: index = self._indexes[index_name] except KeyError: - raise errors.IndexDoesNotExist + raise u1db_errors.IndexDoesNotExist doc_ids = index.lookup(key_values) result = [] for doc_id in doc_ids: @@ -405,7 +406,7 @@ class IndexedCouchDatabase(couch.CouchDatabase): try: index = self._indexes[index_name] except KeyError: - raise errors.IndexDoesNotExist + raise u1db_errors.IndexDoesNotExist if isinstance(start_value, basestring): start_value = (start_value,) if isinstance(end_value, basestring): @@ -420,7 +421,7 @@ class IndexedCouchDatabase(couch.CouchDatabase): try: index = self._indexes[index_name] except KeyError: - raise errors.IndexDoesNotExist + raise u1db_errors.IndexDoesNotExist keys = index.keys() # XXX inefficiency warning return list(set([tuple(key.split('\x01')) for key in keys])) @@ -461,4 +462,208 @@ class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase): test_sync.DatabaseSyncTests.tearDown(self) +class CouchDatabaseExceptionsTests(CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + self.db = couch.CouchDatabase( + 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + ensure_ddocs=False) # note that we don't enforce ddocs here + + def tearDown(self): + self.db.delete_database() + + def test_missing_design_doc_raises(self): + """ + Test that all methods that access design documents will raise if the + design docs are not present. + """ + # _get_generation() + self.assertRaises( + errors.MissingDesignDocError, + self.db._get_generation) + # _get_generation_info() + self.assertRaises( + errors.MissingDesignDocError, + self.db._get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocError, + self.db._get_trans_id_for_gen, 1) + # _get_transaction_log() + self.assertRaises( + errors.MissingDesignDocError, + self.db._get_transaction_log) + # create_doc() + self.assertRaises( + errors.MissingDesignDocError, + self.db.create_doc, {}) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocError, + self.db.whats_changed) + # _do_set_replica_gen_and_trans_id() + self.assertRaises( + errors.MissingDesignDocError, + self.db._do_set_replica_gen_and_trans_id, 1, 2, 3) + # fake a conflict so we can test resolve_doc() + first_rev = self.db._allocate_doc_rev(None) + doc = couch.CouchDocument( + doc_id='mydoc', rev=self.db._allocate_doc_rev(first_rev)) + self.db._get_doc = Mock(return_value=doc) + self.assertRaises( + errors.MissingDesignDocError, + self.db.resolve_doc, doc, [first_rev]) + + def test_missing_design_doc_functions_raises(self): + """ + Test that all methods that access design documents list functions + will raise if the functions are not present. + """ + self.db = couch.CouchDatabase( + 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + ensure_ddocs=True) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + transactions['lists'] = {} + self.db._database.save(transactions) + # _get_generation() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db._get_generation) + # _get_generation_info() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db._get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db._get_trans_id_for_gen, 1) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.whats_changed) + + def test_absent_design_doc_functions_raises(self): + """ + Test that all methods that access design documents list functions + will raise if the functions are not present. + """ + self.db = couch.CouchDatabase( + 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + ensure_ddocs=True) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + del transactions['lists'] + self.db._database.save(transactions) + # _get_generation() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db._get_generation) + # _get_generation_info() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db._get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db._get_trans_id_for_gen, 1) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.whats_changed) + + def test_missing_design_doc_named_views_raises(self): + """ + Test that all methods that access design documents' named views will + raise if the views are not present. + """ + self.db = couch.CouchDatabase( + 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + ensure_ddocs=True) + # erase views from _design/docs + docs = self.db._database['_design/docs'] + del docs['views'] + self.db._database.save(docs) + # erase views from _design/syncs + syncs = self.db._database['_design/syncs'] + del syncs['views'] + self.db._database.save(syncs) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + del transactions['views'] + self.db._database.save(transactions) + # _get_generation() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db._get_generation) + # _get_generation_info() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db._get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db._get_trans_id_for_gen, 1) + # _get_transaction_log() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db._get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.whats_changed) + + def test_deleted_design_doc_raises(self): + """ + Test that all methods that access design documents will raise if the + design docs are not present. + """ + self.db = couch.CouchDatabase( + 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + ensure_ddocs=True) + # delete _design/docs + del self.db._database['_design/docs'] + # delete _design/syncs + del self.db._database['_design/syncs'] + # delete _design/transactions + del self.db._database['_design/transactions'] + # _get_generation() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db._get_generation) + # _get_generation_info() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db._get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db._get_trans_id_for_gen, 1) + # _get_transaction_log() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db._get_transaction_log) + # create_doc() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.create_doc, {}) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.whats_changed) + # _do_set_replica_gen_and_trans_id() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db._do_set_replica_gen_and_trans_id, 1, 2, 3) + # fake a conflict so we can test resolve_doc() + first_rev = self.db._allocate_doc_rev(None) + doc = couch.CouchDocument( + doc_id='mydoc', rev=self.db._allocate_doc_rev(first_rev)) + self.db._get_doc = Mock(return_value=doc) + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.resolve_doc, doc, [first_rev]) + + load_tests = tests.load_with_scenarios -- cgit v1.2.3 From f125180609c3e97f56eedfe534f7f1c6f985f8f6 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 25 Dec 2013 21:02:58 -0200 Subject: Allow sync of large files (~100MB) (#4836). --- .../changes/feature_4836_allow-sync-of-large-files | 1 + common/src/leap/soledad/common/couch.py | 7 +++- .../leap/soledad/common/tests/couchdb.ini.template | 2 +- .../tests/test_couch_operations_atomicity.py | 1 + .../src/leap/soledad/common/tests/test_server.py | 37 ++++++++++++++++++++++ 5 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 common/changes/feature_4836_allow-sync-of-large-files (limited to 'common') diff --git a/common/changes/feature_4836_allow-sync-of-large-files b/common/changes/feature_4836_allow-sync-of-large-files new file mode 100644 index 00000000..f124e899 --- /dev/null +++ b/common/changes/feature_4836_allow-sync-of-large-files @@ -0,0 +1 @@ + o Allow sync of large files (~100MB) (#4836). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 45ca4282..3ca3f408 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -27,7 +27,7 @@ import socket from couchdb.client import Server -from couchdb.http import ResourceNotFound, Unauthorized, ServerError +from couchdb.http import ResourceNotFound, Unauthorized, ServerError, Session from u1db import query_parser, vectorclock from u1db.errors import ( DatabaseDoesNotExist, @@ -50,6 +50,9 @@ from leap.soledad.common.document import SoledadDocument logger = logging.getLogger(__name__) +COUCH_TIMEOUT = 120 # timeout for transfers between Soledad server and Couch + + class InvalidURLError(Exception): """ Exception raised when Soledad encounters a malformed URL. @@ -275,6 +278,8 @@ class CouchDatabase(CommonBackend): # save params self._url = url self._full_commit = full_commit + if session is None: + session = Session(timeout=COUCH_TIMEOUT) self._session = session self._factory = CouchDocument self._real_replica_uid = None diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template index 217ae201..1fc2205b 100644 --- a/common/src/leap/soledad/common/tests/couchdb.ini.template +++ b/common/src/leap/soledad/common/tests/couchdb.ini.template @@ -6,7 +6,7 @@ database_dir = %(tempdir)s/lib view_index_dir = %(tempdir)s/lib max_document_size = 4294967296 ; 4 GB -os_process_timeout = 5000 ; 5 seconds. for view and external servers. +os_process_timeout = 120000 ; 120 seconds. for view and external servers. max_dbs_open = 100 delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned uri_file = %(tempdir)s/lib/couch.uri diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 8b001859..5384d465 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -100,6 +100,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") def tearDown(self): + self.db.delete_database() CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 83df192b..ff699486 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -25,6 +25,7 @@ import tempfile import simplejson as json import mock import time +import binascii from leap.common.testing.basetest import BaseLeapTest @@ -436,6 +437,42 @@ class EncryptedSyncTestCase( self.assertEqual(doc1, doc2) + def test_sync_very_large_files(self): + """ + Test if Soledad can sync very large files. + """ + # define the size of the "very large file" + length = 100*(10**6) # 100 MB + self.startServer() + # instantiate soledad and create a document + sol1 = self._soledad_instance( + # token is verified in test_target.make_token_soledad_app + auth_token='auth-token' + ) + _, doclist = sol1.get_all_docs() + self.assertEqual([], doclist) + content = binascii.hexlify(os.urandom(length/2)) # len() == length + doc1 = sol1.create_doc({'data': content}) + # sync with server + sol1._server_url = self.getURL() + sol1.sync() + # instantiate soledad with empty db, but with same secrets path + sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') + _, doclist = sol2.get_all_docs() + self.assertEqual([], doclist) + sol2._secrets_path = sol1.secrets_path + sol2._load_secrets() + sol2._set_secret_id(sol1._secret_id) + # sync the new instance + sol2._server_url = self.getURL() + sol2.sync() + _, doclist = sol2.get_all_docs() + self.assertEqual(1, len(doclist)) + doc2 = doclist[0] + # assert incoming doc is equal to the first sent doc + self.assertEqual(doc1, doc2) + + class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): """ -- cgit v1.2.3 From 8f755c7cfcad8da2e6446f878c3df1f8e9cbae5b Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Dec 2013 17:30:58 -0200 Subject: Add test for syncing many small documents (#4836). --- common/src/leap/soledad/common/tests/test_couch.py | 5 ++-- .../src/leap/soledad/common/tests/test_server.py | 35 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) (limited to 'common') diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 0e07575d..c833680e 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -81,9 +81,10 @@ class CouchDBWrapper(object): mkdir_p(os.path.join(self.tempdir, 'lib')) mkdir_p(os.path.join(self.tempdir, 'log')) args = ['couchdb', '-n', '-a', confPath] - #null = open('/dev/null', 'w') + null = open('/dev/null', 'w') + self.process = subprocess.Popen( - args, env=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + args, env=None, stdout=null.fileno(), stderr=null.fileno(), close_fds=True) # find port logPath = os.path.join(self.tempdir, 'log', 'couch.log') diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index ff699486..922f16a2 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -473,6 +473,41 @@ class EncryptedSyncTestCase( self.assertEqual(doc1, doc2) + def test_sync_many_small_files(self): + """ + Test if Soledad can sync many smallfiles. + """ + number_of_docs = 100 + self.startServer() + # instantiate soledad and create a document + sol1 = self._soledad_instance( + # token is verified in test_target.make_token_soledad_app + auth_token='auth-token' + ) + _, doclist = sol1.get_all_docs() + self.assertEqual([], doclist) + # create many small files + for i in range(0, number_of_docs): + sol1.create_doc(json.loads(simple_doc)) + # sync with server + sol1._server_url = self.getURL() + sol1.sync() + # instantiate soledad with empty db, but with same secrets path + sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') + _, doclist = sol2.get_all_docs() + self.assertEqual([], doclist) + sol2._secrets_path = sol1.secrets_path + sol2._load_secrets() + sol2._set_secret_id(sol1._secret_id) + # sync the new instance + sol2._server_url = self.getURL() + sol2.sync() + _, doclist = sol2.get_all_docs() + self.assertEqual(number_of_docs, len(doclist)) + # assert incoming docs are equal to sent docs + for doc in doclist: + self.assertEqual(sol1.get_doc(doc.doc_id), doc) + class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): """ -- cgit v1.2.3 From dc35d67834edde4e0f927c35e0459c27b575f08d Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 26 Dec 2013 17:41:41 -0200 Subject: Make couch tests use only one couch instance. --- common/src/leap/soledad/common/tests/test_couch.py | 14 ++++++------- .../src/leap/soledad/common/tests/test_server.py | 23 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) (limited to 'common') diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index c833680e..dc0ea906 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -127,21 +127,21 @@ class CouchDBTestCase(unittest.TestCase): TestCase base class for tests against a real CouchDB server. """ - def setUp(self): + @classmethod + def setUpClass(cls): """ Make sure we have a CouchDB instance for a test. """ - self.wrapper = CouchDBWrapper() - self.wrapper.start() + cls.wrapper = CouchDBWrapper() + cls.wrapper.start() #self.db = self.wrapper.db - unittest.TestCase.setUp(self) - def tearDown(self): + @classmethod + def tearDownClass(cls): """ Stop CouchDB instance for test. """ - self.wrapper.stop() - unittest.TestCase.tearDown(self) + cls.wrapper.stop() #----------------------------------------------------------------------------- diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 922f16a2..06595ed2 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -377,6 +377,7 @@ class EncryptedSyncTestCase( doc2 = doclist[0] # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) + db.delete_database() def test_encrypted_sym_sync_with_unicode_passphrase(self): """ @@ -435,7 +436,7 @@ class EncryptedSyncTestCase( doc2 = doclist[0] # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) - + db.delete_database() def test_sync_very_large_files(self): """ @@ -471,6 +472,13 @@ class EncryptedSyncTestCase( doc2 = doclist[0] # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) + # delete remote database + db = CouchDatabase( + self._couch_url, + # the name of the user database is "user-". + 'user-user-uuid', + ) + db.delete_database() def test_sync_many_small_files(self): @@ -507,6 +515,13 @@ class EncryptedSyncTestCase( # assert incoming docs are equal to sent docs for doc in doclist: self.assertEqual(sol1.get_doc(doc.doc_id), doc) + # delete remote database + db = CouchDatabase( + self._couch_url, + # the name of the user database is "user-". + 'user-user-uuid', + ) + db.delete_database() class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): @@ -533,6 +548,12 @@ class LockResourceTestCase( def tearDown(self): CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) + # delete remote database + db = CouchDatabase( + self._couch_url, + 'shared', + ) + db.delete_database() def test__try_obtain_filesystem_lock(self): responder = mock.Mock() -- cgit v1.2.3 From 4a2c8a111470247d15a5a1771c5ddb18980f37b7 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 22 Jan 2014 15:11:43 -0200 Subject: Improve DesignDocUnknownError error reporting. --- common/src/leap/soledad/common/couch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'common') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 3ca3f408..40d64370 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -199,7 +199,7 @@ def raise_missing_design_doc_error(exc, ddoc_path): elif exc.message[1] == 'deleted': raise errors.MissingDesignDocDeletedError(path) # other errors are unknown for now - raise errors.DesignDocUnknownError(path) + raise errors.DesignDocUnknownError("%s: %s" % (path, str(exc.message))) def raise_server_error(exc, ddoc_path): -- cgit v1.2.3 From 49e7366e3b6dcad7fde6b78ed3c2bffd3b1430e4 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 24 Jan 2014 18:24:55 -0200 Subject: Add decorator to register exceptions. --- common/src/leap/soledad/common/errors.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'common') diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index f241ee06..fc4d106a 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -25,16 +25,29 @@ from u1db import errors from u1db.remote import http_errors +exceptions = [] + + +def register_exception(cls): + """ + A small decorator to make it easier to register exceptions. + """ + exceptions.append(cls) + return cls + + class SoledadError(errors.U1DBError): """ Base Soledad HTTP errors. """ pass + # # LockResource errors # +@register_exception class InvalidTokenError(SoledadError): """ Exception raised when trying to unlock shared database with invalid token. @@ -44,6 +57,7 @@ class InvalidTokenError(SoledadError): status = 401 +@register_exception class NotLockedError(SoledadError): """ Exception raised when trying to unlock shared database when it is not @@ -54,6 +68,7 @@ class NotLockedError(SoledadError): status = 404 +@register_exception class AlreadyLockedError(SoledadError): """ Exception raised when trying to lock shared database but it is already @@ -64,6 +79,7 @@ class AlreadyLockedError(SoledadError): status = 403 +@register_exception class LockTimedOutError(SoledadError): """ Exception raised when timing out while trying to lock the shared database. @@ -73,6 +89,7 @@ class LockTimedOutError(SoledadError): status = 408 +@register_exception class CouldNotObtainLockError(SoledadError): """ Exception raised when timing out while trying to lock the shared database. @@ -86,6 +103,7 @@ class CouldNotObtainLockError(SoledadError): # CouchDatabase errors # +@register_exception class MissingDesignDocError(SoledadError): """ Raised when trying to access a missing couch design document. @@ -95,6 +113,7 @@ class MissingDesignDocError(SoledadError): status = 500 +@register_exception class MissingDesignDocNamedViewError(SoledadError): """ Raised when trying to access a missing named view on a couch design @@ -105,6 +124,7 @@ class MissingDesignDocNamedViewError(SoledadError): status = 500 +@register_exception class MissingDesignDocListFunctionError(SoledadError): """ Raised when trying to access a missing list function on a couch design @@ -115,6 +135,7 @@ class MissingDesignDocListFunctionError(SoledadError): status = 500 +@register_exception class MissingDesignDocDeletedError(SoledadError): """ Raised when trying to access a deleted couch design document. @@ -124,6 +145,7 @@ class MissingDesignDocDeletedError(SoledadError): status = 500 +@register_exception class DesignDocUnknownError(SoledadError): """ Raised when trying to access a couch design document and getting an @@ -136,10 +158,7 @@ class DesignDocUnknownError(SoledadError): # update u1db "wire description to status" and "wire description to exception" # maps. -for e in [InvalidTokenError, NotLockedError, AlreadyLockedError, - LockTimedOutError, CouldNotObtainLockError, MissingDesignDocError, - MissingDesignDocListFunctionError, MissingDesignDocNamedViewError, - MissingDesignDocDeletedError, DesignDocUnknownError]: +for e in exceptions: http_errors.wire_description_to_status.update({ e.wire_description: e.status}) errors.wire_description_to_exc.update({ -- cgit v1.2.3 From c97f48660a1aaad96f7356ac1a5fce6265241e0f Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 24 Jan 2014 19:09:46 -0200 Subject: Improve unauthorized error messages. --- common/changes/feature_4035_improve-error-messages | 3 ++ common/src/leap/soledad/common/errors.py | 46 +++++++++++++++------- 2 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 common/changes/feature_4035_improve-error-messages (limited to 'common') diff --git a/common/changes/feature_4035_improve-error-messages b/common/changes/feature_4035_improve-error-messages new file mode 100644 index 00000000..4f86a17f --- /dev/null +++ b/common/changes/feature_4035_improve-error-messages @@ -0,0 +1,3 @@ + o Improve error messages. Closes #5035. + * Add MissingTokenError and InvalidTokenError as sub exceptions from + Unauthorized. diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index fc4d106a..446c4c75 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -25,14 +25,17 @@ from u1db import errors from u1db.remote import http_errors -exceptions = [] - - def register_exception(cls): """ - A small decorator to make it easier to register exceptions. + A small decorator that registers exceptions in u1db maps. """ - exceptions.append(cls) + # update u1db "wire description to status" and "wire description to + # exception" maps. + http_errors.wire_description_to_status.update({ + cls.wire_description: cls.status}) + errors.wire_description_to_exc.update({ + cls.wire_description: cls}) + # do not modify the exception return cls @@ -43,6 +46,30 @@ class SoledadError(errors.U1DBError): pass +# +# Authorization errors +# + +@register_exception +class MissingAuthTokenError(errors.Unauthorized): + """ + Exception raised when failing to get authorization for some action because + the auth token is missing in the tokens db. + """ + + wire_description = "missing token" + status = 401 + +@register_exception +class InvalidAuthTokenError(errors.Unauthorized): + """ + Exception raised when failing to get authorization for some action because + the provided token is different from the one in the tokens db. + """ + + wire_descrition = "token mismatch" + status = 401 + # # LockResource errors # @@ -156,15 +183,6 @@ class DesignDocUnknownError(SoledadError): status = 500 -# update u1db "wire description to status" and "wire description to exception" -# maps. -for e in exceptions: - http_errors.wire_description_to_status.update({ - e.wire_description: e.status}) - errors.wire_description_to_exc.update({ - e.wire_description: e}) - - # u1db error statuses also have to be updated http_errors.ERROR_STATUSES = set( http_errors.wire_description_to_status.values()) -- cgit v1.2.3 From 7de7b4653b1815818611d2a6302bd38d1e7a7ccc Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 21 Feb 2014 17:49:28 -0300 Subject: Use less memory when putting docs on couch. Closes #5011. --- ...5011-use-less-memory-when-putting-docs-on-couch | 1 + common/src/leap/soledad/common/couch.py | 267 +++++++++++++++------ .../leap/soledad/common/ddocs/docs/updates/put.js | 64 ----- common/src/leap/soledad/common/tests/test_couch.py | 8 - 4 files changed, 199 insertions(+), 141 deletions(-) create mode 100644 common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch delete mode 100644 common/src/leap/soledad/common/ddocs/docs/updates/put.js (limited to 'common') diff --git a/common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch b/common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch new file mode 100644 index 00000000..7d3f6e4f --- /dev/null +++ b/common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch @@ -0,0 +1 @@ + o Use less memory when putting docs on couch (#5011). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 40d64370..09fe1ca3 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -24,10 +24,21 @@ import uuid import logging import binascii import socket +import time +import sys + + +from StringIO import StringIO from couchdb.client import Server -from couchdb.http import ResourceNotFound, Unauthorized, ServerError, Session +from couchdb.http import ( + ResourceConflict, + ResourceNotFound, + Unauthorized, + ServerError, + Session, +) from u1db import query_parser, vectorclock from u1db.errors import ( DatabaseDoesNotExist, @@ -87,9 +98,9 @@ class CouchDocument(SoledadDocument): SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts) self._couch_rev = None self._conflicts = None - self._modified_conflicts = False + self._transactions = None - def ensure_fetch_conflicts(self, get_conflicts_fun): + def _ensure_fetch_conflicts(self, get_conflicts_fun): """ Ensure conflict data has been fetched from the server. @@ -120,8 +131,7 @@ class CouchDocument(SoledadDocument): :type doc: CouchDocument """ if self._conflicts is None: - raise Exception("Run self.ensure_fetch_conflicts first!") - self._modified_conflicts = True + raise Exception("Run self._ensure_fetch_conflicts first!") self._conflicts.append(doc) self.has_conflicts = len(self._conflicts) > 0 @@ -133,25 +143,13 @@ class CouchDocument(SoledadDocument): :type conflict_revs: [str] """ if self._conflicts is None: - raise Exception("Run self.ensure_fetch_conflicts first!") + raise Exception("Run self._ensure_fetch_conflicts first!") conflicts_len = len(self._conflicts) self._conflicts = filter( lambda doc: doc.rev not in conflict_revs, self._conflicts) - if len(self._conflicts) < conflicts_len: - self._modified_conflicts = True self.has_conflicts = len(self._conflicts) > 0 - def modified_conflicts(self): - """ - Return whether this document's conflicts have been modified. - - :return: Whether this document's conflicts have been modified. - :rtype: bool - """ - return self._conflicts is not None and \ - self._modified_conflicts is True - def _get_couch_rev(self): return self._couch_rev @@ -160,6 +158,14 @@ class CouchDocument(SoledadDocument): couch_rev = property(_get_couch_rev, _set_couch_rev) + def _get_transactions(self): + return self._transactions + + def _set_transactions(self, rev): + self._transactions = rev + + transactions = property(_get_transactions, _set_transactions) + # monkey-patch the u1db http app to use CouchDocument http_app.Document = CouchDocument @@ -225,6 +231,94 @@ def raise_server_error(exc, ddoc_path): raise errors.DesignDocUnknownError(path) +class MultipartWriter(object): + """ + A multipart writer adapted from python-couchdb's one so we can PUT + documents using couch's multipart PUT. + + This stripped down version does not allow for nested structures, and + contains only the essential things we need to PUT SoledadDocuments to the + couch backend. + """ + + CRLF = '\r\n' + + def __init__(self, fileobj, headers=None, boundary=None): + """ + Initialize the multipart writer. + """ + self.fileobj = fileobj + if boundary is None: + boundary = self._make_boundary() + self._boundary = boundary + self._build_headers('related', headers) + + def add(self, mimetype, content, headers={}): + """ + Add a part to the multipart stream. + """ + self.fileobj.write('--') + self.fileobj.write(self._boundary) + self.fileobj.write(self.CRLF) + headers['Content-Type'] = mimetype + self._write_headers(headers) + if content: + # XXX: throw an exception if a boundary appears in the content?? + self.fileobj.write(content) + self.fileobj.write(self.CRLF) + + def close(self): + """ + Close the multipart stream. + """ + self.fileobj.write('--') + self.fileobj.write(self._boundary) + # be careful not to have anything after '--', otherwise old couch + # versions (including bigcouch) will fail. + self.fileobj.write('--') + + def _make_boundary(self): + """ + Create a boundary to discern multi parts. + """ + try: + from uuid import uuid4 + return '==' + uuid4().hex + '==' + except ImportError: + from random import randrange + token = randrange(sys.maxint) + format = '%%0%dd' % len(repr(sys.maxint - 1)) + return '===============' + (format % token) + '==' + + def _write_headers(self, headers): + """ + Write a part header in the buffer stream. + """ + if headers: + for name in sorted(headers.keys()): + value = headers[name] + self.fileobj.write(name) + self.fileobj.write(': ') + self.fileobj.write(value) + self.fileobj.write(self.CRLF) + self.fileobj.write(self.CRLF) + + def _build_headers(self, subtype, headers): + """ + Build the main headers of the multipart stream. + + This is here so we can send headers separete from content using + python-couchdb API. + """ + self.headers = {} + self.headers['Content-Type'] = 'multipart/%s; boundary="%s"' % \ + (subtype, self._boundary) + if headers: + for name in sorted(headers.keys()): + value = headers[name] + self.headers[name] = value + + class CouchDatabase(CommonBackend): """ A U1DB implementation that uses CouchDB as its persistence layer. @@ -564,8 +658,12 @@ class CouchDatabase(CommonBackend): and '_attachments' in result \ and 'u1db_conflicts' in result['_attachments']: doc.has_conflicts = True + #doc.conflicts = self._build_conflicts( + # json.loads(result['_attachments']['u1db_conflicts'])) # store couch revision doc.couch_rev = result['_rev'] + # store transactions + doc.transactions = result['u1db_transactions'] return doc def get_doc(self, doc_id, include_deleted=False): @@ -616,6 +714,10 @@ class CouchDatabase(CommonBackend): """ Put the document in the Couch backend database. + Note that C{old_doc} must have been fetched with the parameter + C{check_for_conflicts} equal to True, so we can properly update the + new document using the conflict information from the old one. + :param old_doc: The old document version. :type old_doc: CouchDocument :param doc: The document to be put. @@ -637,45 +739,61 @@ class CouchDatabase(CommonBackend): design document for an yet unknown reason. """ - trans_id = self._allocate_transaction_id() - # encode content - content = doc.get_json() - if content is not None: - content = binascii.b2a_base64(content)[:-1] # exclude trailing \n - # encode conflicts - conflicts = None - update_conflicts = doc.modified_conflicts() - if update_conflicts is True: - if doc.has_conflicts: - conflicts = binascii.b2a_base64( - json.dumps( - map(lambda cdoc: (cdoc.rev, cdoc.content), - doc.get_conflicts())) - )[:-1] # exclude \n - # perform the request - ddoc_path = ['_design', 'docs', '_update', 'put', doc.doc_id] - resource = self._database.resource(*ddoc_path) + attachments = {} # we save content and conflicts as attachments + parts = [] # and we put it using couch's multipart PUT + # save content as attachment + if doc.is_tombstone() is False: + content = doc.get_json() + attachments['u1db_content'] = { + 'follows': True, + 'content_type': 'application/octet-stream', + 'length': len(content), + } + parts.append(content) + # save conflicts as attachment + if doc.has_conflicts is True: + conflicts = json.dumps( + map(lambda cdoc: (cdoc.rev, cdoc.content), + doc.get_conflicts())) + attachments['u1db_conflicts'] = { + 'follows': True, + 'content_type': 'application/octet-stream', + 'length': len(conflicts), + } + parts.append(conflicts) + # store old transactions, if any + transactions = old_doc.transactions[:] if old_doc is not None else [] + # create a new transaction id and timestamp it so the transaction log + # is consistent when querying the database. + transactions.append( + # here we store milliseconds to keep consistent with javascript + # Date.prototype.getTime() which was used before inside a couchdb + # update handler. + (int(time.time() * 1000), + self._allocate_transaction_id())) + # build the couch document + couch_doc = { + '_id': doc.doc_id, + 'u1db_rev': doc.rev, + 'u1db_transactions': transactions, + '_attachments': attachments, + } + # if we are updating a doc we have to add the couch doc revision + if old_doc is not None: + couch_doc['_rev'] = old_doc.couch_rev + # prepare the multipart PUT + buf = StringIO() + envelope = MultipartWriter(buf) + envelope.add('application/json', json.dumps(couch_doc)) + for part in parts: + envelope.add('application/octet-stream', part) + envelope.close() + # try to save and fail if there's a revision conflict try: - response = resource.put_json( - body={ - 'couch_rev': old_doc.couch_rev - if old_doc is not None else None, - 'u1db_rev': doc.rev, - 'content': content, - 'trans_id': trans_id, - 'conflicts': conflicts, - 'update_conflicts': update_conflicts, - }, - headers={'content-type': 'application/json'}) - # the document might have been updated in between, so we check for - # the return message - msg = response[2].read() - if msg == 'ok': - return - elif msg == 'revision conflict': - raise RevisionConflict() - except ResourceNotFound as e: - raise_missing_design_doc_error(e, ddoc_path) + self._database.resource.put_json( + doc.doc_id, body=buf.getvalue(), headers=envelope.headers) + except ResourceConflict: + raise RevisionConflict() def put_doc(self, doc): """ @@ -810,6 +928,25 @@ class CouchDatabase(CommonBackend): self._put_doc(old_doc, doc) return new_rev + def _build_conflicts(self, doc_id, attached_conflicts): + """ + Build the conflicted documents list from the conflicts attachment + fetched from a couch document. + + :param attached_conflicts: The document's conflicts as fetched from a + couch document attachment. + :type attached_conflicts: dict + """ + conflicts = [] + for doc_rev, content in attached_conflicts: + doc = self._factory(doc_id, doc_rev) + if content is None: + doc.make_tombstone() + else: + doc.content = content + conflicts.append(doc) + return conflicts + def _get_conflicts(self, doc_id, couch_rev=None): """ Get the conflicted versions of a document. @@ -830,16 +967,8 @@ class CouchDatabase(CommonBackend): resource = self._database.resource(doc_id, 'u1db_conflicts') try: response = resource.get_json(**params) - conflicts = [] - # build the conflicted versions - for doc_rev, content in json.loads(response[2].read()): - doc = self._factory(doc_id, doc_rev) - if content is None: - doc.make_tombstone() - else: - doc.content = content - conflicts.append(doc) - return conflicts + return self._build_conflicts( + doc_id, json.loads(response[2].read())) except ResourceNotFound: return [] @@ -970,7 +1099,7 @@ class CouchDatabase(CommonBackend): serialized string. :type my_content: str """ - doc.ensure_fetch_conflicts(self._get_conflicts) + doc._ensure_fetch_conflicts(self._get_conflicts) doc.add_conflict( self._factory(doc_id=doc.doc_id, rev=my_doc_rev, json=my_content)) @@ -988,7 +1117,7 @@ class CouchDatabase(CommonBackend): :param conflict_revs: A list of the revisions to be deleted. :param conflict_revs: [str] """ - doc.ensure_fetch_conflicts(self._get_conflicts) + doc._ensure_fetch_conflicts(self._get_conflicts) doc.delete_conflicts(conflict_revs) def _prune_conflicts(self, doc, doc_vcr): @@ -1150,7 +1279,7 @@ class CouchDatabase(CommonBackend): if cur_doc is not None: doc.couch_rev = cur_doc.couch_rev # fetch conflicts because we will eventually manipulate them - doc.ensure_fetch_conflicts(self._get_conflicts) + doc._ensure_fetch_conflicts(self._get_conflicts) # from now on, it works just like u1db sqlite backend doc_vcr = vectorclock.VectorClockRev(doc.rev) if cur_doc is None: diff --git a/common/src/leap/soledad/common/ddocs/docs/updates/put.js b/common/src/leap/soledad/common/ddocs/docs/updates/put.js deleted file mode 100644 index 5a4647de..00000000 --- a/common/src/leap/soledad/common/ddocs/docs/updates/put.js +++ /dev/null @@ -1,64 +0,0 @@ -function(doc, req){ - /* we expect to receive the following in `req.body`: - * { - * 'couch_rev': '', - * 'u1db_rev': '', - * 'content': '', - * 'trans_id': '' - * 'conflicts': '', - * 'update_conflicts': - * } - */ - var body = JSON.parse(req.body); - - // create a new document document - if (!doc) { - doc = {} - doc['_id'] = req['id']; - } - // or fail if couch revisions do not match - else if (doc['_rev'] != body['couch_rev']) { - // of fail if revisions do not match - return [null, 'revision conflict'] - } - - // store u1db rev - doc.u1db_rev = body['u1db_rev']; - - // save content as attachment - if (body['content'] != null) { - // save u1db content as attachment - if (!doc._attachments) - doc._attachments = {}; - doc._attachments.u1db_content = { - content_type: "application/octet-stream", - data: body['content'] // should be base64 encoded - }; - } - // or delete the attachment if document is tombstone - else if (doc._attachments && - doc._attachments.u1db_content) - delete doc._attachments.u1db_content; - - // store the transaction id - if (!doc.u1db_transactions) - doc.u1db_transactions = []; - var d = new Date(); - doc.u1db_transactions.push([d.getTime(), body['trans_id']]); - - // save conflicts as attachment if they were sent - if (body['update_conflicts']) - if (body['conflicts'] != null) { - if (!doc._attachments) - doc._attachments = {}; - doc._attachments.u1db_conflicts = { - content_type: "application/octet-stream", - data: body['conflicts'] // should be base64 encoded - } - } else { - if(doc._attachments && doc._attachments.u1db_conflicts) - delete doc._attachments.u1db_conflicts - } - - return [doc, 'ok']; -} diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index dc0ea906..97d744e4 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -495,10 +495,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): self.assertRaises( errors.MissingDesignDocError, self.db._get_transaction_log) - # create_doc() - self.assertRaises( - errors.MissingDesignDocError, - self.db.create_doc, {}) # whats_changed() self.assertRaises( errors.MissingDesignDocError, @@ -645,10 +641,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): self.assertRaises( errors.MissingDesignDocDeletedError, self.db._get_transaction_log) - # create_doc() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.create_doc, {}) # whats_changed() self.assertRaises( errors.MissingDesignDocDeletedError, -- cgit v1.2.3 From 981eca6e00555e00c2f04621d6caad328ccc0363 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Mar 2014 12:20:52 -0300 Subject: Make resolve_doc() not use a design document. --- common/src/leap/soledad/common/couch.py | 54 ++++++++++++---------- .../common/ddocs/docs/updates/resolve_doc.js | 39 ---------------- common/src/leap/soledad/common/tests/test_couch.py | 16 ------- 3 files changed, 29 insertions(+), 80 deletions(-) delete mode 100644 common/src/leap/soledad/common/ddocs/docs/updates/resolve_doc.js (limited to 'common') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 09fe1ca3..456d4fdf 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -123,6 +123,16 @@ class CouchDocument(SoledadDocument): """ return self._conflicts + def set_conflicts(self, conflicts): + """ + Set the conflicted versions of the document. + + :param conflicts: The conflicted versions of the document. + :type conflicts: list + """ + self._conflicts = conflicts + self.has_conflicts = len(self._conflicts) > 0 + def add_conflict(self, doc): """ Add a conflict to this document. @@ -658,8 +668,11 @@ class CouchDatabase(CommonBackend): and '_attachments' in result \ and 'u1db_conflicts' in result['_attachments']: doc.has_conflicts = True - #doc.conflicts = self._build_conflicts( - # json.loads(result['_attachments']['u1db_conflicts'])) + doc.set_conflicts( + self._build_conflicts( + doc.doc_id, + json.loads(binascii.a2b_base64( + result['_attachments']['u1db_conflicts']['data'])))) # store couch revision doc.couch_rev = result['_rev'] # store transactions @@ -1196,33 +1209,24 @@ class CouchDatabase(CommonBackend): conflicted_doc_revs) superseded_revs = set(conflicted_doc_revs) doc.rev = new_rev + # this backend stores conflicts as properties of the documents, so we + # have to copy these conflicts over to the document being updated. if cur_doc.rev in superseded_revs: + # the newer doc version will supersede the one in the database, so + # we copy conflicts before updating the backend. + doc.set_conflicts(cur_doc.get_conflicts()) # copy conflicts over. self._delete_conflicts(doc, superseded_revs) self._put_doc(cur_doc, doc) else: - self._add_conflict(doc, new_rev, doc.get_json()) - self._delete_conflicts(doc, superseded_revs) - # perform request to resolve document in server - ddoc_path = [ - '_design', 'docs', '_update', 'resolve_doc', doc.doc_id - ] - resource = self._database.resource(*ddoc_path) - conflicts = None - if doc.has_conflicts: - conflicts = binascii.b2a_base64( - json.dumps( - map(lambda cdoc: (cdoc.rev, cdoc.content), - doc.get_conflicts())) - )[:-1] # exclude \n - try: - response = resource.put_json( - body={ - 'couch_rev': cur_doc.couch_rev, - 'conflicts': conflicts, - }, - headers={'content-type': 'application/json'}) - except ResourceNotFound as e: - raise_missing_design_doc_error(e, ddoc_path) + # the newer doc version does not supersede the one in the + # database, so we will add a conflict to the database and copy + # those over to the document the user has in her hands. + self._add_conflict(cur_doc, new_rev, doc.get_json()) + self._delete_conflicts(cur_doc, superseded_revs) + self._put_doc(cur_doc, cur_doc) # just update conflicts + # backend has been updated with current conflicts, now copy them + # to the current document. + doc.set_conflicts(cur_doc.get_conflicts()) def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, replica_trans_id=''): diff --git a/common/src/leap/soledad/common/ddocs/docs/updates/resolve_doc.js b/common/src/leap/soledad/common/ddocs/docs/updates/resolve_doc.js deleted file mode 100644 index 7ba66cf8..00000000 --- a/common/src/leap/soledad/common/ddocs/docs/updates/resolve_doc.js +++ /dev/null @@ -1,39 +0,0 @@ -function(doc, req){ - /* we expect to receive the following in `req.body`: - * { - * 'couch_rev': '', - * 'conflicts': '', - * } - */ - var body = JSON.parse(req.body); - - // fail if no document was given - if (!doc) { - return [null, 'document does not exist'] - } - - // fail if couch revisions do not match - if (body['couch_rev'] != null - && doc['_rev'] != body['couch_rev']) { - return [null, 'revision conflict'] - } - - // fail if conflicts were not sent - if (body['conflicts'] == null) - return [null, 'missing conflicts'] - - // save conflicts as attachment if they were sent - if (body['conflicts'] != null) { - if (!doc._attachments) - doc._attachments = {}; - doc._attachments.u1db_conflicts = { - content_type: "application/octet-stream", - data: body['conflicts'] // should be base64 encoded - } - } - // or delete attachment if there are no conflicts - else if (doc._attachments && doc._attachments.u1db_conflicts) - delete doc._attachments.u1db_conflicts; - - return [doc, 'ok']; -} diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 97d744e4..86bb4b93 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -503,14 +503,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): self.assertRaises( errors.MissingDesignDocError, self.db._do_set_replica_gen_and_trans_id, 1, 2, 3) - # fake a conflict so we can test resolve_doc() - first_rev = self.db._allocate_doc_rev(None) - doc = couch.CouchDocument( - doc_id='mydoc', rev=self.db._allocate_doc_rev(first_rev)) - self.db._get_doc = Mock(return_value=doc) - self.assertRaises( - errors.MissingDesignDocError, - self.db.resolve_doc, doc, [first_rev]) def test_missing_design_doc_functions_raises(self): """ @@ -649,14 +641,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): self.assertRaises( errors.MissingDesignDocDeletedError, self.db._do_set_replica_gen_and_trans_id, 1, 2, 3) - # fake a conflict so we can test resolve_doc() - first_rev = self.db._allocate_doc_rev(None) - doc = couch.CouchDocument( - doc_id='mydoc', rev=self.db._allocate_doc_rev(first_rev)) - self.db._get_doc = Mock(return_value=doc) - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.resolve_doc, doc, [first_rev]) load_tests = tests.load_with_scenarios -- cgit v1.2.3 From c332b78995334a8c49c788e2fd3fa15eb8936ca8 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 12 Mar 2014 19:31:58 -0300 Subject: Prevent Soledad server from creating or deleting couch databases (#3502). --- ...ouch-server-state-from-creating-or-deleting-dbs | 3 ++ common/src/leap/soledad/common/couch.py | 17 ++++--- .../tests/test_couch_operations_atomicity.py | 6 +++ .../src/leap/soledad/common/tests/test_server.py | 54 ++++++++++++++-------- 4 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs (limited to 'common') diff --git a/common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs b/common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs new file mode 100644 index 00000000..acd74529 --- /dev/null +++ b/common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs @@ -0,0 +1,3 @@ + o Prevent CouchServerState from creating or deleting databases. This way, + Soledad remote clients won't ever be able to do these operations when + syncing. Part of #5302. diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 456d4fdf..11b77938 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -35,7 +35,6 @@ from couchdb.client import Server from couchdb.http import ( ResourceConflict, ResourceNotFound, - Unauthorized, ServerError, Session, ) @@ -48,6 +47,7 @@ from u1db.errors import ( ConflictedDoc, DocumentDoesNotExist, DocumentAlreadyDeleted, + Unauthorized, ) from u1db.backends import CommonBackend, CommonSyncTarget from u1db.remote import http_app @@ -1451,7 +1451,6 @@ class CouchServerState(ServerState): :return: The CouchDatabase object. :rtype: CouchDatabase """ - # TODO: open couch return CouchDatabase.open_database( self._couch_url + '/' + dbname, create=False) @@ -1460,16 +1459,20 @@ class CouchServerState(ServerState): """ Ensure couch database exists. + Usually, this method is used by the server to ensure the existence of + a database. In our setup, the Soledad user that accesses the underlying + couch server should never have permission to create (or delete) + databases. But, in case it ever does, by raising an exception here we + have one more guarantee that no modified client will be able to + enforce creation of a database when syncing. + :param dbname: The name of the database to ensure. :type dbname: str :return: The CouchDatabase object and the replica uid. :rtype: (CouchDatabase, str) """ - db = CouchDatabase.open_database( - self._couch_url + '/' + dbname, - create=True) - return db, db._replica_uid + raise Unauthorized() def delete_database(self, dbname): """ @@ -1478,7 +1481,7 @@ class CouchServerState(ServerState): :param dbname: The name of the database to delete. :type dbname: str """ - CouchDatabase.delete_database(self._couch_url + '/' + dbname) + raise Unauthorized() def _set_couch_url(self, url): """ diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 5384d465..3c457cc5 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -33,11 +33,17 @@ from leap.soledad.common.tests.test_target import ( make_leap_document_for_test, token_leap_sync_target, ) +from leap.soledad.common.tests.test_server import _couch_ensure_database REPEAT_TIMES = 20 +# monkey path CouchServerState so it can ensure databases. + +CouchServerState.ensure_database = _couch_ensure_database + + class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): @staticmethod diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 06595ed2..f8d2a64f 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -51,6 +51,17 @@ from leap.soledad.server import SoledadApp, LockResource from leap.soledad.server.auth import URLToAuthorization +# monkey path CouchServerState so it can ensure databases. + +def _couch_ensure_database(self, dbname): + db = CouchDatabase.open_database( + self._couch_url + '/' + dbname, + create=True) + return db, db._replica_uid + +CouchServerState.ensure_database = _couch_ensure_database + + class ServerAuthorizationTestCase(BaseLeapTest): """ Tests related to Soledad server authorization. @@ -340,15 +351,16 @@ class EncryptedSyncTestCase( _, doclist = sol1.get_all_docs() self.assertEqual([], doclist) doc1 = sol1.create_doc(json.loads(simple_doc)) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # assert doc was sent to couch db + # ensure remote db exists before syncing db = CouchDatabase( self._couch_url, # the name of the user database is "user-". 'user-user-uuid', ) + # sync with server + sol1._server_url = self.getURL() + sol1.sync() + # assert doc was sent to couch db _, doclist = db.get_all_docs() self.assertEqual(1, len(doclist)) couchdoc = doclist[0] @@ -395,15 +407,16 @@ class EncryptedSyncTestCase( _, doclist = sol1.get_all_docs() self.assertEqual([], doclist) doc1 = sol1.create_doc(json.loads(simple_doc)) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # assert doc was sent to couch db + # ensure remote db exists before syncing db = CouchDatabase( self._couch_url, # the name of the user database is "user-". 'user-user-uuid', ) + # sync with server + sol1._server_url = self.getURL() + sol1.sync() + # assert doc was sent to couch db _, doclist = db.get_all_docs() self.assertEqual(1, len(doclist)) couchdoc = doclist[0] @@ -454,6 +467,12 @@ class EncryptedSyncTestCase( self.assertEqual([], doclist) content = binascii.hexlify(os.urandom(length/2)) # len() == length doc1 = sol1.create_doc({'data': content}) + # ensure remote db exists before syncing + db = CouchDatabase( + self._couch_url, + # the name of the user database is "user-". + 'user-user-uuid', + ) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -473,11 +492,6 @@ class EncryptedSyncTestCase( # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) # delete remote database - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-". - 'user-user-uuid', - ) db.delete_database() @@ -497,6 +511,12 @@ class EncryptedSyncTestCase( # create many small files for i in range(0, number_of_docs): sol1.create_doc(json.loads(simple_doc)) + # ensure remote db exists before syncing + db = CouchDatabase( + self._couch_url, + # the name of the user database is "user-". + 'user-user-uuid', + ) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -516,11 +536,6 @@ class EncryptedSyncTestCase( for doc in doclist: self.assertEqual(sol1.get_doc(doc.doc_id), doc) # delete remote database - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-". - 'user-user-uuid', - ) db.delete_database() class LockResourceTestCase( @@ -542,6 +557,9 @@ class LockResourceTestCase( CouchDBTestCase.setUp(self) self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) + # create the databases + CouchDatabase(self._couch_url, 'shared') + CouchDatabase(self._couch_url, 'tokens') self._state = CouchServerState( self._couch_url, 'shared', 'tokens') -- cgit v1.2.3 From b85c9797827731b86798adc958cee0c114667918 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 12 Mar 2014 17:33:11 -0300 Subject: Remove check for couch permissions. --- common/changes/remove-check-for-couch-permissions | 4 ++ common/src/leap/soledad/common/couch.py | 67 ----------------------- 2 files changed, 4 insertions(+), 67 deletions(-) create mode 100644 common/changes/remove-check-for-couch-permissions (limited to 'common') diff --git a/common/changes/remove-check-for-couch-permissions b/common/changes/remove-check-for-couch-permissions new file mode 100644 index 00000000..f77f59f1 --- /dev/null +++ b/common/changes/remove-check-for-couch-permissions @@ -0,0 +1,4 @@ + o Remove check for couch permissions when CouchServerState is instantiated. + This is not necessary anymore because platform takes care of giving the + soledad user enough permissions and tapicero takes care of uploading the + needed design documents. diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 11b77938..d16563d3 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -1354,14 +1354,6 @@ class CouchSyncTarget(CommonSyncTarget): source_replica_transaction_id) -class NotEnoughCouchPermissions(Exception): - """ - Raised when failing to assert for enough permissions on underlying Couch - Database. - """ - pass - - class CouchServerState(ServerState): """ Inteface of the WSGI server with the CouchDB backend. @@ -1381,65 +1373,6 @@ class CouchServerState(ServerState): self._couch_url = couch_url self._shared_db_name = shared_db_name self._tokens_db_name = tokens_db_name - try: - self._check_couch_permissions() - except NotEnoughCouchPermissions: - logger.error("Not enough permissions on underlying couch " - "database (%s)." % self._couch_url) - except (socket.error, socket.gaierror, socket.herror, - socket.timeout), e: - logger.error("Socket problem while trying to reach underlying " - "couch database: (%s, %s)." % - (self._couch_url, e)) - - def _check_couch_permissions(self): - """ - Assert that Soledad Server has enough permissions on the underlying - couch database. - - Soledad Server has to be able to do the following in the couch server: - - * Create, read and write from/to 'shared' db. - * Create, read and write from/to 'user-' dbs. - * Read from 'tokens' db. - - This function tries to perform the actions above using the "low level" - couch library to ensure that Soledad Server can do everything it needs - on the underlying couch database. - - :param couch_url: The URL of the couch database. - :type couch_url: str - - @raise NotEnoughCouchPermissions: Raised in case there are not enough - permissions to read/write/create the needed couch databases. - :rtype: bool - """ - - def _open_couch_db(dbname): - server = Server(url=self._couch_url) - try: - server[dbname] - except ResourceNotFound: - server.create(dbname) - return server[dbname] - - def _create_delete_test_doc(db): - doc_id, _ = db.save({'test': 'document'}) - doc = db[doc_id] - db.delete(doc) - - try: - # test read/write auth for shared db - _create_delete_test_doc( - _open_couch_db(self._shared_db_name)) - # test read/write auth for user- db - _create_delete_test_doc( - _open_couch_db('%stest-db' % USER_DB_PREFIX)) - # test read auth for tokens db - tokensdb = _open_couch_db(self._tokens_db_name) - tokensdb.info() - except Unauthorized: - raise NotEnoughCouchPermissions(self._couch_url) def open_database(self, dbname): """ -- cgit v1.2.3 From 35bf9f390db15e74ad4e37b6fc7fac9d6d7ca658 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 14 Mar 2014 15:13:42 -0300 Subject: Parallelize get_docs on couch backend (#5008). --- ...ture_5008_parallelize-get_docs-on-couch-backend | 1 + common/src/leap/soledad/common/couch.py | 78 ++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 common/changes/feature_5008_parallelize-get_docs-on-couch-backend (limited to 'common') diff --git a/common/changes/feature_5008_parallelize-get_docs-on-couch-backend b/common/changes/feature_5008_parallelize-get_docs-on-couch-backend new file mode 100644 index 00000000..d452c888 --- /dev/null +++ b/common/changes/feature_5008_parallelize-get_docs-on-couch-backend @@ -0,0 +1 @@ + o Parallelize get_docs() on couch backend to accelerate sync (#5008). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index d16563d3..a4f30cf3 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -26,6 +26,7 @@ import binascii import socket import time import sys +import threading from StringIO import StringIO @@ -334,6 +335,47 @@ class CouchDatabase(CommonBackend): A U1DB implementation that uses CouchDB as its persistence layer. """ + # We spawn threads to parallelize the CouchDatabase.get_docs() method + MAX_GET_DOCS_THREADS = 20 + + class _GetDocThread(threading.Thread): + """ + A thread that gets a document from a database. + + TODO: switch this for a twisted deferred to thread. This depends on + replacing python-couchdb for paisley in this module. + """ + + def __init__(self, db, doc_id, check_for_conflicts, + release_fun): + """ + :param db: The database from where to get the document. + :type db: u1db.Database + :param doc_id: The doc_id of the document to be retrieved. + :type doc_id: str + :param check_for_conflicts: Whether the get_doc() method should + check for existing conflicts. + :type check_for_conflicts: bool + :param release_fun: A function that releases a semaphore, to be + called after the document is fetched. + :type release_fun: function + """ + threading.Thread.__init__(self) + self._db = db + self._doc_id = doc_id + self._check_for_conflicts = check_for_conflicts + self._release_fun = release_fun + self._doc = None + + def run(self): + """ + Fetch the document, store it as a property, and call the release + function. + """ + self._doc = self._db._get_doc( + self._doc_id, self._check_for_conflicts) + self._release_fun() + @classmethod def open_database(cls, url, create): """ @@ -401,6 +443,9 @@ class CouchDatabase(CommonBackend): self._set_replica_uid(replica_uid) if ensure_ddocs: self.ensure_ddocs_on_db() + # initialize a thread pool for parallelizing get_docs() + self._sem_pool = threading.BoundedSemaphore( + value=self.MAX_GET_DOCS_THREADS) def ensure_ddocs_on_db(self): """ @@ -1331,6 +1376,39 @@ class CouchDatabase(CommonBackend): old_doc.has_conflicts = doc.has_conflicts return state, self._get_generation() + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + """ + Get the JSON content for many documents. + + :param doc_ids: A list of document identifiers. + :type doc_ids: list + :param check_for_conflicts: If set to False, then the conflict check + will be skipped, and 'None' will be + returned instead of True/False. + :type check_for_conflictsa: bool + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted + documents will not be included in the results. + :return: iterable giving the Document object for each document id + in matching doc_ids order. + :rtype: iterable + """ + # spawn threads to retrieve docs + threads = [] + for doc_id in doc_ids: + self._sem_pool.acquire() + t = self._GetDocThread(self, doc_id, check_for_conflicts, + self._sem_pool.release) + t.start() + threads.append(t) + # join threads and yield docs + for t in threads: + t.join() + if t._doc.is_tombstone() and not include_deleted: + continue + yield t._doc + class CouchSyncTarget(CommonSyncTarget): """ -- cgit v1.2.3 From 288f506daed66e4acb08617dc1db127da4d36241 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 13 Mar 2014 15:08:55 -0300 Subject: Fix raising of auth token errors (#5191). --- common/src/leap/soledad/common/errors.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'common') diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 446c4c75..3a7eadd2 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -50,24 +50,16 @@ class SoledadError(errors.U1DBError): # Authorization errors # -@register_exception -class MissingAuthTokenError(errors.Unauthorized): - """ - Exception raised when failing to get authorization for some action because - the auth token is missing in the tokens db. - """ - - wire_description = "missing token" - status = 401 - @register_exception class InvalidAuthTokenError(errors.Unauthorized): """ Exception raised when failing to get authorization for some action because - the provided token is different from the one in the tokens db. + the provided token either does not exist in the tokens database, has a + distinct structure from the expected one, or is associated with a user + with a distinct uuid than the one provided by the client. """ - wire_descrition = "token mismatch" + wire_descrition = "invalid auth token" status = 401 # -- cgit v1.2.3 From f5b30eb5ae4ac5cc61163e0167e92827310d0c13 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 26 Mar 2014 12:09:10 -0300 Subject: Remove check for design docs from couch server state init (#5387). --- .../bug_5387_remove-check-for-ddocs-on-couch-server-state | 1 + common/src/leap/soledad/common/couch.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state (limited to 'common') diff --git a/common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state b/common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state new file mode 100644 index 00000000..71c0cac2 --- /dev/null +++ b/common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state @@ -0,0 +1 @@ + o Remove check for design docs on couch server state initialization (#5387). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index a4f30cf3..c0ebc425 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -377,7 +377,7 @@ class CouchDatabase(CommonBackend): self._release_fun() @classmethod - def open_database(cls, url, create): + def open_database(cls, url, create, ensure_ddocs=False): """ Open a U1DB database using CouchDB as backend. @@ -385,6 +385,8 @@ class CouchDatabase(CommonBackend): :type url: str :param create: should the replica be created if it does not exist? :type create: bool + :param ensure_ddocs: Ensure that the design docs exist on server. + :type ensure_ddocs: bool :return: the database instance :rtype: CouchDatabase @@ -401,7 +403,7 @@ class CouchDatabase(CommonBackend): except ResourceNotFound: if not create: raise DatabaseDoesNotExist() - return cls(url, dbname) + return cls(url, dbname, ensure_ddocs=ensure_ddocs) def __init__(self, url, dbname, replica_uid=None, full_commit=True, session=None, ensure_ddocs=True): @@ -1464,7 +1466,8 @@ class CouchServerState(ServerState): """ return CouchDatabase.open_database( self._couch_url + '/' + dbname, - create=False) + create=False, + ensure_ddocs=False) def ensure_database(self, dbname): """ -- cgit v1.2.3 From e2a3431c13773af43b50797566feedc15fdf94ce Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 27 Mar 2014 10:42:26 -0300 Subject: Retry recording sync info on couch backend (#5388). --- ...88_avoid-concurrent-syncs-update-handler-conflict | 2 ++ common/src/leap/soledad/common/couch.py | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict (limited to 'common') diff --git a/common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict b/common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict new file mode 100644 index 00000000..d0c820db --- /dev/null +++ b/common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict @@ -0,0 +1,2 @@ + o Avoid concurrent syncs problem by adding a lock for PUTting to the sync + log update handler (#5388). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index c0ebc425..ebe8477f 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -30,6 +30,7 @@ import threading from StringIO import StringIO +from collections import defaultdict from couchdb.client import Server @@ -338,6 +339,8 @@ class CouchDatabase(CommonBackend): # We spawn threads to parallelize the CouchDatabase.get_docs() method MAX_GET_DOCS_THREADS = 20 + update_handler_lock = defaultdict(threading.Lock) + class _GetDocThread(threading.Thread): """ A thread that gets a document from a database. @@ -1133,13 +1136,14 @@ class CouchDatabase(CommonBackend): ddoc_path = ['_design', 'syncs', '_update', 'put', 'u1db_sync_log'] res = self._database.resource(*ddoc_path) try: - res.put_json( - body={ - 'other_replica_uid': other_replica_uid, - 'other_generation': other_generation, - 'other_transaction_id': other_transaction_id, - }, - headers={'content-type': 'application/json'}) + with CouchDatabase.update_handler_lock[self._get_replica_uid()]: + res.put_json( + body={ + 'other_replica_uid': other_replica_uid, + 'other_generation': other_generation, + 'other_transaction_id': other_transaction_id, + }, + headers={'content-type': 'application/json'}) except ResourceNotFound as e: raise_missing_design_doc_error(e, ddoc_path) @@ -1367,7 +1371,7 @@ class CouchDatabase(CommonBackend): if save_conflict: self._force_doc_sync_conflict(doc) if replica_uid is not None and replica_gen is not None: - self._do_set_replica_gen_and_trans_id( + self._set_replica_gen_and_trans_id( replica_uid, replica_gen, replica_trans_id) # update info old_doc.rev = doc.rev -- cgit v1.2.3 From c3f5e9a6b340db969844a662c27fcb2b3f7596b9 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 4 Apr 2014 11:55:35 -0300 Subject: Renew couch connection session after multipart PUT (#5448). --- ...449_add-workaround-for-hanging-requests-after-multipart-put | 1 + common/src/leap/soledad/common/couch.py | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put (limited to 'common') diff --git a/common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put b/common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put new file mode 100644 index 00000000..dece564a --- /dev/null +++ b/common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put @@ -0,0 +1 @@ + o Renew HTTP session after multipart PUTs to avoid request hanging (#5449). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index ebe8477f..1bb84985 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -855,6 +855,7 @@ class CouchDatabase(CommonBackend): try: self._database.resource.put_json( doc.doc_id, body=buf.getvalue(), headers=envelope.headers) + self._renew_couch_session() except ResourceConflict: raise RevisionConflict() @@ -1415,6 +1416,15 @@ class CouchDatabase(CommonBackend): continue yield t._doc + def _renew_couch_session(self): + """ + Create a new couch connection session. + + This is a workaround for #5448. Will not be needed once bigcouch is + merged with couchdb. + """ + self._database.resource.session = Session(timeout=COUCH_TIMEOUT) + class CouchSyncTarget(CommonSyncTarget): """ -- cgit v1.2.3 From 86890a496ba05951573dad7f69fa2840087994d2 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 4 Apr 2014 15:35:20 -0300 Subject: Preload time.strptime() to avoid multi thread problem on couch backend get_docs() method (#5449). --- .../bug_5449_preload-strptime-to-avoid-multi-thread-problem | 2 ++ common/src/leap/soledad/common/couch.py | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem (limited to 'common') diff --git a/common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem b/common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem new file mode 100644 index 00000000..17d1f463 --- /dev/null +++ b/common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem @@ -0,0 +1,2 @@ + o Preload time.strptime() to avoid multi-threaded problem on couch backend + get_docs() method (#5449). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 1bb84985..8e8613a1 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -1401,6 +1401,17 @@ class CouchDatabase(CommonBackend): in matching doc_ids order. :rtype: iterable """ + # Workaround for: + # + # http://bugs.python.org/issue7980 + # https://leap.se/code/issues/5449 + # + # python-couchdb uses time.strptime, which is not thread safe. In + # order to avoid the problem described on the issues above, we preload + # strptime here by evaluating the conversion of an arbitrary date. + # This will not be needed when/if we switch from python-couchdb to + # paisley. + time.strptime('Mar 4 1917', '%b %d %Y') # spawn threads to retrieve docs threads = [] for doc_id in doc_ids: -- cgit v1.2.3 From a3fed4d42ab4a7be7bc7ebe86b35805ac73d62de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 4 Apr 2014 16:34:33 -0300 Subject: Fold in changes --- common/changes/bug_4435_add-lock-timeout-error | 1 - common/changes/bug_4475_remodel-couch-backend | 2 -- common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state | 1 - .../bug_5449_add-workaround-for-hanging-requests-after-multipart-put | 1 - .../changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem | 2 -- common/changes/feature_4035_improve-error-messages | 3 --- common/changes/feature_4836_allow-sync-of-large-files | 1 - common/changes/feature_4994_add-missing-design-doc-exception | 1 - common/changes/feature_5008_parallelize-get_docs-on-couch-backend | 1 - .../changes/feature_5011-use-less-memory-when-putting-docs-on-couch | 1 - ...ture_5302_prevent-couch-server-state-from-creating-or-deleting-dbs | 3 --- .../feature_5388_avoid-concurrent-syncs-update-handler-conflict | 2 -- common/changes/remove-check-for-couch-permissions | 4 ---- 13 files changed, 23 deletions(-) delete mode 100644 common/changes/bug_4435_add-lock-timeout-error delete mode 100644 common/changes/bug_4475_remodel-couch-backend delete mode 100644 common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state delete mode 100644 common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put delete mode 100644 common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem delete mode 100644 common/changes/feature_4035_improve-error-messages delete mode 100644 common/changes/feature_4836_allow-sync-of-large-files delete mode 100644 common/changes/feature_4994_add-missing-design-doc-exception delete mode 100644 common/changes/feature_5008_parallelize-get_docs-on-couch-backend delete mode 100644 common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch delete mode 100644 common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs delete mode 100644 common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict delete mode 100644 common/changes/remove-check-for-couch-permissions (limited to 'common') diff --git a/common/changes/bug_4435_add-lock-timeout-error b/common/changes/bug_4435_add-lock-timeout-error deleted file mode 100644 index 8f220b1d..00000000 --- a/common/changes/bug_4435_add-lock-timeout-error +++ /dev/null @@ -1 +0,0 @@ - o Add lock timeout HTTP error (#4435). diff --git a/common/changes/bug_4475_remodel-couch-backend b/common/changes/bug_4475_remodel-couch-backend deleted file mode 100644 index 13a1b121..00000000 --- a/common/changes/bug_4475_remodel-couch-backend +++ /dev/null @@ -1,2 +0,0 @@ - o Remodel couch backend to fix concurrency and scalability. Closes #4475, - #4682, #4683 and #4680. diff --git a/common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state b/common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state deleted file mode 100644 index 71c0cac2..00000000 --- a/common/changes/bug_5387_remove-check-for-ddocs-on-couch-server-state +++ /dev/null @@ -1 +0,0 @@ - o Remove check for design docs on couch server state initialization (#5387). diff --git a/common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put b/common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put deleted file mode 100644 index dece564a..00000000 --- a/common/changes/bug_5449_add-workaround-for-hanging-requests-after-multipart-put +++ /dev/null @@ -1 +0,0 @@ - o Renew HTTP session after multipart PUTs to avoid request hanging (#5449). diff --git a/common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem b/common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem deleted file mode 100644 index 17d1f463..00000000 --- a/common/changes/bug_5449_preload-strptime-to-avoid-multi-thread-problem +++ /dev/null @@ -1,2 +0,0 @@ - o Preload time.strptime() to avoid multi-threaded problem on couch backend - get_docs() method (#5449). diff --git a/common/changes/feature_4035_improve-error-messages b/common/changes/feature_4035_improve-error-messages deleted file mode 100644 index 4f86a17f..00000000 --- a/common/changes/feature_4035_improve-error-messages +++ /dev/null @@ -1,3 +0,0 @@ - o Improve error messages. Closes #5035. - * Add MissingTokenError and InvalidTokenError as sub exceptions from - Unauthorized. diff --git a/common/changes/feature_4836_allow-sync-of-large-files b/common/changes/feature_4836_allow-sync-of-large-files deleted file mode 100644 index f124e899..00000000 --- a/common/changes/feature_4836_allow-sync-of-large-files +++ /dev/null @@ -1 +0,0 @@ - o Allow sync of large files (~100MB) (#4836). diff --git a/common/changes/feature_4994_add-missing-design-doc-exception b/common/changes/feature_4994_add-missing-design-doc-exception deleted file mode 100644 index d5291cc1..00000000 --- a/common/changes/feature_4994_add-missing-design-doc-exception +++ /dev/null @@ -1 +0,0 @@ - o Add exceptions to deal with missing design documents. Fixes #4994. diff --git a/common/changes/feature_5008_parallelize-get_docs-on-couch-backend b/common/changes/feature_5008_parallelize-get_docs-on-couch-backend deleted file mode 100644 index d452c888..00000000 --- a/common/changes/feature_5008_parallelize-get_docs-on-couch-backend +++ /dev/null @@ -1 +0,0 @@ - o Parallelize get_docs() on couch backend to accelerate sync (#5008). diff --git a/common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch b/common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch deleted file mode 100644 index 7d3f6e4f..00000000 --- a/common/changes/feature_5011-use-less-memory-when-putting-docs-on-couch +++ /dev/null @@ -1 +0,0 @@ - o Use less memory when putting docs on couch (#5011). diff --git a/common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs b/common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs deleted file mode 100644 index acd74529..00000000 --- a/common/changes/feature_5302_prevent-couch-server-state-from-creating-or-deleting-dbs +++ /dev/null @@ -1,3 +0,0 @@ - o Prevent CouchServerState from creating or deleting databases. This way, - Soledad remote clients won't ever be able to do these operations when - syncing. Part of #5302. diff --git a/common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict b/common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict deleted file mode 100644 index d0c820db..00000000 --- a/common/changes/feature_5388_avoid-concurrent-syncs-update-handler-conflict +++ /dev/null @@ -1,2 +0,0 @@ - o Avoid concurrent syncs problem by adding a lock for PUTting to the sync - log update handler (#5388). diff --git a/common/changes/remove-check-for-couch-permissions b/common/changes/remove-check-for-couch-permissions deleted file mode 100644 index f77f59f1..00000000 --- a/common/changes/remove-check-for-couch-permissions +++ /dev/null @@ -1,4 +0,0 @@ - o Remove check for couch permissions when CouchServerState is instantiated. - This is not necessary anymore because platform takes care of giving the - soledad user enough permissions and tapicero takes care of uploading the - needed design documents. -- cgit v1.2.3