diff options
Diffstat (limited to 'common/src')
14 files changed, 786 insertions, 157 deletions
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 0aa84170..b51b32f3 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -40,7 +40,9 @@ from couchdb.http import ( ResourceConflict, ResourceNotFound, ServerError, - Session as CouchHTTPSession, + Session, + urljoin as couch_urljoin, + Resource, ) from u1db import query_parser, vectorclock from u1db.errors import ( @@ -333,17 +335,6 @@ class MultipartWriter(object): self.headers[name] = value -class Session(CouchHTTPSession): - """ - An HTTP session that can be closed. - """ - - def close_connections(self): - for key, conns in list(self.conns.items()): - for conn in conns: - conn.close() - - @contextmanager def couch_server(url): """ @@ -359,7 +350,6 @@ def couch_server(url): session = Session(timeout=COUCH_TIMEOUT) server = Server(url=url, session=session) yield server - session.close_connections() class CouchDatabase(CommonBackend): @@ -371,6 +361,7 @@ class CouchDatabase(CommonBackend): MAX_GET_DOCS_THREADS = 20 update_handler_lock = defaultdict(threading.Lock) + sync_info_lock = defaultdict(threading.Lock) class _GetDocThread(threading.Thread): """ @@ -440,7 +431,8 @@ class CouchDatabase(CommonBackend): if not create: raise DatabaseDoesNotExist() server.create(dbname) - return cls(url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) + return cls( + url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): """ @@ -465,6 +457,10 @@ class CouchDatabase(CommonBackend): self._database = Database( urljoin(self._url, self._dbname), self._session) + try: + self._database.info() + except ResourceNotFound: + raise DatabaseDoesNotExist() if replica_uid is not None: self._set_replica_uid(replica_uid) if ensure_ddocs: @@ -505,7 +501,6 @@ class CouchDatabase(CommonBackend): """ with couch_server(self._url) as server: del(server[self._dbname]) - self.close_connections() def close(self): """ @@ -514,20 +509,12 @@ class CouchDatabase(CommonBackend): :return: True if db was succesfully closed. :rtype: bool """ - self.close_connections() self._url = None self._full_commit = None self._session = None self._database = None return True - def close_connections(self): - """ - Close all open connections to the couch server. - """ - if self._session is not None: - self._session.close_connections() - def __del__(self): """ Close the database upon garbage collection. @@ -576,6 +563,8 @@ class CouchDatabase(CommonBackend): _replica_uid = property(_get_replica_uid, _set_replica_uid) + replica_uid = property(_get_replica_uid) + def _get_generation(self): """ Return the current generation. @@ -869,7 +858,7 @@ class CouchDatabase(CommonBackend): # Date.prototype.getTime() which was used before inside a couchdb # update handler. (int(time.time() * 1000), - self._allocate_transaction_id())) + self._allocate_transaction_id())) # build the couch document couch_doc = { '_id': doc.doc_id, @@ -889,11 +878,9 @@ class CouchDatabase(CommonBackend): envelope.close() # try to save and fail if there's a revision conflict try: - self._database.resource.put_json( + resource = self._new_resource() + resource.put_json( doc.doc_id, body=buf.getvalue(), headers=envelope.headers) - # What follows is a workaround for an ugly bug. See: - # https://leap.se/code/issues/5448 - self.close_connections() except ResourceConflict: raise RevisionConflict() @@ -1465,6 +1452,20 @@ class CouchDatabase(CommonBackend): continue yield t._doc + def _new_resource(self, *path): + """ + Return a new resource for accessing a couch database. + + :return: A resource for accessing a couch database. + :rtype: couchdb.http.Resource + """ + # Workaround for: https://leap.se/code/issues/5448 + url = couch_urljoin(self._database.resource.url, *path) + resource = Resource(url, Session(timeout=COUCH_TIMEOUT)) + resource.credentials = self._database.resource.credentials + resource.headers = self._database.resource.headers.copy() + return resource + class CouchSyncTarget(CommonSyncTarget): """ @@ -1537,8 +1538,8 @@ class CouchServerState(ServerState): :param dbname: The name of the database to ensure. :type dbname: str - :return: The CouchDatabase object and the replica uid. - :rtype: (CouchDatabase, str) + :raise Unauthorized: Always, because Soledad server is not allowed to + create databases. """ raise Unauthorized() @@ -1548,6 +1549,9 @@ class CouchServerState(ServerState): :param dbname: The name of the database to delete. :type dbname: str + + :raise Unauthorized: Always, because Soledad server is not allowed to + delete databases. """ raise Unauthorized() diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js new file mode 100644 index 00000000..d62aeb40 --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js @@ -0,0 +1,105 @@ +/** + * This update handler stores information about ongoing synchronization + * attempts from distinct source replicas. + * + * Normally, u1db synchronization occurs during one POST request. In order to + * split that into many serial POST requests, we store the state of each sync + * in the server, using a document with id 'u1db_sync_state'. To identify + * each sync attempt, we use a sync_id sent by the client. If we ever receive + * a new sync_id, we trash current data for that source replica and start + * over. + * + * We expect the following in the document body: + * + * { + * 'source_replica_uid': '<source_replica_uid>', + * 'sync_id': '<sync_id>', + * 'seen_ids': [['<doc_id>', <at_gen>], ...], // optional + * 'changes_to_return': [ // optional + * 'gen': <gen>, + * 'trans_id': '<trans_id>', + * 'changes_to_return': [[<doc_id>', <gen>, '<trans_id>'], ...] + * ], + * } + * + * The format of the final document stored on server is: + * + * { + * '_id': '<str>', + * '_rev' '<str>', + * 'ongoing_syncs': { + * '<source_replica_uid>': { + * 'sync_id': '<sync_id>', + * 'seen_ids': [['<doc_id>', <at_gen>[, ...], + * 'changes_to_return': { + * 'gen': <gen>, + * 'trans_id': '<trans_id>', + * 'changes_to_return': [ + * ['<doc_id>', <gen>, '<trans_id>'], + * ..., + * ], + * }, + * }, + * ... // info about other source replicas here + * } + * } + */ +function(doc, req) { + + // prevent updates to alien documents + if (doc != null && doc['_id'] != 'u1db_sync_state') + return [null, 'invalid data']; + + // create the document if it doesn't exist + if (!doc) + doc = { + '_id': 'u1db_sync_state', + 'ongoing_syncs': {}, + }; + + // parse and validate incoming data + var body = JSON.parse(req.body); + if (body['source_replica_uid'] == null) + return [null, 'invalid data']; + var source_replica_uid = body['source_replica_uid']; + + if (body['sync_id'] == null) + return [null, 'invalid data']; + var sync_id = body['sync_id']; + + // trash outdated sync data for that replica if that exists + if (doc['ongoing_syncs'][source_replica_uid] != null && + doc['ongoing_syncs'][source_replica_uid]['sync_id'] != sync_id) + delete doc['ongoing_syncs'][source_replica_uid]; + + // create an entry for that source replica + if (doc['ongoing_syncs'][source_replica_uid] == null) + doc['ongoing_syncs'][source_replica_uid] = { + 'sync_id': sync_id, + 'seen_ids': {}, + 'changes_to_return': null, + }; + + // incoming meta-data values should be exclusive, so we count how many + // arrived and deny to accomplish the transaction if the count is high. + var incoming_values = 0; + var info = doc['ongoing_syncs'][source_replica_uid] + + // add incoming seen id + if ('seen_id' in body) { + info['seen_ids'][body['seen_id'][0]] = body['seen_id'][1]; + incoming_values += 1; + } + + // add incoming changes_to_return + if ('changes_to_return' in body) { + info['changes_to_return'] = body['changes_to_return']; + incoming_values += 1; + } + + if (incoming_values != 1) + return [null, 'invalid data']; + + return [doc, 'ok']; +} + diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js new file mode 100644 index 00000000..94b7e767 --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js @@ -0,0 +1,20 @@ +function(doc) { + if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) + for (var source_replica_uid in doc['ongoing_syncs']) { + var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return']; + var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id']; + if (changes == null) + emit([source_replica_uid, sync_id, 0], null); + else if (changes.length == 0) + emit([source_replica_uid, sync_id, 0], []); + else + for (var i = 0; i < changes['changes_to_return'].length; i++) + emit( + [source_replica_uid, sync_id, i], + { + 'gen': changes['gen'], + 'trans_id': changes['trans_id'], + 'next_change_to_return': changes['changes_to_return'][i], + }); + } +} diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js new file mode 100644 index 00000000..16118e88 --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js @@ -0,0 +1,11 @@ +function(doc) { + if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) + for (var source_replica_uid in doc['ongoing_syncs']) { + var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id']; + emit( + [source_replica_uid, sync_id], + { + 'seen_ids': doc['ongoing_syncs'][source_replica_uid]['seen_ids'], + }); + } +} diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js new file mode 100644 index 00000000..e88c6ebb --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js @@ -0,0 +1,17 @@ +function(doc) { + if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) + for (var source_replica_uid in doc['ongoing_syncs']) { + var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return']; + var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id']; + if (changes == null) + emit([source_replica_uid, sync_id], null); + else + emit( + [source_replica_uid, sync_id], + { + 'gen': changes['gen'], + 'trans_id': changes['trans_id'], + 'number_of_changes': changes['changes_to_return'].length + }); + } +} diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 3a7eadd2..ea4bf7f6 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -62,6 +62,7 @@ class InvalidAuthTokenError(errors.Unauthorized): wire_descrition = "invalid auth token" status = 401 + # # LockResource errors # diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index 88f98272..a38bdaed 100644 --- a/common/src/leap/soledad/common/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -1,3 +1,21 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + """ Tests to make sure Soledad provides U1DB functionality and more. """ @@ -273,3 +291,29 @@ RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= =JTFu -----END PGP PRIVATE KEY BLOCK----- """ + + +def load_tests(): + """ + Build a test suite that includes all tests in leap.soledad.common.tests + but does not include tests in the u1db_tests/ subfolder. The reason for + not including those tests are: + + 1. they by themselves only test u1db functionality in the u1db module + (despite we use them as basis for testing soledad functionalities). + + 2. they would fail because we monkey patch u1db's remote http server + to add soledad functionality we need. + """ + import unittest + import glob + import imp + tests_prefix = os.path.join( + '.', 'src', 'leap', 'soledad', 'common', 'tests') + suite = unittest.TestSuite() + for testcase in glob.glob(os.path.join(tests_prefix, 'test_*.py')): + modname = os.path.basename(os.path.splitext(testcase)[0]) + f, pathname, description = imp.find_module(modname, [tests_prefix]) + module = imp.load_module(modname, f, pathname, description) + suite.addTest(unittest.TestLoader().loadTestsFromModule(module)) + return suite diff --git a/common/src/leap/soledad/common/tests/server_state.py b/common/src/leap/soledad/common/tests/server_state.py new file mode 100644 index 00000000..2bc15377 --- /dev/null +++ b/common/src/leap/soledad/common/tests/server_state.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# server_state.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +State for servers to be used in tests. +""" + + +import os +import errno +import tempfile + + +from u1db.remote.server_state import ServerState +from leap.soledad.common.tests.util import ( + copy_sqlcipher_database_for_test, +) + + +class ServerStateForTests(ServerState): + """Passed to a Request when it is instantiated. + + This is used to track server-side state, such as working-directory, open + databases, etc. + """ + + def __init__(self): + self._workingdir = tempfile.mkdtemp() + + def _relpath(self, relpath): + return os.path.join(self._workingdir, relpath) + + def open_database(self, path): + """Open a database at the given location.""" + from leap.soledad.client.sqlcipher import SQLCipherDatabase + return SQLCipherDatabase.open_database(path, '123', False) + + def create_database(self, path): + """Create a database at the given location.""" + from leap.soledad.client.sqlcipher import SQLCipherDatabase + return SQLCipherDatabase.open_database(path, '123', True) + + def check_database(self, path): + """Check if the database at the given location exists. + + Simply returns if it does or raises DatabaseDoesNotExist. + """ + db = self.open_database(path) + db.close() + + def ensure_database(self, path): + """Ensure database at the given location.""" + from leap.soledad.client.sqlcipher import SQLCipherDatabase + full_path = self._relpath(path) + db = SQLCipherDatabase.open_database(full_path, '123', False) + return db, db._replica_uid + + def delete_database(self, path): + """Delete database at the given location.""" + from leap.u1db.backends import sqlite_backend + full_path = self._relpath(path) + sqlite_backend.SQLiteDatabase.delete_database(full_path) + + def _copy_database(self, db): + return copy_sqlcipher_database_for_test(None, db) diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 17d4a519..3b1e5a06 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -219,7 +219,6 @@ def copy_couch_database_for_test(test, db): new_couch_db.put_attachment(new_doc, att, filename=att_name) # cleanup connections to prevent file descriptor leaking - session.close_connections() return new_db @@ -249,13 +248,10 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): # if current test is `test_close` we have to use saved objects to # delete the database because the close() method will have removed the # references needed to do it using the CouchDatabase. - if self.id() == \ - 'leap.soledad.common.tests.test_couch.CouchTests.' \ - 'test_close(couch)': + if self.id().endswith('test_couch.CouchTests.test_close(couch)'): session = couch.Session() server = Server(url=self._url, session=session) del(server[self._dbname]) - session.close_connections() else: self.db.delete_database() test_backends.AllDatabaseTests.tearDown(self) @@ -365,8 +361,6 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests, # The following tests need that the database have an index, so we fake one. -old_class = couch.CouchDatabase - from u1db.backends.inmemory import InMemoryIndex @@ -444,7 +438,12 @@ class IndexedCouchDatabase(couch.CouchDatabase): return list(set([tuple(key.split('\x01')) for key in keys])) -couch.CouchDatabase = IndexedCouchDatabase +# monkey patch CouchDatabase (once) to include virtual indexes +if getattr(couch.CouchDatabase, '_old_class', None) is None: + old_class = couch.CouchDatabase + IndexedCouchDatabase._old_class = old_class + couch.CouchDatabase = IndexedCouchDatabase + sync_scenarios = [] for name, scenario in COUCH_SCENARIOS: diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index c79a6045..891aca0f 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -30,6 +30,7 @@ import threading from pysqlcipher import dbapi2 from StringIO import StringIO +from urlparse import urljoin # u1db stuff. @@ -54,19 +55,26 @@ from leap.soledad.common.crypto import ( ENC_JSON_KEY, ENC_SCHEME_KEY, ) -from leap.soledad.client.target import decrypt_doc +from leap.soledad.client.target import ( + decrypt_doc, + SoledadSyncTarget, +) # u1db tests stuff. +from leap.common.testing.basetest import BaseLeapTest from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest from leap.soledad.common.tests.u1db_tests import test_sqlite_backend from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_open from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.client.target import SoledadSyncTarget -from leap.common.testing.basetest import BaseLeapTest - -PASSWORD = '123456' +from leap.soledad.common.tests.util import ( + make_sqlcipher_database_for_test, + copy_sqlcipher_database_for_test, + make_soledad_app, + SoledadWithCouchServerMixin, + PASSWORD, +) #----------------------------------------------------------------------------- @@ -88,32 +96,6 @@ class TestSQLCipherBackendImpl(tests.TestCase): # The following tests come from `u1db.tests.test_backends`. #----------------------------------------------------------------------------- -def make_sqlcipher_database_for_test(test, replica_uid): - db = SQLCipherDatabase(':memory:', PASSWORD) - db._set_replica_uid(replica_uid) - return db - - -def copy_sqlcipher_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = SQLCipherDatabase(':memory:', PASSWORD) - tmpfile = StringIO() - for line in db._db_handle.iterdump(): - if not 'sqlite_sequence' in line: # work around bug in iterdump - tmpfile.write('%s\n' % line) - tmpfile.seek(0) - new_db._db_handle = dbapi2.connect(':memory:') - new_db._db_handle.cursor().executescript(tmpfile.read()) - new_db._db_handle.commit() - new_db._set_replica_uid(db._replica_uid) - new_db._factory = db._factory - return new_db - - def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts) @@ -451,7 +433,7 @@ sync_scenarios.append(('pyleap', { 'copy_database_for_test': test_sync.copy_database_for_http_test, 'make_document_for_test': make_document_for_test, 'make_app_with_state': tests.test_remote_sync_target.make_http_app, - 'do_sync': sync_via_synchronizer_and_leap, + 'do_sync': test_sync.sync_via_synchronizer, })) @@ -616,7 +598,7 @@ class SQLCipherDatabaseSyncTests( # update on 1 doc1.set_json('{"a": 3}') self.db1.put_doc(doc1) - # conflicts + # conflicts self.sync(self.db2, self.db1) self.sync(db3, self.db1) self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) @@ -658,32 +640,35 @@ class SQLCipherDatabaseSyncTests( 'return': {'docs': [], 'last_gen': 1}}) -def _make_local_db_and_leap_target(test, path='test'): +def _make_local_db_and_token_http_target(test, path='test'): test.startServer() db = test.request_state._create_database(os.path.basename(path)) - st = SoledadSyncTarget.connect(test.getURL(path), test._soledad._crypto) + st = SoledadSyncTarget.connect( + test.getURL(path), crypto=test._soledad._crypto) st.set_token_credentials('user-uuid', 'auth-token') return db, st target_scenarios = [ ('leap', { - 'create_db_and_target': _make_local_db_and_leap_target, - 'make_app_with_state': tests.test_remote_sync_target.make_http_app}), + 'create_db_and_target': _make_local_db_and_token_http_target, +# 'make_app_with_state': tests.test_remote_sync_target.make_http_app, + 'make_app_with_state': make_soledad_app, + 'do_sync': test_sync.sync_via_synchronizer}), ] class SQLCipherSyncTargetTests( - test_sync.DatabaseSyncTargetTests, BaseSoledadTest): + SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, target_scenarios)) - def setUp(self): - test_sync.DatabaseSyncTargetTests.setUp(self) + whitebox = False - def tearDown(self): - test_sync.DatabaseSyncTargetTests.tearDown(self) + def setUp(self): + self.main_test_class = test_sync.DatabaseSyncTargetTests + SoledadWithCouchServerMixin.setUp(self) def test_sync_exchange(self): """ diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py new file mode 100644 index 00000000..fd4a2797 --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_sync.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# test_sync.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import mock +import os +import json +import tempfile +import threading +import time +from urlparse import urljoin + +from leap.soledad.common.couch import ( + CouchServerState, + CouchDatabase, +) + +from leap.soledad.common.tests.u1db_tests import ( + TestCaseWithServer, + simple_doc, +) +from leap.soledad.common.tests.test_couch import CouchDBTestCase +from leap.soledad.common.tests.test_target import ( + make_token_soledad_app, + make_leap_document_for_test, + token_leap_sync_target, +) + +from leap.soledad.client import ( + Soledad, + target, +) + + +class InterruptableSyncTestCase( + CouchDBTestCase, TestCaseWithServer): + """ + Tests for encrypted sync using Soledad server backed by a couch database. + """ + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_leap_document_for_test + + sync_target = token_leap_sync_target + + def _soledad_instance(self, user='user-uuid', passphrase=u'123', + prefix='', + secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, + local_db_path='soledad.u1db', server_url='', + cert_file=None, auth_token=None, secret_id=None): + """ + Instantiate Soledad. + """ + + # this callback ensures we save a document which is sent to the shared + # db. + def _put_doc_side_effect(doc): + self._doc_put = doc + + # we need a mocked shared db or else Soledad will try to access the + # network to find if there are uploaded secrets. + class MockSharedDB(object): + + get_doc = mock.Mock(return_value=None) + put_doc = mock.Mock(side_effect=_put_doc_side_effect) + lock = mock.Mock(return_value=('atoken', 300)) + unlock = mock.Mock() + + def __call__(self): + return self + + Soledad._shared_db = MockSharedDB() + return Soledad( + user, + passphrase, + secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, + cert_file=cert_file, + auth_token=auth_token, + secret_id=secret_id) + + def make_app(self): + self.request_state = CouchServerState(self._couch_url, 'shared', + 'tokens') + return self.make_app_with_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self._couch_url = 'http://localhost:' + str(self.wrapper.port) + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def test_interruptable_sync(self): + """ + Test if Soledad can sync many smallfiles. + """ + + class _SyncInterruptor(threading.Thread): + """ + A thread meant to interrupt the sync process. + """ + + def __init__(self, soledad, couchdb): + self._soledad = soledad + self._couchdb = couchdb + threading.Thread.__init__(self) + + def run(self): + while db._get_generation() < 2: + time.sleep(1) + self._soledad.stop_sync() + time.sleep(1) + + number_of_docs = 10 + self.startServer() + + # instantiate soledad and create a document + sol = self._soledad_instance( + # token is verified in test_target.make_token_soledad_app + auth_token='auth-token' + ) + _, doclist = sol.get_all_docs() + self.assertEqual([], doclist) + + # create many small files + for i in range(0, number_of_docs): + sol.create_doc(json.loads(simple_doc)) + + # ensure remote db exists before syncing + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) + + # create interruptor thread + t = _SyncInterruptor(sol, db) + t.start() + + # sync with server + sol._server_url = self.getURL() + sol.sync() # this will be interrupted when couch db gen >= 2 + t.join() + + # recover the sync process + sol.sync() + + gen, doclist = db.get_all_docs() + self.assertEqual(number_of_docs, len(doclist)) + + # delete remote database + db.delete_database() + db.close() + sol.close() diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py index c1e00d52..3457a3e1 100644 --- a/common/src/leap/soledad/common/tests/test_target.py +++ b/common/src/leap/soledad/common/tests/test_target.py @@ -27,6 +27,7 @@ import simplejson as json import cStringIO +from u1db import SyncTarget from u1db.sync import Synchronizer from u1db.remote import ( http_client, @@ -39,14 +40,20 @@ from leap.soledad.client import ( target, auth, VerifiedHTTPSConnection, + sync, ) from leap.soledad.common.document import SoledadDocument -from leap.soledad.server import SoledadApp from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests import BaseSoledadTest +from leap.soledad.common.tests.util import ( + make_sqlcipher_database_for_test, + make_soledad_app, + make_token_soledad_app, + SoledadWithCouchServerMixin, +) from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_http_database from leap.soledad.common.tests.u1db_tests import test_http_client @@ -54,6 +61,10 @@ from leap.soledad.common.tests.u1db_tests import test_document from leap.soledad.common.tests.u1db_tests import test_remote_sync_target from leap.soledad.common.tests.u1db_tests import test_https from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.test_couch import ( + CouchDBTestCase, + CouchDBWrapper, +) #----------------------------------------------------------------------------- @@ -66,28 +77,6 @@ def make_leap_document_for_test(test, doc_id, rev, content, doc_id, rev, content, has_conflicts=has_conflicts) -def make_soledad_app(state): - return SoledadApp(state) - - -def make_token_soledad_app(state): - app = SoledadApp(state) - - def _verify_authentication_data(uuid, auth_data): - if uuid == 'user-uuid' and auth_data == 'auth-token': - return True - return False - - # we test for action authorization in leap.soledad.common.tests.test_server - def _verify_authorization(uuid, environ): - return True - - application = SoledadTokenAuthMiddleware(app) - application._verify_authentication_data = _verify_authentication_data - application._verify_authorization = _verify_authorization - return application - - LEAP_SCENARIOS = [ ('http', { 'make_database_for_test': test_backends.make_http_database_for_test, @@ -362,16 +351,47 @@ def token_leap_sync_target(test, path): return st +def make_local_db_and_soledad_target(test, path='test'): + test.startServer() + db = test.request_state._create_database(os.path.basename(path)) + st = target.SoledadSyncTarget.connect( + test.getURL(path), crypto=test._soledad._crypto) + return db, st + + +def make_local_db_and_token_soledad_target(test): + db, st = make_local_db_and_soledad_target(test, 'test') + st.set_token_credentials('user-uuid', 'auth-token') + return db, st + + class TestSoledadSyncTarget( - test_remote_sync_target.TestRemoteSyncTargets, BaseSoledadTest): + SoledadWithCouchServerMixin, + test_remote_sync_target.TestRemoteSyncTargets): scenarios = [ ('token_soledad', {'make_app_with_state': make_token_soledad_app, 'make_document_for_test': make_leap_document_for_test, + 'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_database_for_test': make_sqlcipher_database_for_test, 'sync_target': token_leap_sync_target}), ] + def setUp(self): + tests.TestCaseWithServer.setUp(self) + self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets + SoledadWithCouchServerMixin.setUp(self) + self.startServer() + self.db1 = make_sqlcipher_database_for_test(self, 'test1') + self.db2 = self.request_state._create_database('test2') + + def tearDown(self): + SoledadWithCouchServerMixin.tearDown(self) + tests.TestCaseWithServer.tearDown(self) + db, _ = self.request_state.ensure_database('test2') + db.delete_database() + def test_sync_exchange_send(self): """ Test for sync exchanging send of document. @@ -383,7 +403,7 @@ class TestSoledadSyncTarget( remote_target = self.getSyncTarget('test') other_docs = [] - def receive_doc(doc): + def receive_doc(doc, gen, trans_id): other_docs.append((doc.doc_id, doc.rev, doc.get_json())) doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') @@ -398,7 +418,10 @@ class TestSoledadSyncTarget( """ Test for sync exchange failure and retry. - This test was adapted to decrypt remote content before assert. + This test was adapted to: + - decrypt remote content before assert. + - not expect a bounced document because soledad has stateful + recoverable sync. """ self.startServer() @@ -412,7 +435,7 @@ class TestSoledadSyncTarget( _put_doc_if_newer = db._put_doc_if_newer trigger_ids = ['doc-here2'] - def bomb_put_doc_if_newer(doc, save_conflict, + def bomb_put_doc_if_newer(self, doc, save_conflict, replica_uid=None, replica_gen=None, replica_trans_id=None): if doc.doc_id in trigger_ids: @@ -421,7 +444,9 @@ class TestSoledadSyncTarget( replica_uid=replica_uid, replica_gen=replica_gen, replica_trans_id=replica_trans_id) - self.patch(db, '_put_doc_if_newer', bomb_put_doc_if_newer) + from leap.soledad.common.tests.test_couch import IndexedCouchDatabase + self.patch( + IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer) remote_target = self.getSyncTarget('test') other_changes = [] @@ -455,10 +480,11 @@ class TestSoledadSyncTarget( self.assertEqual( (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) self.assertEqual(2, new_gen) - # bounced back to us - self.assertEqual( - ('doc-here', 'replica:1', '{"value": "here"}', 1), - other_changes[0][:-1]) + # we do not expect the document to be bounced back because soledad has + # stateful sync + #self.assertEqual( + # ('doc-here', 'replica:1', '{"value": "here"}', 1), + # other_changes[0][:-1]) def test_sync_exchange_send_ensure_callback(self): """ @@ -471,7 +497,7 @@ class TestSoledadSyncTarget( other_docs = [] replica_uid_box = [] - def receive_doc(doc): + def receive_doc(doc, gen, trans_id): other_docs.append((doc.doc_id, doc.rev, doc.get_json())) def ensure_cb(replica_uid): @@ -489,6 +515,11 @@ class TestSoledadSyncTarget( self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) + def test_sync_exchange_in_stream_error(self): + # we bypass this test because our sync_exchange process does not + # return u1db error 503 "unavailable" for now. + pass + #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_https`. @@ -595,42 +626,34 @@ class TestHTTPDatabaseWithCreds( # The following tests come from `u1db.tests.test_sync`. #----------------------------------------------------------------------------- -def _make_local_db_and_leap_target(test, path='test'): - test.startServer() - db = test.request_state._create_database(os.path.basename(path)) - st = target.SoledadSyncTarget.connect( - test.getURL(path), crypto=test._soledad._crypto) - return db, st - - -def _make_local_db_and_token_leap_target(test): - db, st = _make_local_db_and_leap_target(test, 'test') - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - target_scenarios = [ ('token_leap', {'create_db_and_target': - _make_local_db_and_token_leap_target, - 'make_app_with_state': make_token_soledad_app}), + make_local_db_and_token_soledad_target, + 'make_app_with_state': make_soledad_app}), ] class SoledadDatabaseSyncTargetTests( - test_sync.DatabaseSyncTargetTests, BaseSoledadTest): + SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): scenarios = ( tests.multiply_scenarios( tests.DatabaseBaseTests.scenarios, target_scenarios)) + whitebox = False + + def setUp(self): + self.main_test_class = test_sync.DatabaseSyncTargetTests + SoledadWithCouchServerMixin.setUp(self) + def test_sync_exchange(self): """ Test sync exchange. This test was adapted to decrypt remote content before assert. """ - sol = _make_local_db_and_leap_target(self) + sol, _ = make_local_db_and_soledad_target(self) docs_by_gen = [ (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, 'T-sid')] @@ -703,17 +726,15 @@ class SoledadDatabaseSyncTargetTests( [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) -class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest): +class TestSoledadDbSync( + SoledadWithCouchServerMixin, test_sync.TestDbSync): """Test db.sync remote sync shortcut""" scenarios = [ - ('py-http', { - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - }), ('py-token-http', { + 'create_db_and_target': make_local_db_and_token_soledad_target, 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, + 'make_database_for_test': make_sqlcipher_database_for_test, 'token': True }), ] @@ -721,6 +742,10 @@ class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest): oauth = False token = False + def setUp(self): + self.main_test_class = test_sync.TestDbSync + SoledadWithCouchServerMixin.setUp(self) + def do_sync(self, target_name): """ Perform sync using SoledadSyncTarget and Token auth. @@ -748,7 +773,7 @@ class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest): """ doc1 = self.db.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = self.do_sync('test2.db') + local_gen_before_sync = self.do_sync('test2') gen, _, changes = self.db.whats_changed(local_gen_before_sync) self.assertEqual(1, len(changes)) self.assertEqual(doc2.doc_id, changes[0][0]) @@ -760,24 +785,9 @@ class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest): def test_db_sync_autocreate(self): """ - Test sync. - - Adapted to check for encrypted content. + We bypass this test because we never need to autocreate databases. """ - doc1 = self.db.create_doc_from_json(tests.simple_doc) - local_gen_before_sync = self.do_sync('test3.db') - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(0, gen - local_gen_before_sync) - db3 = self.request_state.open_database('test3.db') - gen, _, changes = db3.whats_changed() - self.assertEqual(1, len(changes)) - self.assertEqual(doc1.doc_id, changes[0][0]) - self.assertGetEncryptedDoc( - db3, doc1.doc_id, doc1.rev, tests.simple_doc, False) - t_gen, _ = self.db._get_replica_gen_and_trans_id('test3.db') - s_gen, _ = db3._get_replica_gen_and_trans_id('test1') - self.assertEqual(1, t_gen) - self.assertEqual(1, s_gen) + pass load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py index a37c36db..633fd8dd 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py @@ -1155,12 +1155,12 @@ class TestDbSync(tests.TestCaseWithServer): super(TestDbSync, self).setUp() self.startServer() self.db = self.make_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database('test2.db') + self.db2 = self.request_state._create_database('test2') def test_db_sync(self): doc1 = self.db.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = self.do_sync('test2.db') + local_gen_before_sync = self.do_sync('test2') gen, _, changes = self.db.whats_changed(local_gen_before_sync) self.assertEqual(1, len(changes)) self.assertEqual(doc2.doc_id, changes[0][0]) diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py new file mode 100644 index 00000000..249cbdaa --- /dev/null +++ b/common/src/leap/soledad/common/tests/util.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# util.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Utilities used by multiple test suites. +""" + + +import tempfile +import shutil +from urlparse import urljoin + +from StringIO import StringIO +from pysqlcipher import dbapi2 +from u1db.errors import DatabaseDoesNotExist + + +from leap.soledad.common import soledad_assert +from leap.soledad.common.couch import CouchDatabase, CouchServerState +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + + +from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests.test_couch import CouchDBWrapper, CouchDBTestCase + + +from leap.soledad.client.sqlcipher import SQLCipherDatabase + + +PASSWORD = '123456' + + +def make_sqlcipher_database_for_test(test, replica_uid): + db = SQLCipherDatabase(':memory:', PASSWORD) + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlcipher_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = SQLCipherDatabase(':memory:', PASSWORD) + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if not 'sqlite_sequence' in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +def make_soledad_app(state): + return SoledadApp(state) + + +def make_token_soledad_app(state): + app = SoledadApp(state) + + def _verify_authentication_data(uuid, auth_data): + if uuid == 'user-uuid' and auth_data == 'auth-token': + return True + return False + + # we test for action authorization in leap.soledad.common.tests.test_server + def _verify_authorization(uuid, environ): + return True + + application = SoledadTokenAuthMiddleware(app) + application._verify_authentication_data = _verify_authentication_data + application._verify_authorization = _verify_authorization + return application + + +class CouchServerStateForTests(CouchServerState): + """ + This is a slightly modified CouchDB server state that allows for creating + a database. + + Ordinarily, the CouchDB server state does not allow some operations, + because for security purposes the Soledad Server should not even have + enough permissions to perform them. For tests, we allow database creation, + otherwise we'd have to create those databases in setUp/tearDown methods, + which is less pleasant than allowing the db to be automatically created. + """ + + def _create_database(self, dbname): + return CouchDatabase.open_database( + urljoin(self._couch_url, dbname), + True, + replica_uid=dbname, + ensure_ddocs=True) + + def ensure_database(self, dbname): + db = self._create_database(dbname) + return db, db.replica_uid + + +class SoledadWithCouchServerMixin( + BaseSoledadTest, + CouchDBTestCase): + + @classmethod + def setUpClass(cls): + """ + Make sure we have a CouchDB instance for a test. + """ + # from BaseLeapTest + cls.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + # from CouchDBTestCase + cls.wrapper = CouchDBWrapper() + cls.wrapper.start() + #self.db = self.wrapper.db + + @classmethod + def tearDownClass(cls): + """ + Stop CouchDB instance for test. + """ + # from BaseLeapTest + soledad_assert( + cls.tempdir.startswith('/tmp/leap_tests-'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(cls.tempdir) + # from CouchDBTestCase + cls.wrapper.stop() + + def setUp(self): + BaseSoledadTest.setUp(self) + CouchDBTestCase.setUp(self) + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.setUp(self) + self._couch_url = 'http://localhost:%d' % self.wrapper.port + + def tearDown(self): + BaseSoledadTest.tearDown(self) + CouchDBTestCase.tearDown(self) + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.tearDown(self) + # delete the test database + try: + db = CouchDatabase(self._couch_url, 'test') + db.delete_database() + except DatabaseDoesNotExist: + pass + + def make_app(self): + couch_url = urljoin( + 'http://localhost:' + str(self.wrapper.port), 'tests') + self.request_state = CouchServerStateForTests( + couch_url, 'shared', 'tokens') + return self.make_app_with_state(self.request_state) |