diff options
Diffstat (limited to 'src/leap/soledad/tests/u1db_tests')
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/README | 34 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/__init__.py | 463 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_backends.py | 1896 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_document.py | 150 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_http_app.py | 1134 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_http_client.py | 363 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_http_database.py | 258 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py | 316 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py | 495 | ||||
| -rw-r--r-- | src/leap/soledad/tests/u1db_tests/test_sync.py | 1287 |
10 files changed, 6396 insertions, 0 deletions
diff --git a/src/leap/soledad/tests/u1db_tests/README b/src/leap/soledad/tests/u1db_tests/README new file mode 100644 index 00000000..605f01fa --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/README @@ -0,0 +1,34 @@ +General info +------------ + +Test files in this directory are derived from u1db-0.1.4 tests. The main +difference is that: + + (1) they include the test infrastructure packed with soledad; and + (2) they do not include c_backend_wrapper testing. + +Dependencies +------------ + +u1db tests depend on the following python packages: + + nose2 + unittest2 + mercurial + hgtools + testtools + discover + oauth + testscenarios + dirspec + paste + routes + simplejson + cython + +Running tests +------------- + +Use nose2 to run tests: + + nose2 leap.soledad.tests.u1db_tests diff --git a/src/leap/soledad/tests/u1db_tests/__init__.py b/src/leap/soledad/tests/u1db_tests/__init__.py new file mode 100644 index 00000000..167077f7 --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/__init__.py @@ -0,0 +1,463 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Test infrastructure for U1DB""" + +import copy +import shutil +import socket +import tempfile +import threading + +try: + import simplejson as json +except ImportError: + import json # noqa + +from wsgiref import simple_server + +from oauth import oauth +from sqlite3 import dbapi2 +from StringIO import StringIO + +import testscenarios +import testtools + +from u1db import ( + errors, + Document, + ) +from u1db.backends import ( + inmemory, + sqlite_backend, + ) +from u1db.remote import ( + server_state, + ) + +try: + from leap.soledad.tests.u1db_tests import c_backend_wrapper + c_backend_error = None +except ImportError, e: + c_backend_wrapper = None # noqa + c_backend_error = e + +# Setting this means that failing assertions will not include this module in +# their traceback. However testtools doesn't seem to set it, and we don't want +# this level to be omitted, but the lower levels to be shown. +# __unittest = 1 + + +class TestCase(testtools.TestCase): + + def createTempDir(self, prefix='u1db-tmp-'): + """Create a temporary directory to do some work in. + + This directory will be scheduled for cleanup when the test ends. + """ + tempdir = tempfile.mkdtemp(prefix=prefix) + self.addCleanup(shutil.rmtree, tempdir) + return tempdir + + def make_document(self, doc_id, doc_rev, content, has_conflicts=False): + return self.make_document_for_test( + self, doc_id, doc_rev, content, has_conflicts) + + def make_document_for_test(self, test, doc_id, doc_rev, content, + has_conflicts): + return make_document_for_test( + test, doc_id, doc_rev, content, has_conflicts) + + def assertGetDoc(self, db, doc_id, doc_rev, content, has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id)) + + def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content, + has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True)) + + def assertGetDocConflicts(self, db, doc_id, conflicts): + """Assert what conflicts are stored for a given doc_id. + + :param conflicts: A list of (doc_rev, content) pairs. + The first item must match the first item returned from the + database, however the rest can be returned in any order. + """ + if conflicts: + conflicts = [(rev, (json.loads(cont) if isinstance(cont, basestring) + else cont)) for (rev, cont) in conflicts] + conflicts = conflicts[:1] + sorted(conflicts[1:]) + actual = db.get_doc_conflicts(doc_id) + if actual: + actual = [(doc.rev, (json.loads(doc.get_json()) + if doc.get_json() is not None else None)) for doc in actual] + actual = actual[:1] + sorted(actual[1:]) + self.assertEqual(conflicts, actual) + + +def multiply_scenarios(a_scenarios, b_scenarios): + """Create the cross-product of scenarios.""" + + all_scenarios = [] + for a_name, a_attrs in a_scenarios: + for b_name, b_attrs in b_scenarios: + name = '%s,%s' % (a_name, b_name) + attrs = dict(a_attrs) + attrs.update(b_attrs) + all_scenarios.append((name, attrs)) + return all_scenarios + + +simple_doc = '{"key": "value"}' +nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' + + +def make_memory_database_for_test(test, replica_uid): + return inmemory.InMemoryDatabase(replica_uid) + + +def copy_memory_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = inmemory.InMemoryDatabase(db._replica_uid) + new_db._transaction_log = db._transaction_log[:] + new_db._docs = copy.deepcopy(db._docs) + new_db._conflicts = copy.deepcopy(db._conflicts) + new_db._indexes = copy.deepcopy(db._indexes) + new_db._factory = db._factory + return new_db + + +def make_sqlite_partial_expanded_for_test(test, replica_uid): + db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlite_partial_expanded_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if not 'sqlite_sequence' in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return Document(doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_c_database_for_test(test, replica_uid): + if c_backend_wrapper is None: + test.skipTest('c_backend_wrapper is not available') + db = c_backend_wrapper.CDatabase(':memory:') + db._set_replica_uid(replica_uid) + return db + + +def copy_c_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + if c_backend_wrapper is None: + test.skipTest('c_backend_wrapper is not available') + new_db = db._copy(db) + return new_db + + +def make_c_document_for_test(test, doc_id, rev, content, has_conflicts=False): + if c_backend_wrapper is None: + test.skipTest('c_backend_wrapper is not available') + return c_backend_wrapper.make_document( + doc_id, rev, content, has_conflicts=has_conflicts) + + +LOCAL_DATABASES_SCENARIOS = [ + ('mem', {'make_database_for_test': make_memory_database_for_test, + 'copy_database_for_test': copy_memory_database_for_test, + 'make_document_for_test': make_document_for_test}), + ('sql', {'make_database_for_test': + make_sqlite_partial_expanded_for_test, + 'copy_database_for_test': + copy_sqlite_partial_expanded_for_test, + 'make_document_for_test': make_document_for_test}), + ] + + +C_DATABASE_SCENARIOS = [ + ('c', {'make_database_for_test': make_c_database_for_test, + 'copy_database_for_test': copy_c_database_for_test, + 'make_document_for_test': make_c_document_for_test})] + + +class DatabaseBaseTests(TestCase): + + accept_fixed_trans_id = False # set to True assertTransactionLog + # is happy with all trans ids = '' + + scenarios = LOCAL_DATABASES_SCENARIOS + + def create_database(self, replica_uid): + return self.make_database_for_test(self, replica_uid) + + def copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + return self.copy_database_for_test(self, db) + + def setUp(self): + super(DatabaseBaseTests, self).setUp() + self.db = self.create_database('test') + + def tearDown(self): + # TODO: Add close_database parameterization + # self.close_database(self.db) + super(DatabaseBaseTests, self).tearDown() + + def assertTransactionLog(self, doc_ids, db): + """Assert that the given docs are in the transaction log.""" + log = db._get_transaction_log() + just_ids = [] + seen_transactions = set() + for doc_id, transaction_id in log: + just_ids.append(doc_id) + self.assertIsNot(None, transaction_id, + "Transaction id should not be None") + if transaction_id == '' and self.accept_fixed_trans_id: + continue + self.assertNotEqual('', transaction_id, + "Transaction id should be a unique string") + self.assertTrue(transaction_id.startswith('T-')) + self.assertNotIn(transaction_id, seen_transactions) + seen_transactions.add(transaction_id) + self.assertEqual(doc_ids, just_ids) + + def getLastTransId(self, db): + """Return the transaction id for the last database update.""" + return self.db._get_transaction_log()[-1][-1] + + +class ServerStateForTests(server_state.ServerState): + """Used in the test suite, so we don't have to touch disk, etc.""" + + def __init__(self): + super(ServerStateForTests, self).__init__() + self._dbs = {} + + def open_database(self, path): + try: + return self._dbs[path] + except KeyError: + raise errors.DatabaseDoesNotExist + + def check_database(self, path): + # cares only about the possible exception + self.open_database(path) + + def ensure_database(self, path): + try: + db = self.open_database(path) + except errors.DatabaseDoesNotExist: + db = self._create_database(path) + return db, db._replica_uid + + def _copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + new_db = copy_memory_database_for_test(None, db) + path = db._replica_uid + while path in self._dbs: + path += 'copy' + self._dbs[path] = new_db + return new_db + + def _create_database(self, path): + db = inmemory.InMemoryDatabase(path) + self._dbs[path] = db + return db + + def delete_database(self, path): + del self._dbs[path] + + +class ResponderForTests(object): + """Responder for tests.""" + _started = False + sent_response = False + status = None + + def start_response(self, status='success', **kwargs): + self._started = True + self.status = status + self.kwargs = kwargs + + def send_response(self, status='success', **kwargs): + self.start_response(status, **kwargs) + self.finish_response() + + def finish_response(self): + self.sent_response = True + + +class TestCaseWithServer(TestCase): + + @staticmethod + def server_def(): + # hook point + # should return (ServerClass, "shutdown method name", "url_scheme") + class _RequestHandler(simple_server.WSGIRequestHandler): + def log_request(*args): + pass # suppress + + def make_server(host_port, application): + assert application, "forgot to override make_app(_with_state)?" + srv = simple_server.WSGIServer(host_port, _RequestHandler) + # patch the value in if it's None + if getattr(application, 'base_url', 1) is None: + application.base_url = "http://%s:%s" % srv.server_address + srv.set_app(application) + return srv + + return make_server, "shutdown", "http" + + @staticmethod + def make_app_with_state(state): + # hook point + return None + + def make_app(self): + # potential hook point + self.request_state = ServerStateForTests() + return self.make_app_with_state(self.request_state) + + def setUp(self): + super(TestCaseWithServer, self).setUp() + self.server = self.server_thread = None + + @property + def url_scheme(self): + return self.server_def()[-1] + + def startServer(self): + server_def = self.server_def() + server_class, shutdown_meth, _ = server_def + application = self.make_app() + self.server = server_class(('127.0.0.1', 0), application) + self.server_thread = threading.Thread(target=self.server.serve_forever, + kwargs=dict(poll_interval=0.01)) + self.server_thread.start() + self.addCleanup(self.server_thread.join) + self.addCleanup(getattr(self.server, shutdown_meth)) + + def getURL(self, path=None): + host, port = self.server.server_address + if path is None: + path = '' + return '%s://%s:%s/%s' % (self.url_scheme, host, port, path) + + +def socket_pair(): + """Return a pair of TCP sockets connected to each other. + + Unlike socket.socketpair, this should work on Windows. + """ + sock_pair = getattr(socket, 'socket_pair', None) + if sock_pair: + return sock_pair(socket.AF_INET, socket.SOCK_STREAM) + listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listen_sock.bind(('127.0.0.1', 0)) + listen_sock.listen(1) + client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client_sock.connect(listen_sock.getsockname()) + server_sock, addr = listen_sock.accept() + listen_sock.close() + return server_sock, client_sock + + +# OAuth related testing + +consumer1 = oauth.OAuthConsumer('K1', 'S1') +token1 = oauth.OAuthToken('kkkk1', 'XYZ') +consumer2 = oauth.OAuthConsumer('K2', 'S2') +token2 = oauth.OAuthToken('kkkk2', 'ZYX') +token3 = oauth.OAuthToken('kkkk3', 'ZYX') + + +class TestingOAuthDataStore(oauth.OAuthDataStore): + """In memory predefined OAuthDataStore for testing.""" + + consumers = { + consumer1.key: consumer1, + consumer2.key: consumer2, + } + + tokens = { + token1.key: token1, + token2.key: token2 + } + + def lookup_consumer(self, key): + return self.consumers.get(key) + + def lookup_token(self, token_type, token_token): + return self.tokens.get(token_token) + + def lookup_nonce(self, oauth_consumer, oauth_token, nonce): + return None + +testingOAuthStore = TestingOAuthDataStore() + +sign_meth_HMAC_SHA1 = oauth.OAuthSignatureMethod_HMAC_SHA1() +sign_meth_PLAINTEXT = oauth.OAuthSignatureMethod_PLAINTEXT() + + +def load_with_scenarios(loader, standard_tests, pattern): + """Load the tests in a given module. + + This just applies testscenarios.generate_scenarios to all the tests that + are present. We do it at load time rather than at run time, because it + plays nicer with various tools. + """ + suite = loader.suiteClass() + suite.addTests(testscenarios.generate_scenarios(standard_tests)) + return suite diff --git a/src/leap/soledad/tests/u1db_tests/test_backends.py b/src/leap/soledad/tests/u1db_tests/test_backends.py new file mode 100644 index 00000000..c93589ea --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_backends.py @@ -0,0 +1,1896 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""The backend class for U1DB. This deals with hiding storage details.""" + +try: + import simplejson as json +except ImportError: + import json # noqa +from u1db import ( + DocumentBase, + errors, + vectorclock, + ) + +from leap.soledad.tests import u1db_tests as tests + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + +from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( + make_http_app, + make_oauth_http_app, +) + +from u1db.remote import ( + http_database, + ) + +try: + from u1db.tests import c_backend_wrapper +except ImportError: + c_backend_wrapper = None # noqa + + +def make_http_database_for_test(test, replica_uid, path='test'): + test.startServer() + test.request_state._create_database(replica_uid) + return http_database.HTTPDatabase(test.getURL(path)) + + +def copy_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + return test.request_state._copy_database(db) + + +def make_oauth_http_database_for_test(test, replica_uid): + http_db = make_http_database_for_test(test, replica_uid, '~/test') + http_db.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + return http_db + + +def copy_oauth_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + http_db = test.request_state._copy_database(db) + http_db.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + return http_db + + +class TestAlternativeDocument(DocumentBase): + """A (not very) alternative implementation of Document.""" + + +class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ + ('http', {'make_database_for_test': make_http_database_for_test, + 'copy_database_for_test': copy_http_database_for_test, + 'make_document_for_test': tests.make_document_for_test, + 'make_app_with_state': make_http_app}), + ('oauth_http', {'make_database_for_test': + make_oauth_http_database_for_test, + 'copy_database_for_test': + copy_oauth_http_database_for_test, + 'make_document_for_test': tests.make_document_for_test, + 'make_app_with_state': make_oauth_http_app}) + ] #+ tests.C_DATABASE_SCENARIOS + + def test_close(self): + self.db.close() + + def test_create_doc_allocating_doc_id(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertNotEqual(None, doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_different_ids_same_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + + def test_create_doc_with_id(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my-id') + self.assertEqual('my-id', doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_existing_id(self): + doc = self.db.create_doc_from_json(simple_doc) + new_content = '{"something": "else"}' + self.assertRaises( + errors.RevisionConflict, self.db.create_doc_from_json, + new_content, doc.doc_id) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_put_doc_creating_initial(self): + doc = self.make_document('my_doc_id', None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertIsNot(None, new_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, simple_doc, False) + + def test_put_doc_space_in_id(self): + doc = self.make_document('my doc id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_update(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + orig_rev = doc.rev + doc.set_json('{"updated": "stuff"}') + new_rev = self.db.put_doc(doc) + self.assertNotEqual(new_rev, orig_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, + '{"updated": "stuff"}', False) + self.assertEqual(doc.rev, new_rev) + + def test_put_non_ascii_key(self): + content = json.dumps({u'key\xe5': u'val'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_non_ascii_value(self): + content = json.dumps({'key': u'\xe5'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_doc_refuses_no_id(self): + doc = self.make_document(None, None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document("", None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_refuses_slashes(self): + doc = self.make_document('a/b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document(r'\b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_url_quoting_is_fine(self): + doc_id = "%2F%2Ffoo%2Fbar" + doc = self.make_document(doc_id, None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertGetDoc(self.db, doc_id, new_rev, simple_doc, False) + + def test_put_doc_refuses_non_existing_old_rev(self): + doc = self.make_document('doc-id', 'test:4', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.put_doc, doc) + + def test_put_doc_refuses_non_ascii_doc_id(self): + doc = self.make_document('d\xc3\xa5c-id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_fails_with_bad_old_rev(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + old_rev = doc.rev + bad_doc = self.make_document(doc.doc_id, 'other:1', + '{"something": "else"}') + self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc) + self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False) + + def test_create_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + new_doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(new_doc.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev)) + + def test_put_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + doc2 = self.make_document('my_doc_id', None, simple_doc) + self.db.put_doc(doc2) + self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (doc2.rev, deleted_doc.rev)) + + def test_get_doc_after_put(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False) + + def test_get_doc_nonexisting(self): + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_doc_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertIs(None, self.db.get_doc('my_doc_id')) + + def test_get_doc_include_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_get_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual([doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual( + [doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + include_deleted=True))) + + def test_get_docs_request_ordered(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + self.assertEqual([doc2, doc1], + list(self.db.get_docs([doc2.doc_id, doc1.doc_id]))) + + def test_get_docs_empty_list(self): + self.assertEqual([], list(self.db.get_docs([]))) + + def test_handles_nested_content(self): + doc = self.db.create_doc_from_json(nested_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + def test_handles_doc_with_null(self): + doc = self.db.create_doc_from_json('{"key": null}') + self.assertGetDoc(self.db, doc.doc_id, doc.rev, '{"key": null}', False) + + def test_delete_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + orig_rev = doc.rev + self.db.delete_doc(doc) + self.assertNotEqual(orig_rev, doc.rev) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + self.assertIs(None, self.db.get_doc(doc.doc_id)) + + def test_delete_doc_non_existent(self): + doc = self.make_document('non-existing', 'other:1', simple_doc) + self.assertRaises(errors.DocumentDoesNotExist, self.db.delete_doc, doc) + + def test_delete_doc_already_deleted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertRaises(errors.DocumentAlreadyDeleted, + self.db.delete_doc, doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_delete_doc_bad_rev(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + doc2 = self.make_document(doc1.doc_id, 'other:1', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.delete_doc, doc2) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_delete_doc_sets_content_to_None(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertIs(None, doc.get_json()) + + def test_delete_doc_rev_supersedes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json(nested_doc) + self.db.put_doc(doc) + doc.set_json('{"fishy": "content"}') + self.db.put_doc(doc) + old_rev = doc.rev + self.db.delete_doc(doc) + cur_vc = vectorclock.VectorClockRev(old_rev) + deleted_vc = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(deleted_vc.is_newer(cur_vc), + "%s does not supersede %s" % (doc.rev, old_rev)) + + def test_delete_then_put(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + doc.set_json(nested_doc) + self.db.put_doc(doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + +class DocumentSizeTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS #+ tests.C_DATABASE_SCENARIOS + + def test_put_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + doc = self.make_document('doc-id', None, simple_doc) + self.assertRaises(errors.DocumentTooBig, self.db.put_doc, doc) + + def test_create_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + self.assertRaises( + errors.DocumentTooBig, self.db.create_doc_from_json, simple_doc, + doc_id='my_doc_id') + + def test_set_document_size_limit_zero(self): + self.db.set_document_size_limit(0) + self.assertEqual(0, self.db.document_size_limit) + + def test_set_document_size_limit(self): + self.db.set_document_size_limit(1000000) + self.assertEqual(1000000, self.db.document_size_limit) + + +class LocalDatabaseTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS #+ tests.C_DATABASE_SCENARIOS + + def test_create_doc_different_ids_diff_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + db2 = self.create_database('other-uid') + doc2 = db2.create_doc_from_json(simple_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + + def test_put_doc_refuses_slashes_picky(self): + doc = self.make_document('/a', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_get_all_docs_empty(self): + self.assertEqual([], list(self.db.get_all_docs()[1])) + + def test_get_all_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc1, doc2]), sorted(list(self.db.get_all_docs()[1]))) + + def test_get_all_docs_exclude_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual([doc1], list(self.db.get_all_docs()[1])) + + def test_get_all_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual( + sorted([doc1, doc2]), + sorted(list(self.db.get_all_docs(include_deleted=True)[1]))) + + def test_get_all_docs_generation(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(nested_doc) + self.assertEqual(2, self.db.get_all_docs()[0]) + + def test_simple_put_doc_if_newer(self): + doc = self.make_document('my-doc-id', 'test:1', simple_doc) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 1), state_at_gen) + self.assertGetDoc(self.db, 'my-doc-id', 'test:1', simple_doc, False) + + def test_simple_put_doc_if_newer_deleted(self): + self.db.create_doc_from_json('{}', doc_id='my-doc-id') + doc = self.make_document('my-doc-id', 'test:2', None) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 2), state_at_gen) + self.assertGetDocIncludeDeleted( + self.db, 'my-doc-id', 'test:2', None, False) + + def test_put_doc_if_newer_already_superseded(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + doc1_rev1 = doc1.rev + doc1.set_json(simple_doc) + self.db.put_doc(doc1) + doc1_rev2 = doc1.rev + # Nothing is inserted, because the document is already superseded + doc = self.make_document(doc1.doc_id, doc1_rev1, orig_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + self.assertGetDoc(self.db, doc1.doc_id, doc1_rev2, simple_doc, False) + + def test_put_doc_if_newer_autoresolve(self): + doc1 = self.db.create_doc_from_json(simple_doc) + rev = doc1.rev + doc = self.make_document(doc1.doc_id, "whatever:1", doc1.get_json()) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + doc2 = self.db.get_doc(doc1.doc_id) + v2 = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev("whatever:1"))) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev(rev))) + # strictly newer locally + self.assertTrue(rev not in doc2.rev) + + def test_put_doc_if_newer_already_converged(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + state_at_gen = self.db._put_doc_if_newer( + doc1, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('converged', 1), state_at_gen) + + def test_put_doc_if_newer_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Nothing is inserted, the document id is returned as would-conflict + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + alt_doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database wasn't altered + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_put_doc_if_newer_newer_generation(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:2', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=2, + replica_trans_id='T-irrelevant') + self.assertEqual('inserted', state) + + def test_put_doc_if_newer_same_generation_same_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.db.create_doc_from_json(simple_doc) + self.make_document(doc.doc_id, 'other:1', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=1, + replica_trans_id='T-sid') + self.assertEqual('converged', state) + + def test_put_doc_if_newer_wrong_transaction_id(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidTransactionId, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_old_generation_older_doc(self): + orig_doc = '{"new": "doc"}' + doc = self.db.create_doc_from_json(orig_doc) + doc_rev1 = doc.rev + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.db._set_replica_gen_and_trans_id('other', 3, 'T-sid') + older_doc = self.make_document(doc.doc_id, doc_rev1, simple_doc) + state, _ = self.db._put_doc_if_newer( + older_doc, save_conflict=False, replica_uid='other', replica_gen=8, + replica_trans_id='T-irrelevant') + self.assertEqual('superseded', state) + + def test_put_doc_if_newer_old_generation_newer_doc(self): + self.db._set_replica_gen_and_trans_id('other', 5, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidGeneration, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.assertEqual('inserted', + self.db._put_doc_if_newer(doc2, save_conflict=False, + replica_uid='other', replica_gen=2, + replica_trans_id='T-id2')[0]) + self.assertEqual((2, 'T-id2'), self.db._get_replica_gen_and_trans_id( + 'other')) + # Compare to the old rev, should be superseded + doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc) + self.assertEqual('superseded', + self.db._put_doc_if_newer(doc2, save_conflict=False, + replica_uid='other', replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + # A conflict that isn't saved still records the sync gen, because we + # don't need to see it again + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer(doc2, save_conflict=False, + replica_uid='other', replica_gen=4, + replica_trans_id='T-id4')[0]) + self.assertEqual( + (4, 'T-id4'), self.db._get_replica_gen_and_trans_id('other')) + + def test__get_replica_gen_and_trans_id(self): + self.assertEqual( + (0, ''), self.db._get_replica_gen_and_trans_id('other-db')) + self.db._set_replica_gen_and_trans_id('other-db', 2, 'T-transaction') + self.assertEqual( + (2, 'T-transaction'), + self.db._get_replica_gen_and_trans_id('other-db')) + + def test_put_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + doc.set_json('{"something": "else"}') + self.db.put_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + + def test_delete_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + db_gen, _, _ = self.db.whats_changed() + self.db.delete_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed(db_gen)) + + def test_whats_changed_initial_database(self): + self.assertEqual((0, '', []), self.db.whats_changed()) + + def test_whats_changed_returns_one_id_for_multiple_changes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.put_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + self.assertEqual((2, last_trans_id, []), self.db.whats_changed(2)) + + def test_whats_changed_returns_last_edits_ascending(self): + doc = self.db.create_doc_from_json(simple_doc) + doc1 = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.delete_doc(doc1) + delete_trans_id = self.getLastTransId(self.db) + self.db.put_doc(doc) + put_trans_id = self.getLastTransId(self.db) + self.assertEqual((4, put_trans_id, + [(doc1.doc_id, 3, delete_trans_id), + (doc.doc_id, 4, put_trans_id)]), + self.db.whats_changed()) + + def test_whats_changed_doesnt_include_old_gen(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((3, last_trans_id, [(doc2.doc_id, 3, last_trans_id)]), + self.db.whats_changed(2)) + + +class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS #+ tests.C_DATABASE_SCENARIOS + + def test_validate_gen_and_trans_id(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.db.validate_gen_and_trans_id(gen, trans_id) + + def test_validate_gen_and_trans_id_invalid_txid(self): + self.db.create_doc_from_json(simple_doc) + gen, _ = self.db._get_generation_info() + self.assertRaises( + errors.InvalidTransactionId, + self.db.validate_gen_and_trans_id, gen, 'wrong') + + def test_validate_gen_and_trans_id_invalid_gen(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.assertRaises( + errors.InvalidGeneration, + self.db.validate_gen_and_trans_id, gen + 1, trans_id) + + +class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS #+ tests.C_DATABASE_SCENARIOS + + def test_validate_source_gen_and_trans_id_same(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 1, 'T-sid') + + def test_validate_source_gen_newer(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 2, 'T-whatevs') + + def test_validate_source_wrong_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.assertRaises( + errors.InvalidTransactionId, + self.db._validate_source, 'other', 1, 'T-sad') + + +class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): + # test supporting/functionality around storing conflicts + + scenarios = tests.LOCAL_DATABASES_SCENARIOS #+ tests.C_DATABASE_SCENARIOS + + def test_get_docs_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([doc2], list(self.db.get_docs([doc1.doc_id]))) + + def test_get_docs_conflicts_ignored(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + no_conflict_doc = self.make_document(doc1.doc_id, 'alternate:1', + nested_doc) + self.assertEqual([no_conflict_doc, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + check_for_conflicts=False))) + + def test_get_doc_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([alt_doc, doc], + self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_all_docs_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + _, docs = self.db.get_all_docs() + self.assertTrue(list(docs)[0].has_conflicts) + + def test_get_doc_conflicts_unconflicted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_doc_conflicts_no_such_id(self): + self.assertEqual([], self.db.get_doc_conflicts('doc-id')) + + def test_resolve_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc.doc_id, + [('alternate:1', nested_doc), (doc.rev, simple_doc)]) + orig_rev = doc.rev + self.db.resolve_doc(doc, [alt_doc.rev, doc.rev]) + self.assertNotEqual(orig_rev, doc.rev) + self.assertFalse(doc.has_conflicts) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc.doc_id, []) + + def test_resolve_doc_picks_biggest_vcr(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + orig_doc1_rev = doc1.rev + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertFalse(doc1.has_conflicts) + self.assertNotEqual(orig_doc1_rev, doc1.rev) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + vcr_1 = vectorclock.VectorClockRev(orig_doc1_rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + vcr_new = vectorclock.VectorClockRev(doc1.rev) + self.assertTrue(vcr_new.is_newer(vcr_1)) + self.assertTrue(vcr_new.is_newer(vcr_2)) + + def test_resolve_doc_partial_not_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDoc(self.db, doc1.doc_id, doc3.rev, content3, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc)]) + + def test_resolve_doc_partial_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc3.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + + def test_resolve_doc_with_delete_conflict(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc2, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, False) + + def test_resolve_doc_with_delete_to_delete(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc1, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDocIncludeDeleted( + self.db, doc1.doc_id, doc1.rev, None, False) + + def test_put_doc_if_newer_save_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Document is inserted as a conflict + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database was updated + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, nested_doc, True) + + def test_force_doc_conflict_supersedes_properly(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', '{"b": 1}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'altalt:1', '{"c": 1}') + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + doc22 = self.make_document(doc1.doc_id, 'alternate:2', '{"b": 2}') + self.db._put_doc_if_newer( + doc22, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='zed') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2', doc22.get_json()), + ('altalt:1', doc3.get_json()), + (doc1.rev, simple_doc)]) + + def test_put_doc_if_newer_save_conflict_was_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue(doc2.has_conflicts) + self.assertGetDoc( + self.db, doc1.doc_id, 'alternate:1', nested_doc, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), (doc1.rev, None)]) + + def test_put_doc_if_newer_propagates_full_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + resolved_vcr.maximize(vcr_2) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual('inserted', state) + self.assertFalse(doc_resolved.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + doc3 = self.db.get_doc(doc1.doc_id) + self.assertFalse(doc3.has_conflicts) + + def test_put_doc_if_newer_propagates_partial_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altalt:1', '{}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), ('test:1', simple_doc), + ('altalt:1', '{}')]) + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_3 = vectorclock.VectorClockRev(doc3.rev) + resolved_vcr.maximize(vcr_3) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual('inserted', state) + self.assertTrue(doc_resolved.has_conflicts) + doc4 = self.db.get_doc(doc1.doc_id) + self.assertTrue(doc4.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2|test:1', '{"good": 1}'), ('altalt:1', '{}')]) + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-id') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.db._put_doc_if_newer(doc2, save_conflict=True, + replica_uid='other', replica_gen=2, + replica_trans_id='T-id2') + # Conflict vs the current update + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer(doc2, save_conflict=True, + replica_uid='other', replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + + def test_put_doc_if_newer_autoresolve_2(self): + # this is an ordering variant of _3, but that already works + # adding the test explicitly to catch the regression easily + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', "{}") + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', + '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:2|other:1', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'inserted') + self.assertFalse(self.db.get_doc(doc_a1.doc_id).has_conflicts) + + def test_put_doc_if_newer_autoresolve_3(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', "{}") + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_doc_if_newer_autoresolve_4(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', None) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', None) + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_refuses_to_update_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": "altval"}' + doc2 = self.make_document(doc1.doc_id, 'altrev:1', content2) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, content2, True) + content3 = '{"key": "local"}' + doc2.set_json(content3) + self.assertRaises(errors.ConflictedDoc, self.db.put_doc, doc2) + + def test_delete_refuses_for_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altrev:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, True) + self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) + + +class DatabaseIndexTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS #+ tests.C_DATABASE_SCENARIOS + + def assertParseError(self, definition): + self.db.create_doc_from_json(nested_doc) + self.assertRaises( + errors.IndexDefinitionParseError, self.db.create_index, 'idx', + definition) + + def assertIndexCreatable(self, definition): + name = "idx" + self.db.create_doc_from_json(nested_doc) + self.db.create_index(name, definition) + self.assertEqual( + [(name, [definition])], self.db.list_indexes()) + + def test_create_index(self): + self.db.create_index('test-idx', 'name') + self.assertEqual([('test-idx', ['name'])], + self.db.list_indexes()) + + def test_create_index_on_non_ascii_field_name(self): + doc = self.db.create_doc_from_json(json.dumps({u'\xe5': 'value'})) + self.db.create_index('test-idx', u'\xe5') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_list_indexes_with_non_ascii_field_names(self): + self.db.create_index('test-idx', u'\xe5') + self.assertEqual( + [('test-idx', [u'\xe5'])], self.db.list_indexes()) + + def test_create_index_evaluates_it(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_wildcard_matches_unicode_value(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_retrieve_unicode_value_from_index(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', u"valu\xe5")) + + def test_create_index_fails_if_name_taken(self): + self.db.create_index('test-idx', 'key') + self.assertRaises(errors.IndexNameTakenError, + self.db.create_index, + 'test-idx', 'stuff') + + def test_create_index_does_not_fail_if_name_taken_with_same_index(self): + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + + def test_create_index_does_not_duplicate_indexed_fields(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.delete_index('test-idx') + self.db.create_index('test-idx', 'key') + self.assertEqual(1, len(self.db.get_from_index('test-idx', 'value'))) + + def test_delete_index_does_not_remove_fields_from_other_indexes(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx2', 'key') + self.db.delete_index('test-idx') + self.assertEqual(1, len(self.db.get_from_index('test-idx2', 'value'))) + + def test_create_index_after_deleting_document(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc2) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_delete_index(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + self.db.delete_index('test-idx') + self.assertEqual([], self.db.list_indexes()) + + def test_create_adds_to_index(self): + self.db.create_index('test-idx', 'key') + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_unmatched(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', 'novalue')) + + def test_create_index_multiple_exact_matches(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + + def test_get_from_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_multi(self): + content = '{"key": "value", "key2": "value2"}' + doc = self.db.create_doc_from_json(content) + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2')) + + def test_get_from_index_multi_list(self): + doc = self.db.create_doc_from_json( + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-1')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-2')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-3')) + self.assertEqual( + [('value', 'value2-1'), ('value', 'value2-2'), + ('value', 'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key', 'key2') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_from_index('test-idx', 'value', 'value2-1') + self.assertTrue(docs[0].has_conflicts) + + def test_get_index_keys_multi_list_list(self): + self.db.create_doc_from_json( + '{"key": "value1-1 value1-2 value1-3", ' + '"key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'split_words(key)', 'key2') + self.assertEqual( + [(u'value1-1', u'value2-1'), (u'value1-1', u'value2-2'), + (u'value1-1', u'value2-3'), (u'value1-2', u'value2-1'), + (u'value1-2', u'value2-2'), (u'value1-2', u'value2-3'), + (u'value1-3', u'value2-1'), (u'value1-3', u'value2-2'), + (u'value1-3', u'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_multi_ordered(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2, doc1], + self.db.get_from_index('test-idx', 'v*', '*')) + + def test_get_range_from_index_start_end(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2', 'value3')) + + def test_get_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + doc3 = self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1, doc3], + self.db.get_range_from_index('test-idx', 'value2')) + + def test_get_range_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', '{"key": "valuedepalue"}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_range_from_index('test-idx', 'a') + self.assertTrue(docs[0].has_conflicts) + + def test_get_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + doc4 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2')) + + def test_get_wildcard_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4, doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2*')) + + def test_get_wildcard_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + doc5 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc5, doc3, doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2*')) + + def test_get_wildcard_range_from_index_start_end(self): + self.db.create_doc_from_json('{"key": "a"}') + self.db.create_doc_from_json('{"key": "boo3"}') + doc3 = self.db.create_doc_from_json('{"key": "catalyst"}') + doc4 = self.db.create_doc_from_json('{"key": "whaever"}') + self.db.create_doc_from_json('{"key": "zerg"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4], + self.db.get_range_from_index('test-idx', 'cat*', 'zap*')) + + def test_get_range_from_index_multi_column_start_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2], + self.db.get_range_from_index( + 'test-idx', ('value2', 'value2'), ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + self.db.create_doc_from_json('{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value3'))) + + def test_get_wildcard_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value2*'))) + + def test_get_wildcard_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value2*'))) + + def test_get_glob_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', '*'))) + + def test_get_glob_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index('test-idx', None, ('value2', '*'))) + + def test_get_range_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v*')) + + def test_get_range_from_index_illegal_wildcard_order_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v*')) + + def test_get_from_index_fails_if_no_index(self): + self.assertRaises( + errors.IndexDoesNotExist, self.db.get_from_index, 'foo') + + def test_get_index_keys_fails_if_no_index(self): + self.assertRaises(errors.IndexDoesNotExist, + self.db.get_index_keys, + 'foo') + + def test_get_index_keys_works_if_no_docs(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_index_keys('test-idx')) + + def test_put_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db.put_doc(doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'altval')) + + def test_delete_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + self.db.delete_doc(doc) + self.assertEqual([doc2], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_illegal_number_of_entries(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidValueForIndex, self.db.get_from_index, 'test-idx') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1', 'v2', 'v3') + + def test_get_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v2') + + def test_get_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v*') + + def test_get_all_from_index(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + # This one should not be in the index + self.db.create_doc_from_json('{"no": "key"}') + diff_value_doc = '{"key": "diff value"}' + doc4 = self.db.create_doc_from_json(diff_value_doc) + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + sorted([doc1, doc2, doc4]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_all_from_index_ordered(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json('{"key": "value x"}') + doc2 = self.db.create_doc_from_json('{"key": "value b"}') + doc3 = self.db.create_doc_from_json('{"key": "value a"}') + doc4 = self.db.create_doc_from_json('{"key": "value m"}') + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + [doc3, doc2, doc4, doc1], self.db.get_from_index('test-idx', '*')) + + def test_put_updates_when_adding_key(self): + doc = self.db.create_doc_from_json("{}") + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_get_from_index_empty_string(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": ""}' + doc2 = self.db.create_doc_from_json(content2) + self.assertEqual([doc2], self.db.get_from_index('test-idx', '')) + # Empty string matches the wildcard. + self.assertEqual( + sorted([doc1, doc2]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_from_index_not_null(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json('{"key": null}') + self.assertEqual([doc1], self.db.get_from_index('test-idx', '*')) + + def test_get_partial_from_index(self): + content1 = '{"k1": "v1", "k2": "v2"}' + content2 = '{"k1": "v1", "k2": "x2"}' + content3 = '{"k1": "v1", "k2": "y2"}' + # doc4 has a different k1 value, so it doesn't match the prefix. + content4 = '{"k1": "NN", "k2": "v2"}' + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.db.create_index('test-idx', 'k1', 'k2') + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "*"))) + + def test_get_glob_match(self): + # Note: the exact glob syntax is probably subject to change + content1 = '{"k1": "v1", "k2": "v1"}' + content2 = '{"k1": "v1", "k2": "v2"}' + content3 = '{"k1": "v1", "k2": "v3"}' + # doc4 has a different k2 prefix value, so it doesn't match + content4 = '{"k1": "v1", "k2": "ZZ"}' + self.db.create_index('test-idx', 'k1', 'k2') + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "v*"))) + + def test_nested_index(self): + doc = self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.doc') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'underneath')) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'underneath'))) + + def test_nested_nonexistent(self): + self.db.create_doc_from_json(nested_doc) + # sub exists, but sub.foo does not: + self.db.create_index('test-idx', 'sub.foo') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_nonexistent2(self): + self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_traverses_lists(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": "bar"}, {"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_doc_from_json('{"foo": ["zap", "baz"]}') + self.db.create_index('test-idx', 'foo.zap') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'bar')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'baz')) + + def test_nested_list_traversal(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": [{"qux": "fnord"}, {"qux": "zombo"}]},' + '{"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_index('test-idx', 'foo.zap.qux') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'fnord')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'zombo')) + + def test_index_list1(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_list2(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_case_sensitive(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'V*')) + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'v*')) + + def test_get_from_index_illegal_glob_before_value(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v2') + + def test_get_from_index_illegal_glob_after_glob(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v*') + + def test_get_from_index_with_sql_wildcards(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "va%lue"}' + content2 = '{"key": "value"}' + content3 = '{"key": "va_lue"}' + doc1 = self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + # The '%' in the search should be treated literally, not as a sql + # globbing character. + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'va%*')) + # Same for '_' + self.assertEqual([doc3], self.db.get_from_index('test-idx', 'va_*')) + + def test_get_from_index_with_lower(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_with_lower_matches_same_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_lower_doesnt_match_different_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual([], rows) + + def test_index_lower_doesnt_match_other_index(self): + self.db.create_index("index", "lower(name)") + self.db.create_index("other_index", "name") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual(0, len(rows)) + + def test_index_split_words_match_first(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_match_second(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_match_both(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_double_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_leading_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": " foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_trailing_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar "}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 12}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number_bigger_than_padding(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 123456}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "123456") + self.assertEqual([doc], rows) + + def test_number_mapping_ignores_non_numbers(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 56}' + doc1 = self.db.create_doc_from_json(content) + content = '{"foo": "this is not a maigret painting"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([doc1], rows) + + def test_get_from_index_with_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": true}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "1") + self.assertEqual([doc], rows) + + def test_get_from_index_with_bool_false(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": false}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "0") + self.assertEqual([doc], rows) + + def test_get_from_index_with_non_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": 42}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([], rows) + + def test_get_from_index_with_combine(self): + self.db.create_index("index", "combine(foo, bar)") + content = '{"foo": "value1", "bar": "value2"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "value1") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "value2") + self.assertEqual([doc], rows) + + def test_get_complex_combine(self): + self.db.create_index( + "index", "combine(number(foo, 5), lower(bar), split_words(baz))") + content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}' + doc = self.db.create_doc_from_json(content) + content = '{"foo": "not a number", "bar": "something"}' + doc2 = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "allcaps") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "nox") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "something") + self.assertEqual([doc2], rows) + + def test_get_index_keys_from_index(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "value1"}' + content2 = '{"key": "value2"}' + content3 = '{"key": "value2"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.assertEqual( + [('value1',), ('value2',)], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_index_keys_from_multicolumn_index(self): + self.db.create_index('test-idx', 'key1', 'key2') + content1 = '{"key1": "value1", "key2": "val2-1"}' + content2 = '{"key1": "value2", "key2": "val2-2"}' + content3 = '{"key1": "value2", "key2": "val2-2"}' + content4 = '{"key1": "value2", "key2": "val3"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual([ + ('value1', 'val2-1'), + ('value2', 'val2-2'), + ('value2', 'val3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_empty_expr(self): + self.assertParseError('') + + def test_nested_unknown_operation(self): + self.assertParseError('unknown_operation(field1)') + + def test_parse_missing_close_paren(self): + self.assertParseError("lower(a") + + def test_parse_trailing_close_paren(self): + self.assertParseError("lower(ab))") + + def test_parse_trailing_chars(self): + self.assertParseError("lower(ab)adsf") + + def test_parse_empty_op(self): + self.assertParseError("(ab)") + + def test_parse_top_level_commas(self): + self.assertParseError("a, b") + + def test_invalid_field_name(self): + self.assertParseError("a.") + + def test_invalid_inner_field_name(self): + self.assertParseError("lower(a.)") + + def test_gobbledigook(self): + self.assertParseError("(@#@cc @#!*DFJSXV(()jccd") + + def test_leading_space(self): + self.assertIndexCreatable(" lower(a)") + + def test_trailing_space(self): + self.assertIndexCreatable("lower(a) ") + + def test_spaces_before_open_paren(self): + self.assertIndexCreatable("lower (a)") + + def test_spaces_after_open_paren(self): + self.assertIndexCreatable("lower( a)") + + def test_spaces_before_close_paren(self): + self.assertIndexCreatable("lower(a )") + + def test_spaces_before_comma(self): + self.assertIndexCreatable("combine(a , b , c)") + + def test_spaces_after_comma(self): + self.assertIndexCreatable("combine(a, b, c)") + + def test_all_together_now(self): + self.assertParseError(' (a) ') + + def test_all_together_now2(self): + self.assertParseError('combine(lower(x)x,foo)') + + +class PythonBackendTests(tests.DatabaseBaseTests): + + def setUp(self): + super(PythonBackendTests, self).setUp() + self.simple_doc = json.loads(simple_doc) + + def test_create_doc_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.assertTrue(isinstance(doc, TestAlternativeDocument)) + + def test_get_doc_after_put_with_factory(self): + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.db.set_document_factory(TestAlternativeDocument) + result = self.db.get_doc('my_doc_id') + self.assertTrue(isinstance(result, TestAlternativeDocument)) + self.assertEqual(doc.doc_id, result.doc_id) + self.assertEqual(doc.rev, result.rev) + self.assertEqual(doc.get_json(), result.get_json()) + self.assertEqual(False, result.has_conflicts) + + def test_get_doc_nonexisting_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_all_docs_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.assertTrue(isinstance( + list(self.db.get_all_docs()[1])[0], TestAlternativeDocument)) + + def test_get_docs_conflicted_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc1 = self.db.create_doc(self.simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue( + isinstance( + list(self.db.get_docs([doc1.doc_id]))[0], + TestAlternativeDocument)) + + def test_get_from_index_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + self.assertTrue( + isinstance( + self.db.get_from_index('test-idx', 'value')[0], + TestAlternativeDocument)) + + def test_sync_exchange_updates_indexes(self): + doc = self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + other_rev = 'test:1|z:2' + st = self.db.get_sync_target() + + def ignore(doc_id, doc_rev, doc): + pass + + doc_other = self.make_document(doc.doc_id, other_rev, new_content) + docs_by_gen = [(doc_other, 10, 'T-sid')] + st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=ignore) + self.assertGetDoc(self.db, doc.doc_id, other_rev, new_content, False) + self.assertEqual( + [doc_other], self.db.get_from_index('test-idx', 'altval')) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + + +# Use a custom loader to apply the scenarios at load time. +load_tests = tests.load_with_scenarios diff --git a/src/leap/soledad/tests/u1db_tests/test_document.py b/src/leap/soledad/tests/u1db_tests/test_document.py new file mode 100644 index 00000000..2a0c0294 --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_document.py @@ -0,0 +1,150 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + + +from u1db import errors + +from leap.soledad.tests import u1db_tests as tests + + +class TestDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) #+ + #tests.C_DATABASE_SCENARIOS) + + def test_create_doc(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual('doc-id', doc.doc_id) + self.assertEqual('uid:1', doc.rev) + self.assertEqual(tests.simple_doc, doc.get_json()) + self.assertFalse(doc.has_conflicts) + + def test__repr__(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual( + '%s(doc-id, uid:1, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__repr__conflicted(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc, + has_conflicts=True) + self.assertEqual( + '%s(doc-id, uid:1, conflicted, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__lt__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('b', 'b', '{}') + self.assertTrue(doc_a < doc_b) + self.assertTrue(doc_b > doc_a) + doc_aa = self.make_document('a', 'a', '{}') + self.assertTrue(doc_aa < doc_a) + + def test__eq__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('a', 'b', '{}') + self.assertTrue(doc_a == doc_b) + doc_b = self.make_document('a', 'b', '{}', has_conflicts=True) + self.assertFalse(doc_a == doc_b) + + def test_non_json_dict(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + '"not a json dictionary"') + + def test_non_json(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + 'not a json dictionary') + + def test_get_size(self): + doc_a = self.make_document('a', 'b', '{"some": "content"}') + self.assertEqual( + len('a' + 'b' + '{"some": "content"}'), doc_a.get_size()) + + def test_get_size_empty_document(self): + doc_a = self.make_document('a', 'b', None) + self.assertEqual(len('a' + 'b'), doc_a.get_size()) + + +class TestPyDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) + + def test_get_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertEqual({"content": ""}, doc.content) + doc.set_json('{"content": "new"}') + self.assertEqual({"content": "new"}, doc.content) + + def test_set_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.content = {"content": "new"} + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_bad_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises( + errors.InvalidContent, setattr, doc, 'content', + '{"content": "new"}') + + def test_is_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.set_json(None) + self.assertTrue(doc_a.is_tombstone()) + + def test_make_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.make_tombstone() + self.assertTrue(doc_a.is_tombstone()) + + def test_same_content_as(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('d', 'e', '{}') + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b = self.make_document('p', 'q', '{}', has_conflicts=True) + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b.content['key'] = 'value' + self.assertFalse(doc_a.same_content_as(doc_b)) + + def test_same_content_as_json_order(self): + doc_a = self.make_document( + 'a', 'b', '{"key1": "val1", "key2": "val2"}') + doc_b = self.make_document( + 'c', 'd', '{"key2": "val2", "key1": "val1"}') + self.assertTrue(doc_a.same_content_as(doc_b)) + + def test_set_json(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.set_json('{"content": "new"}') + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_json_non_dict(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, '"is not a dict"') + + def test_set_json_error(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, 'is not json') + + +load_tests = tests.load_with_scenarios diff --git a/src/leap/soledad/tests/u1db_tests/test_http_app.py b/src/leap/soledad/tests/u1db_tests/test_http_app.py new file mode 100644 index 00000000..73838613 --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_http_app.py @@ -0,0 +1,1134 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Test the WSGI app.""" + +import paste.fixture +import sys +try: + import simplejson as json +except ImportError: + import json # noqa +import StringIO + +from u1db import ( + __version__ as _u1db_version, + errors, + sync, + ) + +from leap.soledad.tests import u1db_tests as tests + +from u1db.remote import ( + http_app, + http_errors, + ) + + +class TestFencedReader(tests.TestCase): + + def test_init(self): + reader = http_app._FencedReader(StringIO.StringIO(""), 25, 100) + self.assertEqual(25, reader.remaining) + + def test_read_chunk(self): + inp = StringIO.StringIO("abcdef") + reader = http_app._FencedReader(inp, 5, 10) + data = reader.read_chunk(2) + self.assertEqual("ab", data) + self.assertEqual(2, inp.tell()) + self.assertEqual(3, reader.remaining) + + def test_read_chunk_remaining(self): + inp = StringIO.StringIO("abcdef") + reader = http_app._FencedReader(inp, 4, 10) + data = reader.read_chunk(9999) + self.assertEqual("abcd", data) + self.assertEqual(4, inp.tell()) + self.assertEqual(0, reader.remaining) + + def test_read_chunk_nothing_left(self): + inp = StringIO.StringIO("abc") + reader = http_app._FencedReader(inp, 2, 10) + reader.read_chunk(2) + self.assertEqual(2, inp.tell()) + self.assertEqual(0, reader.remaining) + data = reader.read_chunk(2) + self.assertEqual("", data) + self.assertEqual(2, inp.tell()) + self.assertEqual(0, reader.remaining) + + def test_read_chunk_kept(self): + inp = StringIO.StringIO("abcde") + reader = http_app._FencedReader(inp, 4, 10) + reader._kept = "xyz" + data = reader.read_chunk(2) # atmost ignored + self.assertEqual("xyz", data) + self.assertEqual(0, inp.tell()) + self.assertEqual(4, reader.remaining) + self.assertIsNone(reader._kept) + + def test_getline(self): + inp = StringIO.StringIO("abc\r\nde") + reader = http_app._FencedReader(inp, 6, 10) + reader.MAXCHUNK = 6 + line = reader.getline() + self.assertEqual("abc\r\n", line) + self.assertEqual("d", reader._kept) + + def test_getline_exact(self): + inp = StringIO.StringIO("abcd\r\nef") + reader = http_app._FencedReader(inp, 6, 10) + reader.MAXCHUNK = 6 + line = reader.getline() + self.assertEqual("abcd\r\n", line) + self.assertIs(None, reader._kept) + + def test_getline_no_newline(self): + inp = StringIO.StringIO("abcd") + reader = http_app._FencedReader(inp, 4, 10) + reader.MAXCHUNK = 6 + line = reader.getline() + self.assertEqual("abcd", line) + + def test_getline_many_chunks(self): + inp = StringIO.StringIO("abcde\r\nf") + reader = http_app._FencedReader(inp, 8, 10) + reader.MAXCHUNK = 4 + line = reader.getline() + self.assertEqual("abcde\r\n", line) + self.assertEqual("f", reader._kept) + line = reader.getline() + self.assertEqual("f", line) + + def test_getline_empty(self): + inp = StringIO.StringIO("") + reader = http_app._FencedReader(inp, 0, 10) + reader.MAXCHUNK = 4 + line = reader.getline() + self.assertEqual("", line) + line = reader.getline() + self.assertEqual("", line) + + def test_getline_just_newline(self): + inp = StringIO.StringIO("\r\n") + reader = http_app._FencedReader(inp, 2, 10) + reader.MAXCHUNK = 4 + line = reader.getline() + self.assertEqual("\r\n", line) + line = reader.getline() + self.assertEqual("", line) + + def test_getline_too_large(self): + inp = StringIO.StringIO("x" * 50) + reader = http_app._FencedReader(inp, 50, 25) + reader.MAXCHUNK = 4 + self.assertRaises(http_app.BadRequest, reader.getline) + + def test_getline_too_large_complete(self): + inp = StringIO.StringIO("x" * 25 + "\r\n") + reader = http_app._FencedReader(inp, 50, 25) + reader.MAXCHUNK = 4 + self.assertRaises(http_app.BadRequest, reader.getline) + + +class TestHTTPMethodDecorator(tests.TestCase): + + def test_args(self): + @http_app.http_method() + def f(self, a, b): + return self, a, b + res = f("self", {"a": "x", "b": "y"}, None) + self.assertEqual(("self", "x", "y"), res) + + def test_args_missing(self): + @http_app.http_method() + def f(self, a, b): + return a, b + self.assertRaises(http_app.BadRequest, f, "self", {"a": "x"}, None) + + def test_args_unexpected(self): + @http_app.http_method() + def f(self, a): + return a + self.assertRaises(http_app.BadRequest, f, "self", + {"a": "x", "c": "z"}, None) + + def test_args_default(self): + @http_app.http_method() + def f(self, a, b="z"): + return a, b + res = f("self", {"a": "x"}, None) + self.assertEqual(("x", "z"), res) + + def test_args_conversion(self): + @http_app.http_method(b=int) + def f(self, a, b): + return self, a, b + res = f("self", {"a": "x", "b": "2"}, None) + self.assertEqual(("self", "x", 2), res) + + self.assertRaises(http_app.BadRequest, f, "self", + {"a": "x", "b": "foo"}, None) + + def test_args_conversion_with_default(self): + @http_app.http_method(b=str) + def f(self, a, b=None): + return self, a, b + res = f("self", {"a": "x"}, None) + self.assertEqual(("self", "x", None), res) + + def test_args_content(self): + @http_app.http_method() + def f(self, a, content): + return a, content + res = f(self, {"a": "x"}, "CONTENT") + self.assertEqual(("x", "CONTENT"), res) + + def test_args_content_as_args(self): + @http_app.http_method(b=int, content_as_args=True) + def f(self, a, b): + return self, a, b + res = f("self", {"a": "x"}, '{"b": "2"}') + self.assertEqual(("self", "x", 2), res) + + self.assertRaises(http_app.BadRequest, f, "self", {}, 'not-json') + + def test_args_content_no_query(self): + @http_app.http_method(no_query=True, + content_as_args=True) + def f(self, a='a', b='b'): + return a, b + res = f("self", {}, '{"b": "y"}') + self.assertEqual(('a', 'y'), res) + + self.assertRaises(http_app.BadRequest, f, "self", {'a': 'x'}, + '{"b": "y"}') + + +class TestResource(object): + + @http_app.http_method() + def get(self, a, b): + self.args = dict(a=a, b=b) + return 'Get' + + @http_app.http_method() + def put(self, a, content): + self.args = dict(a=a) + self.content = content + return 'Put' + + @http_app.http_method(content_as_args=True) + def put_args(self, a, b): + self.args = dict(a=a, b=b) + self.order = ['a'] + self.entries = [] + + @http_app.http_method() + def put_stream_entry(self, content): + self.entries.append(content) + self.order.append('s') + + def put_end(self): + self.order.append('e') + return "Put/end" + + +class parameters: + max_request_size = 200000 + max_entry_size = 100000 + + +class TestHTTPInvocationByMethodWithBody(tests.TestCase): + + def test_get(self): + resource = TestResource() + environ = {'QUERY_STRING': 'a=1&b=2', 'REQUEST_METHOD': 'GET'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + res = invoke() + self.assertEqual('Get', res) + self.assertEqual({'a': '1', 'b': '2'}, resource.args) + + def test_put_json(self): + resource = TestResource() + body = '{"body": true}' + environ = {'QUERY_STRING': 'a=1', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO(body), + 'CONTENT_LENGTH': str(len(body)), + 'CONTENT_TYPE': 'application/json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + res = invoke() + self.assertEqual('Put', res) + self.assertEqual({'a': '1'}, resource.args) + self.assertEqual('{"body": true}', resource.content) + + def test_put_sync_stream(self): + resource = TestResource() + body = ( + '[\r\n' + '{"b": 2},\r\n' # args + '{"entry": "x"},\r\n' # stream entry + '{"entry": "y"}\r\n' # stream entry + ']' + ) + environ = {'QUERY_STRING': 'a=1', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO(body), + 'CONTENT_LENGTH': str(len(body)), + 'CONTENT_TYPE': 'application/x-u1db-sync-stream'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + res = invoke() + self.assertEqual('Put/end', res) + self.assertEqual({'a': '1', 'b': 2}, resource.args) + self.assertEqual( + ['{"entry": "x"}', '{"entry": "y"}'], resource.entries) + self.assertEqual(['a', 's', 's', 'e'], resource.order) + + def _put_sync_stream(self, body): + resource = TestResource() + environ = {'QUERY_STRING': 'a=1&b=2', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO(body), + 'CONTENT_LENGTH': str(len(body)), + 'CONTENT_TYPE': 'application/x-u1db-sync-stream'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + invoke() + + def test_put_sync_stream_wrong_start(self): + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "{}\r\n]") + + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "\r\n{}\r\n]") + + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "") + + def test_put_sync_stream_wrong_end(self): + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "[\r\n{}") + + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "[\r\n") + + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "[\r\n{}\r\n]\r\n...") + + def test_put_sync_stream_missing_comma(self): + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "[\r\n{}\r\n{}\r\n]") + + def test_put_sync_stream_extra_comma(self): + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "[\r\n{},\r\n]") + + self.assertRaises(http_app.BadRequest, + self._put_sync_stream, "[\r\n{},\r\n{},\r\n]") + + def test_bad_request_decode_failure(self): + resource = TestResource() + environ = {'QUERY_STRING': 'a=\xff', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO('{}'), + 'CONTENT_LENGTH': '2', + 'CONTENT_TYPE': 'application/json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_unsupported_content_type(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO('{}'), + 'CONTENT_LENGTH': '2', + 'CONTENT_TYPE': 'text/plain'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_content_length_too_large(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO('{}'), + 'CONTENT_LENGTH': '10000', + 'CONTENT_TYPE': 'text/plain'} + + resource.max_request_size = 5000 + resource.max_entry_size = sys.maxint # we don't get to use this + + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_no_content_length(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO('a'), + 'CONTENT_TYPE': 'application/json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_invalid_content_length(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO('abc'), + 'CONTENT_LENGTH': '1unk', + 'CONTENT_TYPE': 'application/json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_empty_body(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO(''), + 'CONTENT_LENGTH': '0', + 'CONTENT_TYPE': 'application/json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_unsupported_method_get_like(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'DELETE'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_unsupported_method_put_like(self): + resource = TestResource() + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO.StringIO('{}'), + 'CONTENT_LENGTH': '2', + 'CONTENT_TYPE': 'application/json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + def test_bad_request_unsupported_method_put_like_multi_json(self): + resource = TestResource() + body = '{}\r\n{}\r\n' + environ = {'QUERY_STRING': '', 'REQUEST_METHOD': 'POST', + 'wsgi.input': StringIO.StringIO(body), + 'CONTENT_LENGTH': str(len(body)), + 'CONTENT_TYPE': 'application/x-u1db-multi-json'} + invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ, + parameters) + self.assertRaises(http_app.BadRequest, invoke) + + +class TestHTTPResponder(tests.TestCase): + + def start_response(self, status, headers): + self.status = status + self.headers = dict(headers) + self.response_body = [] + + def write(data): + self.response_body.append(data) + + return write + + def test_send_response_content_w_headers(self): + responder = http_app.HTTPResponder(self.start_response) + responder.send_response_content('foo', headers={'x-a': '1'}) + self.assertEqual('200 OK', self.status) + self.assertEqual({'content-type': 'application/json', + 'cache-control': 'no-cache', + 'x-a': '1', 'content-length': '3'}, self.headers) + self.assertEqual([], self.response_body) + self.assertEqual(['foo'], responder.content) + + def test_send_response_json(self): + responder = http_app.HTTPResponder(self.start_response) + responder.send_response_json(value='success') + self.assertEqual('200 OK', self.status) + expected_body = '{"value": "success"}\r\n' + self.assertEqual({'content-type': 'application/json', + 'content-length': str(len(expected_body)), + 'cache-control': 'no-cache'}, self.headers) + self.assertEqual([], self.response_body) + self.assertEqual([expected_body], responder.content) + + def test_send_response_json_status_fail(self): + responder = http_app.HTTPResponder(self.start_response) + responder.send_response_json(400) + self.assertEqual('400 Bad Request', self.status) + expected_body = '{}\r\n' + self.assertEqual({'content-type': 'application/json', + 'content-length': str(len(expected_body)), + 'cache-control': 'no-cache'}, self.headers) + self.assertEqual([], self.response_body) + self.assertEqual([expected_body], responder.content) + + def test_start_finish_response_status_fail(self): + responder = http_app.HTTPResponder(self.start_response) + responder.start_response(404, {'error': 'not found'}) + responder.finish_response() + self.assertEqual('404 Not Found', self.status) + self.assertEqual({'content-type': 'application/json', + 'cache-control': 'no-cache'}, self.headers) + self.assertEqual(['{"error": "not found"}\r\n'], self.response_body) + self.assertEqual([], responder.content) + + def test_send_stream_entry(self): + responder = http_app.HTTPResponder(self.start_response) + responder.content_type = "application/x-u1db-multi-json" + responder.start_response(200) + responder.start_stream() + responder.stream_entry({'entry': 1}) + responder.stream_entry({'entry': 2}) + responder.end_stream() + responder.finish_response() + self.assertEqual('200 OK', self.status) + self.assertEqual({'content-type': 'application/x-u1db-multi-json', + 'cache-control': 'no-cache'}, self.headers) + self.assertEqual(['[', + '\r\n', '{"entry": 1}', + ',\r\n', '{"entry": 2}', + '\r\n]\r\n'], self.response_body) + self.assertEqual([], responder.content) + + def test_send_stream_w_error(self): + responder = http_app.HTTPResponder(self.start_response) + responder.content_type = "application/x-u1db-multi-json" + responder.start_response(200) + responder.start_stream() + responder.stream_entry({'entry': 1}) + responder.send_response_json(503, error="unavailable") + self.assertEqual('200 OK', self.status) + self.assertEqual({'content-type': 'application/x-u1db-multi-json', + 'cache-control': 'no-cache'}, self.headers) + self.assertEqual(['[', + '\r\n', '{"entry": 1}'], self.response_body) + self.assertEqual([',\r\n', '{"error": "unavailable"}\r\n'], + responder.content) + + +class TestHTTPApp(tests.TestCase): + + def setUp(self): + super(TestHTTPApp, self).setUp() + self.state = tests.ServerStateForTests() + self.http_app = http_app.HTTPApp(self.state) + self.app = paste.fixture.TestApp(self.http_app) + self.db0 = self.state._create_database('db0') + + def test_bad_request_broken(self): + resp = self.app.put('/db0/doc/doc1', params='{"x": 1}', + headers={'content-type': 'application/foo'}, + expect_errors=True) + self.assertEqual(400, resp.status) + + def test_bad_request_dispatch(self): + resp = self.app.put('/db0/foo/doc1', params='{"x": 1}', + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(400, resp.status) + + def test_version(self): + resp = self.app.get('/') + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({"version": _u1db_version}, json.loads(resp.body)) + + def test_create_database(self): + resp = self.app.put('/db1', params='{}', + headers={'content-type': 'application/json'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({'ok': True}, json.loads(resp.body)) + + resp = self.app.put('/db1', params='{}', + headers={'content-type': 'application/json'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({'ok': True}, json.loads(resp.body)) + + def test_delete_database(self): + resp = self.app.delete('/db0') + self.assertEqual(200, resp.status) + self.assertRaises(errors.DatabaseDoesNotExist, + self.state.check_database, 'db0') + + def test_get_database(self): + resp = self.app.get('/db0') + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({}, json.loads(resp.body)) + + def test_valid_database_names(self): + resp = self.app.get('/a-database', expect_errors=True) + self.assertEqual(404, resp.status) + + resp = self.app.get('/db1', expect_errors=True) + self.assertEqual(404, resp.status) + + resp = self.app.get('/0', expect_errors=True) + self.assertEqual(404, resp.status) + + resp = self.app.get('/0-0', expect_errors=True) + self.assertEqual(404, resp.status) + + resp = self.app.get('/org.future', expect_errors=True) + self.assertEqual(404, resp.status) + + def test_invalid_database_names(self): + resp = self.app.get('/.a', expect_errors=True) + self.assertEqual(400, resp.status) + + resp = self.app.get('/-a', expect_errors=True) + self.assertEqual(400, resp.status) + + resp = self.app.get('/_a', expect_errors=True) + self.assertEqual(400, resp.status) + + def test_put_doc_create(self): + resp = self.app.put('/db0/doc/doc1', params='{"x": 1}', + headers={'content-type': 'application/json'}) + doc = self.db0.get_doc('doc1') + self.assertEqual(201, resp.status) # created + self.assertEqual('{"x": 1}', doc.get_json()) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({'rev': doc.rev}, json.loads(resp.body)) + + def test_put_doc(self): + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + resp = self.app.put('/db0/doc/doc1?old_rev=%s' % doc.rev, + params='{"x": 2}', + headers={'content-type': 'application/json'}) + doc = self.db0.get_doc('doc1') + self.assertEqual(200, resp.status) + self.assertEqual('{"x": 2}', doc.get_json()) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({'rev': doc.rev}, json.loads(resp.body)) + + def test_put_doc_too_large(self): + self.http_app.max_request_size = 15000 + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + resp = self.app.put('/db0/doc/doc1?old_rev=%s' % doc.rev, + params='{"%s": 2}' % ('z' * 16000), + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(400, resp.status) + + def test_delete_doc(self): + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + resp = self.app.delete('/db0/doc/doc1?old_rev=%s' % doc.rev) + doc = self.db0.get_doc('doc1', include_deleted=True) + self.assertEqual(None, doc.content) + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({'rev': doc.rev}, json.loads(resp.body)) + + def test_get_doc(self): + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + resp = self.app.get('/db0/doc/%s' % doc.doc_id) + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual('{"x": 1}', resp.body) + self.assertEqual(doc.rev, resp.header('x-u1db-rev')) + self.assertEqual('false', resp.header('x-u1db-has-conflicts')) + + def test_get_doc_non_existing(self): + resp = self.app.get('/db0/doc/not-there', expect_errors=True) + self.assertEqual(404, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": "document does not exist"}, json.loads(resp.body)) + self.assertEqual('', resp.header('x-u1db-rev')) + self.assertEqual('false', resp.header('x-u1db-has-conflicts')) + + def test_get_doc_deleted(self): + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + self.db0.delete_doc(doc) + resp = self.app.get('/db0/doc/doc1', expect_errors=True) + self.assertEqual(404, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": errors.DocumentDoesNotExist.wire_description}, + json.loads(resp.body)) + + def test_get_doc_deleted_explicit_exclude(self): + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + self.db0.delete_doc(doc) + resp = self.app.get( + '/db0/doc/doc1?include_deleted=false', expect_errors=True) + self.assertEqual(404, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": errors.DocumentDoesNotExist.wire_description}, + json.loads(resp.body)) + + def test_get_deleted_doc(self): + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + self.db0.delete_doc(doc) + resp = self.app.get( + '/db0/doc/doc1?include_deleted=true', expect_errors=True) + self.assertEqual(404, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": errors.DOCUMENT_DELETED}, json.loads(resp.body)) + self.assertEqual(doc.rev, resp.header('x-u1db-rev')) + self.assertEqual('false', resp.header('x-u1db-has-conflicts')) + + def test_get_doc_non_existing_dabase(self): + resp = self.app.get('/not-there/doc/doc1', expect_errors=True) + self.assertEqual(404, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": "database does not exist"}, json.loads(resp.body)) + + def test_get_docs(self): + doc1 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + doc2 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc2') + ids = ','.join([doc1.doc_id, doc2.doc_id]) + resp = self.app.get('/db0/docs?doc_ids=%s' % ids) + self.assertEqual(200, resp.status) + self.assertEqual( + 'application/json', resp.header('content-type')) + expected = [ + {"content": '{"x": 1}', "doc_rev": "db0:1", "doc_id": "doc1", + "has_conflicts": False}, + {"content": '{"x": 1}', "doc_rev": "db0:1", "doc_id": "doc2", + "has_conflicts": False}] + self.assertEqual(expected, json.loads(resp.body)) + + def test_get_docs_missing_doc_ids(self): + resp = self.app.get('/db0/docs', expect_errors=True) + self.assertEqual(400, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": "missing document ids"}, json.loads(resp.body)) + + def test_get_docs_empty_doc_ids(self): + resp = self.app.get('/db0/docs?doc_ids=', expect_errors=True) + self.assertEqual(400, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual( + {"error": "missing document ids"}, json.loads(resp.body)) + + def test_get_docs_percent(self): + doc1 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc%1') + doc2 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc2') + ids = ','.join([doc1.doc_id, doc2.doc_id]) + resp = self.app.get('/db0/docs?doc_ids=%s' % ids) + self.assertEqual(200, resp.status) + self.assertEqual( + 'application/json', resp.header('content-type')) + expected = [ + {"content": '{"x": 1}', "doc_rev": "db0:1", "doc_id": "doc%1", + "has_conflicts": False}, + {"content": '{"x": 1}', "doc_rev": "db0:1", "doc_id": "doc2", + "has_conflicts": False}] + self.assertEqual(expected, json.loads(resp.body)) + + def test_get_docs_deleted(self): + doc1 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + doc2 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc2') + self.db0.delete_doc(doc2) + ids = ','.join([doc1.doc_id, doc2.doc_id]) + resp = self.app.get('/db0/docs?doc_ids=%s' % ids) + self.assertEqual(200, resp.status) + self.assertEqual( + 'application/json', resp.header('content-type')) + expected = [ + {"content": '{"x": 1}', "doc_rev": "db0:1", "doc_id": "doc1", + "has_conflicts": False}] + self.assertEqual(expected, json.loads(resp.body)) + + def test_get_docs_include_deleted(self): + doc1 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + doc2 = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc2') + self.db0.delete_doc(doc2) + ids = ','.join([doc1.doc_id, doc2.doc_id]) + resp = self.app.get('/db0/docs?doc_ids=%s&include_deleted=true' % ids) + self.assertEqual(200, resp.status) + self.assertEqual( + 'application/json', resp.header('content-type')) + expected = [ + {"content": '{"x": 1}', "doc_rev": "db0:1", "doc_id": "doc1", + "has_conflicts": False}, + {"content": None, "doc_rev": "db0:2", "doc_id": "doc2", + "has_conflicts": False}] + self.assertEqual(expected, json.loads(resp.body)) + + def test_get_sync_info(self): + self.db0._set_replica_gen_and_trans_id('other-id', 1, 'T-transid') + resp = self.app.get('/db0/sync-from/other-id') + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual(dict(target_replica_uid='db0', + target_replica_generation=0, + target_replica_transaction_id='', + source_replica_uid='other-id', + source_replica_generation=1, + source_transaction_id='T-transid'), + json.loads(resp.body)) + + def test_record_sync_info(self): + resp = self.app.put('/db0/sync-from/other-id', + params='{"generation": 2, "transaction_id": "T-transid"}', + headers={'content-type': 'application/json'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({'ok': True}, json.loads(resp.body)) + self.assertEqual( + (2, 'T-transid'), + self.db0._get_replica_gen_and_trans_id('other-id')) + + def test_sync_exchange_send(self): + entries = { + 10: {'id': 'doc-here', 'rev': 'replica:1', 'content': + '{"value": "here"}', 'gen': 10, 'trans_id': 'T-sid'}, + 11: {'id': 'doc-here2', 'rev': 'replica:1', 'content': + '{"value": "here2"}', 'gen': 11, 'trans_id': 'T-sed'} + } + + gens = [] + _do_set_replica_gen_and_trans_id = \ + self.db0._do_set_replica_gen_and_trans_id + + def set_sync_generation_witness(other_uid, other_gen, other_trans_id): + gens.append((other_uid, other_gen)) + _do_set_replica_gen_and_trans_id( + other_uid, other_gen, other_trans_id) + self.assertGetDoc(self.db0, entries[other_gen]['id'], + entries[other_gen]['rev'], + entries[other_gen]['content'], False) + + self.patch( + self.db0, '_do_set_replica_gen_and_trans_id', + set_sync_generation_witness) + + args = dict(last_known_generation=0) + body = ("[\r\n" + + "%s,\r\n" % json.dumps(args) + + "%s,\r\n" % json.dumps(entries[10]) + + "%s\r\n" % json.dumps(entries[11]) + + "]\r\n") + resp = self.app.post('/db0/sync-from/replica', + params=body, + headers={'content-type': + 'application/x-u1db-sync-stream'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/x-u1db-sync-stream', + resp.header('content-type')) + bits = resp.body.split('\r\n') + self.assertEqual('[', bits[0]) + last_trans_id = self.db0._get_transaction_log()[-1][1] + self.assertEqual({'new_generation': 2, + 'new_transaction_id': last_trans_id}, + json.loads(bits[1])) + self.assertEqual(']', bits[2]) + self.assertEqual('', bits[3]) + self.assertEqual([('replica', 10), ('replica', 11)], gens) + + def test_sync_exchange_send_ensure(self): + entries = { + 10: {'id': 'doc-here', 'rev': 'replica:1', 'content': + '{"value": "here"}', 'gen': 10, 'trans_id': 'T-sid'}, + 11: {'id': 'doc-here2', 'rev': 'replica:1', 'content': + '{"value": "here2"}', 'gen': 11, 'trans_id': 'T-sed'} + } + + args = dict(last_known_generation=0, ensure=True) + body = ("[\r\n" + + "%s,\r\n" % json.dumps(args) + + "%s,\r\n" % json.dumps(entries[10]) + + "%s\r\n" % json.dumps(entries[11]) + + "]\r\n") + resp = self.app.post('/dbnew/sync-from/replica', + params=body, + headers={'content-type': + 'application/x-u1db-sync-stream'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/x-u1db-sync-stream', + resp.header('content-type')) + bits = resp.body.split('\r\n') + self.assertEqual('[', bits[0]) + dbnew = self.state.open_database("dbnew") + last_trans_id = dbnew._get_transaction_log()[-1][1] + self.assertEqual({'new_generation': 2, + 'new_transaction_id': last_trans_id, + 'replica_uid': dbnew._replica_uid}, + json.loads(bits[1])) + self.assertEqual(']', bits[2]) + self.assertEqual('', bits[3]) + + def test_sync_exchange_send_entry_too_large(self): + self.patch(http_app.SyncResource, 'max_request_size', 20000) + self.patch(http_app.SyncResource, 'max_entry_size', 10000) + entries = { + 10: {'id': 'doc-here', 'rev': 'replica:1', 'content': + '{"value": "%s"}' % ('H' * 11000), 'gen': 10}, + } + args = dict(last_known_generation=0) + body = ("[\r\n" + + "%s,\r\n" % json.dumps(args) + + "%s\r\n" % json.dumps(entries[10]) + + "]\r\n") + resp = self.app.post('/db0/sync-from/replica', + params=body, + headers={'content-type': + 'application/x-u1db-sync-stream'}, + expect_errors=True) + self.assertEqual(400, resp.status) + + def test_sync_exchange_receive(self): + doc = self.db0.create_doc_from_json('{"value": "there"}') + doc2 = self.db0.create_doc_from_json('{"value": "there2"}') + args = dict(last_known_generation=0) + body = "[\r\n%s\r\n]" % json.dumps(args) + resp = self.app.post('/db0/sync-from/replica', + params=body, + headers={'content-type': + 'application/x-u1db-sync-stream'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/x-u1db-sync-stream', + resp.header('content-type')) + parts = resp.body.splitlines() + self.assertEqual(5, len(parts)) + self.assertEqual('[', parts[0]) + last_trans_id = self.db0._get_transaction_log()[-1][1] + self.assertEqual({'new_generation': 2, + 'new_transaction_id': last_trans_id}, + json.loads(parts[1].rstrip(","))) + part2 = json.loads(parts[2].rstrip(",")) + self.assertTrue(part2['trans_id'].startswith('T-')) + self.assertEqual('{"value": "there"}', part2['content']) + self.assertEqual(doc.rev, part2['rev']) + self.assertEqual(doc.doc_id, part2['id']) + self.assertEqual(1, part2['gen']) + part3 = json.loads(parts[3].rstrip(",")) + self.assertTrue(part3['trans_id'].startswith('T-')) + self.assertEqual('{"value": "there2"}', part3['content']) + self.assertEqual(doc2.rev, part3['rev']) + self.assertEqual(doc2.doc_id, part3['id']) + self.assertEqual(2, part3['gen']) + self.assertEqual(']', parts[4]) + + def test_sync_exchange_error_in_stream(self): + args = dict(last_known_generation=0) + body = "[\r\n%s\r\n]" % json.dumps(args) + + def boom(self, return_doc_cb): + raise errors.Unavailable + + self.patch(sync.SyncExchange, 'return_docs', + boom) + resp = self.app.post('/db0/sync-from/replica', + params=body, + headers={'content-type': + 'application/x-u1db-sync-stream'}) + self.assertEqual(200, resp.status) + self.assertEqual('application/x-u1db-sync-stream', + resp.header('content-type')) + parts = resp.body.splitlines() + self.assertEqual(3, len(parts)) + self.assertEqual('[', parts[0]) + self.assertEqual({'new_generation': 0, 'new_transaction_id': ''}, + json.loads(parts[1].rstrip(","))) + self.assertEqual({'error': 'unavailable'}, json.loads(parts[2])) + + +class TestRequestHooks(tests.TestCase): + + def setUp(self): + super(TestRequestHooks, self).setUp() + self.state = tests.ServerStateForTests() + self.http_app = http_app.HTTPApp(self.state) + self.app = paste.fixture.TestApp(self.http_app) + self.db0 = self.state._create_database('db0') + + def test_begin_and_done(self): + calls = [] + + def begin(environ): + self.assertTrue('PATH_INFO' in environ) + calls.append('begin') + + def done(environ): + self.assertTrue('PATH_INFO' in environ) + calls.append('done') + + self.http_app.request_begin = begin + self.http_app.request_done = done + + doc = self.db0.create_doc_from_json('{"x": 1}', doc_id='doc1') + self.app.get('/db0/doc/%s' % doc.doc_id) + + self.assertEqual(['begin', 'done'], calls) + + def test_bad_request(self): + calls = [] + + def begin(environ): + self.assertTrue('PATH_INFO' in environ) + calls.append('begin') + + def bad_request(environ): + self.assertTrue('PATH_INFO' in environ) + calls.append('bad-request') + + self.http_app.request_begin = begin + self.http_app.request_bad_request = bad_request + # shouldn't be called + self.http_app.request_done = lambda env: 1 / 0 + + resp = self.app.put('/db0/foo/doc1', params='{"x": 1}', + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(400, resp.status) + self.assertEqual(['begin', 'bad-request'], calls) + + +class TestHTTPErrors(tests.TestCase): + + def test_wire_description_to_status(self): + self.assertNotIn("error", http_errors.wire_description_to_status) + + +class TestHTTPAppErrorHandling(tests.TestCase): + + def setUp(self): + super(TestHTTPAppErrorHandling, self).setUp() + self.exc = None + self.state = tests.ServerStateForTests() + + class ErroringResource(object): + + def post(_, args, content): + raise self.exc + + def lookup_resource(environ, responder): + return ErroringResource() + + self.http_app = http_app.HTTPApp(self.state) + self.http_app._lookup_resource = lookup_resource + self.app = paste.fixture.TestApp(self.http_app) + + def test_RevisionConflict_etc(self): + self.exc = errors.RevisionConflict() + resp = self.app.post('/req', params='{}', + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(409, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({"error": "revision conflict"}, + json.loads(resp.body)) + + def test_Unavailable(self): + self.exc = errors.Unavailable + resp = self.app.post('/req', params='{}', + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(503, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({"error": "unavailable"}, + json.loads(resp.body)) + + def test_generic_u1db_errors(self): + self.exc = errors.U1DBError() + resp = self.app.post('/req', params='{}', + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(500, resp.status) + self.assertEqual('application/json', resp.header('content-type')) + self.assertEqual({"error": "error"}, + json.loads(resp.body)) + + def test_generic_u1db_errors_hooks(self): + calls = [] + + def begin(environ): + self.assertTrue('PATH_INFO' in environ) + calls.append('begin') + + def u1db_error(environ, exc): + self.assertTrue('PATH_INFO' in environ) + calls.append(('error', exc)) + + self.http_app.request_begin = begin + self.http_app.request_u1db_error = u1db_error + # shouldn't be called + self.http_app.request_done = lambda env: 1 / 0 + + self.exc = errors.U1DBError() + resp = self.app.post('/req', params='{}', + headers={'content-type': 'application/json'}, + expect_errors=True) + self.assertEqual(500, resp.status) + self.assertEqual(['begin', ('error', self.exc)], calls) + + def test_failure(self): + class Failure(Exception): + pass + self.exc = Failure() + self.assertRaises(Failure, self.app.post, '/req', params='{}', + headers={'content-type': 'application/json'}) + + def test_failure_hooks(self): + class Failure(Exception): + pass + calls = [] + + def begin(environ): + calls.append('begin') + + def failed(environ): + self.assertTrue('PATH_INFO' in environ) + calls.append(('failed', sys.exc_info())) + + self.http_app.request_begin = begin + self.http_app.request_failed = failed + # shouldn't be called + self.http_app.request_done = lambda env: 1 / 0 + + self.exc = Failure() + self.assertRaises(Failure, self.app.post, '/req', params='{}', + headers={'content-type': 'application/json'}) + + self.assertEqual(2, len(calls)) + self.assertEqual('begin', calls[0]) + marker, (exc_type, exc, tb) = calls[1] + self.assertEqual('failed', marker) + self.assertEqual(self.exc, exc) + + +class TestPluggableSyncExchange(tests.TestCase): + + def setUp(self): + super(TestPluggableSyncExchange, self).setUp() + self.state = tests.ServerStateForTests() + self.state.ensure_database('foo') + + def test_plugging(self): + + class MySyncExchange(object): + def __init__(self, db, source_replica_uid, last_known_generation): + pass + + class MySyncResource(http_app.SyncResource): + sync_exchange_class = MySyncExchange + + sync_res = MySyncResource('foo', 'src', self.state, None) + sync_res.post_args( + {'last_known_generation': 0, 'last_known_trans_id': None}, '{}') + self.assertIsInstance(sync_res.sync_exch, MySyncExchange) diff --git a/src/leap/soledad/tests/u1db_tests/test_http_client.py b/src/leap/soledad/tests/u1db_tests/test_http_client.py new file mode 100644 index 00000000..b1bb106c --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_http_client.py @@ -0,0 +1,363 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Tests for HTTPDatabase""" + +from oauth import oauth +try: + import simplejson as json +except ImportError: + import json # noqa + +from u1db import ( + errors, + ) + +from leap.soledad.tests import u1db_tests as tests + +from u1db.remote import ( + http_client, + ) + + +class TestEncoder(tests.TestCase): + + def test_encode_string(self): + self.assertEqual("foo", http_client._encode_query_parameter("foo")) + + def test_encode_true(self): + self.assertEqual("true", http_client._encode_query_parameter(True)) + + def test_encode_false(self): + self.assertEqual("false", http_client._encode_query_parameter(False)) + + +class TestHTTPClientBase(tests.TestCaseWithServer): + + def setUp(self): + super(TestHTTPClientBase, self).setUp() + self.errors = 0 + + def app(self, environ, start_response): + if environ['PATH_INFO'].endswith('echo'): + start_response("200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = environ['wsgi.input'].read(content_length) + return [json.dumps(ret)] + elif environ['PATH_INFO'].endswith('error_then_accept'): + if self.errors >= 3: + start_response( + "200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = '{"oki": "doki"}' + return [json.dumps(ret)] + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + elif environ['PATH_INFO'].endswith('error'): + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + elif '/oauth' in environ['PATH_INFO']: + base_url = self.getURL('').rstrip('/') + oauth_req = oauth.OAuthRequest.from_request( + http_method=environ['REQUEST_METHOD'], + http_url=base_url + environ['PATH_INFO'], + headers={'Authorization': environ['HTTP_AUTHORIZATION']}, + query_string=environ['QUERY_STRING'] + ) + oauth_server = oauth.OAuthServer(tests.testingOAuthStore) + oauth_server.add_signature_method(tests.sign_meth_HMAC_SHA1) + try: + consumer, token, params = oauth_server.verify_request( + oauth_req) + except oauth.OAuthError, e: + start_response("401 Unauthorized", + [('Content-Type', 'application/json')]) + return [json.dumps({"error": "unauthorized", + "message": e.message})] + start_response("200 OK", [('Content-Type', 'application/json')]) + return [json.dumps([environ['PATH_INFO'], token.key, params])] + + def make_app(self): + return self.app + + def getClient(self, **kwds): + self.startServer() + return http_client.HTTPClientBase(self.getURL('dbase'), **kwds) + + def test_construct(self): + self.startServer() + url = self.getURL() + cli = http_client.HTTPClientBase(url) + self.assertEqual(url, cli._url.geturl()) + self.assertIs(None, cli._conn) + + def test_parse_url(self): + cli = http_client.HTTPClientBase( + '%s://127.0.0.1:12345/' % self.url_scheme) + self.assertEqual(self.url_scheme, cli._url.scheme) + self.assertEqual('127.0.0.1', cli._url.hostname) + self.assertEqual(12345, cli._url.port) + self.assertEqual('/', cli._url.path) + + def test__ensure_connection(self): + cli = self.getClient() + self.assertIs(None, cli._conn) + cli._ensure_connection() + self.assertIsNot(None, cli._conn) + conn = cli._conn + cli._ensure_connection() + self.assertIs(conn, cli._conn) + + def test_close(self): + cli = self.getClient() + cli._ensure_connection() + cli.close() + self.assertIs(None, cli._conn) + + def test__request(self): + cli = self.getClient() + res, headers = cli._request('PUT', ['echo'], {}, {}) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': '', + 'body': '{}', + 'REQUEST_METHOD': 'PUT'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', '%FFFF', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/%FFFF/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('POST', ['echo'], {'b': 2}, 'Body', + 'application/x-test') + self.assertEqual({'CONTENT_TYPE': 'application/x-test', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': 'Body', + 'REQUEST_METHOD': 'POST'}, json.loads(res)) + + def test__request_json(self): + cli = self.getClient() + res, headers = cli._request_json( + 'POST', ['echo'], {'b': 2}, {'a': 'x'}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': '{"a": "x"}', + 'REQUEST_METHOD': 'POST'}, res) + + def test_unspecified_http_error(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Crash."}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Fail."}) + except errors.HTTPError, e: + pass + + self.assertEqual(500, e.status) + self.assertEqual("Fail.", e.message) + self.assertTrue("content-type" in e.headers) + + def test_revision_conflict(self): + cli = self.getClient() + self.assertRaises(errors.RevisionConflict, + cli._request_json, 'POST', ['error'], {}, + {'status': "409 Conflict", + 'response': {"error": "revision conflict"}}) + + def test_unavailable_proper(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + self.assertRaises(errors.Unavailable, + cli._request_json, 'POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual(5, self.errors) + + def test_unavailable_then_available(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + res, headers = cli._request_json( + 'POST', ['error_then_accept'], {'b': 2}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/error_then_accept', + 'QUERY_STRING': 'b=2', + 'body': '{"oki": "doki"}', + 'REQUEST_METHOD': 'POST'}, res) + self.assertEqual(3, self.errors) + + def test_unavailable_random_source(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + try: + cli._request_json('POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': "random unavailable."}) + except errors.Unavailable, e: + pass + + self.assertEqual(503, e.status) + self.assertEqual("random unavailable.", e.message) + self.assertTrue("content-type" in e.headers) + self.assertEqual(5, self.errors) + + def test_document_too_big(self): + cli = self.getClient() + self.assertRaises(errors.DocumentTooBig, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "document too big"}}) + + def test_user_quota_exceeded(self): + cli = self.getClient() + self.assertRaises(errors.UserQuotaExceeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user quota exceeded"}}) + + def test_user_needs_subscription(self): + cli = self.getClient() + self.assertRaises(errors.SubscriptionNeeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user needs subscription"}}) + + def test_generic_u1db_error(self): + cli = self.getClient() + self.assertRaises(errors.U1DBError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + except errors.U1DBError, e: + pass + self.assertIs(e.__class__, errors.U1DBError) + + def test_unspecified_bad_request(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': "<Bad Request>"}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': "<Bad Request>"}) + except errors.HTTPError, e: + pass + + self.assertEqual(400, e.status) + self.assertEqual("<Bad Request>", e.message) + self.assertTrue("content-type" in e.headers) + + def test_oauth(self): + cli = self.getClient() + cli.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + params = {'x': u'\xf0', 'y': "foo"} + res, headers = cli._request('GET', ['doc', 'oauth'], params) + self.assertEqual( + ['/dbase/doc/oauth', tests.token1.key, params], json.loads(res)) + + # oauth does its own internal quoting + params = {'x': u'\xf0', 'y': "foo"} + res, headers = cli._request('GET', ['doc', 'oauth', 'foo bar'], params) + self.assertEqual( + ['/dbase/doc/oauth/foo bar', tests.token1.key, params], + json.loads(res)) + + def test_oauth_ctr_creds(self): + cli = self.getClient(creds={'oauth': { + 'consumer_key': tests.consumer1.key, + 'consumer_secret': tests.consumer1.secret, + 'token_key': tests.token1.key, + 'token_secret': tests.token1.secret, + }}) + params = {'x': u'\xf0', 'y': "foo"} + res, headers = cli._request('GET', ['doc', 'oauth'], params) + self.assertEqual( + ['/dbase/doc/oauth', tests.token1.key, params], json.loads(res)) + + def test_unknown_creds(self): + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={'foo': {}}) + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={}) + + def test_oauth_Unauthorized(self): + cli = self.getClient() + cli.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, "WRONG") + params = {'y': 'foo'} + self.assertRaises(errors.Unauthorized, cli._request, 'GET', + ['doc', 'oauth'], params) diff --git a/src/leap/soledad/tests/u1db_tests/test_http_database.py b/src/leap/soledad/tests/u1db_tests/test_http_database.py new file mode 100644 index 00000000..dc20b6ec --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_http_database.py @@ -0,0 +1,258 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Tests for HTTPDatabase""" + +import inspect +try: + import simplejson as json +except ImportError: + import json # noqa + +from u1db import ( + errors, + Document, + ) + +from leap.soledad.tests import u1db_tests as tests + +from u1db.remote import ( + http_database, + http_target, + ) +from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( + make_http_app, +) + + +class TestHTTPDatabaseSimpleOperations(tests.TestCase): + + def setUp(self): + super(TestHTTPDatabaseSimpleOperations, self).setUp() + self.db = http_database.HTTPDatabase('dbase') + self.db._conn = object() # crash if used + self.got = None + self.response_val = None + + def _request(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + def _request_json(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + self.db._request = _request + self.db._request_json = _request_json + + def test__sanity_same_signature(self): + my_request_sig = inspect.getargspec(self.db._request) + my_request_sig = (['self'] + my_request_sig[0],) + my_request_sig[1:] + self.assertEqual(my_request_sig, + inspect.getargspec(http_database.HTTPDatabase._request)) + my_request_json_sig = inspect.getargspec(self.db._request_json) + my_request_json_sig = ((['self'] + my_request_json_sig[0],) + + my_request_json_sig[1:]) + self.assertEqual(my_request_json_sig, + inspect.getargspec(http_database.HTTPDatabase._request_json)) + + def test__ensure(self): + self.response_val = {'ok': True}, {} + self.db._ensure() + self.assertEqual(('PUT', [], {}, {}, None), self.got) + + def test__delete(self): + self.response_val = {'ok': True}, {} + self.db._delete() + self.assertEqual(('DELETE', [], {}, {}, None), self.got) + + def test__check(self): + self.response_val = {}, {} + res = self.db._check() + self.assertEqual({}, res) + self.assertEqual(('GET', [], None, None, None), self.got) + + def test_put_doc(self): + self.response_val = {'rev': 'doc-rev'}, {} + doc = Document('doc-id', None, '{"v": 1}') + res = self.db.put_doc(doc) + self.assertEqual('doc-rev', res) + self.assertEqual('doc-rev', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + self.response_val = {'rev': 'doc-rev-2'}, {} + doc.content = {"v": 2} + res = self.db.put_doc(doc) + self.assertEqual('doc-rev-2', res) + self.assertEqual('doc-rev-2', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + '{"v": 2}', 'application/json'), self.got) + + def test_get_doc(self): + self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev', + 'x-u1db-has-conflicts': 'false'} + self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False) + self.assertEqual( + ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None), + self.got) + + def test_get_doc_non_existing(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('not-there')) + self.assertEqual( + ('GET', ['doc', 'not-there'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('deleted')) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted_include_deleted(self): + self.response_val = errors.HTTPError(404, + json.dumps( + {"error": errors.DOCUMENT_DELETED} + ), + {'x-u1db-rev': 'doc-rev-gone', + 'x-u1db-has-conflicts': 'false'}) + doc = self.db.get_doc('deleted', include_deleted=True) + self.assertEqual('deleted', doc.doc_id) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertIs(None, doc.content) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None), + self.got) + + def test_get_doc_pass_through_errors(self): + self.response_val = errors.HTTPError(500, 'Crash.') + self.assertRaises(errors.HTTPError, + self.db.get_doc, 'something-something') + + def test_create_doc_with_id(self): + self.response_val = {'rev': 'doc-rev'}, {} + new_doc = self.db.create_doc_from_json('{"v": 1}', doc_id='doc-id') + self.assertEqual('doc-rev', new_doc.rev) + self.assertEqual('doc-id', new_doc.doc_id) + self.assertEqual('{"v": 1}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + def test_create_doc_without_id(self): + self.response_val = {'rev': 'doc-rev-2'}, {} + new_doc = self.db.create_doc_from_json('{"v": 3}') + self.assertEqual('D-', new_doc.doc_id[:2]) + self.assertEqual('doc-rev-2', new_doc.rev) + self.assertEqual('{"v": 3}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', new_doc.doc_id], {}, + '{"v": 3}', 'application/json'), self.got) + + def test_delete_doc(self): + self.response_val = {'rev': 'doc-rev-gone'}, {} + doc = Document('doc-id', 'doc-rev', None) + self.db.delete_doc(doc) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertEqual(('DELETE', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + None, None), self.got) + + def test_get_sync_target(self): + st = self.db.get_sync_target() + self.assertIsInstance(st, http_target.HTTPSyncTarget) + self.assertEqual(st._url, self.db._url) + + def test_get_sync_target_inherits_oauth_credentials(self): + self.db.set_oauth_credentials(tests.consumer1.key, + tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + st = self.db.get_sync_target() + self.assertEqual(self.db._creds, st._creds) + + +class TestHTTPDatabaseCtrWithCreds(tests.TestCase): + + def test_ctr_with_creds(self): + db1 = http_database.HTTPDatabase('http://dbs/db', creds={'oauth': { + 'consumer_key': tests.consumer1.key, + 'consumer_secret': tests.consumer1.secret, + 'token_key': tests.token1.key, + 'token_secret': tests.token1.secret + }}) + self.assertIn('oauth', db1._creds) + + +class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): + + make_app_with_state = staticmethod(make_http_app) + + def setUp(self): + super(TestHTTPDatabaseIntegration, self).setUp() + self.startServer() + + def test_non_existing_db(self): + db = http_database.HTTPDatabase(self.getURL('not-there')) + self.assertRaises(errors.DatabaseDoesNotExist, db.get_doc, 'doc1') + + def test__ensure(self): + db = http_database.HTTPDatabase(self.getURL('new')) + db._ensure() + self.assertIs(None, db.get_doc('doc1')) + + def test__delete(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase(self.getURL('db0')) + db._delete() + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_open_database_existing(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + self.assertIs(None, db.get_doc('doc1')) + + def test_open_database_non_existing(self): + self.assertRaises(errors.DatabaseDoesNotExist, + http_database.HTTPDatabase.open_database, + self.getURL('not-there'), + create=False) + + def test_open_database_create(self): + db = http_database.HTTPDatabase.open_database(self.getURL('new'), + create=True) + self.assertIs(None, db.get_doc('doc1')) + + def test_delete_database_existing(self): + self.request_state._create_database('db0') + http_database.HTTPDatabase.delete_database(self.getURL('db0')) + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_doc_ids_needing_quoting(self): + db0 = self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + doc = Document('%fff', None, '{}') + db.put_doc(doc) + self.assertGetDoc(db0, '%fff', doc.rev, '{}', False) + self.assertGetDoc(db, '%fff', doc.rev, '{}', False) diff --git a/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py b/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py new file mode 100644 index 00000000..6f69073d --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py @@ -0,0 +1,316 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Tests for the remote sync targets""" + +import cStringIO + +from u1db import ( + errors, + ) + +from leap.soledad.tests import u1db_tests as tests + +from u1db.remote import ( + http_app, + http_target, + oauth_middleware, + ) + + +class TestHTTPSyncTargetBasics(tests.TestCase): + + def test_parse_url(self): + remote_target = http_target.HTTPSyncTarget('http://127.0.0.1:12345/') + self.assertEqual('http', remote_target._url.scheme) + self.assertEqual('127.0.0.1', remote_target._url.hostname) + self.assertEqual(12345, remote_target._url.port) + self.assertEqual('/', remote_target._url.path) + + +class TestParsingSyncStream(tests.TestCase): + + def test_wrong_start(self): + tgt = http_target.HTTPSyncTarget("http://foo/foo") + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "{}\r\n]", None) + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "\r\n{}\r\n]", None) + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "", None) + + def test_wrong_end(self): + tgt = http_target.HTTPSyncTarget("http://foo/foo") + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "[\r\n{}", None) + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "[\r\n", None) + + def test_missing_comma(self): + tgt = http_target.HTTPSyncTarget("http://foo/foo") + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, + '[\r\n{}\r\n{"id": "i", "rev": "r", ' + '"content": "c", "gen": 3}\r\n]', None) + + def test_no_entries(self): + tgt = http_target.HTTPSyncTarget("http://foo/foo") + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "[\r\n]", None) + + def test_extra_comma(self): + tgt = http_target.HTTPSyncTarget("http://foo/foo") + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, "[\r\n{},\r\n]", None) + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, + '[\r\n{},\r\n{"id": "i", "rev": "r", ' + '"content": "{}", "gen": 3, "trans_id": "T-sid"}' + ',\r\n]', + lambda doc, gen, trans_id: None) + + def test_error_in_stream(self): + tgt = http_target.HTTPSyncTarget("http://foo/foo") + + self.assertRaises(errors.Unavailable, + tgt._parse_sync_stream, + '[\r\n{"new_generation": 0},' + '\r\n{"error": "unavailable"}\r\n', None) + + self.assertRaises(errors.Unavailable, + tgt._parse_sync_stream, + '[\r\n{"error": "unavailable"}\r\n', None) + + self.assertRaises(errors.BrokenSyncStream, + tgt._parse_sync_stream, + '[\r\n{"error": "?"}\r\n', None) + + +def make_http_app(state): + return http_app.HTTPApp(state) + + +def http_sync_target(test, path): + return http_target.HTTPSyncTarget(test.getURL(path)) + + +def make_oauth_http_app(state): + app = http_app.HTTPApp(state) + application = oauth_middleware.OAuthMiddleware(app, None, prefix='/~/') + application.get_oauth_data_store = lambda: tests.testingOAuthStore + return application + + +def oauth_http_sync_target(test, path): + st = http_sync_target(test, '~/' + path) + st.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + return st + + +class TestRemoteSyncTargets(tests.TestCaseWithServer): + + scenarios = [ + ('http', {'make_app_with_state': make_http_app, + 'make_document_for_test': tests.make_document_for_test, + 'sync_target': http_sync_target}), + ('oauth_http', {'make_app_with_state': make_oauth_http_app, + 'make_document_for_test': tests.make_document_for_test, + 'sync_target': oauth_http_sync_target}), + ] + + def getSyncTarget(self, path=None): + if self.server is None: + self.startServer() + return self.sync_target(self, path) + + def test_get_sync_info(self): + self.startServer() + db = self.request_state._create_database('test') + db._set_replica_gen_and_trans_id('other-id', 1, 'T-transid') + remote_target = self.getSyncTarget('test') + self.assertEqual(('test', 0, '', 1, 'T-transid'), + remote_target.get_sync_info('other-id')) + + def test_record_sync_info(self): + self.startServer() + db = self.request_state._create_database('test') + remote_target = self.getSyncTarget('test') + remote_target.record_sync_info('other-id', 2, 'T-transid') + self.assertEqual( + (2, 'T-transid'), db._get_replica_gen_and_trans_id('other-id')) + + def test_sync_exchange_send(self): + self.startServer() + db = self.request_state._create_database('test') + remote_target = self.getSyncTarget('test') + other_docs = [] + + def receive_doc(doc): + other_docs.append((doc.doc_id, doc.rev, doc.get_json())) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + new_gen, trans_id = remote_target.sync_exchange( + [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=receive_doc) + self.assertEqual(1, new_gen) + self.assertGetDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', False) + + def test_sync_exchange_send_failure_and_retry_scenario(self): + self.startServer() + + def blackhole_getstderr(inst): + return cStringIO.StringIO() + + self.patch(self.server.RequestHandlerClass, 'get_stderr', + blackhole_getstderr) + db = self.request_state._create_database('test') + _put_doc_if_newer = db._put_doc_if_newer + trigger_ids = ['doc-here2'] + + def bomb_put_doc_if_newer(doc, save_conflict, + replica_uid=None, replica_gen=None, + replica_trans_id=None): + if doc.doc_id in trigger_ids: + raise Exception + return _put_doc_if_newer(doc, save_conflict=save_conflict, + replica_uid=replica_uid, replica_gen=replica_gen, + replica_trans_id=replica_trans_id) + self.patch(db, '_put_doc_if_newer', bomb_put_doc_if_newer) + remote_target = self.getSyncTarget('test') + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc2 = self.make_document('doc-here2', 'replica:1', + '{"value": "here2"}') + self.assertRaises( + errors.HTTPError, + remote_target.sync_exchange, + [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], + 'replica', last_known_generation=0, last_known_trans_id=None, + return_doc_cb=receive_doc) + self.assertGetDoc(db, 'doc-here', 'replica:1', '{"value": "here"}', + False) + self.assertEqual( + (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) + self.assertEqual([], other_changes) + # retry + trigger_ids = [] + new_gen, trans_id = remote_target.sync_exchange( + [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=receive_doc) + self.assertGetDoc(db, 'doc-here2', 'replica:1', '{"value": "here2"}', + False) + self.assertEqual( + (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) + self.assertEqual(2, new_gen) + # bounced back to us + self.assertEqual( + ('doc-here', 'replica:1', '{"value": "here"}', 1), + other_changes[0][:-1]) + + def test_sync_exchange_in_stream_error(self): + self.startServer() + + def blackhole_getstderr(inst): + return cStringIO.StringIO() + + self.patch(self.server.RequestHandlerClass, 'get_stderr', + blackhole_getstderr) + db = self.request_state._create_database('test') + doc = db.create_doc_from_json('{"value": "there"}') + + def bomb_get_docs(doc_ids, check_for_conflicts=None, + include_deleted=False): + yield doc + # delayed failure case + raise errors.Unavailable + + self.patch(db, 'get_docs', bomb_get_docs) + remote_target = self.getSyncTarget('test') + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + self.assertRaises( + errors.Unavailable, remote_target.sync_exchange, [], 'replica', + last_known_generation=0, last_known_trans_id=None, + return_doc_cb=receive_doc) + self.assertEqual( + (doc.doc_id, doc.rev, '{"value": "there"}', 1), + other_changes[0][:-1]) + + def test_sync_exchange_receive(self): + self.startServer() + db = self.request_state._create_database('test') + doc = db.create_doc_from_json('{"value": "there"}') + remote_target = self.getSyncTarget('test') + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + new_gen, trans_id = remote_target.sync_exchange( + [], 'replica', last_known_generation=0, last_known_trans_id=None, + return_doc_cb=receive_doc) + self.assertEqual(1, new_gen) + self.assertEqual( + (doc.doc_id, doc.rev, '{"value": "there"}', 1), + other_changes[0][:-1]) + + def test_sync_exchange_send_ensure_callback(self): + self.startServer() + remote_target = self.getSyncTarget('test') + other_docs = [] + replica_uid_box = [] + + def receive_doc(doc): + other_docs.append((doc.doc_id, doc.rev, doc.get_json())) + + def ensure_cb(replica_uid): + replica_uid_box.append(replica_uid) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + new_gen, trans_id = remote_target.sync_exchange( + [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=receive_doc, + ensure_callback=ensure_cb) + self.assertEqual(1, new_gen) + db = self.request_state.open_database('test') + self.assertEqual(1, len(replica_uid_box)) + self.assertEqual(db._replica_uid, replica_uid_box[0]) + self.assertGetDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', False) + + +load_tests = tests.load_with_scenarios diff --git a/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py b/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py new file mode 100644 index 00000000..081d3ae7 --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py @@ -0,0 +1,495 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Test sqlite backend internals.""" + +import os +import time +import threading + +from sqlite3 import dbapi2 + +from u1db import ( + errors, + query_parser, + ) + +from leap.soledad.tests import u1db_tests as tests + +from u1db.backends import sqlite_backend +from leap.soledad.tests.u1db_tests.test_backends import TestAlternativeDocument + + +simple_doc = '{"key": "value"}' +nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' + + +class TestSQLiteDatabase(tests.TestCase): + + def test_atomic_initialize(self): + tmpdir = self.createTempDir() + dbname = os.path.join(tmpdir, 'atomic.db') + + t2 = None # will be a thread + + class SQLiteDatabaseTesting(sqlite_backend.SQLiteDatabase): + _index_storage_value = "testing" + + def __init__(self, dbname, ntry): + self._try = ntry + self._is_initialized_invocations = 0 + super(SQLiteDatabaseTesting, self).__init__(dbname) + + def _is_initialized(self, c): + res = super(SQLiteDatabaseTesting, self)._is_initialized(c) + if self._try == 1: + self._is_initialized_invocations += 1 + if self._is_initialized_invocations == 2: + t2.start() + # hard to do better and have a generic test + time.sleep(0.05) + return res + + outcome2 = [] + + def second_try(): + try: + db2 = SQLiteDatabaseTesting(dbname, 2) + except Exception, e: + outcome2.append(e) + else: + outcome2.append(db2) + + t2 = threading.Thread(target=second_try) + db1 = SQLiteDatabaseTesting(dbname, 1) + t2.join() + + self.assertIsInstance(outcome2[0], SQLiteDatabaseTesting) + db2 = outcome2[0] + self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor())) + + +class TestSQLitePartialExpandDatabase(tests.TestCase): + + def setUp(self): + super(TestSQLitePartialExpandDatabase, self).setUp() + self.db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + self.db._set_replica_uid('test') + + def test_create_database(self): + raw_db = self.db._get_sqlite_handle() + self.assertNotEqual(None, raw_db) + + def test_default_replica_uid(self): + self.db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + self.assertIsNot(None, self.db._replica_uid) + self.assertEqual(32, len(self.db._replica_uid)) + int(self.db._replica_uid, 16) + + def test__close_sqlite_handle(self): + raw_db = self.db._get_sqlite_handle() + self.db._close_sqlite_handle() + self.assertRaises(dbapi2.ProgrammingError, + raw_db.cursor) + + def test_create_database_initializes_schema(self): + raw_db = self.db._get_sqlite_handle() + c = raw_db.cursor() + c.execute("SELECT * FROM u1db_config") + config = dict([(r[0], r[1]) for r in c.fetchall()]) + self.assertEqual({'sql_schema': '0', 'replica_uid': 'test', + 'index_storage': 'expand referenced'}, config) + + # These tables must exist, though we don't care what is in them yet + c.execute("SELECT * FROM transaction_log") + c.execute("SELECT * FROM document") + c.execute("SELECT * FROM document_fields") + c.execute("SELECT * FROM sync_log") + c.execute("SELECT * FROM conflicts") + c.execute("SELECT * FROM index_definitions") + + def test__parse_index(self): + self.db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + g = self.db._parse_index_definition('fieldname') + self.assertIsInstance(g, query_parser.ExtractField) + self.assertEqual(['fieldname'], g.field) + + def test__update_indexes(self): + self.db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + g = self.db._parse_index_definition('fieldname') + c = self.db._get_sqlite_handle().cursor() + self.db._update_indexes('doc-id', {'fieldname': 'val'}, + [('fieldname', g)], c) + c.execute('SELECT doc_id, field_name, value FROM document_fields') + self.assertEqual([('doc-id', 'fieldname', 'val')], + c.fetchall()) + + def test__set_replica_uid(self): + # Start from scratch, so that replica_uid isn't set. + self.db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + self.assertIsNot(None, self.db._real_replica_uid) + self.assertIsNot(None, self.db._replica_uid) + self.db._set_replica_uid('foo') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") + self.assertEqual(('foo',), c.fetchone()) + self.assertEqual('foo', self.db._real_replica_uid) + self.assertEqual('foo', self.db._replica_uid) + self.db._close_sqlite_handle() + self.assertEqual('foo', self.db._replica_uid) + + def test__get_generation(self): + self.assertEqual(0, self.db._get_generation()) + + def test__get_generation_info(self): + self.assertEqual((0, ''), self.db._get_generation_info()) + + def test_create_index(self): + self.db.create_index('test-idx', "key") + self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) + + def test_create_index_multiple_fields(self): + self.db.create_index('test-idx', "key", "key2") + self.assertEqual([('test-idx', ["key", "key2"])], + self.db.list_indexes()) + + def test__get_index_definition(self): + self.db.create_index('test-idx', "key", "key2") + # TODO: How would you test that an index is getting used for an SQL + # request? + self.assertEqual(["key", "key2"], + self.db._get_index_definition('test-idx')) + + def test_list_index_mixed(self): + # Make sure that we properly order the output + c = self.db._get_sqlite_handle().cursor() + # We intentionally insert the data in weird ordering, to make sure the + # query still gets it back correctly. + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + [('idx-1', 0, 'key10'), + ('idx-2', 2, 'key22'), + ('idx-1', 1, 'key11'), + ('idx-2', 0, 'key20'), + ('idx-2', 1, 'key21')]) + self.assertEqual([('idx-1', ['key10', 'key11']), + ('idx-2', ['key20', 'key21', 'key22'])], + self.db.list_indexes()) + + def test_no_indexes_no_document_fields(self): + self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + + def test_create_extracts_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + self.db.create_index('test', 'key1', 'key2') + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual(sorted( + [(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "val2"), + (doc2.doc_id, "key1", "valx"), + (doc2.doc_id, "key2", "valy"), + ]), sorted(c.fetchall())) + + def test_put_updates_fields(self): + self.db.create_index('test', 'key1', 'key2') + doc1 = self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + doc1.content = {"key1": "val1", "key2": "valy"} + self.db.put_doc(doc1) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "valy"), + ], c.fetchall()) + + def test_put_updates_nested_fields(self): + self.db.create_index('test', 'key', 'sub.doc') + doc1 = self.db.create_doc_from_json(nested_doc) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key", "value"), + (doc1.doc_id, "sub.doc", "underneath"), + ], c.fetchall()) + + def test__ensure_schema_rollback(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/rollback.db' + + class SQLitePartialExpandDbTesting( + sqlite_backend.SQLitePartialExpandDatabase): + + def _set_replica_uid_in_transaction(self, uid): + super(SQLitePartialExpandDbTesting, + self)._set_replica_uid_in_transaction(uid) + if fail: + raise Exception() + + db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) + db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + fail = True + self.assertRaises(Exception, db._ensure_schema) + fail = False + db._initialize(db._db_handle.cursor()) + + def test__open_database(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/test.sqlite' + sqlite_backend.SQLitePartialExpandDatabase(path) + db2 = sqlite_backend.SQLiteDatabase._open_database(path) + self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) + + def test__open_database_with_factory(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/test.sqlite' + sqlite_backend.SQLitePartialExpandDatabase(path) + db2 = sqlite_backend.SQLiteDatabase._open_database( + path, document_factory=TestAlternativeDocument) + self.assertEqual(TestAlternativeDocument, db2._factory) + + def test__open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlite_backend.SQLiteDatabase._open_database, path) + + def test__open_database_during_init(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/initialised.db' + db = sqlite_backend.SQLitePartialExpandDatabase.__new__( + sqlite_backend.SQLitePartialExpandDatabase) + db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + self.addCleanup(db.close) + observed = [] + + class SQLiteDatabaseTesting(sqlite_backend.SQLiteDatabase): + WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1 + + @classmethod + def _which_index_storage(cls, c): + res = super(SQLiteDatabaseTesting, cls)._which_index_storage(c) + db._ensure_schema() # init db + observed.append(res[0]) + return res + + db2 = SQLiteDatabaseTesting._open_database(path) + self.addCleanup(db2.close) + self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) + self.assertEqual([None, + sqlite_backend.SQLitePartialExpandDatabase._index_storage_value], + observed) + + def test__open_database_invalid(self): + class SQLiteDatabaseTesting(sqlite_backend.SQLiteDatabase): + WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1 + temp_dir = self.createTempDir(prefix='u1db-test-') + path1 = temp_dir + '/invalid1.db' + with open(path1, 'wb') as f: + f.write("") + self.assertRaises(dbapi2.OperationalError, + SQLiteDatabaseTesting._open_database, path1) + with open(path1, 'wb') as f: + f.write("invalid") + self.assertRaises(dbapi2.DatabaseError, + SQLiteDatabaseTesting._open_database, path1) + + def test_open_database_existing(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/existing.sqlite' + sqlite_backend.SQLitePartialExpandDatabase(path) + db2 = sqlite_backend.SQLiteDatabase.open_database(path, create=False) + self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) + + def test_open_database_with_factory(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/existing.sqlite' + sqlite_backend.SQLitePartialExpandDatabase(path) + db2 = sqlite_backend.SQLiteDatabase.open_database( + path, create=False, document_factory=TestAlternativeDocument) + self.assertEqual(TestAlternativeDocument, db2._factory) + + def test_open_database_create(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + sqlite_backend.SQLiteDatabase.open_database(path, create=True) + db2 = sqlite_backend.SQLiteDatabase.open_database(path, create=False) + self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) + + def test_open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlite_backend.SQLiteDatabase.open_database, path, + create=False) + + def test_delete_database_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db = sqlite_backend.SQLiteDatabase.open_database(path, create=True) + db.close() + sqlite_backend.SQLiteDatabase.delete_database(path) + self.assertRaises(errors.DatabaseDoesNotExist, + sqlite_backend.SQLiteDatabase.open_database, path, + create=False) + + def test_delete_database_nonexistent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlite_backend.SQLiteDatabase.delete_database, path) + + def test__get_indexed_fields(self): + self.db.create_index('idx1', 'a', 'b') + self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) + self.db.create_index('idx2', 'b', 'c') + self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) + + def test_indexed_fields_expanded(self): + self.db.create_index('idx1', 'key1') + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def test_create_index_updates_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.db.create_index('idx1', 'key1') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def assertFormatQueryEquals(self, exp_statement, exp_args, definition, + values): + statement, args = self.db._format_query(definition, values) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_query(self): + self.assertFormatQueryEquals( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " + "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " + "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " + "ORDER BY d0.value;", ["key1", "a"], + ["key1"], ["a"]) + + def test__format_query2(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b", "key3", "c"], + ["key1", "key2", "key3"], ["a", "b", "c"]) + + def test__format_query_wildcard(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' + 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' + 'ORDER BY d0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], + ["a", "b*", "*"]) + + def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, + start_value, end_value): + statement, args = self.db._format_range_query( + definition, start_value, end_value) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_range_query(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', + 'key3', 'r'], + ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) + + def test__format_range_query_no_start(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], None, ["a", "b", "c"]) + + def test__format_range_query_no_end(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], ["a", "b", "c"], None) + + def test__format_range_query_wildcard(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' + 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' + 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' + 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' + 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', + 'key3'], + ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) diff --git a/src/leap/soledad/tests/u1db_tests/test_sync.py b/src/leap/soledad/tests/u1db_tests/test_sync.py new file mode 100644 index 00000000..7140cfd3 --- /dev/null +++ b/src/leap/soledad/tests/u1db_tests/test_sync.py @@ -0,0 +1,1287 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""The Synchronization class for U1DB.""" + +import os +from wsgiref import simple_server + +from u1db import ( + errors, + sync, + vectorclock, + SyncTarget, + ) + +from leap.soledad.tests import u1db_tests as tests + +from u1db.backends import ( + inmemory, + ) +from u1db.remote import ( + http_target, + ) + +from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( + make_http_app, + make_oauth_http_app, + ) + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +def _make_local_db_and_target(test): + db = test.create_database('test') + st = db.get_sync_target() + return db, st + + +def _make_local_db_and_http_target(test, path='test'): + test.startServer() + db = test.request_state._create_database(os.path.basename(path)) + st = http_target.HTTPSyncTarget.connect(test.getURL(path)) + return db, st + + +def _make_c_db_and_c_http_target(test, path='test'): + test.startServer() + db = test.request_state._create_database(os.path.basename(path)) + url = test.getURL(path) + st = tests.c_backend_wrapper.create_http_sync_target(url) + return db, st + + +def _make_local_db_and_oauth_http_target(test): + db, st = _make_local_db_and_http_target(test, '~/test') + st.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + return db, st + + +def _make_c_db_and_oauth_http_target(test, path='~/test'): + test.startServer() + db = test.request_state._create_database(os.path.basename(path)) + url = test.getURL(path) + st = tests.c_backend_wrapper.create_oauth_http_sync_target(url, + tests.consumer1.key, tests.consumer1.secret, + tests.token1.key, tests.token1.secret) + return db, st + + +target_scenarios = [ + ('local', {'create_db_and_target': _make_local_db_and_target}), + ('http', {'create_db_and_target': _make_local_db_and_http_target, + 'make_app_with_state': make_http_app}), + ('oauth_http', {'create_db_and_target': + _make_local_db_and_oauth_http_target, + 'make_app_with_state': make_oauth_http_app}), + ] + +c_db_scenarios = [ + ('local,c', {'create_db_and_target': _make_local_db_and_target, + 'make_database_for_test': tests.make_c_database_for_test, + 'copy_database_for_test': tests.copy_c_database_for_test, + 'make_document_for_test': tests.make_c_document_for_test, + 'whitebox': False}), + ('http,c', {'create_db_and_target': _make_c_db_and_c_http_target, + 'make_database_for_test': tests.make_c_database_for_test, + 'copy_database_for_test': tests.copy_c_database_for_test, + 'make_document_for_test': tests.make_c_document_for_test, + 'make_app_with_state': make_http_app, + 'whitebox': False}), + ('oauth_http,c', {'create_db_and_target': _make_c_db_and_oauth_http_target, + 'make_database_for_test': tests.make_c_database_for_test, + 'copy_database_for_test': tests.copy_c_database_for_test, + 'make_document_for_test': tests.make_c_document_for_test, + 'make_app_with_state': make_oauth_http_app, + 'whitebox': False}), + ] + + +class DatabaseSyncTargetTests(tests.DatabaseBaseTests, + tests.TestCaseWithServer): + + scenarios = (tests.multiply_scenarios(tests.DatabaseBaseTests.scenarios, + target_scenarios)) + #+ c_db_scenarios) + # whitebox true means self.db is the actual local db object + # against which the sync is performed + whitebox = True + + def setUp(self): + super(DatabaseSyncTargetTests, self).setUp() + self.db, self.st = self.create_db_and_target(self) + self.other_changes = [] + + def tearDown(self): + # We delete them explicitly, so that connections are cleanly closed + del self.st + self.db.close() + del self.db + super(DatabaseSyncTargetTests, self).tearDown() + + def receive_doc(self, doc, gen, trans_id): + self.other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + def set_trace_hook(self, callback, shallow=False): + setter = (self.st._set_trace_hook if not shallow else + self.st._set_trace_hook_shallow) + try: + setter(callback) + except NotImplementedError: + self.skipTest("%s does not implement _set_trace_hook" + % (self.st.__class__.__name__,)) + + def test_get_sync_target(self): + self.assertIsNot(None, self.st) + + def test_get_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + + def test_create_doc_updates_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + self.db.create_doc_from_json(simple_doc) + self.assertEqual(1, self.st.get_sync_info('other')[1]) + + def test_record_sync_info(self): + self.st.record_sync_info('replica', 10, 'T-transid') + self.assertEqual( + ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) + + def test_sync_exchange(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_deleted(self): + doc = self.db.create_doc_from_json('{}') + edit_rev = 'replica:1|' + doc.rev + docs_by_gen = [ + (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, edit_rev, None, False) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), + (self.make_document('doc-id2', 'replica:1', nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_refuses_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_ignores_convergence(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + gen, txid = self.db._get_generation_info() + docs_by_gen = [ + (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=gen, + last_known_trans_id=txid, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual(([], 1), (self.other_changes, new_gen)) + + def test_sync_exchange_returns_new_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_returns_deleted_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) + self.assertEqual(2, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_returns_many_new_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, simple_doc, 1), + (doc2.doc_id, doc2.rev, nested_doc, 2)], + [c[:-1] for c in self.other_changes]) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + + def test_sync_exchange_getting_newer_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): + expected = [] + + def before_whatschanged_cb(state): + if state != 'before whats_changed': + return + cont = '{"key": "cuncurrent"}' + conc_rev = self.db.put_doc( + self.make_document(doc.doc_id, 'test:1|z:2', cont)) + expected.append((doc.doc_id, conc_rev, cont, 3)) + + self.set_trace_hook(before_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(expected, [c[:-1] for c in self.other_changes]) + self.assertEqual(3, new_gen) + + def test_sync_exchange_with_concurrent_updates(self): + + def after_whatschanged_cb(state): + if state != 'after whats_changed': + return + self.db.create_doc_from_json('{"new": "doc"}') + + self.set_trace_hook(after_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_converged_handling(self): + doc = self.db.create_doc_from_json(simple_doc) + docs_by_gen = [ + (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), + (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + 'T-bar')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_detect_incomplete_exchange(self): + def before_get_docs_explode(state): + if state != 'before get_docs': + return + raise errors.U1DBError("fail") + self.set_trace_hook(before_get_docs_explode) + # suppress traceback printing in the wsgiref server + self.patch(simple_server.ServerHandler, + 'log_exception', lambda h, exc_info: None) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertRaises( + (errors.U1DBError, errors.BrokenSyncStream), + self.st.sync_exchange, [], 'other-replica', + last_known_generation=0, last_known_trans_id=None, + return_doc_cb=self.receive_doc) + + def test_sync_exchange_doc_ids(self): + sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) + if sync_exchange_doc_ids is None: + self.skipTest("sync_exchange_doc_ids not implemented") + db2 = self.create_database('test2') + doc = db2.create_doc_from_json(simple_doc) + new_gen, trans_id = sync_exchange_doc_ids( + db2, [(doc.doc_id, 10, 'T-sid')], 0, None, + return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertTransactionLog([doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) + + def test__set_trace_hook(self): + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ], + called) + + def test__set_trace_hook_shallow(self): + if (self.st._set_trace_hook_shallow == self.st._set_trace_hook + or self.st._set_trace_hook_shallow.im_func == + SyncTarget._set_trace_hook_shallow.im_func): + # shallow same as full + expected = ['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ] + else: + expected = ['sync_exchange', 'record_sync_info'] + + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb, shallow=True) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(expected, called) + + +def sync_via_synchronizer(test, db_source, db_target, trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios = [] +for name, scenario in tests.LOCAL_DATABASES_SCENARIOS: + scenario = dict(scenario) + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + scenario = dict(scenario) + + +def make_database_for_http_test(test, replica_uid): + if test.server is None: + test.startServer() + db = test.request_state._create_database(replica_uid) + try: + http_at = test._http_at + except AttributeError: + http_at = test._http_at = {} + http_at[db] = replica_uid + return db + + +def copy_database_for_http_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR HOUSE. + if test.server is None: + test.startServer() + new_db = test.request_state._copy_database(db) + try: + http_at = test._http_at + except AttributeError: + http_at = test._http_at = {} + path = db._replica_uid + while path in http_at.values(): + path += 'copy' + http_at[new_db] = path + return new_db + + +def sync_via_synchronizer_and_http(test, db_source, db_target, + trace_hook=None, trace_hook_shallow=None): + if trace_hook: + test.skipTest("full trace hook unsupported over http") + path = test._http_at[db_target] + target = http_target.HTTPSyncTarget.connect(test.getURL(path)) + if trace_hook_shallow: + target._set_trace_hook_shallow(trace_hook_shallow) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios.append(('pyhttp', { + 'make_database_for_test': make_database_for_http_test, + 'copy_database_for_test': copy_database_for_http_test, + 'make_document_for_test': tests.make_document_for_test, + 'make_app_with_state': make_http_app, + 'do_sync': sync_via_synchronizer_and_http + })) + + +if tests.c_backend_wrapper is not None: + # TODO: We should hook up sync tests with an HTTP target + def sync_via_c_sync(test, db_source, db_target, trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return tests.c_backend_wrapper.sync_db_to_target(db_source, target) + + #for name, scenario in tests.C_DATABASE_SCENARIOS: + # scenario = dict(scenario) + # scenario['do_sync'] = sync_via_synchronizer + # sync_scenarios.append((name + ',pysync', scenario)) + # scenario = dict(scenario) + # scenario['do_sync'] = sync_via_c_sync + # sync_scenarios.append((name + ',csync', scenario)) + + +class DatabaseSyncTests(tests.DatabaseBaseTests, + tests.TestCaseWithServer): + + scenarios = sync_scenarios + do_sync = None # set by scenarios + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return super(DatabaseSyncTests, self).create_database(replica_uid) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = super(DatabaseSyncTests, self).copy_database(db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = (name + '(copy)', sync_role + or orig_sync_role) + return db_copy + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def setUp(self): + self._use_tracking = {} + super(DatabaseSyncTests, self).setUp() + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards(self): + # here we test that when a database that has a conflicted document is + # the target of a sync, and the source database has a revision of the + # conflicted document that is newer than the target database's, and + # that source's database's document's content is the same as the + # target's document's conflict's, the target's document's conflict gets + # autoresolved, and the document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # <-------' + # a3b2 a3b2 (autoresolved and propagated) + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + revc = self.db1.get_doc_conflicts('doc')[-1].rev + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db2, self.db1) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + vecc = vectorclock.VectorClockRev(revc) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + self.assertTrue(vec3.is_newer(vecc)) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards_three(self): + # same as autoresolves_moar_backwards, but with three databases (note + # all the syncs go in the same direction -- this is a more natural + # scenario): + # + # A B C + # a1 - - + # `-------> + # a1 a1 - + # `-------> + # a1 a1 a1 + # v v + # a2 a1b1 a1 + # `-------------------> + # a2 a1b1 a2 + # `-------> + # a2+a1b1 a2 + # v + # a2 a2+a1b1 a2c1 (same as a1b1) + # `-------------------> + # a2c1 a2+a1b1 a2c1 + # `-------> + # a2b2c1 a2b2c1 a2c1 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + self.db3 = self.create_database('test3', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + self.sync(self.db2, self.db3) + for db, content in [(self.db2, '{"hi": 42}'), + (self.db1, '{}'), + ]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + # db2 and db3 now both have a doc of {}, but db2 has a + # conflict + doc = self.db2.get_doc('doc') + self.assertTrue(doc.has_conflicts) + revc = self.db2.get_doc_conflicts('doc')[-1].rev + self.assertEqual('{}', doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) + # set db3 to have a doc of {hi:42} (same as db2 before the conflict) + doc = self.db3.get_doc('doc') + doc.set_json('{"hi": 42}') + self.db3.put_doc(doc) + rev3 = doc.rev + # sync it across to db1 + self.sync(self.db1, self.db3) + # db1 now has hi:42, with a rev that is newer than db2's doc + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertFalse(doc.has_conflicts) + self.assertEqual('{"hi": 42}', doc.get_json()) + VCR = vectorclock.VectorClockRev + self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) + # so sync it to db2 + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db2.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # db2's revision of the document is strictly newer than db1's before + # the sync, and db3's before that sync way back when + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) + # make sure both dbs now have the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_puts_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + self.db1.create_index('test-idx', 'key') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog(self.db1, + {'receive': {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + self.db1.create_index('test-idx', 'key') + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc2.doc_id, from_idx.doc_id) + self.assertEqual(doc2.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc.doc_id, from_idx.doc_id) + self.assertEqual(doc.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + self.db2.create_index('test-idx', 'key') + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog(self.db3, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_resolution(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(db3._replica_uid)) + self.sync(db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(db3._replica_uid)) + self.assertEqual(3, len(db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1, db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + db3 = self.copy_database(self.db2) + db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, db2_copy) + + +class TestDbSync(tests.TestCaseWithServer): + """Test db.sync remote sync shortcut""" + + scenarios = [ + ('py-http', { + 'make_app_with_state': make_http_app, + 'make_database_for_test': tests.make_memory_database_for_test, + }), + #('c-http', { + # 'make_app_with_state': make_http_app, + # 'make_database_for_test': tests.make_c_database_for_test + # }), + ('py-oauth-http', { + 'make_app_with_state': make_oauth_http_app, + 'make_database_for_test': tests.make_memory_database_for_test, + 'oauth': True + }), + #('c-oauth-http', { + # 'make_app_with_state': make_oauth_http_app, + # 'make_database_for_test': tests.make_c_database_for_test, + # 'oauth': True + # }), + ] + + oauth = False + + def do_sync(self, target_name): + if self.oauth: + path = '~/' + target_name + extra = dict(creds={'oauth': { + 'consumer_key': tests.consumer1.key, + 'consumer_secret': tests.consumer1.secret, + 'token_key': tests.token1.key, + 'token_secret': tests.token1.secret + }}) + else: + path = target_name + extra = {} + target_url = self.getURL(path) + return self.db.sync(target_url, **extra) + + def setUp(self): + super(TestDbSync, self).setUp() + self.startServer() + self.db = self.make_database_for_test(self, 'test1') + self.db2 = self.request_state._create_database('test2.db') + + def test_db_sync(self): + doc1 = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db2.create_doc_from_json(tests.nested_doc) + local_gen_before_sync = self.do_sync('test2.db') + gen, _, changes = self.db.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + self.assertGetDoc(self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, + False) + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, tests.nested_doc, + False) + + def test_db_sync_autocreate(self): + doc1 = self.db.create_doc_from_json(tests.simple_doc) + local_gen_before_sync = self.do_sync('test3.db') + gen, _, changes = self.db.whats_changed(local_gen_before_sync) + self.assertEqual(0, gen - local_gen_before_sync) + db3 = self.request_state.open_database('test3.db') + gen, _, changes = db3.whats_changed() + self.assertEqual(1, len(changes)) + self.assertEqual(doc1.doc_id, changes[0][0]) + self.assertGetDoc(db3, doc1.doc_id, doc1.rev, tests.simple_doc, + False) + t_gen, _ = self.db._get_replica_gen_and_trans_id('test3.db') + s_gen, _ = db3._get_replica_gen_and_trans_id('test1') + self.assertEqual(1, t_gen) + self.assertEqual(1, s_gen) + + +class TestRemoteSyncIntegration(tests.TestCaseWithServer): + """Integration tests for the most common sync scenario local -> remote""" + + make_app_with_state = staticmethod(make_http_app) + + def setUp(self): + super(TestRemoteSyncIntegration, self).setUp() + self.startServer() + self.db1 = inmemory.InMemoryDatabase('test1') + self.db2 = self.request_state._create_database('test2') + + def test_sync_tracks_generations_incrementally(self): + doc11 = self.db1.create_doc_from_json('{"a": 1}') + doc12 = self.db1.create_doc_from_json('{"a": 2}') + doc21 = self.db2.create_doc_from_json('{"b": 1}') + doc22 = self.db2.create_doc_from_json('{"b": 2}') + #sanity + self.assertEqual(2, len(self.db1._get_transaction_log())) + self.assertEqual(2, len(self.db2._get_transaction_log())) + progress1 = [] + progress2 = [] + _do_set_replica_gen_and_trans_id = \ + self.db1._do_set_replica_gen_and_trans_id + + def set_sync_generation_witness1(other_uid, other_gen, trans_id): + progress1.append((other_uid, other_gen, + [d for d, t in self.db1._get_transaction_log()[2:]])) + _do_set_replica_gen_and_trans_id(other_uid, other_gen, trans_id) + self.patch(self.db1, '_do_set_replica_gen_and_trans_id', + set_sync_generation_witness1) + _do_set_replica_gen_and_trans_id2 = \ + self.db2._do_set_replica_gen_and_trans_id + + def set_sync_generation_witness2(other_uid, other_gen, trans_id): + progress2.append((other_uid, other_gen, + [d for d, t in self.db2._get_transaction_log()[2:]])) + _do_set_replica_gen_and_trans_id2(other_uid, other_gen, trans_id) + self.patch(self.db2, '_do_set_replica_gen_and_trans_id', + set_sync_generation_witness2) + + db2_url = self.getURL('test2') + self.db1.sync(db2_url) + + self.assertEqual([('test2', 1, [doc21.doc_id]), + ('test2', 2, [doc21.doc_id, doc22.doc_id]), + ('test2', 4, [doc21.doc_id, doc22.doc_id])], + progress1) + self.assertEqual([('test1', 1, [doc11.doc_id]), + ('test1', 2, [doc11.doc_id, doc12.doc_id]), + ('test1', 4, [doc11.doc_id, doc12.doc_id])], + progress2) + + +load_tests = tests.load_with_scenarios |