diff options
Diffstat (limited to 'testing/test_soledad')
| -rw-r--r-- | testing/test_soledad/__init__.py | 5 | ||||
| -rw-r--r-- | testing/test_soledad/fixture_soledad.conf | 11 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/README | 23 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/__init__.py | 415 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/test_backends.py | 1888 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/test_document.py | 153 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/test_http_client.py | 304 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/test_http_database.py | 233 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/test_https.py | 105 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/test_open.py | 72 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/testing-certs/Makefile | 35 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/testing-certs/cacert.pem | 58 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/testing-certs/testing.cert | 61 | ||||
| -rw-r--r-- | testing/test_soledad/u1db_tests/testing-certs/testing.key | 16 | ||||
| -rw-r--r-- | testing/test_soledad/util.py | 430 |
15 files changed, 3809 insertions, 0 deletions
diff --git a/testing/test_soledad/__init__.py b/testing/test_soledad/__init__.py new file mode 100644 index 00000000..c07c8b0e --- /dev/null +++ b/testing/test_soledad/__init__.py @@ -0,0 +1,5 @@ +from test_soledad import util + +__all__ = [ + 'util', +] diff --git a/testing/test_soledad/fixture_soledad.conf b/testing/test_soledad/fixture_soledad.conf new file mode 100644 index 00000000..8d8161c3 --- /dev/null +++ b/testing/test_soledad/fixture_soledad.conf @@ -0,0 +1,11 @@ +[soledad-server] +couch_url = http://soledad:passwd@localhost:5984 +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc +batching = 0 + +[database-security] +members = user1, user2 +members_roles = role1, role2 +admins = user3, user4 +admins_roles = role3, role3 diff --git a/testing/test_soledad/u1db_tests/README b/testing/test_soledad/u1db_tests/README new file mode 100644 index 00000000..546dfdc9 --- /dev/null +++ b/testing/test_soledad/u1db_tests/README @@ -0,0 +1,23 @@ +General info +------------ + +Test files in this directory are derived from u1db-0.1.4 tests. The main +difference is that: + + (1) they include the test infrastructure packed with soledad; and + (2) they do not include c_backend_wrapper testing. + +Dependencies +------------ + +u1db tests depend on the following python packages: + + unittest2 + mercurial + hgtools + testtools + discover + testscenarios + paste + routes + cython diff --git a/testing/test_soledad/u1db_tests/__init__.py b/testing/test_soledad/u1db_tests/__init__.py new file mode 100644 index 00000000..ba776864 --- /dev/null +++ b/testing/test_soledad/u1db_tests/__init__.py @@ -0,0 +1,415 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. +""" +Test infrastructure for U1DB +""" + +import copy +import shutil +import socket +import tempfile +import threading +import json + +from wsgiref import simple_server + +from pysqlcipher import dbapi2 +from StringIO import StringIO + +import testscenarios +from twisted.trial import unittest +from twisted.web.server import Site +from twisted.web.wsgi import WSGIResource +from twisted.internet import reactor + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.backends import inmemory +from leap.soledad.common.l2db.backends import sqlite_backend +from leap.soledad.common.l2db.remote import server_state +from leap.soledad.common.l2db.remote import http_app +from leap.soledad.common.l2db.remote import http_target + + +class TestCase(unittest.TestCase): + + def createTempDir(self, prefix='u1db-tmp-'): + """Create a temporary directory to do some work in. + + This directory will be scheduled for cleanup when the test ends. + """ + tempdir = tempfile.mkdtemp(prefix=prefix) + self.addCleanup(shutil.rmtree, tempdir) + return tempdir + + def make_document(self, doc_id, doc_rev, content, has_conflicts=False): + return self.make_document_for_test( + self, doc_id, doc_rev, content, has_conflicts) + + def make_document_for_test(self, test, doc_id, doc_rev, content, + has_conflicts): + return make_document_for_test( + test, doc_id, doc_rev, content, has_conflicts) + + def assertGetDoc(self, db, doc_id, doc_rev, content, has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id)) + + def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content, + has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True)) + + def assertGetDocConflicts(self, db, doc_id, conflicts): + """Assert what conflicts are stored for a given doc_id. + + :param conflicts: A list of (doc_rev, content) pairs. + The first item must match the first item returned from the + database, however the rest can be returned in any order. + """ + if conflicts: + conflicts = [(rev, + (json.loads(cont) if isinstance(cont, basestring) + else cont)) for (rev, cont) in conflicts] + conflicts = conflicts[:1] + sorted(conflicts[1:]) + actual = db.get_doc_conflicts(doc_id) + if actual: + actual = [ + (doc.rev, (json.loads(doc.get_json()) + if doc.get_json() is not None else None)) + for doc in actual] + actual = actual[:1] + sorted(actual[1:]) + self.assertEqual(conflicts, actual) + + +def multiply_scenarios(a_scenarios, b_scenarios): + """Create the cross-product of scenarios.""" + + all_scenarios = [] + for a_name, a_attrs in a_scenarios: + for b_name, b_attrs in b_scenarios: + name = '%s,%s' % (a_name, b_name) + attrs = dict(a_attrs) + attrs.update(b_attrs) + all_scenarios.append((name, attrs)) + return all_scenarios + + +simple_doc = '{"key": "value"}' +nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' + + +def make_memory_database_for_test(test, replica_uid): + return inmemory.InMemoryDatabase(replica_uid) + + +def copy_memory_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = inmemory.InMemoryDatabase(db._replica_uid) + new_db._transaction_log = db._transaction_log[:] + new_db._docs = copy.deepcopy(db._docs) + new_db._conflicts = copy.deepcopy(db._conflicts) + new_db._indexes = copy.deepcopy(db._indexes) + new_db._factory = db._factory + return new_db + + +def make_sqlite_partial_expanded_for_test(test, replica_uid): + db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlite_partial_expanded_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if 'sqlite_sequence' not in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return Document(doc_id, rev, content, has_conflicts=has_conflicts) + + +LOCAL_DATABASES_SCENARIOS = [ + ('mem', {'make_database_for_test': make_memory_database_for_test, + 'copy_database_for_test': copy_memory_database_for_test, + 'make_document_for_test': make_document_for_test}), + ('sql', {'make_database_for_test': + make_sqlite_partial_expanded_for_test, + 'copy_database_for_test': + copy_sqlite_partial_expanded_for_test, + 'make_document_for_test': make_document_for_test}), +] + + +class DatabaseBaseTests(TestCase): + + # set to True assertTransactionLog + # is happy with all trans ids = '' + accept_fixed_trans_id = False + + scenarios = LOCAL_DATABASES_SCENARIOS + + def make_database_for_test(self, replica_uid): + return make_memory_database_for_test(self, replica_uid) + + def create_database(self, *args): + return self.make_database_for_test(self, *args) + + def copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + return self.copy_database_for_test(self, db) + + def setUp(self): + super(DatabaseBaseTests, self).setUp() + self.db = self.create_database('test') + + def tearDown(self): + if hasattr(self, 'db') and self.db is not None: + self.db.close() + super(DatabaseBaseTests, self).tearDown() + + def assertTransactionLog(self, doc_ids, db): + """Assert that the given docs are in the transaction log.""" + log = db._get_transaction_log() + just_ids = [] + seen_transactions = set() + for doc_id, transaction_id in log: + just_ids.append(doc_id) + self.assertIsNot(None, transaction_id, + "Transaction id should not be None") + if transaction_id == '' and self.accept_fixed_trans_id: + continue + self.assertNotEqual('', transaction_id, + "Transaction id should be a unique string") + self.assertTrue(transaction_id.startswith('T-')) + self.assertNotIn(transaction_id, seen_transactions) + seen_transactions.add(transaction_id) + self.assertEqual(doc_ids, just_ids) + + def getLastTransId(self, db): + """Return the transaction id for the last database update.""" + return self.db._get_transaction_log()[-1][-1] + + +class ServerStateForTests(server_state.ServerState): + + """Used in the test suite, so we don't have to touch disk, etc.""" + + def __init__(self): + super(ServerStateForTests, self).__init__() + self._dbs = {} + + def open_database(self, path): + try: + return self._dbs[path] + except KeyError: + raise errors.DatabaseDoesNotExist + + def check_database(self, path): + # cares only about the possible exception + self.open_database(path) + + def ensure_database(self, path): + try: + db = self.open_database(path) + except errors.DatabaseDoesNotExist: + db = self._create_database(path) + return db, db._replica_uid + + def _copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + new_db = copy_memory_database_for_test(None, db) + path = db._replica_uid + while path in self._dbs: + path += 'copy' + self._dbs[path] = new_db + return new_db + + def _create_database(self, path): + db = inmemory.InMemoryDatabase(path) + self._dbs[path] = db + return db + + def delete_database(self, path): + del self._dbs[path] + + +class ResponderForTests(object): + + """Responder for tests.""" + _started = False + sent_response = False + status = None + + def start_response(self, status='success', **kwargs): + self._started = True + self.status = status + self.kwargs = kwargs + + def send_response(self, status='success', **kwargs): + self.start_response(status, **kwargs) + self.finish_response() + + def finish_response(self): + self.sent_response = True + + +class TestCaseWithServer(TestCase): + + @staticmethod + def server_def(): + # hook point + # should return (ServerClass, "shutdown method name", "url_scheme") + class _RequestHandler(simple_server.WSGIRequestHandler): + + def log_request(*args): + pass # suppress + + def make_server(host_port, application): + assert application, "forgot to override make_app(_with_state)?" + srv = simple_server.WSGIServer(host_port, _RequestHandler) + # patch the value in if it's None + if getattr(application, 'base_url', 1) is None: + application.base_url = "http://%s:%s" % srv.server_address + srv.set_app(application) + return srv + + return make_server, "shutdown", "http" + + @staticmethod + def make_app_with_state(state): + # hook point + return None + + def make_app(self): + # potential hook point + self.request_state = ServerStateForTests() + return self.make_app_with_state(self.request_state) + + def setUp(self): + super(TestCaseWithServer, self).setUp() + self.server = self.server_thread = self.port = None + + def tearDown(self): + if self.server is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + if self.port: + self.port.stopListening() + super(TestCaseWithServer, self).tearDown() + + @property + def url_scheme(self): + return 'http' + + def startTwistedServer(self): + application = self.make_app() + resource = WSGIResource(reactor, reactor.getThreadPool(), application) + site = Site(resource) + self.port = reactor.listenTCP(0, site, interface='127.0.0.1') + host = self.port.getHost() + self.server_address = (host.host, host.port) + self.addCleanup(self.port.stopListening) + + def startServer(self): + server_def = self.server_def() + server_class, shutdown_meth, _ = server_def + application = self.make_app() + self.server = server_class(('127.0.0.1', 0), application) + self.server_thread = threading.Thread(target=self.server.serve_forever, + kwargs=dict(poll_interval=0.01)) + self.server_thread.start() + self.addCleanup(self.server_thread.join) + self.addCleanup(getattr(self.server, shutdown_meth)) + self.server_address = self.server.server_address + + def getURL(self, path=None): + host, port = self.server_address + if path is None: + path = '' + return '%s://%s:%s/%s' % (self.url_scheme, host, port, path) + + +def socket_pair(): + """Return a pair of TCP sockets connected to each other. + + Unlike socket.socketpair, this should work on Windows. + """ + sock_pair = getattr(socket, 'socket_pair', None) + if sock_pair: + return sock_pair(socket.AF_INET, socket.SOCK_STREAM) + listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listen_sock.bind(('127.0.0.1', 0)) + listen_sock.listen(1) + client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client_sock.connect(listen_sock.getsockname()) + server_sock, addr = listen_sock.accept() + listen_sock.close() + return server_sock, client_sock + + +def load_with_scenarios(loader, standard_tests, pattern): + """Load the tests in a given module. + + This just applies testscenarios.generate_scenarios to all the tests that + are present. We do it at load time rather than at run time, because it + plays nicer with various tools. + """ + suite = loader.suiteClass() + suite.addTests(testscenarios.generate_scenarios(standard_tests)) + return suite + + +# from u1db.tests.test_remote_sync_target + +def make_http_app(state): + return http_app.HTTPApp(state) + + +def http_sync_target(test, path): + return http_target.HTTPSyncTarget(test.getURL(path)) diff --git a/testing/test_soledad/u1db_tests/test_backends.py b/testing/test_soledad/u1db_tests/test_backends.py new file mode 100644 index 00000000..10dcdff9 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_backends.py @@ -0,0 +1,1888 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of leap.soledad.common +# +# leap.soledad.common is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +""" +The backend class for L2DB. This deals with hiding storage details. +""" + +import json + +from leap.soledad.common.l2db import DocumentBase +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db.remote import http_database + +from test_soledad import u1db_tests as tests + +from unittest import skip + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +def make_http_database_for_test(test, replica_uid, path='test', *args): + test.startServer() + test.request_state._create_database(replica_uid) + return http_database.HTTPDatabase(test.getURL(path)) + + +def copy_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + return test.request_state._copy_database(db) + + +class TestAlternativeDocument(DocumentBase): + + """A (not very) alternative implementation of Document.""" + + +@skip("Skiping tests imported from U1DB.") +class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ + ('http', {'make_database_for_test': make_http_database_for_test, + 'copy_database_for_test': copy_http_database_for_test, + 'make_document_for_test': tests.make_document_for_test, + 'make_app_with_state': tests.make_http_app}), + ] + + def test_close(self): + self.db.close() + + def test_create_doc_allocating_doc_id(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertNotEqual(None, doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_different_ids_same_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + + def test_create_doc_with_id(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my-id') + self.assertEqual('my-id', doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_existing_id(self): + doc = self.db.create_doc_from_json(simple_doc) + new_content = '{"something": "else"}' + self.assertRaises( + errors.RevisionConflict, self.db.create_doc_from_json, + new_content, doc.doc_id) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_put_doc_creating_initial(self): + doc = self.make_document('my_doc_id', None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertIsNot(None, new_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, simple_doc, False) + + def test_put_doc_space_in_id(self): + doc = self.make_document('my doc id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_update(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + orig_rev = doc.rev + doc.set_json('{"updated": "stuff"}') + new_rev = self.db.put_doc(doc) + self.assertNotEqual(new_rev, orig_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, + '{"updated": "stuff"}', False) + self.assertEqual(doc.rev, new_rev) + + def test_put_non_ascii_key(self): + content = json.dumps({u'key\xe5': u'val'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_non_ascii_value(self): + content = json.dumps({'key': u'\xe5'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_doc_refuses_no_id(self): + doc = self.make_document(None, None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document("", None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_refuses_slashes(self): + doc = self.make_document('a/b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document(r'\b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_url_quoting_is_fine(self): + doc_id = "%2F%2Ffoo%2Fbar" + doc = self.make_document(doc_id, None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertGetDoc(self.db, doc_id, new_rev, simple_doc, False) + + def test_put_doc_refuses_non_existing_old_rev(self): + doc = self.make_document('doc-id', 'test:4', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.put_doc, doc) + + def test_put_doc_refuses_non_ascii_doc_id(self): + doc = self.make_document('d\xc3\xa5c-id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_fails_with_bad_old_rev(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + old_rev = doc.rev + bad_doc = self.make_document(doc.doc_id, 'other:1', + '{"something": "else"}') + self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc) + self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False) + + def test_create_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + new_doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(new_doc.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev)) + + def test_put_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + doc2 = self.make_document('my_doc_id', None, simple_doc) + self.db.put_doc(doc2) + self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (doc2.rev, deleted_doc.rev)) + + def test_get_doc_after_put(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False) + + def test_get_doc_nonexisting(self): + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_doc_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertIs(None, self.db.get_doc('my_doc_id')) + + def test_get_doc_include_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_get_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual([doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual( + [doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + include_deleted=True))) + + def test_get_docs_request_ordered(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + self.assertEqual([doc2, doc1], + list(self.db.get_docs([doc2.doc_id, doc1.doc_id]))) + + def test_get_docs_empty_list(self): + self.assertEqual([], list(self.db.get_docs([]))) + + def test_handles_nested_content(self): + doc = self.db.create_doc_from_json(nested_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + def test_handles_doc_with_null(self): + doc = self.db.create_doc_from_json('{"key": null}') + self.assertGetDoc(self.db, doc.doc_id, doc.rev, '{"key": null}', False) + + def test_delete_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + orig_rev = doc.rev + self.db.delete_doc(doc) + self.assertNotEqual(orig_rev, doc.rev) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + self.assertIs(None, self.db.get_doc(doc.doc_id)) + + def test_delete_doc_non_existent(self): + doc = self.make_document('non-existing', 'other:1', simple_doc) + self.assertRaises(errors.DocumentDoesNotExist, self.db.delete_doc, doc) + + def test_delete_doc_already_deleted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertRaises(errors.DocumentAlreadyDeleted, + self.db.delete_doc, doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_delete_doc_bad_rev(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + doc2 = self.make_document(doc1.doc_id, 'other:1', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.delete_doc, doc2) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_delete_doc_sets_content_to_None(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertIs(None, doc.get_json()) + + def test_delete_doc_rev_supersedes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json(nested_doc) + self.db.put_doc(doc) + doc.set_json('{"fishy": "content"}') + self.db.put_doc(doc) + old_rev = doc.rev + self.db.delete_doc(doc) + cur_vc = vectorclock.VectorClockRev(old_rev) + deleted_vc = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(deleted_vc.is_newer(cur_vc), + "%s does not supersede %s" % (doc.rev, old_rev)) + + def test_delete_then_put(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + doc.set_json(nested_doc) + self.db.put_doc(doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + +@skip("Skiping tests imported from U1DB.") +class DocumentSizeTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_put_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + doc = self.make_document('doc-id', None, simple_doc) + self.assertRaises(errors.DocumentTooBig, self.db.put_doc, doc) + + def test_create_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + self.assertRaises( + errors.DocumentTooBig, self.db.create_doc_from_json, simple_doc, + doc_id='my_doc_id') + + def test_set_document_size_limit_zero(self): + self.db.set_document_size_limit(0) + self.assertEqual(0, self.db.document_size_limit) + + def test_set_document_size_limit(self): + self.db.set_document_size_limit(1000000) + self.assertEqual(1000000, self.db.document_size_limit) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def setUp(self): + tests.DatabaseBaseTests.setUp(self) + + def test_create_doc_different_ids_diff_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + db2 = self.create_database('other-uid') + doc2 = db2.create_doc_from_json(simple_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + db2.close() + + def test_put_doc_refuses_slashes_picky(self): + doc = self.make_document('/a', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_get_all_docs_empty(self): + self.assertEqual([], list(self.db.get_all_docs()[1])) + + def test_get_all_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc1, doc2]), sorted(list(self.db.get_all_docs()[1]))) + + def test_get_all_docs_exclude_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual([doc1], list(self.db.get_all_docs()[1])) + + def test_get_all_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual( + sorted([doc1, doc2]), + sorted(list(self.db.get_all_docs(include_deleted=True)[1]))) + + def test_get_all_docs_generation(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(nested_doc) + self.assertEqual(2, self.db.get_all_docs()[0]) + + def test_simple_put_doc_if_newer(self): + doc = self.make_document('my-doc-id', 'test:1', simple_doc) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 1), state_at_gen) + self.assertGetDoc(self.db, 'my-doc-id', 'test:1', simple_doc, False) + + def test_simple_put_doc_if_newer_deleted(self): + self.db.create_doc_from_json('{}', doc_id='my-doc-id') + doc = self.make_document('my-doc-id', 'test:2', None) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 2), state_at_gen) + self.assertGetDocIncludeDeleted( + self.db, 'my-doc-id', 'test:2', None, False) + + def test_put_doc_if_newer_already_superseded(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + doc1_rev1 = doc1.rev + doc1.set_json(simple_doc) + self.db.put_doc(doc1) + doc1_rev2 = doc1.rev + # Nothing is inserted, because the document is already superseded + doc = self.make_document(doc1.doc_id, doc1_rev1, orig_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + self.assertGetDoc(self.db, doc1.doc_id, doc1_rev2, simple_doc, False) + + def test_put_doc_if_newer_autoresolve(self): + doc1 = self.db.create_doc_from_json(simple_doc) + rev = doc1.rev + doc = self.make_document(doc1.doc_id, "whatever:1", doc1.get_json()) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + doc2 = self.db.get_doc(doc1.doc_id) + v2 = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev("whatever:1"))) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev(rev))) + # strictly newer locally + self.assertTrue(rev not in doc2.rev) + + def test_put_doc_if_newer_already_converged(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + state_at_gen = self.db._put_doc_if_newer( + doc1, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('converged', 1), state_at_gen) + + def test_put_doc_if_newer_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Nothing is inserted, the document id is returned as would-conflict + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + alt_doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database wasn't altered + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_put_doc_if_newer_newer_generation(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:2', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=2, + replica_trans_id='T-irrelevant') + self.assertEqual('inserted', state) + + def test_put_doc_if_newer_same_generation_same_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.db.create_doc_from_json(simple_doc) + self.make_document(doc.doc_id, 'other:1', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=1, + replica_trans_id='T-sid') + self.assertEqual('converged', state) + + def test_put_doc_if_newer_wrong_transaction_id(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidTransactionId, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_old_generation_older_doc(self): + orig_doc = '{"new": "doc"}' + doc = self.db.create_doc_from_json(orig_doc) + doc_rev1 = doc.rev + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.db._set_replica_gen_and_trans_id('other', 3, 'T-sid') + older_doc = self.make_document(doc.doc_id, doc_rev1, simple_doc) + state, _ = self.db._put_doc_if_newer( + older_doc, save_conflict=False, replica_uid='other', replica_gen=8, + replica_trans_id='T-irrelevant') + self.assertEqual('superseded', state) + + def test_put_doc_if_newer_old_generation_newer_doc(self): + self.db._set_replica_gen_and_trans_id('other', 5, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidGeneration, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.assertEqual('inserted', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=2, + replica_trans_id='T-id2')[0]) + self.assertEqual((2, 'T-id2'), self.db._get_replica_gen_and_trans_id( + 'other')) + # Compare to the old rev, should be superseded + doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc) + self.assertEqual('superseded', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + # A conflict that isn't saved still records the sync gen, because we + # don't need to see it again + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=4, + replica_trans_id='T-id4')[0]) + self.assertEqual( + (4, 'T-id4'), self.db._get_replica_gen_and_trans_id('other')) + + def test__get_replica_gen_and_trans_id(self): + self.assertEqual( + (0, ''), self.db._get_replica_gen_and_trans_id('other-db')) + self.db._set_replica_gen_and_trans_id('other-db', 2, 'T-transaction') + self.assertEqual( + (2, 'T-transaction'), + self.db._get_replica_gen_and_trans_id('other-db')) + + def test_put_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + doc.set_json('{"something": "else"}') + self.db.put_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + + def test_delete_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + db_gen, _, _ = self.db.whats_changed() + self.db.delete_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed(db_gen)) + + def test_whats_changed_initial_database(self): + self.assertEqual((0, '', []), self.db.whats_changed()) + + def test_whats_changed_returns_one_id_for_multiple_changes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.put_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + self.assertEqual((2, last_trans_id, []), self.db.whats_changed(2)) + + def test_whats_changed_returns_last_edits_ascending(self): + doc = self.db.create_doc_from_json(simple_doc) + doc1 = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.delete_doc(doc1) + delete_trans_id = self.getLastTransId(self.db) + self.db.put_doc(doc) + put_trans_id = self.getLastTransId(self.db) + self.assertEqual((4, put_trans_id, + [(doc1.doc_id, 3, delete_trans_id), + (doc.doc_id, 4, put_trans_id)]), + self.db.whats_changed()) + + def test_whats_changed_doesnt_include_old_gen(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((3, last_trans_id, [(doc2.doc_id, 3, last_trans_id)]), + self.db.whats_changed(2)) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_validate_gen_and_trans_id(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.db.validate_gen_and_trans_id(gen, trans_id) + + def test_validate_gen_and_trans_id_invalid_txid(self): + self.db.create_doc_from_json(simple_doc) + gen, _ = self.db._get_generation_info() + self.assertRaises( + errors.InvalidTransactionId, + self.db.validate_gen_and_trans_id, gen, 'wrong') + + def test_validate_gen_and_trans_id_invalid_gen(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.assertRaises( + errors.InvalidGeneration, + self.db.validate_gen_and_trans_id, gen + 1, trans_id) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_validate_source_gen_and_trans_id_same(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 1, 'T-sid') + + def test_validate_source_gen_newer(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 2, 'T-whatevs') + + def test_validate_source_wrong_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.assertRaises( + errors.InvalidTransactionId, + self.db._validate_source, 'other', 1, 'T-sad') + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): + # test supporting/functionality around storing conflicts + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_get_docs_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([doc2], list(self.db.get_docs([doc1.doc_id]))) + + def test_get_docs_conflicts_ignored(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + no_conflict_doc = self.make_document(doc1.doc_id, 'alternate:1', + nested_doc) + self.assertEqual([no_conflict_doc, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + check_for_conflicts=False))) + + def test_get_doc_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([alt_doc, doc], + self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_all_docs_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + _, docs = self.db.get_all_docs() + self.assertTrue(list(docs)[0].has_conflicts) + + def test_get_doc_conflicts_unconflicted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_doc_conflicts_no_such_id(self): + self.assertEqual([], self.db.get_doc_conflicts('doc-id')) + + def test_resolve_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc.doc_id, + [('alternate:1', nested_doc), + (doc.rev, simple_doc)]) + orig_rev = doc.rev + self.db.resolve_doc(doc, [alt_doc.rev, doc.rev]) + self.assertNotEqual(orig_rev, doc.rev) + self.assertFalse(doc.has_conflicts) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc.doc_id, []) + + def test_resolve_doc_picks_biggest_vcr(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + orig_doc1_rev = doc1.rev + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertFalse(doc1.has_conflicts) + self.assertNotEqual(orig_doc1_rev, doc1.rev) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + vcr_1 = vectorclock.VectorClockRev(orig_doc1_rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + vcr_new = vectorclock.VectorClockRev(doc1.rev) + self.assertTrue(vcr_new.is_newer(vcr_1)) + self.assertTrue(vcr_new.is_newer(vcr_2)) + + def test_resolve_doc_partial_not_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDoc(self.db, doc1.doc_id, doc3.rev, content3, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc)]) + + def test_resolve_doc_partial_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc3.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + + def test_resolve_doc_with_delete_conflict(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc2, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, False) + + def test_resolve_doc_with_delete_to_delete(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc1, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDocIncludeDeleted( + self.db, doc1.doc_id, doc1.rev, None, False) + + def test_put_doc_if_newer_save_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Document is inserted as a conflict + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database was updated + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, nested_doc, True) + + def test_force_doc_conflict_supersedes_properly(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', '{"b": 1}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'altalt:1', '{"c": 1}') + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + doc22 = self.make_document(doc1.doc_id, 'alternate:2', '{"b": 2}') + self.db._put_doc_if_newer( + doc22, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='zed') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2', doc22.get_json()), + ('altalt:1', doc3.get_json()), + (doc1.rev, simple_doc)]) + + def test_put_doc_if_newer_save_conflict_was_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue(doc2.has_conflicts) + self.assertGetDoc( + self.db, doc1.doc_id, 'alternate:1', nested_doc, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), + (doc1.rev, None)]) + + def test_put_doc_if_newer_propagates_full_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + resolved_vcr.maximize(vcr_2) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual('inserted', state) + self.assertFalse(doc_resolved.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + doc3 = self.db.get_doc(doc1.doc_id) + self.assertFalse(doc3.has_conflicts) + + def test_put_doc_if_newer_propagates_partial_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altalt:1', '{}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), + ('test:1', simple_doc), + ('altalt:1', '{}')]) + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_3 = vectorclock.VectorClockRev(doc3.rev) + resolved_vcr.maximize(vcr_3) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual('inserted', state) + self.assertTrue(doc_resolved.has_conflicts) + doc4 = self.db.get_doc(doc1.doc_id) + self.assertTrue(doc4.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2|test:1', '{"good": 1}'), + ('altalt:1', '{}')]) + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-id') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.db._put_doc_if_newer(doc2, save_conflict=True, + replica_uid='other', replica_gen=2, + replica_trans_id='T-id2') + # Conflict vs the current update + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer( + doc2, + save_conflict=True, + replica_uid='other', + replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + + def test_put_doc_if_newer_autoresolve_2(self): + # this is an ordering variant of _3, but that already works + # adding the test explicitly to catch the regression easily + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', "{}") + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', + '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:2|other:1', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'inserted') + self.assertFalse(self.db.get_doc(doc_a1.doc_id).has_conflicts) + + def test_put_doc_if_newer_autoresolve_3(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', "{}") + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_doc_if_newer_autoresolve_4(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', None) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', None) + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_refuses_to_update_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": "altval"}' + doc2 = self.make_document(doc1.doc_id, 'altrev:1', content2) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, content2, True) + content3 = '{"key": "local"}' + doc2.set_json(content3) + self.assertRaises(errors.ConflictedDoc, self.db.put_doc, doc2) + + def test_delete_refuses_for_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altrev:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, True) + self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) + + +@skip("Skiping tests imported from U1DB.") +class DatabaseIndexTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def assertParseError(self, definition): + self.db.create_doc_from_json(nested_doc) + self.assertRaises( + errors.IndexDefinitionParseError, self.db.create_index, 'idx', + definition) + + def assertIndexCreatable(self, definition): + name = "idx" + self.db.create_doc_from_json(nested_doc) + self.db.create_index(name, definition) + self.assertEqual( + [(name, [definition])], self.db.list_indexes()) + + def test_create_index(self): + self.db.create_index('test-idx', 'name') + self.assertEqual([('test-idx', ['name'])], + self.db.list_indexes()) + + def test_create_index_on_non_ascii_field_name(self): + doc = self.db.create_doc_from_json(json.dumps({u'\xe5': 'value'})) + self.db.create_index('test-idx', u'\xe5') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_list_indexes_with_non_ascii_field_names(self): + self.db.create_index('test-idx', u'\xe5') + self.assertEqual( + [('test-idx', [u'\xe5'])], self.db.list_indexes()) + + def test_create_index_evaluates_it(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_wildcard_matches_unicode_value(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_retrieve_unicode_value_from_index(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', u"valu\xe5")) + + def test_create_index_fails_if_name_taken(self): + self.db.create_index('test-idx', 'key') + self.assertRaises(errors.IndexNameTakenError, + self.db.create_index, + 'test-idx', 'stuff') + + def test_create_index_does_not_fail_if_name_taken_with_same_index(self): + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + + def test_create_index_does_not_duplicate_indexed_fields(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.delete_index('test-idx') + self.db.create_index('test-idx', 'key') + self.assertEqual(1, len(self.db.get_from_index('test-idx', 'value'))) + + def test_delete_index_does_not_remove_fields_from_other_indexes(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx2', 'key') + self.db.delete_index('test-idx') + self.assertEqual(1, len(self.db.get_from_index('test-idx2', 'value'))) + + def test_create_index_after_deleting_document(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc2) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_delete_index(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + self.db.delete_index('test-idx') + self.assertEqual([], self.db.list_indexes()) + + def test_create_adds_to_index(self): + self.db.create_index('test-idx', 'key') + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_unmatched(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', 'novalue')) + + def test_create_index_multiple_exact_matches(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + + def test_get_from_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_multi(self): + content = '{"key": "value", "key2": "value2"}' + doc = self.db.create_doc_from_json(content) + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2')) + + def test_get_from_index_multi_list(self): + doc = self.db.create_doc_from_json( + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-1')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-2')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-3')) + self.assertEqual( + [('value', 'value2-1'), ('value', 'value2-2'), + ('value', 'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key', 'key2') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_from_index('test-idx', 'value', 'value2-1') + self.assertTrue(docs[0].has_conflicts) + + def test_get_index_keys_multi_list_list(self): + self.db.create_doc_from_json( + '{"key": "value1-1 value1-2 value1-3", ' + '"key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'split_words(key)', 'key2') + self.assertEqual( + [(u'value1-1', u'value2-1'), (u'value1-1', u'value2-2'), + (u'value1-1', u'value2-3'), (u'value1-2', u'value2-1'), + (u'value1-2', u'value2-2'), (u'value1-2', u'value2-3'), + (u'value1-3', u'value2-1'), (u'value1-3', u'value2-2'), + (u'value1-3', u'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_multi_ordered(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2, doc1], + self.db.get_from_index('test-idx', 'v*', '*')) + + def test_get_range_from_index_start_end(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2', 'value3')) + + def test_get_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + doc3 = self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1, doc3], + self.db.get_range_from_index('test-idx', 'value2')) + + def test_get_range_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', '{"key": "valuedepalue"}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_range_from_index('test-idx', 'a') + self.assertTrue(docs[0].has_conflicts) + + def test_get_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + doc4 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2')) + + def test_get_wildcard_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4, doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2*')) + + def test_get_wildcard_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + doc5 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc5, doc3, doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2*')) + + def test_get_wildcard_range_from_index_start_end(self): + self.db.create_doc_from_json('{"key": "a"}') + self.db.create_doc_from_json('{"key": "boo3"}') + doc3 = self.db.create_doc_from_json('{"key": "catalyst"}') + doc4 = self.db.create_doc_from_json('{"key": "whaever"}') + self.db.create_doc_from_json('{"key": "zerg"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4], + self.db.get_range_from_index('test-idx', 'cat*', 'zap*')) + + def test_get_range_from_index_multi_column_start_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2], + self.db.get_range_from_index( + 'test-idx', ('value2', 'value2'), ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + self.db.create_doc_from_json('{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value3'))) + + def test_get_wildcard_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value2*'))) + + def test_get_wildcard_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value2*'))) + + def test_get_glob_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', '*'))) + + def test_get_glob_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index('test-idx', None, ('value2', '*'))) + + def test_get_range_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v*')) + + def test_get_range_from_index_illegal_wildcard_order_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v*')) + + def test_get_from_index_fails_if_no_index(self): + self.assertRaises( + errors.IndexDoesNotExist, self.db.get_from_index, 'foo') + + def test_get_index_keys_fails_if_no_index(self): + self.assertRaises(errors.IndexDoesNotExist, + self.db.get_index_keys, + 'foo') + + def test_get_index_keys_works_if_no_docs(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_index_keys('test-idx')) + + def test_put_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db.put_doc(doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'altval')) + + def test_delete_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + self.db.delete_doc(doc) + self.assertEqual([doc2], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_illegal_number_of_entries(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidValueForIndex, self.db.get_from_index, 'test-idx') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1', 'v2', 'v3') + + def test_get_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v2') + + def test_get_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v*') + + def test_get_all_from_index(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + # This one should not be in the index + self.db.create_doc_from_json('{"no": "key"}') + diff_value_doc = '{"key": "diff value"}' + doc4 = self.db.create_doc_from_json(diff_value_doc) + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + sorted([doc1, doc2, doc4]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_all_from_index_ordered(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json('{"key": "value x"}') + doc2 = self.db.create_doc_from_json('{"key": "value b"}') + doc3 = self.db.create_doc_from_json('{"key": "value a"}') + doc4 = self.db.create_doc_from_json('{"key": "value m"}') + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + [doc3, doc2, doc4, doc1], self.db.get_from_index('test-idx', '*')) + + def test_put_updates_when_adding_key(self): + doc = self.db.create_doc_from_json("{}") + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_get_from_index_empty_string(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": ""}' + doc2 = self.db.create_doc_from_json(content2) + self.assertEqual([doc2], self.db.get_from_index('test-idx', '')) + # Empty string matches the wildcard. + self.assertEqual( + sorted([doc1, doc2]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_from_index_not_null(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json('{"key": null}') + self.assertEqual([doc1], self.db.get_from_index('test-idx', '*')) + + def test_get_partial_from_index(self): + content1 = '{"k1": "v1", "k2": "v2"}' + content2 = '{"k1": "v1", "k2": "x2"}' + content3 = '{"k1": "v1", "k2": "y2"}' + # doc4 has a different k1 value, so it doesn't match the prefix. + content4 = '{"k1": "NN", "k2": "v2"}' + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.db.create_index('test-idx', 'k1', 'k2') + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "*"))) + + def test_get_glob_match(self): + # Note: the exact glob syntax is probably subject to change + content1 = '{"k1": "v1", "k2": "v1"}' + content2 = '{"k1": "v1", "k2": "v2"}' + content3 = '{"k1": "v1", "k2": "v3"}' + # doc4 has a different k2 prefix value, so it doesn't match + content4 = '{"k1": "v1", "k2": "ZZ"}' + self.db.create_index('test-idx', 'k1', 'k2') + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "v*"))) + + def test_nested_index(self): + doc = self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.doc') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'underneath')) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'underneath'))) + + def test_nested_nonexistent(self): + self.db.create_doc_from_json(nested_doc) + # sub exists, but sub.foo does not: + self.db.create_index('test-idx', 'sub.foo') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_nonexistent2(self): + self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_traverses_lists(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": "bar"}, {"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_doc_from_json('{"foo": ["zap", "baz"]}') + self.db.create_index('test-idx', 'foo.zap') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'bar')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'baz')) + + def test_nested_list_traversal(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": [{"qux": "fnord"}, {"qux": "zombo"}]},' + '{"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_index('test-idx', 'foo.zap.qux') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'fnord')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'zombo')) + + def test_index_list1(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_list2(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_case_sensitive(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'V*')) + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'v*')) + + def test_get_from_index_illegal_glob_before_value(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v2') + + def test_get_from_index_illegal_glob_after_glob(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v*') + + def test_get_from_index_with_sql_wildcards(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "va%lue"}' + content2 = '{"key": "value"}' + content3 = '{"key": "va_lue"}' + doc1 = self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + # The '%' in the search should be treated literally, not as a sql + # globbing character. + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'va%*')) + # Same for '_' + self.assertEqual([doc3], self.db.get_from_index('test-idx', 'va_*')) + + def test_get_from_index_with_lower(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_with_lower_matches_same_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_lower_doesnt_match_different_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual([], rows) + + def test_index_lower_doesnt_match_other_index(self): + self.db.create_index("index", "lower(name)") + self.db.create_index("other_index", "name") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual(0, len(rows)) + + def test_index_split_words_match_first(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_match_second(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_match_both(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_double_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_leading_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": " foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_trailing_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar "}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 12}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number_bigger_than_padding(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 123456}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "123456") + self.assertEqual([doc], rows) + + def test_number_mapping_ignores_non_numbers(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 56}' + doc1 = self.db.create_doc_from_json(content) + content = '{"foo": "this is not a maigret painting"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([doc1], rows) + + def test_get_from_index_with_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": true}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "1") + self.assertEqual([doc], rows) + + def test_get_from_index_with_bool_false(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": false}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "0") + self.assertEqual([doc], rows) + + def test_get_from_index_with_non_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": 42}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([], rows) + + def test_get_from_index_with_combine(self): + self.db.create_index("index", "combine(foo, bar)") + content = '{"foo": "value1", "bar": "value2"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "value1") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "value2") + self.assertEqual([doc], rows) + + def test_get_complex_combine(self): + self.db.create_index( + "index", "combine(number(foo, 5), lower(bar), split_words(baz))") + content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}' + doc = self.db.create_doc_from_json(content) + content = '{"foo": "not a number", "bar": "something"}' + doc2 = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "allcaps") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "nox") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "something") + self.assertEqual([doc2], rows) + + def test_get_index_keys_from_index(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "value1"}' + content2 = '{"key": "value2"}' + content3 = '{"key": "value2"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.assertEqual( + [('value1',), ('value2',)], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_index_keys_from_multicolumn_index(self): + self.db.create_index('test-idx', 'key1', 'key2') + content1 = '{"key1": "value1", "key2": "val2-1"}' + content2 = '{"key1": "value2", "key2": "val2-2"}' + content3 = '{"key1": "value2", "key2": "val2-2"}' + content4 = '{"key1": "value2", "key2": "val3"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual([ + ('value1', 'val2-1'), + ('value2', 'val2-2'), + ('value2', 'val3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_empty_expr(self): + self.assertParseError('') + + def test_nested_unknown_operation(self): + self.assertParseError('unknown_operation(field1)') + + def test_parse_missing_close_paren(self): + self.assertParseError("lower(a") + + def test_parse_trailing_close_paren(self): + self.assertParseError("lower(ab))") + + def test_parse_trailing_chars(self): + self.assertParseError("lower(ab)adsf") + + def test_parse_empty_op(self): + self.assertParseError("(ab)") + + def test_parse_top_level_commas(self): + self.assertParseError("a, b") + + def test_invalid_field_name(self): + self.assertParseError("a.") + + def test_invalid_inner_field_name(self): + self.assertParseError("lower(a.)") + + def test_gobbledigook(self): + self.assertParseError("(@#@cc @#!*DFJSXV(()jccd") + + def test_leading_space(self): + self.assertIndexCreatable(" lower(a)") + + def test_trailing_space(self): + self.assertIndexCreatable("lower(a) ") + + def test_spaces_before_open_paren(self): + self.assertIndexCreatable("lower (a)") + + def test_spaces_after_open_paren(self): + self.assertIndexCreatable("lower( a)") + + def test_spaces_before_close_paren(self): + self.assertIndexCreatable("lower(a )") + + def test_spaces_before_comma(self): + self.assertIndexCreatable("combine(a , b , c)") + + def test_spaces_after_comma(self): + self.assertIndexCreatable("combine(a, b, c)") + + def test_all_together_now(self): + self.assertParseError(' (a) ') + + def test_all_together_now2(self): + self.assertParseError('combine(lower(x)x,foo)') + + +@skip("Skiping tests imported from U1DB.") +class PythonBackendTests(tests.DatabaseBaseTests): + + def setUp(self): + super(PythonBackendTests, self).setUp() + self.simple_doc = json.loads(simple_doc) + + def test_create_doc_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.assertTrue(isinstance(doc, TestAlternativeDocument)) + + def test_get_doc_after_put_with_factory(self): + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.db.set_document_factory(TestAlternativeDocument) + result = self.db.get_doc('my_doc_id') + self.assertTrue(isinstance(result, TestAlternativeDocument)) + self.assertEqual(doc.doc_id, result.doc_id) + self.assertEqual(doc.rev, result.rev) + self.assertEqual(doc.get_json(), result.get_json()) + self.assertEqual(False, result.has_conflicts) + + def test_get_doc_nonexisting_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_all_docs_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.assertTrue(isinstance( + list(self.db.get_all_docs()[1])[0], TestAlternativeDocument)) + + def test_get_docs_conflicted_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc1 = self.db.create_doc(self.simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue( + isinstance( + list(self.db.get_docs([doc1.doc_id]))[0], + TestAlternativeDocument)) + + def test_get_from_index_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + self.assertTrue( + isinstance( + self.db.get_from_index('test-idx', 'value')[0], + TestAlternativeDocument)) + + def test_sync_exchange_updates_indexes(self): + doc = self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + other_rev = 'test:1|z:2' + st = self.db.get_sync_target() + + def ignore(doc_id, doc_rev, doc): + pass + + doc_other = self.make_document(doc.doc_id, other_rev, new_content) + docs_by_gen = [(doc_other, 10, 'T-sid')] + st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=ignore) + self.assertGetDoc(self.db, doc.doc_id, other_rev, new_content, False) + self.assertEqual( + [doc_other], self.db.get_from_index('test-idx', 'altval')) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + + +# Use a custom loader to apply the scenarios at load time. +load_tests = tests.load_with_scenarios diff --git a/testing/test_soledad/u1db_tests/test_document.py b/testing/test_soledad/u1db_tests/test_document.py new file mode 100644 index 00000000..a7ead2d1 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_document.py @@ -0,0 +1,153 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of leap.soledad.common +# +# leap.soledad.common is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. +from unittest import skip + +from leap.soledad.common.l2db import errors + +from test_soledad import u1db_tests as tests + + +@skip("Skiping tests imported from U1DB.") +class TestDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) # + + # tests.C_DATABASE_SCENARIOS) + + def test_create_doc(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual('doc-id', doc.doc_id) + self.assertEqual('uid:1', doc.rev) + self.assertEqual(tests.simple_doc, doc.get_json()) + self.assertFalse(doc.has_conflicts) + + def test__repr__(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual( + '%s(doc-id, uid:1, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__repr__conflicted(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc, + has_conflicts=True) + self.assertEqual( + '%s(doc-id, uid:1, conflicted, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__lt__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('b', 'b', '{}') + self.assertTrue(doc_a < doc_b) + self.assertTrue(doc_b > doc_a) + doc_aa = self.make_document('a', 'a', '{}') + self.assertTrue(doc_aa < doc_a) + + def test__eq__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('a', 'b', '{}') + self.assertTrue(doc_a == doc_b) + doc_b = self.make_document('a', 'b', '{}', has_conflicts=True) + self.assertFalse(doc_a == doc_b) + + def test_non_json_dict(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + '"not a json dictionary"') + + def test_non_json(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + 'not a json dictionary') + + def test_get_size(self): + doc_a = self.make_document('a', 'b', '{"some": "content"}') + self.assertEqual( + len('a' + 'b' + '{"some": "content"}'), doc_a.get_size()) + + def test_get_size_empty_document(self): + doc_a = self.make_document('a', 'b', None) + self.assertEqual(len('a' + 'b'), doc_a.get_size()) + + +@skip("Skiping tests imported from U1DB.") +class TestPyDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) + + def test_get_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertEqual({"content": ""}, doc.content) + doc.set_json('{"content": "new"}') + self.assertEqual({"content": "new"}, doc.content) + + def test_set_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.content = {"content": "new"} + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_bad_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises( + errors.InvalidContent, setattr, doc, 'content', + '{"content": "new"}') + + def test_is_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.set_json(None) + self.assertTrue(doc_a.is_tombstone()) + + def test_make_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.make_tombstone() + self.assertTrue(doc_a.is_tombstone()) + + def test_same_content_as(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('d', 'e', '{}') + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b = self.make_document('p', 'q', '{}', has_conflicts=True) + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b.content['key'] = 'value' + self.assertFalse(doc_a.same_content_as(doc_b)) + + def test_same_content_as_json_order(self): + doc_a = self.make_document( + 'a', 'b', '{"key1": "val1", "key2": "val2"}') + doc_b = self.make_document( + 'c', 'd', '{"key2": "val2", "key1": "val1"}') + self.assertTrue(doc_a.same_content_as(doc_b)) + + def test_set_json(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.set_json('{"content": "new"}') + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_json_non_dict(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, '"is not a dict"') + + def test_set_json_error(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, 'is not json') + + +load_tests = tests.load_with_scenarios diff --git a/testing/test_soledad/u1db_tests/test_http_client.py b/testing/test_soledad/u1db_tests/test_http_client.py new file mode 100644 index 00000000..e9516236 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_http_client.py @@ -0,0 +1,304 @@ +# Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +""" +Tests for HTTPDatabase +""" +import json + +from unittest import skip + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_client + +from test_soledad import u1db_tests as tests + + +@skip("Skiping tests imported from U1DB.") +class TestEncoder(tests.TestCase): + + def test_encode_string(self): + self.assertEqual("foo", http_client._encode_query_parameter("foo")) + + def test_encode_true(self): + self.assertEqual("true", http_client._encode_query_parameter(True)) + + def test_encode_false(self): + self.assertEqual("false", http_client._encode_query_parameter(False)) + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPClientBase(tests.TestCaseWithServer): + + def setUp(self): + super(TestHTTPClientBase, self).setUp() + self.errors = 0 + + def app(self, environ, start_response): + if environ['PATH_INFO'].endswith('echo'): + start_response("200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = environ['wsgi.input'].read(content_length) + return [json.dumps(ret)] + elif environ['PATH_INFO'].endswith('error_then_accept'): + if self.errors >= 3: + start_response( + "200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = '{"oki": "doki"}' + return [json.dumps(ret)] + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + elif environ['PATH_INFO'].endswith('error'): + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + + def make_app(self): + return self.app + + def getClient(self, **kwds): + self.startServer() + return http_client.HTTPClientBase(self.getURL('dbase'), **kwds) + + def test_construct(self): + self.startServer() + url = self.getURL() + cli = http_client.HTTPClientBase(url) + self.assertEqual(url, cli._url.geturl()) + self.assertIs(None, cli._conn) + + def test_parse_url(self): + cli = http_client.HTTPClientBase( + '%s://127.0.0.1:12345/' % self.url_scheme) + self.assertEqual(self.url_scheme, cli._url.scheme) + self.assertEqual('127.0.0.1', cli._url.hostname) + self.assertEqual(12345, cli._url.port) + self.assertEqual('/', cli._url.path) + + def test__ensure_connection(self): + cli = self.getClient() + self.assertIs(None, cli._conn) + cli._ensure_connection() + self.assertIsNot(None, cli._conn) + conn = cli._conn + cli._ensure_connection() + self.assertIs(conn, cli._conn) + + def test_close(self): + cli = self.getClient() + cli._ensure_connection() + cli.close() + self.assertIs(None, cli._conn) + + def test__request(self): + cli = self.getClient() + res, headers = cli._request('PUT', ['echo'], {}, {}) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': '', + 'body': '{}', + 'REQUEST_METHOD': 'PUT'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', '%FFFF', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/%FFFF/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('POST', ['echo'], {'b': 2}, 'Body', + 'application/x-test') + self.assertEqual({'CONTENT_TYPE': 'application/x-test', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': 'Body', + 'REQUEST_METHOD': 'POST'}, json.loads(res)) + + def test__request_json(self): + cli = self.getClient() + res, headers = cli._request_json( + 'POST', ['echo'], {'b': 2}, {'a': 'x'}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': '{"a": "x"}', + 'REQUEST_METHOD': 'POST'}, res) + + def test_unspecified_http_error(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Crash."}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Fail."}) + except errors.HTTPError, e: + pass + + self.assertEqual(500, e.status) + self.assertEqual("Fail.", e.message) + self.assertTrue("content-type" in e.headers) + + def test_revision_conflict(self): + cli = self.getClient() + self.assertRaises(errors.RevisionConflict, + cli._request_json, 'POST', ['error'], {}, + {'status': "409 Conflict", + 'response': {"error": "revision conflict"}}) + + def test_unavailable_proper(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + self.assertRaises(errors.Unavailable, + cli._request_json, 'POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual(5, self.errors) + + def test_unavailable_then_available(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + res, headers = cli._request_json( + 'POST', ['error_then_accept'], {'b': 2}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/error_then_accept', + 'QUERY_STRING': 'b=2', + 'body': '{"oki": "doki"}', + 'REQUEST_METHOD': 'POST'}, res) + self.assertEqual(3, self.errors) + + def test_unavailable_random_source(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + try: + cli._request_json('POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': "random unavailable."}) + except errors.Unavailable, e: + pass + + self.assertEqual(503, e.status) + self.assertEqual("random unavailable.", e.message) + self.assertTrue("content-type" in e.headers) + self.assertEqual(5, self.errors) + + def test_document_too_big(self): + cli = self.getClient() + self.assertRaises(errors.DocumentTooBig, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "document too big"}}) + + def test_user_quota_exceeded(self): + cli = self.getClient() + self.assertRaises(errors.UserQuotaExceeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user quota exceeded"}}) + + def test_user_needs_subscription(self): + cli = self.getClient() + self.assertRaises(errors.SubscriptionNeeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user needs subscription"}}) + + def test_generic_u1db_error(self): + cli = self.getClient() + self.assertRaises(errors.U1DBError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + except errors.U1DBError, e: + pass + self.assertIs(e.__class__, errors.U1DBError) + + def test_unspecified_bad_request(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': "<Bad Request>"}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': "<Bad Request>"}) + except errors.HTTPError, e: + pass + + self.assertEqual(400, e.status) + self.assertEqual("<Bad Request>", e.message) + self.assertTrue("content-type" in e.headers) + + def test_unknown_creds(self): + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={'foo': {}}) + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={}) diff --git a/testing/test_soledad/u1db_tests/test_http_database.py b/testing/test_soledad/u1db_tests/test_http_database.py new file mode 100644 index 00000000..a3ed9361 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_http_database.py @@ -0,0 +1,233 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Tests for HTTPDatabase""" + +import inspect +import json + +from unittest import skip + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.remote import http_database +from leap.soledad.common.l2db.remote import http_target +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import make_http_app + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPDatabaseSimpleOperations(tests.TestCase): + + def setUp(self): + super(TestHTTPDatabaseSimpleOperations, self).setUp() + self.db = http_database.HTTPDatabase('dbase') + self.db._conn = object() # crash if used + self.got = None + self.response_val = None + + def _request(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + def _request_json(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + self.db._request = _request + self.db._request_json = _request_json + + def test__sanity_same_signature(self): + my_request_sig = inspect.getargspec(self.db._request) + my_request_sig = (['self'] + my_request_sig[0],) + my_request_sig[1:] + self.assertEqual( + my_request_sig, + inspect.getargspec(http_database.HTTPDatabase._request)) + my_request_json_sig = inspect.getargspec(self.db._request_json) + my_request_json_sig = ((['self'] + my_request_json_sig[0],) + + my_request_json_sig[1:]) + self.assertEqual( + my_request_json_sig, + inspect.getargspec(http_database.HTTPDatabase._request_json)) + + def test__ensure(self): + self.response_val = {'ok': True}, {} + self.db._ensure() + self.assertEqual(('PUT', [], {}, {}, None), self.got) + + def test__delete(self): + self.response_val = {'ok': True}, {} + self.db._delete() + self.assertEqual(('DELETE', [], {}, {}, None), self.got) + + def test__check(self): + self.response_val = {}, {} + res = self.db._check() + self.assertEqual({}, res) + self.assertEqual(('GET', [], None, None, None), self.got) + + def test_put_doc(self): + self.response_val = {'rev': 'doc-rev'}, {} + doc = Document('doc-id', None, '{"v": 1}') + res = self.db.put_doc(doc) + self.assertEqual('doc-rev', res) + self.assertEqual('doc-rev', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + self.response_val = {'rev': 'doc-rev-2'}, {} + doc.content = {"v": 2} + res = self.db.put_doc(doc) + self.assertEqual('doc-rev-2', res) + self.assertEqual('doc-rev-2', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + '{"v": 2}', 'application/json'), self.got) + + def test_get_doc(self): + self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev', + 'x-u1db-has-conflicts': 'false'} + self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False) + self.assertEqual( + ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None), + self.got) + + def test_get_doc_non_existing(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('not-there')) + self.assertEqual( + ('GET', ['doc', 'not-there'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('deleted')) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted_include_deleted(self): + self.response_val = errors.HTTPError( + 404, + json.dumps({"error": errors.DOCUMENT_DELETED}), + {'x-u1db-rev': 'doc-rev-gone', + 'x-u1db-has-conflicts': 'false'}) + doc = self.db.get_doc('deleted', include_deleted=True) + self.assertEqual('deleted', doc.doc_id) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertIs(None, doc.content) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None), + self.got) + + def test_get_doc_pass_through_errors(self): + self.response_val = errors.HTTPError(500, 'Crash.') + self.assertRaises(errors.HTTPError, + self.db.get_doc, 'something-something') + + def test_create_doc_with_id(self): + self.response_val = {'rev': 'doc-rev'}, {} + new_doc = self.db.create_doc_from_json('{"v": 1}', doc_id='doc-id') + self.assertEqual('doc-rev', new_doc.rev) + self.assertEqual('doc-id', new_doc.doc_id) + self.assertEqual('{"v": 1}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + def test_create_doc_without_id(self): + self.response_val = {'rev': 'doc-rev-2'}, {} + new_doc = self.db.create_doc_from_json('{"v": 3}') + self.assertEqual('D-', new_doc.doc_id[:2]) + self.assertEqual('doc-rev-2', new_doc.rev) + self.assertEqual('{"v": 3}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', new_doc.doc_id], {}, + '{"v": 3}', 'application/json'), self.got) + + def test_delete_doc(self): + self.response_val = {'rev': 'doc-rev-gone'}, {} + doc = Document('doc-id', 'doc-rev', None) + self.db.delete_doc(doc) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertEqual(('DELETE', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + None, None), self.got) + + def test_get_sync_target(self): + st = self.db.get_sync_target() + self.assertIsInstance(st, http_target.HTTPSyncTarget) + self.assertEqual(st._url, self.db._url) + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): + + make_app_with_state = staticmethod(make_http_app) + + def setUp(self): + super(TestHTTPDatabaseIntegration, self).setUp() + self.startServer() + + def test_non_existing_db(self): + db = http_database.HTTPDatabase(self.getURL('not-there')) + self.assertRaises(errors.DatabaseDoesNotExist, db.get_doc, 'doc1') + + def test__ensure(self): + db = http_database.HTTPDatabase(self.getURL('new')) + db._ensure() + self.assertIs(None, db.get_doc('doc1')) + + def test__delete(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase(self.getURL('db0')) + db._delete() + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_open_database_existing(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + self.assertIs(None, db.get_doc('doc1')) + + def test_open_database_non_existing(self): + self.assertRaises(errors.DatabaseDoesNotExist, + http_database.HTTPDatabase.open_database, + self.getURL('not-there'), + create=False) + + def test_open_database_create(self): + db = http_database.HTTPDatabase.open_database(self.getURL('new'), + create=True) + self.assertIs(None, db.get_doc('doc1')) + + def test_delete_database_existing(self): + self.request_state._create_database('db0') + http_database.HTTPDatabase.delete_database(self.getURL('db0')) + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_doc_ids_needing_quoting(self): + db0 = self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + doc = Document('%fff', None, '{}') + db.put_doc(doc) + self.assertGetDoc(db0, '%fff', doc.rev, '{}', False) + self.assertGetDoc(db, '%fff', doc.rev, '{}', False) diff --git a/testing/test_soledad/u1db_tests/test_https.py b/testing/test_soledad/u1db_tests/test_https.py new file mode 100644 index 00000000..baffa723 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_https.py @@ -0,0 +1,105 @@ +"""Test support for client-side https support.""" + +import os +import ssl +import sys + +from paste import httpserver +from unittest import skip + +from leap.soledad.common.l2db.remote import http_client + +from leap import soledad +from test_soledad import u1db_tests as tests + + +def https_server_def(): + def make_server(host_port, application): + from OpenSSL import SSL + cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + 'testing.cert') + key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + 'testing.key') + ssl_context = SSL.Context(SSL.SSLv23_METHOD) + ssl_context.use_privatekey_file(key_file) + ssl_context.use_certificate_chain_file(cert_file) + srv = httpserver.WSGIServerBase(application, host_port, + httpserver.WSGIHandler, + ssl_context=ssl_context + ) + + def shutdown_request(req): + req.shutdown() + srv.close_request(req) + + srv.shutdown_request = shutdown_request + application.base_url = "https://localhost:%s" % srv.server_address[1] + return srv + return make_server, "shutdown", "https" + + +@skip("Skiping tests imported from U1DB.") +class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): + + scenarios = [] + + def setUp(self): + try: + import OpenSSL # noqa + except ImportError: + self.skipTest("Requires pyOpenSSL") + self.cacert_pem = os.path.join(os.path.dirname(__file__), + 'testing-certs', 'cacert.pem') + # The default u1db http_client class for doing HTTPS only does HTTPS + # if the platform is linux. Because of this, soledad replaces that + # class with one that will do HTTPS independent of the platform. In + # order to maintain the compatibility with u1db default tests, we undo + # that replacement here. + http_client._VerifiedHTTPSConnection = \ + soledad.client.api.old__VerifiedHTTPSConnection + super(TestHttpSyncTargetHttpsSupport, self).setUp() + + def getSyncTarget(self, host, path=None, cert_file=None): + if self.server is None: + self.startServer() + return self.sync_target(self, host, path, cert_file=cert_file) + + def test_working(self): + self.startServer() + db = self.request_state._create_database('test') + self.patch(http_client, 'CA_CERTS', self.cacert_pem) + remote_target = self.getSyncTarget('localhost', 'test') + remote_target.record_sync_info('other-id', 2, 'T-id') + self.assertEqual( + (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) + + def test_cannot_verify_cert(self): + if not sys.platform.startswith('linux'): + self.skipTest( + "XXX certificate verification happens on linux only for now") + self.startServer() + # don't print expected traceback server-side + self.server.handle_error = lambda req, cli_addr: None + self.request_state._create_database('test') + remote_target = self.getSyncTarget('localhost', 'test') + try: + remote_target.record_sync_info('other-id', 2, 'T-id') + except ssl.SSLError, e: + self.assertIn("certificate verify failed", str(e)) + else: + self.fail("certificate verification should have failed.") + + def test_host_mismatch(self): + if not sys.platform.startswith('linux'): + self.skipTest( + "XXX certificate verification happens on linux only for now") + self.startServer() + self.request_state._create_database('test') + self.patch(http_client, 'CA_CERTS', self.cacert_pem) + remote_target = self.getSyncTarget('127.0.0.1', 'test') + self.assertRaises( + http_client.CertificateError, remote_target.record_sync_info, + 'other-id', 2, 'T-id') + + +load_tests = tests.load_with_scenarios diff --git a/testing/test_soledad/u1db_tests/test_open.py b/testing/test_soledad/u1db_tests/test_open.py new file mode 100644 index 00000000..30d4de00 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_open.py @@ -0,0 +1,72 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Test u1db.open""" + +import os +from unittest import skip + +from leap.soledad.common.l2db import ( + errors, open as u1db_open, +) +from test_soledad import u1db_tests as tests +from leap.soledad.common.l2db.backends import sqlite_backend +from test_soledad.u1db_tests.test_backends \ + import TestAlternativeDocument + + +@skip("Skiping tests imported from U1DB.") +class TestU1DBOpen(tests.TestCase): + + def setUp(self): + super(TestU1DBOpen, self).setUp() + tmpdir = self.createTempDir() + self.db_path = tmpdir + '/test.db' + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + u1db_open, self.db_path, create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = u1db_open(self.db_path, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, sqlite_backend.SQLiteDatabase) + + def test_open_with_factory(self): + db = u1db_open(self.db_path, create=True, + document_factory=TestAlternativeDocument) + self.addCleanup(db.close) + self.assertEqual(TestAlternativeDocument, db._factory) + + def test_open_existing(self): + db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = u1db_open(self.db_path, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) + self.addCleanup(db.close) + db2 = u1db_open(self.db_path, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) diff --git a/testing/test_soledad/u1db_tests/testing-certs/Makefile b/testing/test_soledad/u1db_tests/testing-certs/Makefile new file mode 100644 index 00000000..2385e75b --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/Makefile @@ -0,0 +1,35 @@ +CATOP=./demoCA +ORIG_CONF=/usr/lib/ssl/openssl.cnf +ELEVEN_YEARS=-days 4015 + +init: + cp $(ORIG_CONF) ca.conf + install -d $(CATOP) + install -d $(CATOP)/certs + install -d $(CATOP)/crl + install -d $(CATOP)/newcerts + install -d $(CATOP)/private + touch $(CATOP)/index.txt + echo 01>$(CATOP)/crlnumber + @echo '**** Making CA certificate ...' + openssl req -nodes -new \ + -newkey rsa -keyout $(CATOP)/private/cakey.pem \ + -out $(CATOP)/careq.pem \ + -multivalue-rdn \ + -subj "/C=UK/ST=-/O=u1db LOCAL TESTING ONLY, DO NO TRUST/CN=u1db testing CA" + openssl ca -config ./ca.conf -create_serial \ + -out $(CATOP)/cacert.pem $(ELEVEN_YEARS) -batch \ + -keyfile $(CATOP)/private/cakey.pem -selfsign \ + -extensions v3_ca -infiles $(CATOP)/careq.pem + +pems: + cp ./demoCA/cacert.pem . + openssl req -new -config ca.conf \ + -multivalue-rdn \ + -subj "/O=u1db LOCAL TESTING ONLY, DO NOT TRUST/CN=localhost" \ + -nodes -keyout testing.key -out newreq.pem $(ELEVEN_YEARS) + openssl ca -batch -config ./ca.conf $(ELEVEN_YEARS) \ + -policy policy_anything \ + -out testing.cert -infiles newreq.pem + +.PHONY: init pems diff --git a/testing/test_soledad/u1db_tests/testing-certs/cacert.pem b/testing/test_soledad/u1db_tests/testing-certs/cacert.pem new file mode 100644 index 00000000..c019a730 --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/cacert.pem @@ -0,0 +1,58 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e4:de:01:76:c4:78:78:7e + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Validity + Not Before: May 3 11:11:11 2012 GMT + Not After : May 1 11:11:11 2023 GMT + Subject: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:bc:91:a5:7f:7d:37:f7:06:c7:db:5b:83:6a:6b: + 63:c3:8b:5c:f7:84:4d:97:6d:d4:be:bf:e7:79:a8: + c1:03:57:ec:90:d4:20:e7:02:95:d9:a6:49:e3:f9: + 9a:ea:37:b9:b2:02:62:ab:40:d3:42:bb:4a:4e:a2: + 47:71:0f:1d:a2:c5:94:a1:cf:35:d3:23:32:42:c0: + 1e:8d:cb:08:58:fb:8a:5c:3e:ea:eb:d5:2c:ed:d6: + aa:09:b4:b5:7d:e3:45:c9:ae:c2:82:b2:ae:c0:81: + bc:24:06:65:a9:e7:e0:61:ac:25:ee:53:d3:d7:be: + 22:f7:00:a2:ad:c6:0e:3a:39 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + X509v3 Authority Key Identifier: + keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 72:9b:c1:f7:07:65:83:36:25:4e:01:2f:b7:4a:f2:a4:00:28: + 80:c7:56:2c:32:39:90:13:61:4b:bb:12:c5:44:9d:42:57:85: + 28:19:70:69:e1:43:c8:bd:11:f6:94:df:91:2d:c3:ea:82:8d: + b4:8f:5d:47:a3:00:99:53:29:93:27:6c:c5:da:c1:20:6f:ab: + ec:4a:be:34:f3:8f:02:e5:0c:c0:03:ac:2b:33:41:71:4f:0a: + 72:5a:b4:26:1a:7f:81:bc:c0:95:8a:06:87:a8:11:9f:5c:73: + 38:df:5a:69:40:21:29:ad:46:23:56:75:e1:e9:8b:10:18:4c: + 7b:54 +-----BEGIN CERTIFICATE----- +MIICkjCCAfugAwIBAgIJAOTeAXbEeHh+MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg +T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x +MjA1MDMxMTExMTFaFw0yMzA1MDExMTExMTFaMGIxCzAJBgNVBAYTAlVLMQowCAYD +VQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcgT05MWSwgRE8gTk8g +VFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAvJGlf3039wbH21uDamtjw4tc94RNl23Uvr/neajBA1fskNQg +5wKV2aZJ4/ma6je5sgJiq0DTQrtKTqJHcQ8dosWUoc810yMyQsAejcsIWPuKXD7q +69Us7daqCbS1feNFya7CgrKuwIG8JAZlqefgYawl7lPT174i9wCircYOOjkCAwEA +AaNQME4wHQYDVR0OBBYEFNs9k1FsMhVUjxBQ/ElPNhUou5VtMB8GA1UdIwQYMBaA +FNs9k1FsMhVUjxBQ/ElPNhUou5VtMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF +BQADgYEAcpvB9wdlgzYlTgEvt0rypAAogMdWLDI5kBNhS7sSxUSdQleFKBlwaeFD +yL0R9pTfkS3D6oKNtI9dR6MAmVMpkydsxdrBIG+r7Eq+NPOPAuUMwAOsKzNBcU8K +clq0Jhp/gbzAlYoGh6gRn1xzON9aaUAhKa1GI1Z14emLEBhMe1Q= +-----END CERTIFICATE----- diff --git a/testing/test_soledad/u1db_tests/testing-certs/testing.cert b/testing/test_soledad/u1db_tests/testing-certs/testing.cert new file mode 100644 index 00000000..985684fb --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/testing.cert @@ -0,0 +1,61 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e4:de:01:76:c4:78:78:7f + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Validity + Not Before: May 3 11:11:14 2012 GMT + Not After : May 1 11:11:14 2023 GMT + Subject: O=u1db LOCAL TESTING ONLY, DO NOT TRUST, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c6:1d:72:d3:c5:e4:fc:d1:4c:d9:e4:08:3e:90: + 10:ce:3f:1f:87:4a:1d:4f:7f:2a:5a:52:c9:65:4f: + d9:2c:bf:69:75:18:1a:b5:c9:09:32:00:47:f5:60: + aa:c6:dd:3a:87:37:5f:16:be:de:29:b5:ea:fc:41: + 7e:eb:77:bb:df:63:c3:06:1e:ed:e9:a0:67:1a:f1: + ec:e1:9d:f7:9c:8f:1c:fa:c3:66:7b:39:dc:70:ae: + 09:1b:9c:c0:9a:c4:90:77:45:8e:39:95:a9:2f:92: + 43:bd:27:07:5a:99:51:6e:76:a0:af:dd:b1:2c:8f: + ca:8b:8c:47:0d:f6:6e:fc:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + OpenSSL Generated Certificate + X509v3 Subject Key Identifier: + 1C:63:85:E1:1D:F3:89:2E:6C:4E:3F:FB:D0:10:64:5A:C1:22:6A:2A + X509v3 Authority Key Identifier: + keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + + Signature Algorithm: sha1WithRSAEncryption + 1d:6d:3e:bd:93:fd:bd:3e:17:b8:9f:f0:99:7f:db:50:5c:b2: + 01:42:03:b5:d5:94:05:d3:f6:8e:80:82:55:47:1f:58:f2:18: + 6c:ab:ef:43:2c:2f:10:e1:7c:c4:5c:cc:ac:50:50:22:42:aa: + 35:33:f5:b9:f3:a6:66:55:d9:36:f4:f2:e4:d4:d9:b5:2c:52: + 66:d4:21:17:97:22:b8:9b:d7:0e:7c:3d:ce:85:19:ca:c4:d2: + 58:62:31:c6:18:3e:44:fc:f4:30:b6:95:87:ee:21:4a:08:f0: + af:3c:8f:c4:ba:5e:a1:5c:37:1a:7d:7b:fe:66:ae:62:50:17: + 31:ca +-----BEGIN CERTIFICATE----- +MIICnzCCAgigAwIBAgIJAOTeAXbEeHh/MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg +T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x +MjA1MDMxMTExMTRaFw0yMzA1MDExMTExMTRaMEQxLjAsBgNVBAoMJXUxZGIgTE9D +QUwgVEVTVElORyBPTkxZLCBETyBOT1QgVFJVU1QxEjAQBgNVBAMMCWxvY2FsaG9z +dDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxh1y08Xk/NFM2eQIPpAQzj8f +h0odT38qWlLJZU/ZLL9pdRgatckJMgBH9WCqxt06hzdfFr7eKbXq/EF+63e732PD +Bh7t6aBnGvHs4Z33nI8c+sNmeznccK4JG5zAmsSQd0WOOZWpL5JDvScHWplRbnag +r92xLI/Ki4xHDfZu/GkCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0E +HxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBxjheEd +84kubE4/+9AQZFrBImoqMB8GA1UdIwQYMBaAFNs9k1FsMhVUjxBQ/ElPNhUou5Vt +MA0GCSqGSIb3DQEBBQUAA4GBAB1tPr2T/b0+F7if8Jl/21BcsgFCA7XVlAXT9o6A +glVHH1jyGGyr70MsLxDhfMRczKxQUCJCqjUz9bnzpmZV2Tb08uTU2bUsUmbUIReX +Irib1w58Pc6FGcrE0lhiMcYYPkT89DC2lYfuIUoI8K88j8S6XqFcNxp9e/5mrmJQ +FzHK +-----END CERTIFICATE----- diff --git a/testing/test_soledad/u1db_tests/testing-certs/testing.key b/testing/test_soledad/u1db_tests/testing-certs/testing.key new file mode 100644 index 00000000..d83d4920 --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/testing.key @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMYdctPF5PzRTNnk +CD6QEM4/H4dKHU9/KlpSyWVP2Sy/aXUYGrXJCTIAR/VgqsbdOoc3Xxa+3im16vxB +fut3u99jwwYe7emgZxrx7OGd95yPHPrDZns53HCuCRucwJrEkHdFjjmVqS+SQ70n +B1qZUW52oK/dsSyPyouMRw32bvxpAgMBAAECgYBs3lXxhjg1rhabTjIxnx19GTcM +M3Az9V+izweZQu3HJ1CeZiaXauhAr+LbNsniCkRVddotN6oCJdQB10QVxXBZc9Jz +HPJ4zxtZfRZlNMTMmG7eLWrfxpgWnb/BUjDb40yy1nhr9yhDUnI/8RoHDRHnAEHZ +/CnHGUrqcVcrY5zJAQJBAPLhBJg9W88JVmcOKdWxRgs7dLHnZb999Kv1V5mczmAi +jvGvbUmucqOqke6pTUHNYyNHqU6pySzGUi2cH+BAkFECQQDQ0VoAOysg6FVoT15v +tGh57t5sTiCZZ7PS8jwvtThsgA+vcf6c16XWzXgjGXSap4r2QDOY2rI5lsWLaQ8T ++fyZAkAfyFJRmbXp4c7srW3MCOahkaYzoZQu+syJtBFCiMJ40gzik5I5khpuUGPI +V19EvRu8AiSlppIsycb3MPb64XgBAkEAy7DrUf5le5wmc7G4NM6OeyJ+5LbxJbL6 +vnJ8My1a9LuWkVVpQCU7J+UVo2dZTuLPspW9vwTVhUeFOxAoHRxlQQJAFem93f7m +el2BkB2EFqU3onPejkZ5UrDmfmeOQR1axMQNSXqSxcJxqa16Ru1BWV2gcWRbwajQ +oc+kuJThu/r/Ug== +-----END PRIVATE KEY----- diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py new file mode 100644 index 00000000..f81001b9 --- /dev/null +++ b/testing/test_soledad/util.py @@ -0,0 +1,430 @@ +# -*- CODING: UTF-8 -*- +# util.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Utilities used by multiple test suites. +""" + + +import os +import tempfile +import shutil +import random +import string +import couchdb + +from uuid import uuid4 +from mock import Mock +from urlparse import urljoin +from StringIO import StringIO +from pysqlcipher import dbapi2 + +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest + +from leap.soledad.common import l2db +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db.remote import http_database + +from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common.couch.state import CouchServerState + +from leap.soledad.common.crypto import ENC_SCHEME_KEY + +from leap.soledad.client import Soledad +from leap.soledad.client import http_target +from leap.soledad.client import auth +from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import SQLCipherOptions + +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + + +PASSWORD = '123456' +ADDRESS = 'leap@leap.se' + + +def make_local_db_and_target(test): + db = test.create_database('test') + st = db.get_sync_target() + return db, st + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_sqlcipher_database_for_test(test, replica_uid): + db = SQLCipherDatabase( + SQLCipherOptions(':memory:', PASSWORD)) + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlcipher_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = make_sqlcipher_database_for_test(test, None) + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if 'sqlite_sequence' not in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +SQLCIPHER_SCENARIOS = [ + ('sqlcipher', {'make_database_for_test': make_sqlcipher_database_for_test, + 'copy_database_for_test': copy_sqlcipher_database_for_test, + 'make_document_for_test': make_document_for_test, }), +] + + +def make_soledad_app(state): + return SoledadApp(state) + + +def make_token_soledad_app(state): + app = SoledadApp(state) + + def _verify_authentication_data(uuid, auth_data): + if uuid.startswith('user-') and auth_data == 'auth-token': + return True + return False + + # we test for action authorization in leap.soledad.common.tests.test_server + def _verify_authorization(uuid, environ): + return True + + application = SoledadTokenAuthMiddleware(app) + application._verify_authentication_data = _verify_authentication_data + application._verify_authorization = _verify_authorization + return application + + +def make_soledad_document_for_test(test, doc_id, rev, content, + has_conflicts=False): + return SoledadDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_token_http_database_for_test(test, replica_uid): + test.startServer() + test.request_state._create_database(replica_uid) + + class _HTTPDatabaseWithToken( + http_database.HTTPDatabase, auth.TokenBasedAuth): + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + http_db = _HTTPDatabaseWithToken(test.getURL('test')) + http_db.set_token_credentials('user-uuid', 'auth-token') + return http_db + + +def copy_token_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + http_db = test.request_state._copy_database(db) + http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') + return http_db + + +def sync_via_synchronizer(test, db_source, db_target, trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +class MockedSharedDBTest(object): + + def get_default_shared_mock(self, put_doc_side_effect=None, + get_doc_return_value=None): + """ + Get a default class for mocking the shared DB + """ + class defaultMockSharedDB(object): + get_doc = Mock(return_value=get_doc_return_value) + put_doc = Mock(side_effect=put_doc_side_effect) + open = Mock(return_value=None) + close = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + return defaultMockSharedDB + + +def soledad_sync_target( + test, path, source_replica_uid=uuid4().hex, + sync_db=None, sync_enc_pool=None): + creds = {'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }} + return http_target.SoledadHTTPSyncTarget( + test.getURL(path), + source_replica_uid, + creds, + test._soledad._crypto, + None, # cert_file + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + + +# redefine the base leap test class so it inherits from twisted trial's +# TestCase. This is needed so trial knows that it has to manage a reactor and +# wait for deferreds returned by tests to be fired. +BaseLeapTest = type( + 'BaseLeapTest', (unittest.TestCase,), dict(BaseLeapTest.__dict__)) + + +class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): + + """ + Instantiates Soledad for usage in tests. + """ + defer_sync_encryption = False + + def setUp(self): + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + self.db2_file = os.path.join(self.tempdir, "db2.u1db") + self.email = ADDRESS + # open test dbs + self._db1 = l2db.open(self.db1_file, create=True, + document_factory=SoledadDocument) + self._db2 = l2db.open(self.db2_file, create=True, + document_factory=SoledadDocument) + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # initialize soledad by hand so we can control keys + # XXX check if this soledad is actually used + self._soledad = self._soledad_instance( + prefix=self.rand_prefix, user=self.email) + + def tearDown(self): + self._db1.close() + self._db2.close() + self._soledad.close() + + # restore paths + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + + def _delete_temporary_dirs(): + # XXX should not access "private" attrs + for f in [self._soledad.local_db_path, + self._soledad.secrets.secrets_path]: + if os.path.isfile(f): + os.unlink(f) + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + from twisted.internet import reactor + reactor.addSystemEventTrigger( + "after", "shutdown", _delete_temporary_dirs) + + def _soledad_instance(self, user=ADDRESS, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='https://127.0.0.1/', + cert_file=None, + shared_db_class=None, + auth_token='auth-token'): + + def _put_doc_side_effect(doc): + self._doc_put = doc + + if shared_db_class is not None: + MockSharedDB = shared_db_class + else: + MockSharedDB = self.get_default_shared_mock( + _put_doc_side_effect) + + soledad = Soledad( + user, + passphrase, + secrets_path=os.path.join( + self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, # Soledad will fail if not given an url. + cert_file=cert_file, + defer_encryption=self.defer_sync_encryption, + shared_db=MockSharedDB(), + auth_token=auth_token) + self.addCleanup(soledad.close) + return soledad + + def assertGetEncryptedDoc( + self, db, doc_id, doc_rev, content, has_conflicts): + """ + Assert that the document in the database looks correct. + """ + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + doc = db.get_doc(doc_id) + + if ENC_SCHEME_KEY in doc.content: + # XXX check for SYM_KEY too + key = self._soledad._crypto.doc_passphrase(doc.doc_id) + secret = self._soledad._crypto.secret + decrypted = decrypt_doc_dict( + doc.content, doc.doc_id, doc.rev, + key, secret) + doc.set_json(decrypted) + self.assertEqual(exp_doc.doc_id, doc.doc_id) + self.assertEqual(exp_doc.rev, doc.rev) + self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) + self.assertEqual(exp_doc.content, doc.content) + + +class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): + + """ + TestCase base class for tests against a real CouchDB server. + """ + + def setUp(self): + """ + Make sure we have a CouchDB instance for a test. + """ + self.couch_port = 5984 + self.couch_url = 'http://localhost:%d' % self.couch_port + self.couch_server = couchdb.Server(self.couch_url) + + def delete_db(self, name): + try: + self.couch_server.delete(name) + except: + # ignore if already missing + pass + + +class CouchServerStateForTests(CouchServerState): + + """ + This is a slightly modified CouchDB server state that allows for creating + a database. + + Ordinarily, the CouchDB server state does not allow some operations, + because for security purposes the Soledad Server should not even have + enough permissions to perform them. For tests, we allow database creation, + otherwise we'd have to create those databases in setUp/tearDown methods, + which is less pleasant than allowing the db to be automatically created. + """ + + def __init__(self, *args, **kwargs): + self.dbs = [] + super(CouchServerStateForTests, self).__init__(*args, **kwargs) + + def _create_database(self, replica_uid=None, dbname=None): + """ + Create db and append to a list, allowing test to close it later + """ + dbname = dbname or ('test-%s' % uuid4().hex) + db = CouchDatabase.open_database( + urljoin(self.couch_url, dbname), + True, + replica_uid=replica_uid or 'test', + ensure_ddocs=True) + self.dbs.append(db) + return db + + def ensure_database(self, dbname): + db = self._create_database(dbname=dbname) + return db, db.replica_uid + + +class SoledadWithCouchServerMixin( + BaseSoledadTest, + CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + BaseSoledadTest.setUp(self) + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.setUp(self) + + def tearDown(self): + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.tearDown(self) + # delete the test database + BaseSoledadTest.tearDown(self) + CouchDBTestCase.tearDown(self) + + def make_app(self): + self.request_state = CouchServerStateForTests(self.couch_url) + self.addCleanup(self.delete_dbs) + return self.make_app_with_state(self.request_state) + + def delete_dbs(self): + for db in self.request_state.dbs: + self.delete_db(db._dbname) |