summaryrefslogtreecommitdiff
path: root/common
diff options
context:
space:
mode:
Diffstat (limited to 'common')
-rw-r--r--common/setup.py2
-rw-r--r--common/src/leap/soledad/common/couch.py19
-rw-r--r--common/src/leap/soledad/common/ddocs/syncs/updates/state.js99
-rw-r--r--common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js19
-rw-r--r--common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js9
-rw-r--r--common/src/leap/soledad/common/ddocs/syncs/views/state/map.js16
-rw-r--r--common/src/leap/soledad/common/errors.py1
-rw-r--r--common/src/leap/soledad/common/tests/__init__.py44
-rw-r--r--common/src/leap/soledad/common/tests/server_state.py80
-rw-r--r--common/src/leap/soledad/common/tests/test_couch.py13
-rw-r--r--common/src/leap/soledad/common/tests/test_sqlcipher.py69
-rw-r--r--common/src/leap/soledad/common/tests/test_sync.py176
-rw-r--r--common/src/leap/soledad/common/tests/test_target.py160
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_sync.py4
-rw-r--r--common/src/leap/soledad/common/tests/util.py177
15 files changed, 758 insertions, 130 deletions
diff --git a/common/setup.py b/common/setup.py
index e142d958..6ee166ef 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -285,7 +285,7 @@ setup(
namespace_packages=["leap", "leap.soledad"],
packages=find_packages('src', exclude=['leap.soledad.common.tests']),
package_dir={'': 'src'},
- test_suite='leap.soledad.common.tests',
+ test_suite='leap.soledad.common.tests.load_tests',
install_requires=utils.parse_requirements(),
tests_require=utils.parse_requirements(
reqfiles=['pkg/requirements-testing.pip']),
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 0aa84170..3bc1f543 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -371,6 +371,7 @@ class CouchDatabase(CommonBackend):
MAX_GET_DOCS_THREADS = 20
update_handler_lock = defaultdict(threading.Lock)
+ sync_info_lock = defaultdict(threading.Lock)
class _GetDocThread(threading.Thread):
"""
@@ -440,7 +441,8 @@ class CouchDatabase(CommonBackend):
if not create:
raise DatabaseDoesNotExist()
server.create(dbname)
- return cls(url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs)
+ return cls(
+ url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs)
def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True):
"""
@@ -465,6 +467,10 @@ class CouchDatabase(CommonBackend):
self._database = Database(
urljoin(self._url, self._dbname),
self._session)
+ try:
+ self._database.info()
+ except ResourceNotFound:
+ raise DatabaseDoesNotExist()
if replica_uid is not None:
self._set_replica_uid(replica_uid)
if ensure_ddocs:
@@ -576,6 +582,8 @@ class CouchDatabase(CommonBackend):
_replica_uid = property(_get_replica_uid, _set_replica_uid)
+ replica_uid = property(_get_replica_uid)
+
def _get_generation(self):
"""
Return the current generation.
@@ -869,7 +877,7 @@ class CouchDatabase(CommonBackend):
# Date.prototype.getTime() which was used before inside a couchdb
# update handler.
(int(time.time() * 1000),
- self._allocate_transaction_id()))
+ self._allocate_transaction_id()))
# build the couch document
couch_doc = {
'_id': doc.doc_id,
@@ -1537,8 +1545,8 @@ class CouchServerState(ServerState):
:param dbname: The name of the database to ensure.
:type dbname: str
- :return: The CouchDatabase object and the replica uid.
- :rtype: (CouchDatabase, str)
+ :raise Unauthorized: Always, because Soledad server is not allowed to
+ create databases.
"""
raise Unauthorized()
@@ -1548,6 +1556,9 @@ class CouchServerState(ServerState):
:param dbname: The name of the database to delete.
:type dbname: str
+
+ :raise Unauthorized: Always, because Soledad server is not allowed to
+ delete databases.
"""
raise Unauthorized()
diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js
new file mode 100644
index 00000000..cb2b6b7b
--- /dev/null
+++ b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js
@@ -0,0 +1,99 @@
+/**
+ * This update handler stores information about ongoing synchronization
+ * attempts from distinct source replicas.
+ *
+ * Normally, u1db synchronization occurs during one POST request. In order to
+ * split that into many serial POST requests, we store the state of each sync
+ * in the server, using a document with id 'u1db_sync_state'. To identify
+ * each sync attempt, we use a sync_id sent by the client. If we ever receive
+ * a new sync_id, we trash current data for that source replica and start
+ * over.
+ *
+ * We expect the following in the document body:
+ *
+ * {
+ * 'source_replica_uid': '<source_replica_uid>',
+ * 'sync_id': '<sync_id>',
+ * 'seen_ids': [['<doc_id>', <at_gen>], ...], // optional
+ * 'changes_to_return': [ // optional
+ * 'gen': <gen>,
+ * 'trans_id': '<trans_id>',
+ * 'changes_to_return': [[<doc_id>', <gen>, '<trans_id>'], ...]
+ * ],
+ * }
+ *
+ * The format of the final document stored on server is:
+ *
+ * {
+ * '_id': '<str>',
+ * '_rev' '<str>',
+ * 'ongoing_syncs': {
+ * '<source_replica_uid>': {
+ * 'seen_ids': [['<doc_id>', <at_gen>[, ...],
+ * 'changes_to_return': {
+ * 'gen': <gen>,
+ * 'trans_id': '<trans_id>',
+ * 'changes_to_return': [
+ * ['<doc_id>', <gen>, '<trans_id>'],
+ * ...,
+ * ],
+ * },
+ * },
+ * ... // info about other source replicas here
+ * }
+ * }
+ */
+function(doc, req) {
+
+ // prevent updates to alien documents
+ if (doc != null && doc['_id'] != 'u1db_sync_state')
+ return [null, 'invalid data'];
+
+ // create the document if it doesn't exist
+ if (!doc)
+ doc = {
+ '_id': 'u1db_sync_state',
+ 'ongoing_syncs': {},
+ };
+
+ // parse and validate incoming data
+ var body = JSON.parse(req.body);
+ if (body['source_replica_uid'] == null)
+ return [null, 'invalid data']
+ var source_replica_uid = body['source_replica_uid'];
+
+ // trash outdated sync data for that replica if that exists
+ if (doc['ongoing_syncs'][source_replica_uid] != null &&
+ doc['ongoing_syncs'][source_replica_uid] == null)
+ delete doc['ongoing_syncs'][source_replica_uid];
+
+ // create an entry for that source replica
+ if (doc['ongoing_syncs'][source_replica_uid] == null)
+ doc['ongoing_syncs'][source_replica_uid] = {
+ 'seen_ids': {},
+ 'changes_to_return': null,
+ };
+
+ // incoming meta-data values should be exclusive, so we count how many
+ // arrived and deny to accomplish the transaction if the count is high.
+ var incoming_values = 0;
+ var info = doc['ongoing_syncs'][source_replica_uid]
+
+ // add incoming seen id
+ if ('seen_id' in body) {
+ info['seen_ids'][body['seen_id'][0]] = body['seen_id'][1];
+ incoming_values += 1;
+ }
+
+ // add incoming changes_to_return
+ if ('changes_to_return' in body) {
+ info['changes_to_return'] = body['changes_to_return'];
+ incoming_values += 1;
+ }
+
+ if (incoming_values != 1)
+ return [null, 'invalid data'];
+
+ return [doc, 'ok'];
+}
+
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js
new file mode 100644
index 00000000..04ceb2ec
--- /dev/null
+++ b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js
@@ -0,0 +1,19 @@
+function(doc) {
+ if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null)
+ for (var source_replica_uid in doc['ongoing_syncs']) {
+ var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return'];
+ if (changes == null)
+ emit([source_replica_uid, 0], null);
+ else if (changes.length == 0)
+ emit([source_replica_uid, 0], []);
+ else
+ for (var i = 0; i < changes['changes_to_return'].length; i++)
+ emit(
+ [source_replica_uid, i],
+ {
+ 'gen': changes['gen'],
+ 'trans_id': changes['trans_id'],
+ 'next_change_to_return': changes['changes_to_return'][i],
+ });
+ }
+}
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js
new file mode 100644
index 00000000..34c65b3f
--- /dev/null
+++ b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js
@@ -0,0 +1,9 @@
+function(doc) {
+ if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null)
+ for (var source_replica_uid in doc['ongoing_syncs'])
+ emit(
+ source_replica_uid,
+ {
+ 'seen_ids': doc['ongoing_syncs'][source_replica_uid]['seen_ids'],
+ });
+}
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js
new file mode 100644
index 00000000..1d8f8e84
--- /dev/null
+++ b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js
@@ -0,0 +1,16 @@
+function(doc) {
+ if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null)
+ for (var source_replica_uid in doc['ongoing_syncs']) {
+ var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return'];
+ if (changes == null)
+ emit(source_replica_uid, null);
+ else
+ emit(
+ source_replica_uid,
+ {
+ 'gen': changes['gen'],
+ 'trans_id': changes['trans_id'],
+ 'number_of_changes': changes['changes_to_return'].length
+ });
+ }
+}
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index 3a7eadd2..ea4bf7f6 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -62,6 +62,7 @@ class InvalidAuthTokenError(errors.Unauthorized):
wire_descrition = "invalid auth token"
status = 401
+
#
# LockResource errors
#
diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py
index 88f98272..a38bdaed 100644
--- a/common/src/leap/soledad/common/tests/__init__.py
+++ b/common/src/leap/soledad/common/tests/__init__.py
@@ -1,3 +1,21 @@
+# -*- coding: utf-8 -*-
+# __init__.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
"""
Tests to make sure Soledad provides U1DB functionality and more.
"""
@@ -273,3 +291,29 @@ RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
=JTFu
-----END PGP PRIVATE KEY BLOCK-----
"""
+
+
+def load_tests():
+ """
+ Build a test suite that includes all tests in leap.soledad.common.tests
+ but does not include tests in the u1db_tests/ subfolder. The reason for
+ not including those tests are:
+
+ 1. they by themselves only test u1db functionality in the u1db module
+ (despite we use them as basis for testing soledad functionalities).
+
+ 2. they would fail because we monkey patch u1db's remote http server
+ to add soledad functionality we need.
+ """
+ import unittest
+ import glob
+ import imp
+ tests_prefix = os.path.join(
+ '.', 'src', 'leap', 'soledad', 'common', 'tests')
+ suite = unittest.TestSuite()
+ for testcase in glob.glob(os.path.join(tests_prefix, 'test_*.py')):
+ modname = os.path.basename(os.path.splitext(testcase)[0])
+ f, pathname, description = imp.find_module(modname, [tests_prefix])
+ module = imp.load_module(modname, f, pathname, description)
+ suite.addTest(unittest.TestLoader().loadTestsFromModule(module))
+ return suite
diff --git a/common/src/leap/soledad/common/tests/server_state.py b/common/src/leap/soledad/common/tests/server_state.py
new file mode 100644
index 00000000..2bc15377
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/server_state.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# server_state.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+State for servers to be used in tests.
+"""
+
+
+import os
+import errno
+import tempfile
+
+
+from u1db.remote.server_state import ServerState
+from leap.soledad.common.tests.util import (
+ copy_sqlcipher_database_for_test,
+)
+
+
+class ServerStateForTests(ServerState):
+ """Passed to a Request when it is instantiated.
+
+ This is used to track server-side state, such as working-directory, open
+ databases, etc.
+ """
+
+ def __init__(self):
+ self._workingdir = tempfile.mkdtemp()
+
+ def _relpath(self, relpath):
+ return os.path.join(self._workingdir, relpath)
+
+ def open_database(self, path):
+ """Open a database at the given location."""
+ from leap.soledad.client.sqlcipher import SQLCipherDatabase
+ return SQLCipherDatabase.open_database(path, '123', False)
+
+ def create_database(self, path):
+ """Create a database at the given location."""
+ from leap.soledad.client.sqlcipher import SQLCipherDatabase
+ return SQLCipherDatabase.open_database(path, '123', True)
+
+ def check_database(self, path):
+ """Check if the database at the given location exists.
+
+ Simply returns if it does or raises DatabaseDoesNotExist.
+ """
+ db = self.open_database(path)
+ db.close()
+
+ def ensure_database(self, path):
+ """Ensure database at the given location."""
+ from leap.soledad.client.sqlcipher import SQLCipherDatabase
+ full_path = self._relpath(path)
+ db = SQLCipherDatabase.open_database(full_path, '123', False)
+ return db, db._replica_uid
+
+ def delete_database(self, path):
+ """Delete database at the given location."""
+ from leap.u1db.backends import sqlite_backend
+ full_path = self._relpath(path)
+ sqlite_backend.SQLiteDatabase.delete_database(full_path)
+
+ def _copy_database(self, db):
+ return copy_sqlcipher_database_for_test(None, db)
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 17d4a519..a1fa9568 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -249,9 +249,7 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase):
# if current test is `test_close` we have to use saved objects to
# delete the database because the close() method will have removed the
# references needed to do it using the CouchDatabase.
- if self.id() == \
- 'leap.soledad.common.tests.test_couch.CouchTests.' \
- 'test_close(couch)':
+ if self.id().endswith('test_couch.CouchTests.test_close(couch)'):
session = couch.Session()
server = Server(url=self._url, session=session)
del(server[self._dbname])
@@ -365,8 +363,6 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests,
# The following tests need that the database have an index, so we fake one.
-old_class = couch.CouchDatabase
-
from u1db.backends.inmemory import InMemoryIndex
@@ -444,7 +440,12 @@ class IndexedCouchDatabase(couch.CouchDatabase):
return list(set([tuple(key.split('\x01')) for key in keys]))
-couch.CouchDatabase = IndexedCouchDatabase
+# monkey patch CouchDatabase (once) to include virtual indexes
+if getattr(couch.CouchDatabase, '_old_class', None) is None:
+ old_class = couch.CouchDatabase
+ IndexedCouchDatabase._old_class = old_class
+ couch.CouchDatabase = IndexedCouchDatabase
+
sync_scenarios = []
for name, scenario in COUCH_SCENARIOS:
diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py
index c79a6045..891aca0f 100644
--- a/common/src/leap/soledad/common/tests/test_sqlcipher.py
+++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py
@@ -30,6 +30,7 @@ import threading
from pysqlcipher import dbapi2
from StringIO import StringIO
+from urlparse import urljoin
# u1db stuff.
@@ -54,19 +55,26 @@ from leap.soledad.common.crypto import (
ENC_JSON_KEY,
ENC_SCHEME_KEY,
)
-from leap.soledad.client.target import decrypt_doc
+from leap.soledad.client.target import (
+ decrypt_doc,
+ SoledadSyncTarget,
+)
# u1db tests stuff.
+from leap.common.testing.basetest import BaseLeapTest
from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest
from leap.soledad.common.tests.u1db_tests import test_sqlite_backend
from leap.soledad.common.tests.u1db_tests import test_backends
from leap.soledad.common.tests.u1db_tests import test_open
from leap.soledad.common.tests.u1db_tests import test_sync
-from leap.soledad.client.target import SoledadSyncTarget
-from leap.common.testing.basetest import BaseLeapTest
-
-PASSWORD = '123456'
+from leap.soledad.common.tests.util import (
+ make_sqlcipher_database_for_test,
+ copy_sqlcipher_database_for_test,
+ make_soledad_app,
+ SoledadWithCouchServerMixin,
+ PASSWORD,
+)
#-----------------------------------------------------------------------------
@@ -88,32 +96,6 @@ class TestSQLCipherBackendImpl(tests.TestCase):
# The following tests come from `u1db.tests.test_backends`.
#-----------------------------------------------------------------------------
-def make_sqlcipher_database_for_test(test, replica_uid):
- db = SQLCipherDatabase(':memory:', PASSWORD)
- db._set_replica_uid(replica_uid)
- return db
-
-
-def copy_sqlcipher_database_for_test(test, db):
- # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
- # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
- # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
- # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
- # HOUSE.
- new_db = SQLCipherDatabase(':memory:', PASSWORD)
- tmpfile = StringIO()
- for line in db._db_handle.iterdump():
- if not 'sqlite_sequence' in line: # work around bug in iterdump
- tmpfile.write('%s\n' % line)
- tmpfile.seek(0)
- new_db._db_handle = dbapi2.connect(':memory:')
- new_db._db_handle.cursor().executescript(tmpfile.read())
- new_db._db_handle.commit()
- new_db._set_replica_uid(db._replica_uid)
- new_db._factory = db._factory
- return new_db
-
-
def make_document_for_test(test, doc_id, rev, content, has_conflicts=False):
return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts)
@@ -451,7 +433,7 @@ sync_scenarios.append(('pyleap', {
'copy_database_for_test': test_sync.copy_database_for_http_test,
'make_document_for_test': make_document_for_test,
'make_app_with_state': tests.test_remote_sync_target.make_http_app,
- 'do_sync': sync_via_synchronizer_and_leap,
+ 'do_sync': test_sync.sync_via_synchronizer,
}))
@@ -616,7 +598,7 @@ class SQLCipherDatabaseSyncTests(
# update on 1
doc1.set_json('{"a": 3}')
self.db1.put_doc(doc1)
- # conflicts
+ # conflicts
self.sync(self.db2, self.db1)
self.sync(db3, self.db1)
self.assertTrue(self.db2.get_doc('the-doc').has_conflicts)
@@ -658,32 +640,35 @@ class SQLCipherDatabaseSyncTests(
'return': {'docs': [], 'last_gen': 1}})
-def _make_local_db_and_leap_target(test, path='test'):
+def _make_local_db_and_token_http_target(test, path='test'):
test.startServer()
db = test.request_state._create_database(os.path.basename(path))
- st = SoledadSyncTarget.connect(test.getURL(path), test._soledad._crypto)
+ st = SoledadSyncTarget.connect(
+ test.getURL(path), crypto=test._soledad._crypto)
st.set_token_credentials('user-uuid', 'auth-token')
return db, st
target_scenarios = [
('leap', {
- 'create_db_and_target': _make_local_db_and_leap_target,
- 'make_app_with_state': tests.test_remote_sync_target.make_http_app}),
+ 'create_db_and_target': _make_local_db_and_token_http_target,
+# 'make_app_with_state': tests.test_remote_sync_target.make_http_app,
+ 'make_app_with_state': make_soledad_app,
+ 'do_sync': test_sync.sync_via_synchronizer}),
]
class SQLCipherSyncTargetTests(
- test_sync.DatabaseSyncTargetTests, BaseSoledadTest):
+ SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests):
scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS,
target_scenarios))
- def setUp(self):
- test_sync.DatabaseSyncTargetTests.setUp(self)
+ whitebox = False
- def tearDown(self):
- test_sync.DatabaseSyncTargetTests.tearDown(self)
+ def setUp(self):
+ self.main_test_class = test_sync.DatabaseSyncTargetTests
+ SoledadWithCouchServerMixin.setUp(self)
def test_sync_exchange(self):
"""
diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py
new file mode 100644
index 00000000..fd4a2797
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_sync.py
@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+# test_sync.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import mock
+import os
+import json
+import tempfile
+import threading
+import time
+from urlparse import urljoin
+
+from leap.soledad.common.couch import (
+ CouchServerState,
+ CouchDatabase,
+)
+
+from leap.soledad.common.tests.u1db_tests import (
+ TestCaseWithServer,
+ simple_doc,
+)
+from leap.soledad.common.tests.test_couch import CouchDBTestCase
+from leap.soledad.common.tests.test_target import (
+ make_token_soledad_app,
+ make_leap_document_for_test,
+ token_leap_sync_target,
+)
+
+from leap.soledad.client import (
+ Soledad,
+ target,
+)
+
+
+class InterruptableSyncTestCase(
+ CouchDBTestCase, TestCaseWithServer):
+ """
+ Tests for encrypted sync using Soledad server backed by a couch database.
+ """
+
+ @staticmethod
+ def make_app_with_state(state):
+ return make_token_soledad_app(state)
+
+ make_document_for_test = make_leap_document_for_test
+
+ sync_target = token_leap_sync_target
+
+ def _soledad_instance(self, user='user-uuid', passphrase=u'123',
+ prefix='',
+ secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
+ local_db_path='soledad.u1db', server_url='',
+ cert_file=None, auth_token=None, secret_id=None):
+ """
+ Instantiate Soledad.
+ """
+
+ # this callback ensures we save a document which is sent to the shared
+ # db.
+ def _put_doc_side_effect(doc):
+ self._doc_put = doc
+
+ # we need a mocked shared db or else Soledad will try to access the
+ # network to find if there are uploaded secrets.
+ class MockSharedDB(object):
+
+ get_doc = mock.Mock(return_value=None)
+ put_doc = mock.Mock(side_effect=_put_doc_side_effect)
+ lock = mock.Mock(return_value=('atoken', 300))
+ unlock = mock.Mock()
+
+ def __call__(self):
+ return self
+
+ Soledad._shared_db = MockSharedDB()
+ return Soledad(
+ user,
+ passphrase,
+ secrets_path=os.path.join(self.tempdir, prefix, secrets_path),
+ local_db_path=os.path.join(
+ self.tempdir, prefix, local_db_path),
+ server_url=server_url,
+ cert_file=cert_file,
+ auth_token=auth_token,
+ secret_id=secret_id)
+
+ def make_app(self):
+ self.request_state = CouchServerState(self._couch_url, 'shared',
+ 'tokens')
+ return self.make_app_with_state(self.request_state)
+
+ def setUp(self):
+ TestCaseWithServer.setUp(self)
+ CouchDBTestCase.setUp(self)
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ self._couch_url = 'http://localhost:' + str(self.wrapper.port)
+
+ def tearDown(self):
+ CouchDBTestCase.tearDown(self)
+ TestCaseWithServer.tearDown(self)
+
+ def test_interruptable_sync(self):
+ """
+ Test if Soledad can sync many smallfiles.
+ """
+
+ class _SyncInterruptor(threading.Thread):
+ """
+ A thread meant to interrupt the sync process.
+ """
+
+ def __init__(self, soledad, couchdb):
+ self._soledad = soledad
+ self._couchdb = couchdb
+ threading.Thread.__init__(self)
+
+ def run(self):
+ while db._get_generation() < 2:
+ time.sleep(1)
+ self._soledad.stop_sync()
+ time.sleep(1)
+
+ number_of_docs = 10
+ self.startServer()
+
+ # instantiate soledad and create a document
+ sol = self._soledad_instance(
+ # token is verified in test_target.make_token_soledad_app
+ auth_token='auth-token'
+ )
+ _, doclist = sol.get_all_docs()
+ self.assertEqual([], doclist)
+
+ # create many small files
+ for i in range(0, number_of_docs):
+ sol.create_doc(json.loads(simple_doc))
+
+ # ensure remote db exists before syncing
+ db = CouchDatabase.open_database(
+ urljoin(self._couch_url, 'user-user-uuid'),
+ create=True,
+ ensure_ddocs=True)
+
+ # create interruptor thread
+ t = _SyncInterruptor(sol, db)
+ t.start()
+
+ # sync with server
+ sol._server_url = self.getURL()
+ sol.sync() # this will be interrupted when couch db gen >= 2
+ t.join()
+
+ # recover the sync process
+ sol.sync()
+
+ gen, doclist = db.get_all_docs()
+ self.assertEqual(number_of_docs, len(doclist))
+
+ # delete remote database
+ db.delete_database()
+ db.close()
+ sol.close()
diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py
index c1e00d52..3457a3e1 100644
--- a/common/src/leap/soledad/common/tests/test_target.py
+++ b/common/src/leap/soledad/common/tests/test_target.py
@@ -27,6 +27,7 @@ import simplejson as json
import cStringIO
+from u1db import SyncTarget
from u1db.sync import Synchronizer
from u1db.remote import (
http_client,
@@ -39,14 +40,20 @@ from leap.soledad.client import (
target,
auth,
VerifiedHTTPSConnection,
+ sync,
)
from leap.soledad.common.document import SoledadDocument
-from leap.soledad.server import SoledadApp
from leap.soledad.server.auth import SoledadTokenAuthMiddleware
from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests import BaseSoledadTest
+from leap.soledad.common.tests.util import (
+ make_sqlcipher_database_for_test,
+ make_soledad_app,
+ make_token_soledad_app,
+ SoledadWithCouchServerMixin,
+)
from leap.soledad.common.tests.u1db_tests import test_backends
from leap.soledad.common.tests.u1db_tests import test_http_database
from leap.soledad.common.tests.u1db_tests import test_http_client
@@ -54,6 +61,10 @@ from leap.soledad.common.tests.u1db_tests import test_document
from leap.soledad.common.tests.u1db_tests import test_remote_sync_target
from leap.soledad.common.tests.u1db_tests import test_https
from leap.soledad.common.tests.u1db_tests import test_sync
+from leap.soledad.common.tests.test_couch import (
+ CouchDBTestCase,
+ CouchDBWrapper,
+)
#-----------------------------------------------------------------------------
@@ -66,28 +77,6 @@ def make_leap_document_for_test(test, doc_id, rev, content,
doc_id, rev, content, has_conflicts=has_conflicts)
-def make_soledad_app(state):
- return SoledadApp(state)
-
-
-def make_token_soledad_app(state):
- app = SoledadApp(state)
-
- def _verify_authentication_data(uuid, auth_data):
- if uuid == 'user-uuid' and auth_data == 'auth-token':
- return True
- return False
-
- # we test for action authorization in leap.soledad.common.tests.test_server
- def _verify_authorization(uuid, environ):
- return True
-
- application = SoledadTokenAuthMiddleware(app)
- application._verify_authentication_data = _verify_authentication_data
- application._verify_authorization = _verify_authorization
- return application
-
-
LEAP_SCENARIOS = [
('http', {
'make_database_for_test': test_backends.make_http_database_for_test,
@@ -362,16 +351,47 @@ def token_leap_sync_target(test, path):
return st
+def make_local_db_and_soledad_target(test, path='test'):
+ test.startServer()
+ db = test.request_state._create_database(os.path.basename(path))
+ st = target.SoledadSyncTarget.connect(
+ test.getURL(path), crypto=test._soledad._crypto)
+ return db, st
+
+
+def make_local_db_and_token_soledad_target(test):
+ db, st = make_local_db_and_soledad_target(test, 'test')
+ st.set_token_credentials('user-uuid', 'auth-token')
+ return db, st
+
+
class TestSoledadSyncTarget(
- test_remote_sync_target.TestRemoteSyncTargets, BaseSoledadTest):
+ SoledadWithCouchServerMixin,
+ test_remote_sync_target.TestRemoteSyncTargets):
scenarios = [
('token_soledad',
{'make_app_with_state': make_token_soledad_app,
'make_document_for_test': make_leap_document_for_test,
+ 'create_db_and_target': make_local_db_and_token_soledad_target,
+ 'make_database_for_test': make_sqlcipher_database_for_test,
'sync_target': token_leap_sync_target}),
]
+ def setUp(self):
+ tests.TestCaseWithServer.setUp(self)
+ self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets
+ SoledadWithCouchServerMixin.setUp(self)
+ self.startServer()
+ self.db1 = make_sqlcipher_database_for_test(self, 'test1')
+ self.db2 = self.request_state._create_database('test2')
+
+ def tearDown(self):
+ SoledadWithCouchServerMixin.tearDown(self)
+ tests.TestCaseWithServer.tearDown(self)
+ db, _ = self.request_state.ensure_database('test2')
+ db.delete_database()
+
def test_sync_exchange_send(self):
"""
Test for sync exchanging send of document.
@@ -383,7 +403,7 @@ class TestSoledadSyncTarget(
remote_target = self.getSyncTarget('test')
other_docs = []
- def receive_doc(doc):
+ def receive_doc(doc, gen, trans_id):
other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
@@ -398,7 +418,10 @@ class TestSoledadSyncTarget(
"""
Test for sync exchange failure and retry.
- This test was adapted to decrypt remote content before assert.
+ This test was adapted to:
+ - decrypt remote content before assert.
+ - not expect a bounced document because soledad has stateful
+ recoverable sync.
"""
self.startServer()
@@ -412,7 +435,7 @@ class TestSoledadSyncTarget(
_put_doc_if_newer = db._put_doc_if_newer
trigger_ids = ['doc-here2']
- def bomb_put_doc_if_newer(doc, save_conflict,
+ def bomb_put_doc_if_newer(self, doc, save_conflict,
replica_uid=None, replica_gen=None,
replica_trans_id=None):
if doc.doc_id in trigger_ids:
@@ -421,7 +444,9 @@ class TestSoledadSyncTarget(
replica_uid=replica_uid,
replica_gen=replica_gen,
replica_trans_id=replica_trans_id)
- self.patch(db, '_put_doc_if_newer', bomb_put_doc_if_newer)
+ from leap.soledad.common.tests.test_couch import IndexedCouchDatabase
+ self.patch(
+ IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer)
remote_target = self.getSyncTarget('test')
other_changes = []
@@ -455,10 +480,11 @@ class TestSoledadSyncTarget(
self.assertEqual(
(11, 'T-sud'), db._get_replica_gen_and_trans_id('replica'))
self.assertEqual(2, new_gen)
- # bounced back to us
- self.assertEqual(
- ('doc-here', 'replica:1', '{"value": "here"}', 1),
- other_changes[0][:-1])
+ # we do not expect the document to be bounced back because soledad has
+ # stateful sync
+ #self.assertEqual(
+ # ('doc-here', 'replica:1', '{"value": "here"}', 1),
+ # other_changes[0][:-1])
def test_sync_exchange_send_ensure_callback(self):
"""
@@ -471,7 +497,7 @@ class TestSoledadSyncTarget(
other_docs = []
replica_uid_box = []
- def receive_doc(doc):
+ def receive_doc(doc, gen, trans_id):
other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
def ensure_cb(replica_uid):
@@ -489,6 +515,11 @@ class TestSoledadSyncTarget(
self.assertGetEncryptedDoc(
db, 'doc-here', 'replica:1', '{"value": "here"}', False)
+ def test_sync_exchange_in_stream_error(self):
+ # we bypass this test because our sync_exchange process does not
+ # return u1db error 503 "unavailable" for now.
+ pass
+
#-----------------------------------------------------------------------------
# The following tests come from `u1db.tests.test_https`.
@@ -595,42 +626,34 @@ class TestHTTPDatabaseWithCreds(
# The following tests come from `u1db.tests.test_sync`.
#-----------------------------------------------------------------------------
-def _make_local_db_and_leap_target(test, path='test'):
- test.startServer()
- db = test.request_state._create_database(os.path.basename(path))
- st = target.SoledadSyncTarget.connect(
- test.getURL(path), crypto=test._soledad._crypto)
- return db, st
-
-
-def _make_local_db_and_token_leap_target(test):
- db, st = _make_local_db_and_leap_target(test, 'test')
- st.set_token_credentials('user-uuid', 'auth-token')
- return db, st
-
-
target_scenarios = [
('token_leap', {'create_db_and_target':
- _make_local_db_and_token_leap_target,
- 'make_app_with_state': make_token_soledad_app}),
+ make_local_db_and_token_soledad_target,
+ 'make_app_with_state': make_soledad_app}),
]
class SoledadDatabaseSyncTargetTests(
- test_sync.DatabaseSyncTargetTests, BaseSoledadTest):
+ SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests):
scenarios = (
tests.multiply_scenarios(
tests.DatabaseBaseTests.scenarios,
target_scenarios))
+ whitebox = False
+
+ def setUp(self):
+ self.main_test_class = test_sync.DatabaseSyncTargetTests
+ SoledadWithCouchServerMixin.setUp(self)
+
def test_sync_exchange(self):
"""
Test sync exchange.
This test was adapted to decrypt remote content before assert.
"""
- sol = _make_local_db_and_leap_target(self)
+ sol, _ = make_local_db_and_soledad_target(self)
docs_by_gen = [
(self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
'T-sid')]
@@ -703,17 +726,15 @@ class SoledadDatabaseSyncTargetTests(
[(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
-class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest):
+class TestSoledadDbSync(
+ SoledadWithCouchServerMixin, test_sync.TestDbSync):
"""Test db.sync remote sync shortcut"""
scenarios = [
- ('py-http', {
- 'make_app_with_state': make_soledad_app,
- 'make_database_for_test': tests.make_memory_database_for_test,
- }),
('py-token-http', {
+ 'create_db_and_target': make_local_db_and_token_soledad_target,
'make_app_with_state': make_token_soledad_app,
- 'make_database_for_test': tests.make_memory_database_for_test,
+ 'make_database_for_test': make_sqlcipher_database_for_test,
'token': True
}),
]
@@ -721,6 +742,10 @@ class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest):
oauth = False
token = False
+ def setUp(self):
+ self.main_test_class = test_sync.TestDbSync
+ SoledadWithCouchServerMixin.setUp(self)
+
def do_sync(self, target_name):
"""
Perform sync using SoledadSyncTarget and Token auth.
@@ -748,7 +773,7 @@ class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest):
"""
doc1 = self.db.create_doc_from_json(tests.simple_doc)
doc2 = self.db2.create_doc_from_json(tests.nested_doc)
- local_gen_before_sync = self.do_sync('test2.db')
+ local_gen_before_sync = self.do_sync('test2')
gen, _, changes = self.db.whats_changed(local_gen_before_sync)
self.assertEqual(1, len(changes))
self.assertEqual(doc2.doc_id, changes[0][0])
@@ -760,24 +785,9 @@ class TestSoledadDbSync(test_sync.TestDbSync, BaseSoledadTest):
def test_db_sync_autocreate(self):
"""
- Test sync.
-
- Adapted to check for encrypted content.
+ We bypass this test because we never need to autocreate databases.
"""
- doc1 = self.db.create_doc_from_json(tests.simple_doc)
- local_gen_before_sync = self.do_sync('test3.db')
- gen, _, changes = self.db.whats_changed(local_gen_before_sync)
- self.assertEqual(0, gen - local_gen_before_sync)
- db3 = self.request_state.open_database('test3.db')
- gen, _, changes = db3.whats_changed()
- self.assertEqual(1, len(changes))
- self.assertEqual(doc1.doc_id, changes[0][0])
- self.assertGetEncryptedDoc(
- db3, doc1.doc_id, doc1.rev, tests.simple_doc, False)
- t_gen, _ = self.db._get_replica_gen_and_trans_id('test3.db')
- s_gen, _ = db3._get_replica_gen_and_trans_id('test1')
- self.assertEqual(1, t_gen)
- self.assertEqual(1, s_gen)
+ pass
load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
index a37c36db..633fd8dd 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
@@ -1155,12 +1155,12 @@ class TestDbSync(tests.TestCaseWithServer):
super(TestDbSync, self).setUp()
self.startServer()
self.db = self.make_database_for_test(self, 'test1')
- self.db2 = self.request_state._create_database('test2.db')
+ self.db2 = self.request_state._create_database('test2')
def test_db_sync(self):
doc1 = self.db.create_doc_from_json(tests.simple_doc)
doc2 = self.db2.create_doc_from_json(tests.nested_doc)
- local_gen_before_sync = self.do_sync('test2.db')
+ local_gen_before_sync = self.do_sync('test2')
gen, _, changes = self.db.whats_changed(local_gen_before_sync)
self.assertEqual(1, len(changes))
self.assertEqual(doc2.doc_id, changes[0][0])
diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py
new file mode 100644
index 00000000..249cbdaa
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/util.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# util.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Utilities used by multiple test suites.
+"""
+
+
+import tempfile
+import shutil
+from urlparse import urljoin
+
+from StringIO import StringIO
+from pysqlcipher import dbapi2
+from u1db.errors import DatabaseDoesNotExist
+
+
+from leap.soledad.common import soledad_assert
+from leap.soledad.common.couch import CouchDatabase, CouchServerState
+from leap.soledad.server import SoledadApp
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+
+from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest
+from leap.soledad.common.tests.test_couch import CouchDBWrapper, CouchDBTestCase
+
+
+from leap.soledad.client.sqlcipher import SQLCipherDatabase
+
+
+PASSWORD = '123456'
+
+
+def make_sqlcipher_database_for_test(test, replica_uid):
+ db = SQLCipherDatabase(':memory:', PASSWORD)
+ db._set_replica_uid(replica_uid)
+ return db
+
+
+def copy_sqlcipher_database_for_test(test, db):
+ # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
+ # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
+ # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
+ # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
+ # HOUSE.
+ new_db = SQLCipherDatabase(':memory:', PASSWORD)
+ tmpfile = StringIO()
+ for line in db._db_handle.iterdump():
+ if not 'sqlite_sequence' in line: # work around bug in iterdump
+ tmpfile.write('%s\n' % line)
+ tmpfile.seek(0)
+ new_db._db_handle = dbapi2.connect(':memory:')
+ new_db._db_handle.cursor().executescript(tmpfile.read())
+ new_db._db_handle.commit()
+ new_db._set_replica_uid(db._replica_uid)
+ new_db._factory = db._factory
+ return new_db
+
+
+def make_soledad_app(state):
+ return SoledadApp(state)
+
+
+def make_token_soledad_app(state):
+ app = SoledadApp(state)
+
+ def _verify_authentication_data(uuid, auth_data):
+ if uuid == 'user-uuid' and auth_data == 'auth-token':
+ return True
+ return False
+
+ # we test for action authorization in leap.soledad.common.tests.test_server
+ def _verify_authorization(uuid, environ):
+ return True
+
+ application = SoledadTokenAuthMiddleware(app)
+ application._verify_authentication_data = _verify_authentication_data
+ application._verify_authorization = _verify_authorization
+ return application
+
+
+class CouchServerStateForTests(CouchServerState):
+ """
+ This is a slightly modified CouchDB server state that allows for creating
+ a database.
+
+ Ordinarily, the CouchDB server state does not allow some operations,
+ because for security purposes the Soledad Server should not even have
+ enough permissions to perform them. For tests, we allow database creation,
+ otherwise we'd have to create those databases in setUp/tearDown methods,
+ which is less pleasant than allowing the db to be automatically created.
+ """
+
+ def _create_database(self, dbname):
+ return CouchDatabase.open_database(
+ urljoin(self._couch_url, dbname),
+ True,
+ replica_uid=dbname,
+ ensure_ddocs=True)
+
+ def ensure_database(self, dbname):
+ db = self._create_database(dbname)
+ return db, db.replica_uid
+
+
+class SoledadWithCouchServerMixin(
+ BaseSoledadTest,
+ CouchDBTestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Make sure we have a CouchDB instance for a test.
+ """
+ # from BaseLeapTest
+ cls.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ # from CouchDBTestCase
+ cls.wrapper = CouchDBWrapper()
+ cls.wrapper.start()
+ #self.db = self.wrapper.db
+
+ @classmethod
+ def tearDownClass(cls):
+ """
+ Stop CouchDB instance for test.
+ """
+ # from BaseLeapTest
+ soledad_assert(
+ cls.tempdir.startswith('/tmp/leap_tests-'),
+ "beware! tried to remove a dir which does not "
+ "live in temporal folder!")
+ shutil.rmtree(cls.tempdir)
+ # from CouchDBTestCase
+ cls.wrapper.stop()
+
+ def setUp(self):
+ BaseSoledadTest.setUp(self)
+ CouchDBTestCase.setUp(self)
+ main_test_class = getattr(self, 'main_test_class', None)
+ if main_test_class is not None:
+ main_test_class.setUp(self)
+ self._couch_url = 'http://localhost:%d' % self.wrapper.port
+
+ def tearDown(self):
+ BaseSoledadTest.tearDown(self)
+ CouchDBTestCase.tearDown(self)
+ main_test_class = getattr(self, 'main_test_class', None)
+ if main_test_class is not None:
+ main_test_class.tearDown(self)
+ # delete the test database
+ try:
+ db = CouchDatabase(self._couch_url, 'test')
+ db.delete_database()
+ except DatabaseDoesNotExist:
+ pass
+
+ def make_app(self):
+ couch_url = urljoin(
+ 'http://localhost:' + str(self.wrapper.port), 'tests')
+ self.request_state = CouchServerStateForTests(
+ couch_url, 'shared', 'tokens')
+ return self.make_app_with_state(self.request_state)