summaryrefslogtreecommitdiff
path: root/common/src
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2014-10-01 15:39:03 -0300
committerdrebs <drebs@leap.se>2014-10-01 15:39:03 -0300
commit3de445b289f9373e617f216fccbf1cfcff139270 (patch)
tree580f8a7384b2ad91c12f53aa2af5034aa9304944 /common/src
parenta7568bd38bdc63de5f074861862390281f49abf0 (diff)
parent365fa1603a977040a1891880a66118f196a54ac0 (diff)
Merge tag '0.6.0' into deb-0.6.0
Tag leap.soledad version 0.6.0 Conflicts: client/pkg/requirements.pip
Diffstat (limited to 'common/src')
-rw-r--r--common/src/leap/soledad/common/couch.py55
-rw-r--r--common/src/leap/soledad/common/ddocs/syncs/updates/put.js141
-rw-r--r--common/src/leap/soledad/common/tests/__init__.py51
-rw-r--r--common/src/leap/soledad/common/tests/test_couch.py9
-rw-r--r--common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py18
-rw-r--r--common/src/leap/soledad/common/tests/test_crypto.py53
-rw-r--r--common/src/leap/soledad/common/tests/test_http.py64
-rw-r--r--common/src/leap/soledad/common/tests/test_http_client.py116
-rw-r--r--common/src/leap/soledad/common/tests/test_https.py108
-rw-r--r--common/src/leap/soledad/common/tests/test_server.py40
-rw-r--r--common/src/leap/soledad/common/tests/test_soledad.py5
-rw-r--r--common/src/leap/soledad/common/tests/test_soledad_doc.py44
-rw-r--r--common/src/leap/soledad/common/tests/test_sqlcipher.py27
-rw-r--r--common/src/leap/soledad/common/tests/test_sync.py138
-rw-r--r--common/src/leap/soledad/common/tests/test_sync_deferred.py227
-rw-r--r--common/src/leap/soledad/common/tests/test_sync_target.py589
-rw-r--r--common/src/leap/soledad/common/tests/test_target.py21
-rw-r--r--common/src/leap/soledad/common/tests/test_target_soledad.py102
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/__init__.py5
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_backends.py2
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_sync.py3
21 files changed, 1667 insertions, 151 deletions
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index b51b32f3..5658f4ce 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -1106,7 +1106,9 @@ class CouchDatabase(CommonBackend):
)
def _set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation, other_transaction_id):
+ other_generation, other_transaction_id,
+ number_of_docs=None, doc_idx=None,
+ sync_id=None):
"""
Set the last-known generation and transaction id for the other
database replica.
@@ -1122,12 +1124,21 @@ class CouchDatabase(CommonBackend):
:param other_transaction_id: The transaction id associated with the
generation.
:type other_transaction_id: str
+ :param number_of_docs: The total amount of documents sent on this sync
+ session.
+ :type number_of_docs: int
+ :param doc_idx: The index of the current document being sent.
+ :type doc_idx: int
+ :param sync_id: The id of the current sync session.
+ :type sync_id: str
"""
self._do_set_replica_gen_and_trans_id(
- other_replica_uid, other_generation, other_transaction_id)
+ other_replica_uid, other_generation, other_transaction_id,
+ number_of_docs=number_of_docs, doc_idx=doc_idx, sync_id=sync_id)
def _do_set_replica_gen_and_trans_id(
- self, other_replica_uid, other_generation, other_transaction_id):
+ self, other_replica_uid, other_generation, other_transaction_id,
+ number_of_docs=None, doc_idx=None, sync_id=None):
"""
Set the last-known generation and transaction id for the other
database replica.
@@ -1143,6 +1154,13 @@ class CouchDatabase(CommonBackend):
:param other_transaction_id: The transaction id associated with the
generation.
:type other_transaction_id: str
+ :param number_of_docs: The total amount of documents sent on this sync
+ session.
+ :type number_of_docs: int
+ :param doc_idx: The index of the current document being sent.
+ :type doc_idx: int
+ :param sync_id: The id of the current sync session.
+ :type sync_id: str
:raise MissingDesignDocError: Raised when tried to access a missing
design document.
@@ -1163,12 +1181,19 @@ class CouchDatabase(CommonBackend):
res = self._database.resource(*ddoc_path)
try:
with CouchDatabase.update_handler_lock[self._get_replica_uid()]:
+ body={
+ 'other_replica_uid': other_replica_uid,
+ 'other_generation': other_generation,
+ 'other_transaction_id': other_transaction_id,
+ }
+ if number_of_docs is not None:
+ body['number_of_docs'] = number_of_docs
+ if doc_idx is not None:
+ body['doc_idx'] = doc_idx
+ if sync_id is not None:
+ body['sync_id'] = sync_id
res.put_json(
- body={
- 'other_replica_uid': other_replica_uid,
- 'other_generation': other_generation,
- 'other_transaction_id': other_transaction_id,
- },
+ body=body,
headers={'content-type': 'application/json'})
except ResourceNotFound as e:
raise_missing_design_doc_error(e, ddoc_path)
@@ -1306,7 +1331,8 @@ class CouchDatabase(CommonBackend):
doc.set_conflicts(cur_doc.get_conflicts())
def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen,
- replica_trans_id=''):
+ replica_trans_id='', number_of_docs=None,
+ doc_idx=None, sync_id=None):
"""
Insert/update document into the database with a given revision.
@@ -1339,6 +1365,13 @@ class CouchDatabase(CommonBackend):
:param replica_trans_id: The transaction_id associated with the
generation.
:type replica_trans_id: str
+ :param number_of_docs: The total amount of documents sent on this sync
+ session.
+ :type number_of_docs: int
+ :param doc_idx: The index of the current document being sent.
+ :type doc_idx: int
+ :param sync_id: The id of the current sync session.
+ :type sync_id: str
:return: (state, at_gen) - If we don't have doc_id already, or if
doc_rev supersedes the existing document revision, then the
@@ -1398,7 +1431,9 @@ class CouchDatabase(CommonBackend):
self._force_doc_sync_conflict(doc)
if replica_uid is not None and replica_gen is not None:
self._set_replica_gen_and_trans_id(
- replica_uid, replica_gen, replica_trans_id)
+ replica_uid, replica_gen, replica_trans_id,
+ number_of_docs=number_of_docs, doc_idx=doc_idx,
+ sync_id=sync_id)
# update info
old_doc.rev = doc.rev
if doc.is_tombstone():
diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/put.js b/common/src/leap/soledad/common/ddocs/syncs/updates/put.js
index 722f695a..b0ae2de6 100644
--- a/common/src/leap/soledad/common/ddocs/syncs/updates/put.js
+++ b/common/src/leap/soledad/common/ddocs/syncs/updates/put.js
@@ -1,22 +1,151 @@
+/**
+ * The u1db_sync_log document stores both the actual sync log and a list of
+ * pending updates to the log, in case we receive incoming documents out of
+ * the correct order (i.e. if there are parallel PUTs during the sync
+ * process).
+ *
+ * The structure of the document is the following:
+ *
+ * {
+ * 'syncs': [
+ * ['<replica_uid>', <gen>, '<trans_id>'],
+ * ...
+ * ],
+ * 'pending': {
+ * 'other_replica_uid': {
+ * 'sync_id': '<sync_id>',
+ * 'log': [[<gen>, '<trans_id>'], ...]
+ * },
+ * ...
+ * }
+ * }
+ *
+ * The update function below does the following:
+ *
+ * 0. If we do not receive a sync_id, we just update the 'syncs' list with
+ * the incoming info about the source replica state.
+ *
+ * 1. Otherwise, if the incoming sync_id differs from current stored
+ * sync_id, then we assume that the previous sync session for that source
+ * replica was interrupted and discard all pending data.
+ *
+ * 2. Then we append incoming info as pending data for that source replica
+ * and current sync_id, and sort the pending data by generation.
+ *
+ * 3. Then we go through pending data and find the most recent generation
+ * that we can use to update the actual sync log.
+ *
+ * 4. Finally, we insert the most up to date information into the sync log.
+ */
function(doc, req){
+
+ // create the document if it doesn't exist
if (!doc) {
doc = {}
doc['_id'] = 'u1db_sync_log';
doc['syncs'] = [];
}
- body = JSON.parse(req.body);
+
+ // get and validate incoming info
+ var body = JSON.parse(req.body);
+ var other_replica_uid = body['other_replica_uid'];
+ var other_generation = parseInt(body['other_generation']);
+ var other_transaction_id = body['other_transaction_id']
+ var sync_id = body['sync_id'];
+ var number_of_docs = body['number_of_docs'];
+ var doc_idx = body['doc_idx'];
+
+ // parse integers
+ if (number_of_docs != null)
+ number_of_docs = parseInt(number_of_docs);
+ if (doc_idx != null)
+ doc_idx = parseInt(doc_idx);
+
+ if (other_replica_uid == null
+ || other_generation == null
+ || other_transaction_id == null)
+ return [null, 'invalid data'];
+
+ // create slot for pending logs
+ if (doc['pending'] == null)
+ doc['pending'] = {};
+
+ // these are the values that will be actually inserted
+ var current_gen = other_generation;
+ var current_trans_id = other_transaction_id;
+
+ /*------------- Wait for sequential values before storing -------------*/
+
+ // we just try to obtain pending log if we received a sync_id
+ if (sync_id != null) {
+
+ // create slot for current source and sync_id pending log
+ if (doc['pending'][other_replica_uid] == null
+ || doc['pending'][other_replica_uid]['sync_id'] != sync_id) {
+ doc['pending'][other_replica_uid] = {
+ 'sync_id': sync_id,
+ 'log': [],
+ 'last_doc_idx': 0,
+ }
+ }
+
+ // append incoming data to pending log
+ doc['pending'][other_replica_uid]['log'].push([
+ other_generation,
+ other_transaction_id,
+ doc_idx,
+ ])
+
+ // sort pending log according to generation
+ doc['pending'][other_replica_uid]['log'].sort(function(a, b) {
+ return a[0] - b[0];
+ });
+
+ // get most up-to-date information from pending log
+ var last_doc_idx = doc['pending'][other_replica_uid]['last_doc_idx'];
+ var pending_idx = doc['pending'][other_replica_uid]['log'][0][2];
+
+ current_gen = null;
+ current_trans_id = null;
+
+ while (last_doc_idx + 1 == pending_idx) {
+ pending = doc['pending'][other_replica_uid]['log'].shift()
+ current_gen = pending[0];
+ current_trans_id = pending[1];
+ last_doc_idx = pending[2]
+ if (doc['pending'][other_replica_uid]['log'].length == 0)
+ break;
+ pending_idx = doc['pending'][other_replica_uid]['log'][0][2];
+ }
+
+ // leave the sync log untouched if we still did not receive enough docs
+ if (current_gen == null)
+ return [doc, 'ok'];
+
+ // update last index of received doc
+ doc['pending'][other_replica_uid]['last_doc_idx'] = last_doc_idx;
+
+ // eventually remove all pending data from that replica
+ if (last_doc_idx == number_of_docs)
+ delete doc['pending'][other_replica_uid]
+ }
+
+ /*--------------- Store source replica info on sync log ---------------*/
+
// remove outdated info
doc['syncs'] = doc['syncs'].filter(
function (entry) {
- return entry[0] != body['other_replica_uid'];
+ return entry[0] != other_replica_uid;
}
);
- // store u1db rev
+
+ // store in log
doc['syncs'].push([
- body['other_replica_uid'],
- body['other_generation'],
- body['other_transaction_id']
+ other_replica_uid,
+ current_gen,
+ current_trans_id
]);
+
return [doc, 'ok'];
}
diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py
index a38bdaed..3081683b 100644
--- a/common/src/leap/soledad/common/tests/__init__.py
+++ b/common/src/leap/soledad/common/tests/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# __init__.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -19,7 +19,6 @@
"""
Tests to make sure Soledad provides U1DB functionality and more.
"""
-
import os
import random
import string
@@ -29,11 +28,8 @@ from mock import Mock
from leap.soledad.common.document import SoledadDocument
from leap.soledad.client import Soledad
-from leap.soledad.client.crypto import SoledadCrypto
-from leap.soledad.client.target import (
- decrypt_doc,
- ENC_SCHEME_KEY,
-)
+from leap.soledad.client.crypto import decrypt_doc_dict
+from leap.soledad.client.crypto import ENC_SCHEME_KEY
from leap.common.testing.basetest import BaseLeapTest
@@ -49,6 +45,7 @@ class BaseSoledadTest(BaseLeapTest):
"""
Instantiates Soledad for usage in tests.
"""
+ defer_sync_encryption = False
def setUp(self):
# config info
@@ -73,11 +70,26 @@ class BaseSoledadTest(BaseLeapTest):
self._db1.close()
self._db2.close()
self._soledad.close()
+
# XXX should not access "private" attrs
for f in [self._soledad._local_db_path, self._soledad._secrets_path]:
if os.path.isfile(f):
os.unlink(f)
+ def get_default_shared_mock(self, put_doc_side_effect):
+ """
+ Get a default class for mocking the shared DB
+ """
+ class defaultMockSharedDB(object):
+ get_doc = Mock(return_value=None)
+ put_doc = Mock(side_effect=put_doc_side_effect)
+ lock = Mock(return_value=('atoken', 300))
+ unlock = Mock(return_value=True)
+
+ def __call__(self):
+ return self
+ return defaultMockSharedDB
+
def _soledad_instance(self, user=ADDRESS, passphrase=u'123',
prefix='',
secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
@@ -88,18 +100,11 @@ class BaseSoledadTest(BaseLeapTest):
def _put_doc_side_effect(doc):
self._doc_put = doc
- class MockSharedDB(object):
-
- get_doc = Mock(return_value=None)
- put_doc = Mock(side_effect=_put_doc_side_effect)
- lock = Mock(return_value=('atoken', 300))
- unlock = Mock(return_value=True)
-
- def __call__(self):
- return self
-
if shared_db_class is not None:
MockSharedDB = shared_db_class
+ else:
+ MockSharedDB = self.get_default_shared_mock(
+ _put_doc_side_effect)
Soledad._shared_db = MockSharedDB()
return Soledad(
@@ -111,7 +116,8 @@ class BaseSoledadTest(BaseLeapTest):
self.tempdir, prefix, local_db_path),
server_url=server_url, # Soledad will fail if not given an url.
cert_file=cert_file,
- secret_id=secret_id)
+ secret_id=secret_id,
+ defer_encryption=self.defer_sync_encryption)
def assertGetEncryptedDoc(
self, db, doc_id, doc_rev, content, has_conflicts):
@@ -121,8 +127,15 @@ class BaseSoledadTest(BaseLeapTest):
exp_doc = self.make_document(doc_id, doc_rev, content,
has_conflicts=has_conflicts)
doc = db.get_doc(doc_id)
+
if ENC_SCHEME_KEY in doc.content:
- doc.set_json(decrypt_doc(self._soledad._crypto, doc))
+ # XXX check for SYM_KEY too
+ key = self._soledad._crypto.doc_passphrase(doc.doc_id)
+ secret = self._soledad._crypto.secret
+ decrypted = decrypt_doc_dict(
+ doc.content, doc.doc_id, doc.rev,
+ key, secret)
+ doc.set_json(decrypted)
self.assertEqual(exp_doc.doc_id, doc.doc_id)
self.assertEqual(exp_doc.rev, doc.rev)
self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts)
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 3b1e5a06..10d6c136 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -91,14 +91,19 @@ class CouchDBWrapper(object):
logPath = os.path.join(self.tempdir, 'log', 'couch.log')
while not os.path.exists(logPath):
if self.process.poll() is not None:
+ got_stdout, got_stderr = "", ""
+ if self.process.stdout is not None:
+ got_stdout = self.process.stdout.read()
+
+ if self.process.stderr is not None:
+ got_stderr = self.process.stderr.read()
raise Exception("""
couchdb exited with code %d.
stdout:
%s
stderr:
%s""" % (
- self.process.returncode, self.process.stdout.read(),
- self.process.stderr.read()))
+ self.process.returncode, got_stdout, got_stderr))
time.sleep(0.01)
while os.stat(logPath).st_size == 0:
time.sleep(0.01)
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index b03f79e7..6465eb80 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# test_soledad.py
-# Copyright (C) 2013 LEAP
+# test_couch_operations_atomicity.py
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -14,11 +14,9 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
+Test atomocity for couch operations.
"""
-
import os
import mock
import tempfile
@@ -32,7 +30,7 @@ from leap.soledad.client import Soledad
from leap.soledad.common.couch import CouchDatabase, CouchServerState
from leap.soledad.common.tests.test_couch import CouchDBTestCase
from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
-from leap.soledad.common.tests.test_target import (
+from leap.soledad.common.tests.test_sync_target import (
make_token_soledad_app,
make_leap_document_for_test,
token_leap_sync_target,
@@ -224,9 +222,9 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
#
# Concurrency tests
#
-
+
class _WorkerThread(threading.Thread):
-
+
def __init__(self, params, run_method):
threading.Thread.__init__(self)
self._params = params
@@ -260,7 +258,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
for thread in threads:
thread.join()
-
+
# assert length of transaction_log
transaction_log = self.db._get_transaction_log()
self.assertEqual(
@@ -341,7 +339,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
# wait for threads to finish
for thread in threads:
thread.join()
-
+
# do the sync!
sol.sync()
diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py
index 4b2470ba..1071af14 100644
--- a/common/src/leap/soledad/common/tests/test_crypto.py
+++ b/common/src/leap/soledad/common/tests/test_crypto.py
@@ -14,37 +14,17 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
Tests for cryptographic related stuff.
"""
-
import os
-import shutil
-import tempfile
-import simplejson as json
import hashlib
import binascii
-
-from leap.common.testing.basetest import BaseLeapTest
-from leap.soledad.client import (
- Soledad,
- crypto,
- target,
-)
+from leap.soledad.client import crypto
from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.tests import (
- BaseSoledadTest,
- KEY_FINGERPRINT,
- PRIVATE_KEY,
-)
+from leap.soledad.common.tests import BaseSoledadTest
from leap.soledad.common.crypto import WrongMac, UnknownMacMethod
-from leap.soledad.common.tests.u1db_tests import (
- simple_doc,
- nested_doc,
-)
class EncryptedSyncTestCase(BaseSoledadTest):
@@ -59,16 +39,17 @@ class EncryptedSyncTestCase(BaseSoledadTest):
simpledoc = {'key': 'val'}
doc1 = SoledadDocument(doc_id='id')
doc1.content = simpledoc
+
# encrypt doc
- doc1.set_json(target.encrypt_doc(self._soledad._crypto, doc1))
+ doc1.set_json(crypto.encrypt_doc(self._soledad._crypto, doc1))
# assert content is different and includes keys
self.assertNotEqual(
simpledoc, doc1.content,
'incorrect document encryption')
- self.assertTrue(target.ENC_JSON_KEY in doc1.content)
- self.assertTrue(target.ENC_SCHEME_KEY in doc1.content)
+ self.assertTrue(crypto.ENC_JSON_KEY in doc1.content)
+ self.assertTrue(crypto.ENC_SCHEME_KEY in doc1.content)
# decrypt doc
- doc1.set_json(target.decrypt_doc(self._soledad._crypto, doc1))
+ doc1.set_json(crypto.decrypt_doc(self._soledad._crypto, doc1))
self.assertEqual(
simpledoc, doc1.content, 'incorrect document encryption')
@@ -159,15 +140,15 @@ class MacAuthTestCase(BaseSoledadTest):
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
- doc.set_json(target.encrypt_doc(self._soledad._crypto, doc))
- self.assertTrue(target.MAC_KEY in doc.content)
- self.assertTrue(target.MAC_METHOD_KEY in doc.content)
+ doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
+ self.assertTrue(crypto.MAC_KEY in doc.content)
+ self.assertTrue(crypto.MAC_METHOD_KEY in doc.content)
# mess with MAC
- doc.content[target.MAC_KEY] = '1234567890ABCDEF'
+ doc.content[crypto.MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(
WrongMac,
- target.decrypt_doc, self._soledad._crypto, doc)
+ crypto.decrypt_doc, self._soledad._crypto, doc)
def test_decrypt_with_unknown_mac_method_raises(self):
"""
@@ -177,15 +158,15 @@ class MacAuthTestCase(BaseSoledadTest):
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
- doc.set_json(target.encrypt_doc(self._soledad._crypto, doc))
- self.assertTrue(target.MAC_KEY in doc.content)
- self.assertTrue(target.MAC_METHOD_KEY in doc.content)
+ doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
+ self.assertTrue(crypto.MAC_KEY in doc.content)
+ self.assertTrue(crypto.MAC_METHOD_KEY in doc.content)
# mess with MAC method
- doc.content[target.MAC_METHOD_KEY] = 'mymac'
+ doc.content[crypto.MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(
UnknownMacMethod,
- target.decrypt_doc, self._soledad._crypto, doc)
+ crypto.decrypt_doc, self._soledad._crypto, doc)
class SoledadCryptoAESTestCase(BaseSoledadTest):
diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py
new file mode 100644
index 00000000..d21470e0
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_http.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# test_http.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test Leap backend bits: test http database
+"""
+from u1db.remote import http_database
+
+from leap.soledad.client import auth
+
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests.u1db_tests import test_http_database
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_http_database`.
+#-----------------------------------------------------------------------------
+
+class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth):
+ """
+ Wraps our token auth implementation.
+ """
+
+ def set_token_credentials(self, uuid, token):
+ auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
+
+ def _sign_request(self, method, url_query, params):
+ return auth.TokenBasedAuth._sign_request(
+ self, method, url_query, params)
+
+
+class TestHTTPDatabaseWithCreds(
+ test_http_database.TestHTTPDatabaseCtrWithCreds):
+
+ def test_get_sync_target_inherits_token_credentials(self):
+ # this test was from TestDatabaseSimpleOperations but we put it here
+ # for convenience.
+ self.db = _HTTPDatabase('dbase')
+ self.db.set_token_credentials('user-uuid', 'auth-token')
+ st = self.db.get_sync_target()
+ self.assertEqual(self.db._creds, st._creds)
+
+ def test_ctr_with_creds(self):
+ db1 = _HTTPDatabase('http://dbs/db', creds={'token': {
+ 'uuid': 'user-uuid',
+ 'token': 'auth-token',
+ }})
+ self.assertIn('token', db1._creds)
+
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py
new file mode 100644
index 00000000..3169398b
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_http_client.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# test_http_client.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test Leap backend bits: sync target
+"""
+import json
+
+from u1db.remote import http_client
+
+from leap.soledad.client import auth
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests.u1db_tests import test_http_client
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_http_client`.
+#-----------------------------------------------------------------------------
+
+class TestSoledadClientBase(test_http_client.TestHTTPClientBase):
+ """
+ This class should be used to test Token auth.
+ """
+
+ def getClientWithToken(self, **kwds):
+ self.startServer()
+
+ class _HTTPClientWithToken(
+ http_client.HTTPClientBase, auth.TokenBasedAuth):
+
+ def set_token_credentials(self, uuid, token):
+ auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
+
+ def _sign_request(self, method, url_query, params):
+ return auth.TokenBasedAuth._sign_request(
+ self, method, url_query, params)
+
+ return _HTTPClientWithToken(self.getURL('dbase'), **kwds)
+
+ def test_oauth(self):
+ """
+ Suppress oauth test (we test for token auth here).
+ """
+ pass
+
+ def test_oauth_ctr_creds(self):
+ """
+ Suppress oauth test (we test for token auth here).
+ """
+ pass
+
+ def test_oauth_Unauthorized(self):
+ """
+ Suppress oauth test (we test for token auth here).
+ """
+ pass
+
+ def app(self, environ, start_response):
+ res = test_http_client.TestHTTPClientBase.app(
+ self, environ, start_response)
+ if res is not None:
+ return res
+ # mime solead application here.
+ if '/token' in environ['PATH_INFO']:
+ auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY)
+ if not auth:
+ start_response("401 Unauthorized",
+ [('Content-Type', 'application/json')])
+ return [json.dumps({"error": "unauthorized",
+ "message": e.message})]
+ scheme, encoded = auth.split(None, 1)
+ if scheme.lower() != 'token':
+ start_response("401 Unauthorized",
+ [('Content-Type', 'application/json')])
+ return [json.dumps({"error": "unauthorized",
+ "message": e.message})]
+ uuid, token = encoded.decode('base64').split(':', 1)
+ if uuid != 'user-uuid' and token != 'auth-token':
+ return unauth_err("Incorrect address or token.")
+ start_response("200 OK", [('Content-Type', 'application/json')])
+ return [json.dumps([environ['PATH_INFO'], uuid, token])]
+
+ def test_token(self):
+ """
+ Test if token is sent correctly.
+ """
+ cli = self.getClientWithToken()
+ cli.set_token_credentials('user-uuid', 'auth-token')
+ res, headers = cli._request('GET', ['doc', 'token'])
+ self.assertEqual(
+ ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res))
+
+ def test_token_ctr_creds(self):
+ cli = self.getClientWithToken(creds={'token': {
+ 'uuid': 'user-uuid',
+ 'token': 'auth-token',
+ }})
+ res, headers = cli._request('GET', ['doc', 'token'])
+ self.assertEqual(
+ ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res))
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py
new file mode 100644
index 00000000..b6288188
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_https.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+# test_sync_target.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test Leap backend bits: https
+"""
+from leap.soledad.common.tests import BaseSoledadTest
+from leap.soledad.common.tests import test_sync_target as test_st
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests.u1db_tests import test_backends
+from leap.soledad.common.tests.u1db_tests import test_https
+
+from leap.soledad import client
+from leap.soledad.server import SoledadApp
+
+from u1db.remote import http_client
+
+
+def make_soledad_app(state):
+ return SoledadApp(state)
+
+LEAP_SCENARIOS = [
+ ('http', {
+ 'make_database_for_test': test_backends.make_http_database_for_test,
+ 'copy_database_for_test': test_backends.copy_http_database_for_test,
+ 'make_document_for_test': test_st.make_leap_document_for_test,
+ 'make_app_with_state': test_st.make_soledad_app}),
+]
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_https`.
+#-----------------------------------------------------------------------------
+
+def token_leap_https_sync_target(test, host, path):
+ _, port = test.server.server_address
+ st = client.target.SoledadSyncTarget(
+ 'https://%s:%d/%s' % (host, port, path),
+ crypto=test._soledad._crypto)
+ st.set_token_credentials('user-uuid', 'auth-token')
+ return st
+
+
+class TestSoledadSyncTargetHttpsSupport(
+ test_https.TestHttpSyncTargetHttpsSupport,
+ BaseSoledadTest):
+
+ scenarios = [
+ ('token_soledad_https',
+ {'server_def': test_https.https_server_def,
+ 'make_app_with_state': test_st.make_token_soledad_app,
+ 'make_document_for_test': test_st.make_leap_document_for_test,
+ 'sync_target': token_leap_https_sync_target}),
+ ]
+
+ def setUp(self):
+ # the parent constructor undoes our SSL monkey patch to ensure tests
+ # run smoothly with standard u1db.
+ test_https.TestHttpSyncTargetHttpsSupport.setUp(self)
+ # so here monkey patch again to test our functionality.
+ http_client._VerifiedHTTPSConnection = client.VerifiedHTTPSConnection
+ client.SOLEDAD_CERT = http_client.CA_CERTS
+
+ def test_working(self):
+ """
+ Test that SSL connections work well.
+
+ This test was adapted to patch Soledad's HTTPS connection custom class
+ with the intended CA certificates.
+ """
+ self.startServer()
+ db = self.request_state._create_database('test')
+ self.patch(client, 'SOLEDAD_CERT', self.cacert_pem)
+ remote_target = self.getSyncTarget('localhost', 'test')
+ remote_target.record_sync_info('other-id', 2, 'T-id')
+ self.assertEqual(
+ (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id'))
+
+ def test_host_mismatch(self):
+ """
+ Test that SSL connections to a hostname different than the one in the
+ certificate raise CertificateError.
+
+ This test was adapted to patch Soledad's HTTPS connection custom class
+ with the intended CA certificates.
+ """
+ self.startServer()
+ self.request_state._create_database('test')
+ self.patch(client, 'SOLEDAD_CERT', self.cacert_pem)
+ remote_target = self.getSyncTarget('127.0.0.1', 'test')
+ self.assertRaises(
+ http_client.CertificateError, remote_target.record_sync_info,
+ 'other-id', 2, 'T-id')
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index 1c5a7407..cb5348b4 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -14,12 +14,9 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
Tests for server-related functionality.
"""
-
import os
import tempfile
import simplejson as json
@@ -39,16 +36,13 @@ from leap.soledad.common.tests.u1db_tests import (
simple_doc,
)
from leap.soledad.common.tests.test_couch import CouchDBTestCase
-from leap.soledad.common.tests.test_target import (
+from leap.soledad.common.tests.test_target_soledad import (
make_token_soledad_app,
make_leap_document_for_test,
- token_leap_sync_target,
)
-from leap.soledad.client import (
- Soledad,
- target,
-)
-from leap.soledad.server import SoledadApp, LockResource
+from leap.soledad.common.tests.test_sync_target import token_leap_sync_target
+from leap.soledad.client import Soledad, crypto
+from leap.soledad.server import LockResource
from leap.soledad.server.auth import URLToAuthorization
@@ -369,12 +363,12 @@ class EncryptedSyncTestCase(
self.assertEqual(doc1.doc_id, couchdoc.doc_id)
self.assertEqual(doc1.rev, couchdoc.rev)
self.assertEqual(6, len(couchdoc.content))
- self.assertTrue(target.ENC_JSON_KEY in couchdoc.content)
- self.assertTrue(target.ENC_SCHEME_KEY in couchdoc.content)
- self.assertTrue(target.ENC_METHOD_KEY in couchdoc.content)
- self.assertTrue(target.ENC_IV_KEY in couchdoc.content)
- self.assertTrue(target.MAC_KEY in couchdoc.content)
- self.assertTrue(target.MAC_METHOD_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content)
+ self.assertTrue(crypto.MAC_KEY in couchdoc.content)
+ self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content)
# instantiate soledad with empty db, but with same secrets path
sol2 = self._soledad_instance(prefix='x', auth_token='auth-token')
_, doclist = sol2.get_all_docs()
@@ -427,12 +421,12 @@ class EncryptedSyncTestCase(
self.assertEqual(doc1.doc_id, couchdoc.doc_id)
self.assertEqual(doc1.rev, couchdoc.rev)
self.assertEqual(6, len(couchdoc.content))
- self.assertTrue(target.ENC_JSON_KEY in couchdoc.content)
- self.assertTrue(target.ENC_SCHEME_KEY in couchdoc.content)
- self.assertTrue(target.ENC_METHOD_KEY in couchdoc.content)
- self.assertTrue(target.ENC_IV_KEY in couchdoc.content)
- self.assertTrue(target.MAC_KEY in couchdoc.content)
- self.assertTrue(target.MAC_METHOD_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content)
+ self.assertTrue(crypto.MAC_KEY in couchdoc.content)
+ self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content)
# instantiate soledad with empty db, but with same secrets path
sol2 = self._soledad_instance(
prefix='x',
@@ -502,7 +496,6 @@ class EncryptedSyncTestCase(
sol1.close()
sol2.close()
-
def test_sync_many_small_files(self):
"""
Test if Soledad can sync many smallfiles.
@@ -548,6 +541,7 @@ class EncryptedSyncTestCase(
sol1.close()
sol2.close()
+
class LockResourceTestCase(
CouchDBTestCase, TestCaseWithServer):
"""
diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py
index 5a3bf2b0..11e43423 100644
--- a/common/src/leap/soledad/common/tests/test_soledad.py
+++ b/common/src/leap/soledad/common/tests/test_soledad.py
@@ -14,18 +14,13 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
Tests for general Soledad functionality.
"""
-
-
import os
from mock import Mock
-from pysqlcipher.dbapi2 import DatabaseError
from leap.common.events import events_pb2 as proto
from leap.soledad.common.tests import (
BaseSoledadTest,
diff --git a/common/src/leap/soledad/common/tests/test_soledad_doc.py b/common/src/leap/soledad/common/tests/test_soledad_doc.py
new file mode 100644
index 00000000..0952de6d
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_soledad_doc.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# test_soledad_doc.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test Leap backend bits: soledad docs
+"""
+from leap.soledad.common.tests import BaseSoledadTest
+from leap.soledad.common.tests.u1db_tests import test_document
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests import test_sync_target as st
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_document`.
+#-----------------------------------------------------------------------------
+
+
+class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest):
+
+ scenarios = ([(
+ 'leap', {
+ 'make_document_for_test': st.make_leap_document_for_test})])
+
+
+class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest):
+
+ scenarios = ([(
+ 'leap', {
+ 'make_document_for_test': st.make_leap_document_for_test})])
+
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py
index 891aca0f..595966ec 100644
--- a/common/src/leap/soledad/common/tests/test_sqlcipher.py
+++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py
@@ -14,16 +14,11 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
Test sqlcipher backend internals.
"""
-
-
import os
import time
-import unittest
import simplejson as json
import threading
@@ -50,15 +45,9 @@ from leap.soledad.client.sqlcipher import (
DatabaseIsNotEncrypted,
open as u1db_open,
)
-from leap.soledad.common.crypto import (
- EncryptionSchemes,
- ENC_JSON_KEY,
- ENC_SCHEME_KEY,
-)
-from leap.soledad.client.target import (
- decrypt_doc,
- SoledadSyncTarget,
-)
+from leap.soledad.client.target import SoledadSyncTarget
+from leap.soledad.common.crypto import ENC_SCHEME_KEY
+from leap.soledad.client.crypto import decrypt_doc_dict
# u1db tests stuff.
@@ -269,6 +258,7 @@ class TestSQLCipherPartialExpandDatabase(
db = SQLCipherDatabase.__new__(
SQLCipherDatabase)
db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed
+ db._syncers = {}
c = db._db_handle.cursor()
c.execute('PRAGMA key="%s"' % PASSWORD)
self.addCleanup(db.close)
@@ -614,7 +604,12 @@ class SQLCipherDatabaseSyncTests(
self.sync(self.db2, db3)
doc3 = db3.get_doc('the-doc')
if ENC_SCHEME_KEY in doc3.content:
- doc3.set_json(decrypt_doc(self._soledad._crypto, doc3))
+ _crypto = self._soledad._crypto
+ key = _crypto.doc_passphrase(doc3.doc_id)
+ secret = _crypto.secret
+ doc3.set_json(decrypt_doc_dict(
+ doc3.content,
+ doc3.doc_id, doc3.rev, key, secret))
self.assertEqual(doc4.get_json(), doc3.get_json())
self.assertFalse(doc3.has_conflicts)
@@ -796,7 +791,7 @@ class SQLCipherEncryptionTest(BaseLeapTest):
# trying to open the a non-encrypted database with sqlcipher
# backend should raise a DatabaseIsNotEncrypted exception.
SQLCipherDatabase(self.DB_FILE, PASSWORD)
- raise db1pi2.DatabaseError(
+ raise dbapi2.DatabaseError(
"SQLCipher backend should not be able to open non-encrypted "
"dbs.")
except DatabaseIsNotEncrypted:
diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py
index fd4a2797..0433fac9 100644
--- a/common/src/leap/soledad/common/tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/test_sync.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# test_sync.py
-# Copyright (C) 2014 LEAP
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -24,26 +24,31 @@ import threading
import time
from urlparse import urljoin
-from leap.soledad.common.couch import (
- CouchServerState,
- CouchDatabase,
-)
+from leap.soledad.common import couch
+from leap.soledad.common.tests import BaseSoledadTest
+from leap.soledad.common.tests import test_sync_target
+from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests.u1db_tests import (
TestCaseWithServer,
simple_doc,
+ test_backends,
+ test_sync
)
from leap.soledad.common.tests.test_couch import CouchDBTestCase
-from leap.soledad.common.tests.test_target import (
+from leap.soledad.common.tests.test_target_soledad import (
make_token_soledad_app,
make_leap_document_for_test,
- token_leap_sync_target,
)
-
+from leap.soledad.common.tests.test_sync_target import token_leap_sync_target
from leap.soledad.client import (
Soledad,
target,
)
+from leap.soledad.common.tests.util import SoledadWithCouchServerMixin
+from leap.soledad.client.sync import SoledadSynchronizer
+from leap.soledad.server import SoledadApp
+
class InterruptableSyncTestCase(
@@ -99,8 +104,8 @@ class InterruptableSyncTestCase(
secret_id=secret_id)
def make_app(self):
- self.request_state = CouchServerState(self._couch_url, 'shared',
- 'tokens')
+ self.request_state = couch.CouchServerState(
+ self._couch_url, 'shared', 'tokens')
return self.make_app_with_state(self.request_state)
def setUp(self):
@@ -150,7 +155,7 @@ class InterruptableSyncTestCase(
sol.create_doc(json.loads(simple_doc))
# ensure remote db exists before syncing
- db = CouchDatabase.open_database(
+ db = couch.CouchDatabase.open_database(
urljoin(self._couch_url, 'user-user-uuid'),
create=True,
ensure_ddocs=True)
@@ -174,3 +179,114 @@ class InterruptableSyncTestCase(
db.delete_database()
db.close()
sol.close()
+
+
+def make_soledad_app(state):
+ return SoledadApp(state)
+
+
+class TestSoledadDbSync(
+ SoledadWithCouchServerMixin,
+ test_sync.TestDbSync):
+ """
+ Test db.sync remote sync shortcut
+ """
+
+ scenarios = [
+ ('py-http', {
+ 'make_app_with_state': make_soledad_app,
+ 'make_database_for_test': tests.make_memory_database_for_test,
+ }),
+ ('py-token-http', {
+ 'make_app_with_state': test_sync_target.make_token_soledad_app,
+ 'make_database_for_test': tests.make_memory_database_for_test,
+ 'token': True
+ }),
+ ]
+
+ oauth = False
+ token = False
+
+ def setUp(self):
+ """
+ Need to explicitely invoke inicialization on all bases.
+ """
+ tests.TestCaseWithServer.setUp(self)
+ self.main_test_class = test_sync.TestDbSync
+ SoledadWithCouchServerMixin.setUp(self)
+ self.startServer()
+ self.db2 = couch.CouchDatabase.open_database(
+ urljoin(
+ 'http://localhost:' + str(self.wrapper.port), 'test'),
+ create=True,
+ ensure_ddocs=True)
+
+ def tearDown(self):
+ """
+ Need to explicitely invoke destruction on all bases.
+ """
+ self.db2.delete_database()
+ SoledadWithCouchServerMixin.tearDown(self)
+ tests.TestCaseWithServer.tearDown(self)
+
+ def do_sync(self, target_name):
+ """
+ Perform sync using SoledadSynchronizer, SoledadSyncTarget
+ and Token auth.
+ """
+ extra = {}
+ extra = dict(creds={'token': {
+ 'uuid': 'user-uuid',
+ 'token': 'auth-token',
+ }})
+ target_url = self.getURL(target_name)
+ return SoledadSynchronizer(
+ self.db,
+ target.SoledadSyncTarget(
+ target_url,
+ crypto=self._soledad._crypto,
+ **extra)).sync(autocreate=True,
+ defer_decryption=False)
+
+ def test_db_sync(self):
+ """
+ Test sync.
+
+ Adapted to check for encrypted content.
+ """
+ doc1 = self.db.create_doc_from_json(tests.simple_doc)
+ doc2 = self.db2.create_doc_from_json(tests.nested_doc)
+ local_gen_before_sync = self.do_sync('test')
+ gen, _, changes = self.db.whats_changed(local_gen_before_sync)
+ self.assertEqual(1, len(changes))
+ self.assertEqual(doc2.doc_id, changes[0][0])
+ self.assertEqual(1, gen - local_gen_before_sync)
+ self.assertGetEncryptedDoc(
+ self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False)
+
+ def test_db_sync_autocreate(self):
+ """
+ Test sync.
+
+ Adapted to check for encrypted content.
+ """
+ doc1 = self.db.create_doc_from_json(tests.simple_doc)
+ local_gen_before_sync = self.do_sync('test')
+ gen, _, changes = self.db.whats_changed(local_gen_before_sync)
+ self.assertEqual(0, gen - local_gen_before_sync)
+ db3 = self.request_state.open_database('test')
+ gen, _, changes = db3.whats_changed()
+ self.assertEqual(1, len(changes))
+ self.assertEqual(doc1.doc_id, changes[0][0])
+ self.assertGetEncryptedDoc(
+ db3, doc1.doc_id, doc1.rev, tests.simple_doc, False)
+ t_gen, _ = self.db._get_replica_gen_and_trans_id(
+ db3.replica_uid)
+ s_gen, _ = db3._get_replica_gen_and_trans_id('test1')
+ self.assertEqual(1, t_gen)
+ self.assertEqual(1, s_gen)
+
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py
new file mode 100644
index 00000000..48e3150f
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py
@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+# test_sync_deferred.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test Leap backend bits: sync with deferred encryption/decryption.
+"""
+import time
+import os
+import random
+import string
+from urlparse import urljoin
+
+from leap.soledad.common.tests import u1db_tests as tests, ADDRESS
+from leap.soledad.common.tests.u1db_tests import test_sync
+
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common import couch
+from leap.soledad.client import target
+from leap.soledad.client.sync import SoledadSynchronizer
+
+# Just to make clear how this test is different... :)
+DEFER_DECRYPTION = True
+
+WAIT_STEP = 1
+MAX_WAIT = 10
+
+from leap.soledad.common.tests import test_sqlcipher as ts
+from leap.soledad.server import SoledadApp
+
+
+from leap.soledad.client.sqlcipher import open as open_sqlcipher
+from leap.soledad.common.tests.util import SoledadWithCouchServerMixin
+from leap.soledad.common.tests.util import make_soledad_app
+
+
+DBPASS = "pass"
+
+
+class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin):
+ """
+ Another base class for testing the deferred encryption/decryption during
+ the syncs, using the intermediate database.
+ """
+ defer_sync_encryption = True
+
+ def setUp(self):
+ # config info
+ self.db1_file = os.path.join(self.tempdir, "db1.u1db")
+ self.db_pass = DBPASS
+ self.email = ADDRESS
+
+ # get a random prefix for each test, so we do not mess with
+ # concurrency during initialization and shutting down of
+ # each local db.
+ self.rand_prefix = ''.join(
+ map(lambda x: random.choice(string.ascii_letters), range(6)))
+ # initialize soledad by hand so we can control keys
+ self._soledad = self._soledad_instance(
+ prefix=self.rand_prefix, user=self.email)
+
+ # open test dbs: db1 will be the local sqlcipher db
+ # (which instantiates a syncdb)
+ self.db1 = open_sqlcipher(self.db1_file, DBPASS, create=True,
+ document_factory=SoledadDocument,
+ crypto=self._soledad._crypto,
+ defer_encryption=True)
+ self.db2 = couch.CouchDatabase.open_database(
+ urljoin(
+ 'http://localhost:' + str(self.wrapper.port), 'test'),
+ create=True,
+ ensure_ddocs=True)
+
+ def tearDown(self):
+ self.db1.close()
+ self.db2.close()
+ self._soledad.close()
+
+ # XXX should not access "private" attrs
+ for f in [self._soledad._local_db_path,
+ self._soledad._secrets_path,
+ self.db1._sync_db_path]:
+ if os.path.isfile(f):
+ os.unlink(f)
+
+
+#SQLCIPHER_SCENARIOS = [
+# ('http', {
+# #'make_app_with_state': test_sync_target.make_token_soledad_app,
+# 'make_app_with_state': make_soledad_app,
+# 'make_database_for_test': ts.make_sqlcipher_database_for_test,
+# 'copy_database_for_test': ts.copy_sqlcipher_database_for_test,
+# 'make_document_for_test': ts.make_document_for_test,
+# 'token': True
+# }),
+#]
+
+
+class SyncTimeoutError(Exception):
+ """
+ Dummy exception to notify timeout during sync.
+ """
+ pass
+
+
+class TestSoledadDbSyncDeferredEncDecr(
+ BaseSoledadDeferredEncTest,
+ test_sync.TestDbSync):
+ """
+ Test db.sync remote sync shortcut.
+ Case with deferred encryption and decryption: using the intermediate
+ syncdb.
+ """
+
+ scenarios = [
+ ('http', {
+ 'make_app_with_state': make_soledad_app,
+ 'make_database_for_test': tests.make_memory_database_for_test,
+ }),
+ ]
+
+ oauth = False
+ token = True
+
+ def setUp(self):
+ """
+ Need to explicitely invoke inicialization on all bases.
+ """
+ tests.TestCaseWithServer.setUp(self)
+ self.main_test_class = test_sync.TestDbSync
+ BaseSoledadDeferredEncTest.setUp(self)
+ self.startServer()
+ self.syncer = None
+
+ def tearDown(self):
+ """
+ Need to explicitely invoke destruction on all bases.
+ """
+ BaseSoledadDeferredEncTest.tearDown(self)
+ tests.TestCaseWithServer.tearDown(self)
+
+ def do_sync(self, target_name):
+ """
+ Perform sync using SoledadSynchronizer, SoledadSyncTarget
+ and Token auth.
+ """
+ if self.token:
+ extra = dict(creds={'token': {
+ 'uuid': 'user-uuid',
+ 'token': 'auth-token',
+ }})
+ target_url = self.getURL(target_name)
+ syncdb = getattr(self.db1, "_sync_db", None)
+
+ syncer = SoledadSynchronizer(
+ self.db1,
+ target.SoledadSyncTarget(
+ target_url,
+ crypto=self._soledad._crypto,
+ sync_db=syncdb,
+ **extra))
+ # Keep a reference to be able to know when the sync
+ # has finished.
+ self.syncer = syncer
+ return syncer.sync(
+ autocreate=True, defer_decryption=DEFER_DECRYPTION)
+ else:
+ return test_sync.TestDbSync.do_sync(self, target_name)
+
+ def wait_for_sync(self):
+ """
+ Wait for sync to finish.
+ """
+ wait = 0
+ syncer = self.syncer
+ if syncer is not None:
+ while syncer.syncing:
+ time.sleep(WAIT_STEP)
+ wait += WAIT_STEP
+ if wait >= MAX_WAIT:
+ raise SyncTimeoutError
+
+ def test_db_sync(self):
+ """
+ Test sync.
+
+ Adapted to check for encrypted content.
+ """
+ doc1 = self.db1.create_doc_from_json(tests.simple_doc)
+ doc2 = self.db2.create_doc_from_json(tests.nested_doc)
+
+ import time
+ # need to give time to the encryption to proceed
+ # TODO should implement a defer list to subscribe to the all-decrypted
+ # event
+ time.sleep(2)
+
+ local_gen_before_sync = self.do_sync('test')
+ self.wait_for_sync()
+
+ gen, _, changes = self.db1.whats_changed(local_gen_before_sync)
+ self.assertEqual(1, len(changes))
+
+ self.assertEqual(doc2.doc_id, changes[0][0])
+ self.assertEqual(1, gen - local_gen_before_sync)
+
+ self.assertGetEncryptedDoc(
+ self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False)
+
+ def test_db_sync_autocreate(self):
+ pass
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py
new file mode 100644
index 00000000..edc4589b
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_sync_target.py
@@ -0,0 +1,589 @@
+# -*- coding: utf-8 -*-
+# test_sync_target.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test Leap backend bits: sync target
+"""
+import cStringIO
+import os
+
+import simplejson as json
+import u1db
+
+from uuid import uuid4
+
+from u1db.remote import http_database
+
+from u1db import SyncTarget
+from u1db.sync import Synchronizer
+from u1db.remote import (
+ http_client,
+ http_database,
+ http_target,
+)
+
+from leap.soledad import client
+from leap.soledad.client import (
+ target,
+ auth,
+ crypto,
+ VerifiedHTTPSConnection,
+ sync,
+)
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests import BaseSoledadTest
+from leap.soledad.common.tests.util import (
+ make_sqlcipher_database_for_test,
+ make_soledad_app,
+ make_token_soledad_app,
+ SoledadWithCouchServerMixin,
+)
+from leap.soledad.common.tests.u1db_tests import test_backends
+from leap.soledad.common.tests.u1db_tests import test_remote_sync_target
+from leap.soledad.common.tests.u1db_tests import test_sync
+from leap.soledad.common.tests.test_couch import (
+ CouchDBTestCase,
+ CouchDBWrapper,
+)
+
+from leap.soledad.server import SoledadApp
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_backends`.
+#-----------------------------------------------------------------------------
+
+def make_leap_document_for_test(test, doc_id, rev, content,
+ has_conflicts=False):
+ return SoledadDocument(
+ doc_id, rev, content, has_conflicts=has_conflicts)
+
+
+LEAP_SCENARIOS = [
+ ('http', {
+ 'make_database_for_test': test_backends.make_http_database_for_test,
+ 'copy_database_for_test': test_backends.copy_http_database_for_test,
+ 'make_document_for_test': make_leap_document_for_test,
+ 'make_app_with_state': make_soledad_app}),
+]
+
+
+def make_token_http_database_for_test(test, replica_uid):
+ test.startServer()
+ test.request_state._create_database(replica_uid)
+
+ class _HTTPDatabaseWithToken(
+ http_database.HTTPDatabase, auth.TokenBasedAuth):
+
+ def set_token_credentials(self, uuid, token):
+ auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
+
+ def _sign_request(self, method, url_query, params):
+ return auth.TokenBasedAuth._sign_request(
+ self, method, url_query, params)
+
+ http_db = _HTTPDatabaseWithToken(test.getURL('test'))
+ http_db.set_token_credentials('user-uuid', 'auth-token')
+ return http_db
+
+
+def copy_token_http_database_for_test(test, db):
+ # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
+ # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
+ # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
+ # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
+ # HOUSE.
+ http_db = test.request_state._copy_database(db)
+ http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token')
+ return http_db
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_remote_sync_target`.
+#-----------------------------------------------------------------------------
+
+class TestSoledadSyncTargetBasics(
+ test_remote_sync_target.TestHTTPSyncTargetBasics):
+ """
+ Some tests had to be copied to this class so we can instantiate our own
+ target.
+ """
+
+ def test_parse_url(self):
+ remote_target = target.SoledadSyncTarget('http://127.0.0.1:12345/')
+ self.assertEqual('http', remote_target._url.scheme)
+ self.assertEqual('127.0.0.1', remote_target._url.hostname)
+ self.assertEqual(12345, remote_target._url.port)
+ self.assertEqual('/', remote_target._url.path)
+
+
+class TestSoledadParsingSyncStream(
+ test_remote_sync_target.TestParsingSyncStream,
+ BaseSoledadTest):
+ """
+ Some tests had to be copied to this class so we can instantiate our own
+ target.
+ """
+
+ def setUp(self):
+ test_remote_sync_target.TestParsingSyncStream.setUp(self)
+
+ def tearDown(self):
+ test_remote_sync_target.TestParsingSyncStream.tearDown(self)
+
+ def test_extra_comma(self):
+ """
+ Test adapted to use encrypted content.
+ """
+ doc = SoledadDocument('i', rev='r')
+ doc.content = {}
+ _crypto = self._soledad._crypto
+ key = _crypto.doc_passphrase(doc.doc_id)
+ secret = _crypto.secret
+
+ enc_json = crypto.encrypt_docstr(
+ doc.get_json(), doc.doc_id, doc.rev,
+ key, secret)
+ tgt = target.SoledadSyncTarget(
+ "http://foo/foo", crypto=self._soledad._crypto)
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "[\r\n{},\r\n]", None)
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream,
+ '[\r\n{},\r\n{"id": "i", "rev": "r", '
+ '"content": %s, "gen": 3, "trans_id": "T-sid"}'
+ ',\r\n]' % json.dumps(enc_json),
+ lambda doc, gen, trans_id: None)
+
+ def test_wrong_start(self):
+ tgt = target.SoledadSyncTarget("http://foo/foo")
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "{}\r\n]", None)
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "\r\n{}\r\n]", None)
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "", None)
+
+ def test_wrong_end(self):
+ tgt = target.SoledadSyncTarget("http://foo/foo")
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "[\r\n{}", None)
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "[\r\n", None)
+
+ def test_missing_comma(self):
+ tgt = target.SoledadSyncTarget("http://foo/foo")
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream,
+ '[\r\n{}\r\n{"id": "i", "rev": "r", '
+ '"content": "c", "gen": 3}\r\n]', None)
+
+ def test_no_entries(self):
+ tgt = target.SoledadSyncTarget("http://foo/foo")
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream, "[\r\n]", None)
+
+ def test_error_in_stream(self):
+ tgt = target.SoledadSyncTarget("http://foo/foo")
+
+ self.assertRaises(u1db.errors.Unavailable,
+ tgt._parse_sync_stream,
+ '[\r\n{"new_generation": 0},'
+ '\r\n{"error": "unavailable"}\r\n', None)
+
+ self.assertRaises(u1db.errors.Unavailable,
+ tgt._parse_sync_stream,
+ '[\r\n{"error": "unavailable"}\r\n', None)
+
+ self.assertRaises(u1db.errors.BrokenSyncStream,
+ tgt._parse_sync_stream,
+ '[\r\n{"error": "?"}\r\n', None)
+
+
+#
+# functions for TestRemoteSyncTargets
+#
+
+def leap_sync_target(test, path):
+ return target.SoledadSyncTarget(
+ test.getURL(path), crypto=test._soledad._crypto)
+
+
+def token_leap_sync_target(test, path):
+ st = leap_sync_target(test, path)
+ st.set_token_credentials('user-uuid', 'auth-token')
+ return st
+
+
+def make_local_db_and_soledad_target(test, path='test'):
+ test.startServer()
+ db = test.request_state._create_database(os.path.basename(path))
+ st = target.SoledadSyncTarget.connect(
+ test.getURL(path), crypto=test._soledad._crypto)
+ return db, st
+
+
+def make_local_db_and_token_soledad_target(test):
+ db, st = make_local_db_and_soledad_target(test, 'test')
+ st.set_token_credentials('user-uuid', 'auth-token')
+ return db, st
+
+
+class TestSoledadSyncTarget(
+ SoledadWithCouchServerMixin,
+ test_remote_sync_target.TestRemoteSyncTargets):
+
+ scenarios = [
+ ('token_soledad',
+ {'make_app_with_state': make_token_soledad_app,
+ 'make_document_for_test': make_leap_document_for_test,
+ 'create_db_and_target': make_local_db_and_token_soledad_target,
+ 'make_database_for_test': make_sqlcipher_database_for_test,
+ 'sync_target': token_leap_sync_target}),
+ ]
+
+ def setUp(self):
+ tests.TestCaseWithServer.setUp(self)
+ self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets
+ SoledadWithCouchServerMixin.setUp(self)
+ self.startServer()
+ self.db1 = make_sqlcipher_database_for_test(self, 'test1')
+ self.db2 = self.request_state._create_database('test2')
+
+ def tearDown(self):
+ SoledadWithCouchServerMixin.tearDown(self)
+ tests.TestCaseWithServer.tearDown(self)
+ db, _ = self.request_state.ensure_database('test2')
+ db.delete_database()
+
+ def test_sync_exchange_send(self):
+ """
+ Test for sync exchanging send of document.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ self.startServer()
+ db = self.request_state._create_database('test')
+ remote_target = self.getSyncTarget('test')
+ other_docs = []
+
+ def receive_doc(doc, gen, trans_id):
+ other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
+
+ doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
+ new_gen, trans_id = remote_target.sync_exchange(
+ [(doc, 10, 'T-sid')], 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=receive_doc,
+ defer_decryption=False)
+ self.assertEqual(1, new_gen)
+ self.assertGetEncryptedDoc(
+ db, 'doc-here', 'replica:1', '{"value": "here"}', False)
+
+ def test_sync_exchange_send_failure_and_retry_scenario(self):
+ """
+ Test for sync exchange failure and retry.
+
+ This test was adapted to:
+ - decrypt remote content before assert.
+ - not expect a bounced document because soledad has stateful
+ recoverable sync.
+ """
+
+ self.startServer()
+
+ def blackhole_getstderr(inst):
+ return cStringIO.StringIO()
+
+ self.patch(self.server.RequestHandlerClass, 'get_stderr',
+ blackhole_getstderr)
+ db = self.request_state._create_database('test')
+ _put_doc_if_newer = db._put_doc_if_newer
+ trigger_ids = ['doc-here2']
+
+ def bomb_put_doc_if_newer(self, doc, save_conflict,
+ replica_uid=None, replica_gen=None,
+ replica_trans_id=None, number_of_docs=None,
+ doc_idx=None, sync_id=None):
+ if doc.doc_id in trigger_ids:
+ raise Exception
+ return _put_doc_if_newer(doc, save_conflict=save_conflict,
+ replica_uid=replica_uid,
+ replica_gen=replica_gen,
+ replica_trans_id=replica_trans_id,
+ number_of_docs=number_of_docs,
+ doc_idx=doc_idx, sync_id=sync_id)
+ from leap.soledad.common.tests.test_couch import IndexedCouchDatabase
+ self.patch(
+ IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer)
+ remote_target = self.getSyncTarget('test')
+ other_changes = []
+
+ def receive_doc(doc, gen, trans_id):
+ other_changes.append(
+ (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id))
+
+ doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
+ doc2 = self.make_document('doc-here2', 'replica:1',
+ '{"value": "here2"}')
+
+ # we do not expect an HTTPError because soledad sync fails gracefully
+ remote_target.sync_exchange(
+ [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')],
+ 'replica', last_known_generation=0, last_known_trans_id=None,
+ return_doc_cb=receive_doc)
+ self.assertGetEncryptedDoc(
+ db, 'doc-here', 'replica:1', '{"value": "here"}',
+ False)
+ self.assertEqual(
+ (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica'))
+ self.assertEqual([], other_changes)
+ # retry
+ trigger_ids = []
+ new_gen, trans_id = remote_target.sync_exchange(
+ [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=receive_doc)
+ self.assertGetEncryptedDoc(
+ db, 'doc-here2', 'replica:1', '{"value": "here2"}',
+ False)
+ self.assertEqual(
+ (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica'))
+ self.assertEqual(2, new_gen)
+ self.assertEqual(
+ ('doc-here', 'replica:1', '{"value": "here"}', 1),
+ other_changes[0][:-1])
+
+ def test_sync_exchange_send_ensure_callback(self):
+ """
+ Test for sync exchange failure and retry.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ self.startServer()
+ remote_target = self.getSyncTarget('test')
+ other_docs = []
+ replica_uid_box = []
+
+ def receive_doc(doc, gen, trans_id):
+ other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
+
+ def ensure_cb(replica_uid):
+ replica_uid_box.append(replica_uid)
+
+ doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
+ new_gen, trans_id = remote_target.sync_exchange(
+ [(doc, 10, 'T-sid')], 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=receive_doc,
+ ensure_callback=ensure_cb, defer_decryption=False)
+ self.assertEqual(1, new_gen)
+ db = self.request_state.open_database('test')
+ self.assertEqual(1, len(replica_uid_box))
+ self.assertEqual(db._replica_uid, replica_uid_box[0])
+ self.assertGetEncryptedDoc(
+ db, 'doc-here', 'replica:1', '{"value": "here"}', False)
+
+ def test_sync_exchange_in_stream_error(self):
+ # we bypass this test because our sync_exchange process does not
+ # return u1db error 503 "unavailable" for now.
+ pass
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_sync`.
+#-----------------------------------------------------------------------------
+
+target_scenarios = [
+ ('token_leap', {'create_db_and_target':
+ make_local_db_and_token_soledad_target,
+ 'make_app_with_state': make_soledad_app}),
+]
+
+
+class SoledadDatabaseSyncTargetTests(
+ SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests):
+
+ scenarios = (
+ tests.multiply_scenarios(
+ tests.DatabaseBaseTests.scenarios,
+ target_scenarios))
+
+ whitebox = False
+
+ def setUp(self):
+ self.main_test_class = test_sync.DatabaseSyncTargetTests
+ SoledadWithCouchServerMixin.setUp(self)
+
+ def test_sync_exchange(self):
+ """
+ Test sync exchange.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ sol, _ = make_local_db_and_soledad_target(self)
+ docs_by_gen = [
+ (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
+ 'T-sid')]
+ new_gen, trans_id = self.st.sync_exchange(
+ docs_by_gen, 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=self.receive_doc,
+ defer_decryption=False)
+ self.assertGetEncryptedDoc(
+ self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
+ self.assertTransactionLog(['doc-id'], self.db)
+ last_trans_id = self.getLastTransId(self.db)
+ self.assertEqual(([], 1, last_trans_id),
+ (self.other_changes, new_gen, last_trans_id))
+ self.assertEqual(10, self.st.get_sync_info('replica')[3])
+ sol.close()
+
+ def test_sync_exchange_push_many(self):
+ """
+ Test sync exchange.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ docs_by_gen = [
+ (self.make_document(
+ 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'),
+ (self.make_document(
+ 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')]
+ new_gen, trans_id = self.st.sync_exchange(
+ docs_by_gen, 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=self.receive_doc,
+ defer_decryption=False)
+ self.assertGetEncryptedDoc(
+ self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db, 'doc-id2', 'replica:1', tests.nested_doc, False)
+ self.assertTransactionLog(['doc-id', 'doc-id2'], self.db)
+ last_trans_id = self.getLastTransId(self.db)
+ self.assertEqual(([], 2, last_trans_id),
+ (self.other_changes, new_gen, trans_id))
+ self.assertEqual(11, self.st.get_sync_info('replica')[3])
+
+ def test_sync_exchange_returns_many_new_docs(self):
+ """
+ Test sync exchange.
+
+ This test was adapted to avoid JSON serialization comparison as local
+ and remote representations might differ. It looks directly at the
+ doc's contents instead.
+ """
+ doc = self.db.create_doc_from_json(tests.simple_doc)
+ doc2 = self.db.create_doc_from_json(tests.nested_doc)
+ self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
+ new_gen, _ = self.st.sync_exchange(
+ [], 'other-replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=self.receive_doc,
+ defer_decryption=False)
+ self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
+ self.assertEqual(2, new_gen)
+ self.assertEqual(
+ [(doc.doc_id, doc.rev, 1),
+ (doc2.doc_id, doc2.rev, 2)],
+ [c[:-3] + c[-2:-1] for c in self.other_changes])
+ self.assertEqual(
+ json.loads(tests.simple_doc),
+ json.loads(self.other_changes[0][2]))
+ self.assertEqual(
+ json.loads(tests.nested_doc),
+ json.loads(self.other_changes[1][2]))
+ if self.whitebox:
+ self.assertEqual(
+ self.db._last_exchange_log['return'],
+ {'last_gen': 2, 'docs':
+ [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
+
+
+class TestSoledadDbSync(
+ SoledadWithCouchServerMixin, test_sync.TestDbSync):
+ """Test db.sync remote sync shortcut"""
+
+ scenarios = [
+ ('py-token-http', {
+ 'create_db_and_target': make_local_db_and_token_soledad_target,
+ 'make_app_with_state': make_token_soledad_app,
+ 'make_database_for_test': make_sqlcipher_database_for_test,
+ 'token': True
+ }),
+ ]
+
+ oauth = False
+ token = False
+
+ def setUp(self):
+ self.main_test_class = test_sync.TestDbSync
+ SoledadWithCouchServerMixin.setUp(self)
+
+ def do_sync(self, target_name):
+ """
+ Perform sync using SoledadSynchronizer, SoledadSyncTarget
+ and Token auth.
+ """
+ if self.token:
+ extra = dict(creds={'token': {
+ 'uuid': 'user-uuid',
+ 'token': 'auth-token',
+ }})
+ target_url = self.getURL(target_name)
+ return sync.SoledadSynchronizer(
+ self.db,
+ target.SoledadSyncTarget(
+ target_url,
+ crypto=self._soledad._crypto,
+ **extra)).sync(autocreate=True,
+ defer_decryption=False)
+ else:
+ return test_sync.TestDbSync.do_sync(self, target_name)
+
+ def test_db_sync(self):
+ """
+ Test sync.
+
+ Adapted to check for encrypted content.
+ """
+ doc1 = self.db.create_doc_from_json(tests.simple_doc)
+ doc2 = self.db2.create_doc_from_json(tests.nested_doc)
+ local_gen_before_sync = self.do_sync('test2')
+ gen, _, changes = self.db.whats_changed(local_gen_before_sync)
+ self.assertEqual(1, len(changes))
+ self.assertEqual(doc2.doc_id, changes[0][0])
+ self.assertEqual(1, gen - local_gen_before_sync)
+ self.assertGetEncryptedDoc(
+ self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False)
+
+ def test_db_sync_autocreate(self):
+ """
+ We bypass this test because we never need to autocreate databases.
+ """
+ pass
+
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py
index 3457a3e1..6242099d 100644
--- a/common/src/leap/soledad/common/tests/test_target.py
+++ b/common/src/leap/soledad/common/tests/test_target.py
@@ -437,13 +437,17 @@ class TestSoledadSyncTarget(
def bomb_put_doc_if_newer(self, doc, save_conflict,
replica_uid=None, replica_gen=None,
- replica_trans_id=None):
+ replica_trans_id=None, number_of_docs=None,
+ doc_idx=None, sync_id=None):
if doc.doc_id in trigger_ids:
raise Exception
return _put_doc_if_newer(doc, save_conflict=save_conflict,
replica_uid=replica_uid,
replica_gen=replica_gen,
- replica_trans_id=replica_trans_id)
+ replica_trans_id=replica_trans_id,
+ number_of_docs=number_of_docs,
+ doc_idx=doc_idx,
+ sync_id=sync_id)
from leap.soledad.common.tests.test_couch import IndexedCouchDatabase
self.patch(
IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer)
@@ -457,9 +461,8 @@ class TestSoledadSyncTarget(
doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
doc2 = self.make_document('doc-here2', 'replica:1',
'{"value": "here2"}')
- self.assertRaises(
- u1db.errors.HTTPError,
- remote_target.sync_exchange,
+ # We do not expect an exception here because the sync fails gracefully
+ remote_target.sync_exchange(
[(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')],
'replica', last_known_generation=0, last_known_trans_id=None,
return_doc_cb=receive_doc)
@@ -480,11 +483,9 @@ class TestSoledadSyncTarget(
self.assertEqual(
(11, 'T-sud'), db._get_replica_gen_and_trans_id('replica'))
self.assertEqual(2, new_gen)
- # we do not expect the document to be bounced back because soledad has
- # stateful sync
- #self.assertEqual(
- # ('doc-here', 'replica:1', '{"value": "here"}', 1),
- # other_changes[0][:-1])
+ self.assertEqual(
+ ('doc-here', 'replica:1', '{"value": "here"}', 1),
+ other_changes[0][:-1])
def test_sync_exchange_send_ensure_callback(self):
"""
diff --git a/common/src/leap/soledad/common/tests/test_target_soledad.py b/common/src/leap/soledad/common/tests/test_target_soledad.py
new file mode 100644
index 00000000..899203b8
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_target_soledad.py
@@ -0,0 +1,102 @@
+from u1db.remote import (
+ http_database,
+)
+
+from leap.soledad.client import (
+ auth,
+ VerifiedHTTPSConnection,
+)
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.server import SoledadApp
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests import BaseSoledadTest
+from leap.soledad.common.tests.u1db_tests import test_backends
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_backends`.
+#-----------------------------------------------------------------------------
+
+def make_leap_document_for_test(test, doc_id, rev, content,
+ has_conflicts=False):
+ return SoledadDocument(
+ doc_id, rev, content, has_conflicts=has_conflicts)
+
+
+def make_soledad_app(state):
+ return SoledadApp(state)
+
+
+def make_token_soledad_app(state):
+ app = SoledadApp(state)
+
+ def _verify_authentication_data(uuid, auth_data):
+ if uuid == 'user-uuid' and auth_data == 'auth-token':
+ return True
+ return False
+
+ # we test for action authorization in leap.soledad.common.tests.test_server
+ def _verify_authorization(uuid, environ):
+ return True
+
+ application = SoledadTokenAuthMiddleware(app)
+ application._verify_authentication_data = _verify_authentication_data
+ application._verify_authorization = _verify_authorization
+ return application
+
+
+LEAP_SCENARIOS = [
+ ('http', {
+ 'make_database_for_test': test_backends.make_http_database_for_test,
+ 'copy_database_for_test': test_backends.copy_http_database_for_test,
+ 'make_document_for_test': make_leap_document_for_test,
+ 'make_app_with_state': make_soledad_app}),
+]
+
+
+def make_token_http_database_for_test(test, replica_uid):
+ test.startServer()
+ test.request_state._create_database(replica_uid)
+
+ class _HTTPDatabaseWithToken(
+ http_database.HTTPDatabase, auth.TokenBasedAuth):
+
+ def set_token_credentials(self, uuid, token):
+ auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
+
+ def _sign_request(self, method, url_query, params):
+ return auth.TokenBasedAuth._sign_request(
+ self, method, url_query, params)
+
+ http_db = _HTTPDatabaseWithToken(test.getURL('test'))
+ http_db.set_token_credentials('user-uuid', 'auth-token')
+ return http_db
+
+
+def copy_token_http_database_for_test(test, db):
+ # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
+ # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
+ # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
+ # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
+ # HOUSE.
+ http_db = test.request_state._copy_database(db)
+ http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token')
+ return http_db
+
+
+class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest):
+
+ scenarios = LEAP_SCENARIOS + [
+ ('token_http', {'make_database_for_test':
+ make_token_http_database_for_test,
+ 'copy_database_for_test':
+ copy_token_http_database_for_test,
+ 'make_document_for_test': make_leap_document_for_test,
+ 'make_app_with_state': make_token_soledad_app,
+ })
+ ]
+
+load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py
index 99ff77b4..ad66fb06 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py
@@ -13,8 +13,9 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""Test infrastructure for U1DB"""
+"""
+Test infrastructure for U1DB
+"""
import copy
import shutil
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
index c0a7e1f7..86e76fad 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
@@ -41,7 +41,7 @@ from u1db.remote import (
)
-def make_http_database_for_test(test, replica_uid, path='test'):
+def make_http_database_for_test(test, replica_uid, path='test', *args):
test.startServer()
test.request_state._create_database(replica_uid)
return http_database.HTTPDatabase(test.getURL(path))
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
index 633fd8dd..5e2bec86 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
@@ -1151,6 +1151,9 @@ class TestDbSync(tests.TestCaseWithServer):
target_url = self.getURL(path)
return self.db.sync(target_url, **extra)
+ def sync(self, callback=None, autocreate=False, defer_decryption=False):
+ return super(TestDbSync, self).sync(callback, autocreate)
+
def setUp(self):
super(TestDbSync, self).setUp()
self.startServer()