diff options
| -rw-r--r-- | src/leap/soledad/__init__.py | 3 | ||||
| -rw-r--r-- | src/leap/soledad/backends/couch.py | 5 | ||||
| -rw-r--r-- | src/leap/soledad/backends/leap_backend.py | 267 | ||||
| -rw-r--r-- | src/leap/soledad/backends/sqlcipher.py | 19 | ||||
| -rw-r--r-- | src/leap/soledad/tests/test_couch.py | 26 | ||||
| -rw-r--r-- | src/leap/soledad/tests/test_crypto.py | 43 | ||||
| -rw-r--r-- | src/leap/soledad/tests/test_leap_backend.py | 55 | ||||
| -rw-r--r-- | src/leap/soledad/tests/test_sqlcipher.py | 324 |
8 files changed, 506 insertions, 236 deletions
diff --git a/src/leap/soledad/__init__.py b/src/leap/soledad/__init__.py index 78888ead..a736d32b 100644 --- a/src/leap/soledad/__init__.py +++ b/src/leap/soledad/__init__.py @@ -421,8 +421,7 @@ class Soledad(object): content = { '_symkey': self.encrypt_sym(self._symkey, self._passphrase), } - doc = LeapDocument(doc_id=self._address_hash(), - crypto=self._crypto) + doc = LeapDocument(doc_id=self._address_hash()) doc.content = content self._shared_db.put_doc(doc) events.signal( diff --git a/src/leap/soledad/backends/couch.py b/src/leap/soledad/backends/couch.py index 13b6733c..ad8d10e3 100644 --- a/src/leap/soledad/backends/couch.py +++ b/src/leap/soledad/backends/couch.py @@ -144,8 +144,7 @@ class CouchDatabase(ObjectStoreDatabase): doc = self._factory( doc_id=doc_id, rev=cdoc['u1db_rev'], - has_conflicts=has_conflicts, - encryption_scheme=cdoc['encryption_scheme']) + has_conflicts=has_conflicts) contents = self._database.get_attachment(cdoc, 'u1db_json') if contents: doc.content = json.loads(contents.read()) @@ -201,8 +200,6 @@ class CouchDatabase(ObjectStoreDatabase): cdoc['_rev'] = old_cdoc['_rev'] # store u1db's rev cdoc['u1db_rev'] = doc.rev - # store document's encryption scheme - cdoc['encryption_scheme'] = doc.encryption_scheme # save doc in db self._database.save(cdoc) # store u1db's content as json string diff --git a/src/leap/soledad/backends/leap_backend.py b/src/leap/soledad/backends/leap_backend.py index 1c0d5a7d..05c27bc1 100644 --- a/src/leap/soledad/backends/leap_backend.py +++ b/src/leap/soledad/backends/leap_backend.py @@ -36,6 +36,7 @@ from u1db.errors import BrokenSyncStream from leap.common.keymanager import KeyManager +from leap.common.check import leap_assert class NoDefaultKey(Exception): @@ -59,6 +60,16 @@ class DocumentNotEncrypted(Exception): pass +class UnknownEncryptionSchemes(Exception): + """ + Raised when trying to decrypt from unknown encryption schemes. + """ + + +# +# Encryption schemes used for encryption. +# + class EncryptionSchemes(object): """ Representation of encryption schemes used to encrypt documents. @@ -69,21 +80,103 @@ class EncryptionSchemes(object): PUBKEY = 'pubkey' +# +# Crypto utilities for a LeapDocument. +# + +def encrypt_doc_json(crypto, doc_id, doc_json): + """ + Return a valid JSON string containing the C{doc} content encrypted to + a symmetric key and the encryption scheme. + + The returned JSON string is the serialization of the following dictionary: + + { + '_encrypted_json': encrypt_sym(doc_content), + '_encryption_scheme: 'symkey', + } + + @param crypto: A SoledadCryto instance to perform the encryption. + @type crypto: leap.soledad.crypto.SoledadCrypto + @param doc_id: The unique id of the document. + @type doc_id: str + @param doc_json: The JSON serialization of the document's contents. + @type doc_json: str + + @return: The JSON serialization representing the encrypted content. + @rtype: str + """ + ciphertext = crypto.encrypt_sym( + doc_json, + crypto.passphrase_hash(doc_id)) + if not crypto.is_encrypted_sym(ciphertext): + raise DocumentNotEncrypted('Failed encrypting document.') + return json.dumps({ + '_encrypted_json': ciphertext, + '_encryption_scheme': EncryptionSchemes.SYMKEY, + }) + + +def decrypt_doc_json(crypto, doc_id, doc_json): + """ + Return a JSON serialization of the decrypted content contained in + C{encrypted_json}. + + The C{encrypted_json} parameter is the JSON serialization of the + following dictionary: + + { + '_encrypted_json': enc_blob, + '_encryption_scheme': enc_scheme, + } + + C{enc_blob} is the encryption of the JSON serialization of the document's + content. For now Soledad just deals with documents whose C{enc_scheme} is + EncryptionSchemes.SYMKEY. + + @param crypto: A SoledadCryto instance to perform the encryption. + @type crypto: leap.soledad.crypto.SoledadCrypto + @param doc_id: The unique id of the document. + @type doc_id: str + @param doc_json: The JSON serialization representation of the encrypted + document's contents. + @type doc_json: str + + @return: The JSON serialization of the decrypted content. + @rtype: str + """ + leap_assert(isinstance(doc_id, str)) + leap_assert(doc_id != '') + leap_assert(isinstance(doc_json, str)) + leap_assert(doc_json != '') + content = json.loads(doc_json) + ciphertext = content['_encrypted_json'] + enc_scheme = content['_encryption_scheme'] + plainjson = None + if enc_scheme == EncryptionSchemes.SYMKEY: + if not crypto.is_encrypted_sym(ciphertext): + raise DocumentNotEncrypted( + 'Unable to identify document encryption for incoming ' + 'document, although it is marked as being encrypted with a ' + 'symmetric key.') + plainjson = crypto.decrypt_sym( + ciphertext, + crypto.passphrase_hash(doc_id)) + else: + raise UnknownEncryptionScheme(enc_scheme) + return plainjson + + class LeapDocument(Document): """ Encryptable and syncable document. - LEAP Documents are standard u1db documents with cabability of returning an - encrypted version of the document json string as well as setting document - content based on an encrypted version of json string. - - Also, LEAP Documents can be flagged as syncable or not, so the replicas + LEAP Documents can be flagged as syncable or not, so the replicas might not sync every document. """ def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False, - encrypted_json=None, crypto=None, syncable=True, - encryption_scheme=EncryptionSchemes.NONE): + syncable=True): """ Container for handling an encryptable document. @@ -95,68 +188,11 @@ class LeapDocument(Document): @type json: str @param has_conflicts: Boolean indicating if this document has conflicts @type has_conflicts: bool - @param encrypted_json: The encrypted JSON string for this document. If - given, the decrypted value supersedes any raw json string given. - @type encrypted_json: str - @param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - @type crypto: soledad.Soledad @param syncable: Should this document be synced with remote replicas? @type syncable: bool - @param encryption_scheme: The encryption scheme for this objects' - contents. - @type encryption_scheme: str """ Document.__init__(self, doc_id, rev, json, has_conflicts) - self._crypto = crypto self._syncable = syncable - self._encryption_scheme = encryption_scheme - if encrypted_json: - self.set_encrypted_json(encrypted_json) - - def get_encrypted_content(self): - """ - Return an encrypted JSON serialization of document's contents. - - @return: The encrypted JSON serialization of document's contents. - @rtype: str - """ - if not self._crypto: - raise NoSoledadCryptoInstance() - return self._crypto.encrypt_sym( - self.get_json(), - self._crypto.passphrase_hash(self.doc_id)) - - def set_encrypted_content(self, cyphertext): - """ - Decrypt C{cyphertext} and set document's content. - contents. - """ - plaintext = self._crypto.decrypt_sym( - cyphertext, - self._crypto.passphrase_hash(self.doc_id)) - self.set_json(plaintext) - self.encryption_scheme = EncryptionSchemes.NONE - - def get_encrypted_json(self): - """ - Return a valid JSON string containing document's content encrypted to - the user's public key. - - @return: The encrypted JSON string. - @rtype: str - """ - return json.dumps({'_encrypted_json': self.get_encrypted_content()}) - - def set_encrypted_json(self, encrypted_json): - """ - Set document's content based on a valid JSON string containing the - encrypted document's contents. - """ - if not self._crypto: - raise NoSoledadCryptoInstance() - cyphertext = json.loads(encrypted_json)['_encrypted_json'] - self.set_encrypted_content(cyphertext) def _get_syncable(self): """ @@ -182,31 +218,6 @@ class LeapDocument(Document): doc="Determine if document should be synced with server." ) - def _get_encryption_scheme(self): - """ - Return the encryption scheme used to encrypt this document's contents. - - @return: The encryption scheme used to encrypt this document's - contents. - @rtype: str - """ - return self._encryption_scheme - - def _set_encryption_scheme(self, encryption_scheme=True): - """ - Set the encryption scheme used to encrypt this document's contents. - - @param encryption_scheme: The encryption scheme. - @type encryption_scheme: str - """ - self._encryption_scheme = encryption_scheme - - encryption_scheme = property( - _get_encryption_scheme, - _set_encryption_scheme, - doc="The encryption scheme used to encrypt this document's contents." - ) - def _get_rev(self): """ Get the document revision. @@ -237,12 +248,20 @@ class LeapDocument(Document): doc="Wrapper to ensure `doc.rev` is always returned as bytes.") +# +# LeapSyncTarget +# + class LeapSyncTarget(HTTPSyncTarget): """ A SyncTarget that encrypts data before sending and decrypts data after receiving. """ + @staticmethod + def connect(url, crypto=None): + return LeapSyncTarget(url, crypto=crypto) + def __init__(self, url, creds=None, crypto=None): """ Initialize the LeapSyncTarget. @@ -290,34 +309,21 @@ class LeapSyncTarget(HTTPSyncTarget): raise BrokenSyncStream line, comma = utils.check_and_strip_comma(entry) entry = json.loads(line) - # decrypt after receiving from server. - if not self._crypto: - raise NoSoledadCryptoInstance() - # all content arriving should be encrypted either with the - # user's symmetric key or with the user's public key. - enc_json = json.loads(entry['content'])['_encrypted_json'] - plain_json = None - if entry['encryption_scheme'] == EncryptionScheme.SYMKEY: - if not self._crypto.is_encrypted_sym(enc_json): - raise DocumentNotEncrypted( - 'Incoming document\'s contents should be ' - 'encrypted with a symmetric key.') - plain_json = self._crypto.decrypt_sym( - enc_json, self._crypto._symkey) - elif entry['encryption_scheme'] == EncryptionScheme.PUBKEY: - if not self._crypto.is_encrypted_asym(enc_json): - raise DocumentNotEncrypted( - 'Incoming document\'s contents should be ' - 'encrypted to the user\'s public key.') - plain_json = self._crypto.decrypt_asym(enc_json) - else: - raise DocumentNotEncrypted( - "Incoming document from sync is not encrypted.") - # if decryption was OK, then create the document. - doc = LeapDocument(entry['id'], entry['rev'], - json=plain_json, - crypto=self._crypto, - encryption_scheme=EncryptionScheme.NONE) + #------------------------------------------------------------- + # symmetric decryption of document's contents + #------------------------------------------------------------- + # if arriving content was symmetrically encrypted, we decrypt + # it. + doc = LeapDocument(entry['id'], entry['rev'], entry['content']) + if doc.content and '_encryption_scheme' in doc.content: + if doc.content['_encryption_scheme'] == \ + EncryptionSchemes.SYMKEY: + doc.set_json( + decrypt_doc_json( + self._crypto, doc.doc_id, entry['content'])) + #------------------------------------------------------------- + # end of symmetric decryption + #------------------------------------------------------------- return_doc_cb(doc, entry['gen'], entry['trans_id']) if parts[-1] != ']': try: @@ -361,17 +367,22 @@ class LeapSyncTarget(HTTPSyncTarget): ensure=ensure_callback is not None) comma = ',' for doc, gen, trans_id in docs_by_generations: - if doc.syncable: - # encrypt and verify before sending to server. - enc_json = json.loads( - doc.get_encrypted_json())['_encrypted_json'] - if not self._crypto.is_encrypted_sym(enc_json): - raise DocumentNotEncrypted( - "Could not encrypt document before sync.") - size += prepare(id=doc.doc_id, rev=doc.rev, - content=enc_json, - gen=gen, trans_id=trans_id, - encryption_scheme=EncryptionSchemes.SYMKEY) + # skip non-syncable docs + if isinstance(doc, LeapDocument) and not doc.syncable: + continue + #------------------------------------------------------------- + # symmetric encryption of document's contents + #------------------------------------------------------------- + enc_json = doc.get_json() + if not doc.is_tombstone(): + enc_json = encrypt_doc_json( + self._crypto, doc.doc_id, doc.get_json()) + #------------------------------------------------------------- + # end of symmetric encryption + #------------------------------------------------------------- + size += prepare(id=doc.doc_id, rev=doc.rev, + content=enc_json, + gen=gen, trans_id=trans_id) entries.append('\r\n]') size += len(entries[-1]) self._conn.putheader('content-length', str(size)) diff --git a/src/leap/soledad/backends/sqlcipher.py b/src/leap/soledad/backends/sqlcipher.py index fb5c3e79..f840d809 100644 --- a/src/leap/soledad/backends/sqlcipher.py +++ b/src/leap/soledad/backends/sqlcipher.py @@ -100,14 +100,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._crypto = crypto def factory(doc_id=None, rev=None, json='{}', has_conflicts=False, - encrypted_json=None, syncable=True, - encryption_scheme=EncryptionSchemes.NONE): + syncable=True): return LeapDocument(doc_id=doc_id, rev=rev, json=json, has_conflicts=has_conflicts, - encrypted_json=encrypted_json, - crypto=self._crypto, - syncable=syncable, - encryption_scheme=encryption_scheme) + syncable=syncable) self.set_document_factory(factory) def _check_if_db_is_encrypted(self, sqlcipher_file): @@ -248,10 +244,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): c.execute( 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') - c.execute( - 'ALTER TABLE document ' - 'ADD COLUMN encryption_scheme TEXT NOT NULL DEFAULT \'%s\'' % - EncryptionSchemes.NONE) def _put_and_update_indexes(self, old_doc, doc): """ @@ -265,9 +257,9 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( self, old_doc, doc) c = self._db_handle.cursor() - c.execute('UPDATE document SET syncable=?, encryption_scheme=? ' + c.execute('UPDATE document SET syncable=? ' 'WHERE doc_id=?', - (doc.syncable, doc.encryption_scheme, doc.doc_id)) + (doc.syncable, doc.doc_id)) def _get_doc(self, doc_id, check_for_conflicts=False): """ @@ -287,12 +279,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self, doc_id, check_for_conflicts) if doc: c = self._db_handle.cursor() - c.execute('SELECT syncable, encryption_scheme FROM document ' + c.execute('SELECT syncable FROM document ' 'WHERE doc_id=?', (doc.doc_id,)) result = c.fetchone() doc.syncable = bool(result[0]) - doc.encryption_scheme = result[1] return doc sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) diff --git a/src/leap/soledad/tests/test_couch.py b/src/leap/soledad/tests/test_couch.py index cdf9c9ff..456bc080 100644 --- a/src/leap/soledad/tests/test_couch.py +++ b/src/leap/soledad/tests/test_couch.py @@ -17,7 +17,6 @@ except ImportError: import json # noqa from leap.soledad.backends.leap_backend import ( LeapDocument, - EncryptionSchemes, ) @@ -412,30 +411,5 @@ class CouchDatabaseStorageTests(CouchDBTestCase): content = self._fetch_u1db_data(db) self.assertEqual(db._replica_uid, content['replica_uid']) - def test_store_encryption_scheme(self): - db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), - 'u1db_tests') - doc = db.create_doc_from_json(tests.simple_doc) - # assert that docs have no encryption_scheme by default - self.assertEqual(EncryptionSchemes.NONE, doc.encryption_scheme) - # assert that we can store a different value - doc.encryption_scheme = EncryptionSchemes.PUBKEY - db.put_doc(doc) - self.assertEqual( - EncryptionSchemes.PUBKEY, - db.get_doc(doc.doc_id).encryption_scheme) - # assert that we can store another different value - doc.encryption_scheme = EncryptionSchemes.SYMKEY - db.put_doc(doc) - self.assertEqual( - EncryptionSchemes.SYMKEY, - db.get_doc(doc.doc_id).encryption_scheme) - # assert that we can store the default value - doc.encryption_scheme = EncryptionSchemes.NONE - db.put_doc(doc) - self.assertEqual( - EncryptionSchemes.NONE, - db.get_doc(doc.doc_id).encryption_scheme) - load_tests = tests.load_with_scenarios diff --git a/src/leap/soledad/tests/test_crypto.py b/src/leap/soledad/tests/test_crypto.py index 039d2f3c..676c13b0 100644 --- a/src/leap/soledad/tests/test_crypto.py +++ b/src/leap/soledad/tests/test_crypto.py @@ -22,22 +22,26 @@ Tests for cryptographic related stuff. import os +try: + import simplejson as json +except ImportError: + import json # noqa +from leap.soledad.backends.leap_backend import ( + LeapDocument, + encrypt_doc_json, + decrypt_doc_json, + EncryptionSchemes, +) +from leap.soledad import KeyAlreadyExists +from leap.soledad.crypto import SoledadCrypto from leap.common.testing.basetest import BaseLeapTest -from leap.soledad.backends.leap_backend import LeapDocument from leap.soledad.tests import BaseSoledadTest from leap.soledad.tests import ( KEY_FINGERPRINT, PRIVATE_KEY, ) -from leap.soledad import KeyAlreadyExists -from leap.soledad.crypto import SoledadCrypto - -try: - import simplejson as json -except ImportError: - import json # noqa class EncryptedSyncTestCase(BaseSoledadTest): @@ -45,26 +49,31 @@ class EncryptedSyncTestCase(BaseSoledadTest): Tests that guarantee that data will always be encrypted when syncing. """ - def test_get_set_encrypted_json(self): + def test_encrypt_decrypt_json(self): """ - Test getting and setting encrypted content. + Test encrypting and decrypting documents. """ - doc1 = LeapDocument(crypto=self._soledad._crypto) + doc1 = LeapDocument(doc_id='id') doc1.content = {'key': 'val'} - doc2 = LeapDocument(doc_id=doc1.doc_id, - encrypted_json=doc1.get_encrypted_json(), - crypto=self._soledad._crypto) + enc_json = encrypt_doc_json( + self._soledad._crypto, doc1.doc_id, doc1.get_json()) + plain_json = decrypt_doc_json( + self._soledad._crypto, doc1.doc_id, enc_json) + doc2 = LeapDocument(doc_id=doc1.doc_id, json=plain_json) res1 = doc1.get_json() res2 = doc2.get_json() self.assertEqual(res1, res2, 'incorrect document encryption') - def test_successful_symmetric_encryption(self): + def test_encrypt_sym(self): """ Test for successful symmetric encryption. """ - doc1 = LeapDocument(crypto=self._soledad._crypto) + doc1 = LeapDocument() doc1.content = {'key': 'val'} - enc_json = json.loads(doc1.get_encrypted_json())['_encrypted_json'] + enc_json = json.loads( + encrypt_doc_json( + self._soledad._crypto, + doc1.doc_id, doc1.get_json()))['_encrypted_json'] self.assertEqual( True, self._soledad._crypto.is_encrypted_sym(enc_json), diff --git a/src/leap/soledad/tests/test_leap_backend.py b/src/leap/soledad/tests/test_leap_backend.py index 0bf70f9e..7509af0e 100644 --- a/src/leap/soledad/tests/test_leap_backend.py +++ b/src/leap/soledad/tests/test_leap_backend.py @@ -5,6 +5,10 @@ For these tests to run, a leap server has to be running on (default) port """ import u1db +try: + import simplejson as json +except ImportError: + import json # noqa from leap.soledad.backends import leap_backend @@ -29,16 +33,7 @@ from leap.soledad.tests.u1db_tests import test_https def make_leap_document_for_test(test, doc_id, rev, content, has_conflicts=False): return leap_backend.LeapDocument( - doc_id, rev, content, has_conflicts=has_conflicts, - crypto=test._soledad._crypto) - - -def make_leap_encrypted_document_for_test(test, doc_id, rev, encrypted_content, - has_conflicts=False): - return leap_backend.LeapDocument( - doc_id, rev, encrypted_json=encrypted_content, - has_conflicts=has_conflicts, - crypto=test._soledad.crypto) + doc_id, rev, content, has_conflicts=has_conflicts) LEAP_SCENARIOS = [ @@ -94,9 +89,22 @@ class TestLeapSyncTargetBasics( self.assertEqual('/', remote_target._url.path) -class TestLeapParsingSyncStream(test_remote_sync_target.TestParsingSyncStream): +class TestLeapParsingSyncStream( + test_remote_sync_target.TestParsingSyncStream, + BaseSoledadTest): + + def setUp(self): + test_remote_sync_target.TestParsingSyncStream.setUp(self) + BaseSoledadTest.setUp(self) + + def tearDown(self): + test_remote_sync_target.TestParsingSyncStream.tearDown(self) + BaseSoledadTest.tearDown(self) def test_wrong_start(self): + """ + Test adapted to use a LeapSyncTarget. + """ tgt = leap_backend.LeapSyncTarget("http://foo/foo") self.assertRaises(u1db.errors.BrokenSyncStream, @@ -109,6 +117,9 @@ class TestLeapParsingSyncStream(test_remote_sync_target.TestParsingSyncStream): tgt._parse_sync_stream, "", None) def test_wrong_end(self): + """ + Test adapted to use a LeapSyncTarget. + """ tgt = leap_backend.LeapSyncTarget("http://foo/foo") self.assertRaises(u1db.errors.BrokenSyncStream, @@ -118,6 +129,9 @@ class TestLeapParsingSyncStream(test_remote_sync_target.TestParsingSyncStream): tgt._parse_sync_stream, "[\r\n", None) def test_missing_comma(self): + """ + Test adapted to use a LeapSyncTarget. + """ tgt = leap_backend.LeapSyncTarget("http://foo/foo") self.assertRaises(u1db.errors.BrokenSyncStream, @@ -126,21 +140,32 @@ class TestLeapParsingSyncStream(test_remote_sync_target.TestParsingSyncStream): '"content": "c", "gen": 3}\r\n]', None) def test_no_entries(self): + """ + Test adapted to use a LeapSyncTarget. + """ tgt = leap_backend.LeapSyncTarget("http://foo/foo") self.assertRaises(u1db.errors.BrokenSyncStream, tgt._parse_sync_stream, "[\r\n]", None) def test_extra_comma(self): - tgt = leap_backend.LeapSyncTarget("http://foo/foo") + """ + Test adapted to use encrypted content. + """ + doc = leap_backend.LeapDocument('i') + doc.content = {} + enc_json = leap_backend.encrypt_doc_json( + self._soledad._crypto, doc.doc_id, doc.get_json()) + tgt = leap_backend.LeapSyncTarget( + "http://foo/foo", crypto=self._soledad._crypto) self.assertRaises(u1db.errors.BrokenSyncStream, tgt._parse_sync_stream, "[\r\n{},\r\n]", None) - self.assertRaises(leap_backend.NoSoledadCryptoInstance, + self.assertRaises(u1db.errors.BrokenSyncStream, tgt._parse_sync_stream, '[\r\n{},\r\n{"id": "i", "rev": "r", ' - '"content": "{}", "gen": 3, "trans_id": "T-sid"}' - ',\r\n]', + '"content": %s, "gen": 3, "trans_id": "T-sid"}' + ',\r\n]' % json.dumps(enc_json), lambda doc, gen, trans_id: None) def test_error_in_stream(self): diff --git a/src/leap/soledad/tests/test_sqlcipher.py b/src/leap/soledad/tests/test_sqlcipher.py index 337ab4ee..73388202 100644 --- a/src/leap/soledad/tests/test_sqlcipher.py +++ b/src/leap/soledad/tests/test_sqlcipher.py @@ -2,16 +2,21 @@ import os import time -from pysqlcipher import dbapi2 import unittest -from StringIO import StringIO +try: + import simplejson as json +except ImportError: + import json # noqa import threading +from pysqlcipher import dbapi2 +from StringIO import StringIO # u1db stuff. from u1db import ( errors, query_parser, sync, + vectorclock, ) from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase @@ -24,10 +29,12 @@ from leap.soledad.backends.sqlcipher import open as u1db_open from leap.soledad.backends.leap_backend import ( LeapDocument, EncryptionSchemes, + decrypt_doc_json, + encrypt_doc_json, ) # u1db tests stuff. -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.tests import u1db_tests as tests, BaseSoledadTest from leap.soledad.tests.u1db_tests import test_sqlite_backend from leap.soledad.tests.u1db_tests import test_backends from leap.soledad.tests.u1db_tests import test_open @@ -345,29 +352,6 @@ class TestSQLCipherPartialExpandDatabase( self.db.put_doc(doc) self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) - def test_store_encryption_scheme(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - # assert that docs have no encryption_scheme by default - self.assertEqual(EncryptionSchemes.NONE, doc.encryption_scheme) - # assert that we can store a different value - doc.encryption_scheme = EncryptionSchemes.PUBKEY - self.db.put_doc(doc) - self.assertEqual( - EncryptionSchemes.PUBKEY, - self.db.get_doc(doc.doc_id).encryption_scheme) - # assert that we can store another different value - doc.encryption_scheme = EncryptionSchemes.SYMKEY - self.db.put_doc(doc) - self.assertEqual( - EncryptionSchemes.SYMKEY, - self.db.get_doc(doc.doc_id).encryption_scheme) - # assert that we can store the default value - doc.encryption_scheme = EncryptionSchemes.NONE - self.db.put_doc(doc) - self.assertEqual( - EncryptionSchemes.NONE, - self.db.get_doc(doc.doc_id).encryption_scheme) - #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_open`. @@ -430,7 +414,7 @@ def sync_via_synchronizer_and_leap(test, db_source, db_target, if trace_hook: test.skipTest("full trace hook unsupported over http") path = test._http_at[db_target] - target = LeapSyncTarget.connect(test.getURL(path)) + target = LeapSyncTarget.connect(test.getURL(path), test._soledad._crypto) if trace_hook_shallow: target._set_trace_hook_shallow(trace_hook_shallow) return sync.Synchronizer(db_source, target).sync() @@ -445,15 +429,192 @@ sync_scenarios.append(('pyleap', { })) -class SQLCipherDatabaseSyncTests(test_sync.DatabaseSyncTests): +class SQLCipherDatabaseSyncTests( + test_sync.DatabaseSyncTests, BaseSoledadTest): scenarios = sync_scenarios + def setUp(self): + test_sync.DatabaseSyncTests.setUp(self) + BaseSoledadTest.setUp(self) + + def tearDown(self): + test_sync.DatabaseSyncTests.tearDown(self) + BaseSoledadTest.tearDown(self) + + def test_sync_autoresolves(self): + # The remote database can't autoresolve conflicts based on magic + # content convergence, so we modify this test to leave the possibility + # of the remode document ending up in conflicted state. + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # if remote content is in conflicted state, then document revisions + # will be different. + #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + doc1 = self.db1.get_doc('doc') + doc2 = self.db1.get_doc('doc') + if '_encryption_scheme' in doc2.content: + doc2.set_json( + decrypt_doc_json( + self._soledad._crypto, doc2, doc2.get_json())) + self.assertEqual(doc1, doc2) + + def test_sync_autoresolves_moar_backwards(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # Despite that, in Soledad we suppose that the server never syncs, so + # it never has conflicted documents. Also, if it had, convergence + # would not be possible in terms of document's contents. + # + # Therefore we suppress this test. + pass + + def test_sync_autoresolves_moar_backwards_three(self): + # We use the same reasoning from the last test to suppress this one. + pass + + def test_sync_propagates_resolution(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + if '_encryption_scheme' in doc3.content: + doc3.set_json( + decrypt_doc_json( + self._soledad._crypto, doc3.doc_id, doc3.get_json())) + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + + def test_sync_puts_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + exp_doc = self.make_document( + doc.doc_id, doc.rev, tests.simple_doc, False) + doc2 = self.db2.get_doc(doc.doc_id) + # decrypt to compare it it is the case + if '_encryption_scheme' in doc2.content: + doc2 = self.db2.get_doc(doc.doc_id) + doc2.set_json( + decrypt_doc_json( + self._soledad._crypto, doc.doc_id, doc2.get_json())) + self.assertEqual(exp_doc, doc2) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + def _make_local_db_and_leap_target(test, path='test'): test.startServer() db = test.request_state._create_database(os.path.basename(path)) - st = LeapSyncTarget.connect(test.getURL(path)) + st = LeapSyncTarget.connect(test.getURL(path), test._soledad._crypto) return db, st @@ -464,11 +625,114 @@ target_scenarios = [ ] -class SQLCipherSyncTargetTests(test_sync.DatabaseSyncTargetTests): +class SQLCipherSyncTargetTests( + test_sync.DatabaseSyncTargetTests, BaseSoledadTest): scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, target_scenarios)) + def setUp(self): + test_sync.DatabaseSyncTargetTests.setUp(self) + BaseSoledadTest.setUp(self) + + def tearDown(self): + test_sync.DatabaseSyncTargetTests.tearDown(self) + BaseSoledadTest.tearDown(self) + + def test_sync_exchange(self): + """ + Modified to account for possibly receiving encrypted documents from + sever-side. + """ + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + # decrypt doc1 for comparison if needed + tmpdoc = self.db.get_doc('doc-id') + if '_encryption_scheme' in tmpdoc.content: + tmpdoc.set_json( + decrypt_doc_json( + self._soledad._crypto, tmpdoc.doc_id, + tmpdoc.get_json())) + self.assertEqual(tmpdoc.rev, 'replica:1') + self.assertEqual(tmpdoc.content, json.loads(tests.simple_doc)) + self.assertFalse(tmpdoc.has_conflicts) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + """ + Modified to account for possibly receiving encrypted documents from + sever-side. + """ + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, + 'T-1'), + (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + # decrypt doc1 for comparison if needed + tmpdoc1 = self.db.get_doc('doc-id') + if '_encryption_scheme' in tmpdoc1.content: + tmpdoc1.set_json( + decrypt_doc_json( + self._soledad._crypto, tmpdoc1.doc_id, + tmpdoc1.get_json())) + self.assertEqual(tmpdoc1.rev, 'replica:1') + self.assertEqual(tmpdoc1.content, json.loads(tests.simple_doc)) + self.assertFalse(tmpdoc1.has_conflicts) + # decrypt doc2 for comparison if needed + tmpdoc2 = self.db.get_doc('doc-id2') + if '_encryption_scheme' in tmpdoc2.content: + tmpdoc2.set_json( + decrypt_doc_json( + self._soledad._crypto, tmpdoc2.doc_id, + tmpdoc2.get_json())) + self.assertEqual(tmpdoc2.rev, 'replica:1') + self.assertEqual(tmpdoc2.content, json.loads(tests.nested_doc)) + self.assertFalse(tmpdoc2.has_conflicts) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_returns_many_new_docs(self): + """ + Modified to account for JSON serialization differences. + """ + doc = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db.create_doc_from_json(tests.nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, 1), + (doc2.doc_id, doc2.rev, 2)], + [c[:2]+c[3:4] for c in self.other_changes]) + self.assertEqual( + json.dumps(tests.simple_doc), + json.dumps(self.other_changes[0][2])) + self.assertEqual( + json.loads(tests.nested_doc), + json.loads(self.other_changes[1][2])) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + #----------------------------------------------------------------------------- # Tests for actual encryption of the database |