diff options
19 files changed, 61 insertions, 47 deletions
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index c560f661..da6eec66 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -935,6 +935,7 @@ def create_path_if_not_exists(path): # Monkey patching u1db to be able to provide a custom SSL cert # ---------------------------------------------------------------------------- + # We need a more reasonable timeout (in seconds) SOLEDAD_TIMEOUT = 120 diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py index 4fc91d9d..92bc85d6 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py @@ -58,6 +58,7 @@ def debug(*args): if not silent: print(*args) + debug("[+] db path:", tmpdb) debug("[+] num docs", numdocs) @@ -74,6 +75,7 @@ dbpool = adbapi.getConnectionPool(opts) def createDoc(doc): return dbpool.runU1DBQuery("create_doc", doc) + db_indexes = { 'by-chash': ['chash'], 'by-number': ['number']} @@ -168,6 +170,7 @@ def insert_docs(_): deferreds.append(d) return defer.gatherResults(deferreds, consumeErrors=True) + d = create_indexes(None) d.addCallback(insert_docs) d.addCallback(get_from_index) diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py index 38ea18a3..429566c7 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py @@ -58,6 +58,7 @@ def debug(*args): if not silent: print(*args) + debug("[+] db path:", tmpdb) debug("[+] num docs", numdocs) @@ -74,6 +75,7 @@ dbpool = adbapi.getConnectionPool(opts) def createDoc(doc, doc_id): return dbpool.runU1DBQuery("create_doc", doc, doc_id=doc_id) + db_indexes = { 'by-chash': ['chash'], 'by-number': ['number']} @@ -168,6 +170,7 @@ def insert_docs(_): deferreds.append(d) return defer.gatherResults(deferreds, consumeErrors=True) + d = create_indexes(None) d.addCallback(insert_docs) d.addCallback(get_from_index) diff --git a/client/src/leap/soledad/client/examples/run_benchmark.py b/client/src/leap/soledad/client/examples/run_benchmark.py index 61621e89..ddedf433 100644 --- a/client/src/leap/soledad/client/examples/run_benchmark.py +++ b/client/src/leap/soledad/client/examples/run_benchmark.py @@ -14,6 +14,7 @@ cmd = "SILENT=1 TIMES={times} TMPDIR={tmpdir} python ./use_{version}api.py" def parse_time(r): return r.split('\n')[-1] + with open(CSVFILE, 'w') as log: for times in range(0, 10000, 500): diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py index a2683836..39301b41 100644 --- a/client/src/leap/soledad/client/examples/use_adbapi.py +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -39,6 +39,7 @@ def debug(*args): if not silent: print(*args) + debug("[+] db path:", tmpdb) debug("[+] times", times) @@ -87,6 +88,7 @@ def allDone(_): print((end_time - start_time).total_seconds()) reactor.stop() + deferreds = [] payload = open('manifest.phk').read() diff --git a/client/src/leap/soledad/client/examples/use_api.py b/client/src/leap/soledad/client/examples/use_api.py index e2501c98..db77c4b3 100644 --- a/client/src/leap/soledad/client/examples/use_api.py +++ b/client/src/leap/soledad/client/examples/use_api.py @@ -36,6 +36,7 @@ def debug(*args): if not silent: print(*args) + debug("[+] db path:", tmpdb) debug("[+] times", times) @@ -52,6 +53,7 @@ db = sqlcipher.SQLCipherDatabase(opts) def allDone(): debug("ALL DONE!") + payload = open('manifest.phk').read() for i in range(times): diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 6caa39cd..b9db3674 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -594,6 +594,7 @@ def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False, return SoledadDocument(doc_id=doc_id, rev=rev, json=json, has_conflicts=has_conflicts, syncable=syncable) + sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py index 295f3132..27db65af 100644 --- a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py +++ b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py @@ -923,4 +923,5 @@ class SQLitePartialExpandDatabase(SQLiteDatabase): raw_doc = json.loads(doc) self._update_indexes(doc_id, raw_doc, getters, c) + SQLiteDatabase.register_implementation(SQLitePartialExpandDatabase) diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py index a9680890..496274b2 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_app.py +++ b/common/src/leap/soledad/common/l2db/remote/http_app.py @@ -194,6 +194,7 @@ class URLToResource(object): resource_cls = params.pop('resource_cls') return resource_cls, params + url_to_resource = URLToResource() @@ -1,7 +1,7 @@ [pep8] -exclude = versioneer.py,_version.py,*.egg,build,docs,scripts,ddocs.py -ignore = E731 +exclude = versioneer.py,_version.py,*.egg,build,docs,scripts,ddocs.py,.tox +ignore = F812,E731 [flake8] -exclude = versioneer.py,_version.py,*.egg,build,docs,scripts,ddocs.py -ignore = E731 +exclude = versioneer.py,_version.py,*.egg,build,docs,scripts,ddocs.py,.tox +ignore = F812,E731 diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py index dc3054f2..483c7803 100644 --- a/testing/tests/client/test_crypto.py +++ b/testing/tests/client/test_crypto.py @@ -59,8 +59,8 @@ class AESTest(unittest.TestCase): data = snowden1 block = 16 - for i in range(len(data)/block): - chunk = data[i * block:(i+1)*block] + for i in range(len(data) / block): + chunk = data[i * block:(i + 1) * block] aes.write(chunk) aes.end() @@ -69,7 +69,6 @@ class AESTest(unittest.TestCase): assert ciphertext_chunked == ciphertext - def test_decrypt(self): key = 'A' * 32 iv = 'A' * 16 @@ -82,8 +81,8 @@ class AESTest(unittest.TestCase): fd = BytesIO() aes = _crypto.AESDecryptor(key, iv, fd) - for i in range(len(ciphertext)/block): - chunk = ciphertext[i * block:(i+1)*block] + for i in range(len(ciphertext) / block): + chunk = ciphertext[i * block:(i + 1) * block] aes.write(chunk) aes.end() @@ -91,7 +90,6 @@ class AESTest(unittest.TestCase): assert cleartext_chunked == data - class BlobTestCase(unittest.TestCase): class doc_info: @@ -108,13 +106,13 @@ class BlobTestCase(unittest.TestCase): blob = _crypto.BlobEncryptor( self.doc_info, inf, result=outf, - secret='A' * 96, iv='B'*16) + secret='A' * 96, iv='B' * 16) encrypted = yield blob.encrypt() data = base64.urlsafe_b64decode(encrypted.getvalue()) assert data[0] == '\x80' - ts, sch, meth = struct.unpack( + ts, sch, meth = struct.unpack( 'Qbb', data[1:11]) assert sch == 1 assert meth == 1 @@ -128,13 +126,12 @@ class BlobTestCase(unittest.TestCase): ciphertext = data[71:-64] aes_key = _crypto._get_sym_key_for_doc( - self.doc_info.doc_id, 'A'*96) - assert ciphertext == _aes_encrypt(aes_key, 'B'*16, snowden1) + self.doc_info.doc_id, 'A' * 96) + assert ciphertext == _aes_encrypt(aes_key, 'B' * 16, snowden1) - decrypted = _aes_decrypt(aes_key, 'B'*16, ciphertext) + decrypted = _aes_decrypt(aes_key, 'B' * 16, ciphertext) assert str(decrypted) == snowden1 - @defer.inlineCallbacks def test_blob_decryptor(self): @@ -154,7 +151,6 @@ class BlobTestCase(unittest.TestCase): decrypted = yield decryptor.decrypt() assert decrypted.getvalue() == snowden1 - @defer.inlineCallbacks def test_encrypt_and_decrypt(self): """ @@ -173,7 +169,6 @@ class BlobTestCase(unittest.TestCase): assert len(decrypted) != 0 assert json.loads(decrypted) == payload - @defer.inlineCallbacks def test_decrypt_with_wrong_mac_raises(self): """ @@ -193,8 +188,7 @@ class BlobTestCase(unittest.TestCase): doc2.set_json(json.dumps({"raw": str(newraw)})) with pytest.raises(_crypto.InvalidBlob): - decrypted = yield crypto.decrypt_doc(doc2) - + yield crypto.decrypt_doc(doc2) class RecoveryDocumentTestCase(BaseSoledadTest): @@ -283,7 +277,6 @@ class SoledadSecretsTestCase(BaseSoledadTest): "Should have a secret at this point") - class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): diff --git a/testing/tests/couch/test_command.py b/testing/tests/couch/test_command.py index 68097fb1..9fb2c153 100644 --- a/testing/tests/couch/test_command.py +++ b/testing/tests/couch/test_command.py @@ -25,6 +25,7 @@ class CommandBasedDBCreationTest(unittest.TestCase): state.ensure_database, "user-1337") def test_raises_unauthorized_by_default(self): - state = couch_state.CouchServerState("url", check_schema_versions=False) + state = couch_state.CouchServerState("url", + check_schema_versions=False) self.assertRaises(u1db_errors.Unauthorized, state.ensure_database, "user-1337") diff --git a/testing/tests/perf/test_crypto.py b/testing/tests/perf/test_crypto.py index 9ce418ba..367c3b5b 100644 --- a/testing/tests/perf/test_crypto.py +++ b/testing/tests/perf/test_crypto.py @@ -5,7 +5,7 @@ SIZE_LIMT environment variable. For instance, to keep the maximum payload at 1MB: -SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py """ import pytest import os @@ -45,7 +45,7 @@ def create_doc_decryption(size): doc = SoledadDocument( doc_id=uuid4().hex, rev='rev', json=json.dumps(DOC_CONTENT)) - + encrypted_doc = yield crypto.encrypt_doc(doc) doc.set_json(encrypted_doc) @@ -74,16 +74,16 @@ def create_raw_decryption(size): # plugin. encryption_tests = [ - ('10k', 1E4), + ('10k', 1E4), ('100k', 1E5), ('500k', 5E5), - ('1M', 1E6), - ('10M', 1E7), - ('50M', 5E7), + ('1M', 1E6), + ('10M', 1E7), + ('50M', 5E7), ] for name, size in encryption_tests: - if size < LIMIT: + if size < LIMIT: sz = int(size) globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) diff --git a/testing/tests/perf/test_sqlcipher.py b/testing/tests/perf/test_sqlcipher.py index e7a54228..39c9e3ad 100644 --- a/testing/tests/perf/test_sqlcipher.py +++ b/testing/tests/perf/test_sqlcipher.py @@ -29,10 +29,10 @@ def build_test_sqlcipher_create(amount, size): return test -test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500*1000) -test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100*1000) -test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10*1000) +test_async_create_20_500k = build_test_sqlcipher_async_create(20, 500 * 1000) +test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) +test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000) # synchronous -test_create_20_500k = build_test_sqlcipher_create(20, 500*1000) -test_create_100_100k = build_test_sqlcipher_create(100, 100*1000) -test_create_1000_10k = build_test_sqlcipher_create(1000, 10*1000) +test_create_20_500k = build_test_sqlcipher_create(20, 500 * 1000) +test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) +test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/testing/tests/perf/test_sync.py b/testing/tests/perf/test_sync.py index 7b3c4bf0..1bf6cc21 100644 --- a/testing/tests/perf/test_sync.py +++ b/testing/tests/perf/test_sync.py @@ -21,9 +21,9 @@ def create_upload(uploads, size): return test -test_upload_20_500k = create_upload(20, 500*1000) -test_upload_100_100k = create_upload(100, 100*1000) -test_upload_1000_10k = create_upload(1000, 10*1000) +test_upload_20_500k = create_upload(20, 500 * 1000) +test_upload_100_100k = create_upload(100, 100 * 1000) +test_upload_1000_10k = create_upload(1000, 10 * 1000) def create_download(downloads, size): @@ -46,9 +46,9 @@ def create_download(downloads, size): return test -test_download_20_500k = create_download(20, 500*1000) -test_download_100_100k = create_download(100, 100*1000) -test_download_1000_10k = create_download(1000, 10*1000) +test_download_20_500k = create_download(20, 500 * 1000) +test_download_100_100k = create_download(100, 100 * 1000) +test_download_1000_10k = create_download(1000, 10 * 1000) @pytest.inlineCallbacks diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py index 029164eb..26f63a40 100644 --- a/testing/tests/sync/test_sqlcipher_sync.py +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -710,6 +710,7 @@ def make_local_db_and_soledad_target( source_replica_uid=source_replica_uid) return db, st + target_scenarios = [ ('leap', { 'create_db_and_target': make_local_db_and_soledad_target, diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py index 2bcb3aec..432a3cd2 100644 --- a/testing/tests/sync/test_sync_mutex.py +++ b/testing/tests/sync/test_sync_mutex.py @@ -66,6 +66,7 @@ def _timed_sync(self): d.addBoth(_store_finish_time) return d + SoledadSynchronizer.sync = _timed_sync # -- end of monkey-patching diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index e32f08b3..dd69ffa1 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -412,7 +412,8 @@ class SoledadDatabaseSyncTargetTests( This test was adapted to decrypt remote content before assert. """ docs_by_gen = [ - ((self.make_document, ('doc-id', 'replica:1', tests.simple_doc,), {}), + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc,), {}), 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, @@ -519,7 +520,8 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json('{}') edit_rev = 'replica:1|' + doc.rev docs_by_gen = [ - ((self.make_document, (doc.doc_id, edit_rev, None), {}), 10, 'T-sid')] + ((self.make_document, (doc.doc_id, edit_rev, None), {}), + 10, 'T-sid')] new_gen, trans_id = yield self.st.sync_exchange( docs_by_gen, 'replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -662,8 +664,8 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json(tests.simple_doc) docs_by_gen = [ ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), - ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), 5, - 'T-bar')] + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), + 5, 'T-bar')] new_gen, _ = yield self.st.sync_exchange( docs_by_gen, 'other-replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) @@ -746,6 +748,7 @@ class SoledadDatabaseSyncTargetTests( yield self.st.record_sync_info('replica', 0, 'T-sid') self.assertEqual(expected, called) + WAIT_STEP = 1 MAX_WAIT = 10 DBPASS = "pass" diff --git a/testing/tox.ini b/testing/tox.ini index 0eeeab9e..d84566ca 100644 --- a/testing/tox.ini +++ b/testing/tox.ini @@ -42,8 +42,8 @@ deps = pep8 flake8 commands = - pep8 client server common - flake8 --ignore=F812,E731 client server common + pep8 + flake8 [testenv:parallel] deps = |