diff options
24 files changed, 293 insertions, 180 deletions
diff --git a/client/changes/bug_5507_close-connection-after-sync b/client/changes/bug_5507_close-connection-after-sync new file mode 100644 index 00000000..0e77ab25 --- /dev/null +++ b/client/changes/bug_5507_close-connection-after-sync @@ -0,0 +1,2 @@ + o Close connection with server after syncing to avoid client hanging on exit + (#5507). diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 3aea340d..04f8ebf9 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -43,12 +43,15 @@ So, as the statements above were introduced for backwards compatibility with SLCipher 1.1 databases, we do not implement them as all SQLCipher databases handled by Soledad should be created by SQLCipher >= 2.0. """ -import httplib import logging import os import string import threading import time +import json + +from hashlib import sha256 +from contextlib import contextmanager from pysqlcipher import dbapi2 from u1db.backends import sqlite_backend @@ -149,7 +152,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): k_lock = threading.Lock() create_doc_lock = threading.Lock() update_indexes_lock = threading.Lock() - _syncer = None def __init__(self, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', @@ -211,6 +213,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): has_conflicts=has_conflicts, syncable=syncable) self.set_document_factory(factory) + self._syncers = {} @classmethod def _open_database(cls, sqlcipher_file, password, document_factory=None, @@ -351,46 +354,46 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :return: The local generation before the synchronisation was performed. :rtype: int """ - if not self.syncer: - self._create_syncer(url, creds=creds) - - try: - res = self.syncer.sync(autocreate=autocreate) - except httplib.CannotSendRequest: - # raised when you reuse httplib.HTTP object for new request - # while you havn't called its getresponse() - # this catch works for the current connclass used - # by our HTTPClientBase, since it uses httplib. - # we will have to replace it if it changes. - logger.info("Replacing connection and trying again...") - self._syncer = None - self._create_syncer(url, creds=creds) - res = self.syncer.sync(autocreate=autocreate) + res = None + with self.syncer(url, creds=creds) as syncer: + res = syncer.sync(autocreate=autocreate) return res - @property - def syncer(self): + @contextmanager + def syncer(self, url, creds=None): """ Accesor for synchronizer. """ - return self._syncer + syncer = self._get_syncer(url, creds=creds) + yield syncer + syncer.sync_target.close() - def _create_syncer(self, url, creds=None): + def _get_syncer(self, url, creds=None): """ - Creates a synchronizer + Get a synchronizer for C{url} using C{creds}. :param url: The url of the target replica to sync with. :type url: str :param creds: optional dictionary giving credentials. - to authorize the operation with the server. + to authorize the operation with the server. :type creds: dict + + :return: A synchronizer. + :rtype: u1db.sync.Synchronizer """ - if self._syncer is None: - self._syncer = Synchronizer( + # we want to store at most one syncer for each url, so we also store a + # hash of the connection credentials and replace the stored syncer for + # a certain url if credentials have changed. + h = sha256(json.dumps([url, creds])).hexdigest() + cur_h, syncer = self._syncers.get(url, (None, None)) + if syncer is None or h != cur_h: + syncer = Synchronizer( self, SoledadSyncTarget(url, creds=creds, crypto=self._crypto)) + self._syncers[url] = (h, syncer) + return syncer def _extra_schema_init(self, c): """ diff --git a/common/changes/bug_5493_properly-close-connections-on-couch-backend b/common/changes/bug_5493_properly-close-connections-on-couch-backend new file mode 100644 index 00000000..3cb55168 --- /dev/null +++ b/common/changes/bug_5493_properly-close-connections-on-couch-backend @@ -0,0 +1,2 @@ + o Properly close connections on couch backend. Also prevent file descriptor + leaks on tests. Closes #5493. diff --git a/common/changes/feature_5386_prevent-couch-backend-from-trying-to-create-db b/common/changes/feature_5386_prevent-couch-backend-from-trying-to-create-db new file mode 100644 index 00000000..9bbac329 --- /dev/null +++ b/common/changes/feature_5386_prevent-couch-backend-from-trying-to-create-db @@ -0,0 +1 @@ + o Prevent couch backend from always trying to create the database (#5386). diff --git a/common/changes/feature_5386_prevent-uneeded-get-on-couch-server-state b/common/changes/feature_5386_prevent-uneeded-get-on-couch-server-state new file mode 100644 index 00000000..95919873 --- /dev/null +++ b/common/changes/feature_5386_prevent-uneeded-get-on-couch-server-state @@ -0,0 +1,2 @@ + o Prevent Couch Server State from making one uneeded GET request on + instantiation (#5386). diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index b836c997..0aa84170 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -31,14 +31,16 @@ import threading from StringIO import StringIO from collections import defaultdict +from urlparse import urljoin +from contextlib import contextmanager -from couchdb.client import Server +from couchdb.client import Server, Database from couchdb.http import ( ResourceConflict, ResourceNotFound, ServerError, - Session, + Session as CouchHTTPSession, ) from u1db import query_parser, vectorclock from u1db.errors import ( @@ -331,6 +333,35 @@ class MultipartWriter(object): self.headers[name] = value +class Session(CouchHTTPSession): + """ + An HTTP session that can be closed. + """ + + def close_connections(self): + for key, conns in list(self.conns.items()): + for conn in conns: + conn.close() + + +@contextmanager +def couch_server(url): + """ + Provide a connection to a couch server and cleanup after use. + + For database creation and deletion we use an ephemeral connection to the + couch server. That connection has to be properly closed, so we provide it + as a context manager. + + :param url: The URL of the Couch server. + :type url: str + """ + session = Session(timeout=COUCH_TIMEOUT) + server = Server(url=url, session=session) + yield server + session.close_connections() + + class CouchDatabase(CommonBackend): """ A U1DB implementation that uses CouchDB as its persistence layer. @@ -353,7 +384,7 @@ class CouchDatabase(CommonBackend): release_fun): """ :param db: The database from where to get the document. - :type db: u1db.Database + :type db: CouchDatabase :param doc_id: The doc_id of the document to be retrieved. :type doc_id: str :param check_for_conflicts: Whether the get_doc() method should @@ -380,7 +411,7 @@ class CouchDatabase(CommonBackend): self._release_fun() @classmethod - def open_database(cls, url, create, ensure_ddocs=False): + def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False): """ Open a U1DB database using CouchDB as backend. @@ -388,6 +419,8 @@ class CouchDatabase(CommonBackend): :type url: str :param create: should the replica be created if it does not exist? :type create: bool + :param replica_uid: an optional unique replica identifier + :type replica_uid: str :param ensure_ddocs: Ensure that the design docs exist on server. :type ensure_ddocs: bool @@ -400,16 +433,16 @@ class CouchDatabase(CommonBackend): raise InvalidURLError url = m.group(1) dbname = m.group(2) - server = Server(url=url) - try: - server[dbname] - except ResourceNotFound: - if not create: - raise DatabaseDoesNotExist() - return cls(url, dbname, ensure_ddocs=ensure_ddocs) + with couch_server(url) as server: + try: + server[dbname] + except ResourceNotFound: + if not create: + raise DatabaseDoesNotExist() + server.create(dbname) + return cls(url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) - def __init__(self, url, dbname, replica_uid=None, full_commit=True, - session=None, ensure_ddocs=True): + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): """ Create a new Couch data container. @@ -419,31 +452,19 @@ class CouchDatabase(CommonBackend): :type dbname: str :param replica_uid: an optional unique replica identifier :type replica_uid: str - :param full_commit: turn on the X-Couch-Full-Commit header - :type full_commit: bool - :param session: an http.Session instance or None for a default session - :type session: http.Session :param ensure_ddocs: Ensure that the design docs exist on server. :type ensure_ddocs: bool """ # save params self._url = url - self._full_commit = full_commit - if session is None: - session = Session(timeout=COUCH_TIMEOUT) - self._session = session + self._session = Session(timeout=COUCH_TIMEOUT) self._factory = CouchDocument self._real_replica_uid = None # configure couch - self._server = Server(url=self._url, - full_commit=self._full_commit, - session=self._session) self._dbname = dbname - try: - self._database = self._server[self._dbname] - except ResourceNotFound: - self._server.create(self._dbname) - self._database = self._server[self._dbname] + self._database = Database( + urljoin(self._url, self._dbname), + self._session) if replica_uid is not None: self._set_replica_uid(replica_uid) if ensure_ddocs: @@ -482,7 +503,9 @@ class CouchDatabase(CommonBackend): """ Delete a U1DB CouchDB database. """ - del(self._server[self._dbname]) + with couch_server(self._url) as server: + del(server[self._dbname]) + self.close_connections() def close(self): """ @@ -491,13 +514,26 @@ class CouchDatabase(CommonBackend): :return: True if db was succesfully closed. :rtype: bool """ + self.close_connections() self._url = None self._full_commit = None self._session = None - self._server = None self._database = None return True + def close_connections(self): + """ + Close all open connections to the couch server. + """ + if self._session is not None: + self._session.close_connections() + + def __del__(self): + """ + Close the database upon garbage collection. + """ + self.close() + def _set_replica_uid(self, replica_uid): """ Force the replica uid to be set. @@ -855,7 +891,9 @@ class CouchDatabase(CommonBackend): try: self._database.resource.put_json( doc.doc_id, body=buf.getvalue(), headers=envelope.headers) - self._renew_couch_session() + # What follows is a workaround for an ugly bug. See: + # https://leap.se/code/issues/5448 + self.close_connections() except ResourceConflict: raise RevisionConflict() @@ -1427,15 +1465,6 @@ class CouchDatabase(CommonBackend): continue yield t._doc - def _renew_couch_session(self): - """ - Create a new couch connection session. - - This is a workaround for #5448. Will not be needed once bigcouch is - merged with couchdb. - """ - self._database.resource.session = Session(timeout=COUCH_TIMEOUT) - class CouchSyncTarget(CommonSyncTarget): """ @@ -1489,9 +1518,9 @@ class CouchServerState(ServerState): :return: The CouchDatabase object. :rtype: CouchDatabase """ - return CouchDatabase.open_database( - self._couch_url + '/' + dbname, - create=False, + return CouchDatabase( + self._couch_url, + dbname, ensure_ddocs=False) def ensure_database(self, dbname): diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 86bb4b93..17d4a519 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -25,9 +25,10 @@ import copy import shutil from base64 import b64decode from mock import Mock +from urlparse import urljoin -from couchdb.client import Server from u1db import errors as u1db_errors +from couchdb.client import Server from leap.common.files import mkdir_p @@ -151,8 +152,11 @@ class CouchDBTestCase(unittest.TestCase): class TestCouchBackendImpl(CouchDBTestCase): def test__allocate_doc_id(self): - db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), - 'u1db_tests', ensure_ddocs=True) + db = couch.CouchDatabase.open_database( + urljoin( + 'http://localhost:' + str(self.wrapper.port), 'u1db_tests'), + create=True, + ensure_ddocs=True) doc_id1 = db._allocate_doc_id() self.assertTrue(doc_id1.startswith('D-')) self.assertEqual(34, len(doc_id1)) @@ -166,28 +170,36 @@ class TestCouchBackendImpl(CouchDBTestCase): def make_couch_database_for_test(test, replica_uid): port = str(test.wrapper.port) - return couch.CouchDatabase('http://localhost:' + port, replica_uid, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) + return couch.CouchDatabase.open_database( + urljoin('http://localhost:' + port, replica_uid), + create=True, + replica_uid=replica_uid or 'test', + ensure_ddocs=True) def copy_couch_database_for_test(test, db): port = str(test.wrapper.port) couch_url = 'http://localhost:' + port new_dbname = db._replica_uid + '_copy' - new_db = couch.CouchDatabase(couch_url, - new_dbname, - replica_uid=db._replica_uid or 'test') + new_db = couch.CouchDatabase.open_database( + urljoin(couch_url, new_dbname), + create=True, + replica_uid=db._replica_uid or 'test') # copy all docs - old_couch_db = Server(couch_url)[db._replica_uid] - new_couch_db = Server(couch_url)[new_dbname] + session = couch.Session() + old_couch_db = Server(couch_url, session=session)[db._replica_uid] + new_couch_db = Server(couch_url, session=session)[new_dbname] for doc_id in old_couch_db: doc = old_couch_db.get(doc_id) + # bypass u1db_config document + if doc_id == 'u1db_config': + pass # copy design docs - if ('u1db_rev' not in doc): + elif doc_id.startswith('_design'): + del doc['_rev'] new_couch_db.save(doc) # copy u1db docs - else: + elif 'u1db_rev' in doc: new_doc = { '_id': doc['_id'], 'u1db_transactions': doc['u1db_transactions'], @@ -206,6 +218,8 @@ def copy_couch_database_for_test(test, db): if (att is not None): new_couch_db.put_attachment(new_doc, att, filename=att_name) + # cleanup connections to prevent file descriptor leaking + session.close_connections() return new_db @@ -228,7 +242,7 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): def setUp(self): test_backends.AllDatabaseTests.setUp(self) # save db info because of test_close - self._server = self.db._server + self._url = self.db._url self._dbname = self.db._dbname def tearDown(self): @@ -238,7 +252,10 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): if self.id() == \ 'leap.soledad.common.tests.test_couch.CouchTests.' \ 'test_close(couch)': - del(self._server[self._dbname]) + session = couch.Session() + server = Server(url=self._url, session=session) + del(server[self._dbname]) + session.close_connections() else: self.db.delete_database() test_backends.AllDatabaseTests.tearDown(self) @@ -355,10 +372,9 @@ from u1db.backends.inmemory import InMemoryIndex class IndexedCouchDatabase(couch.CouchDatabase): - def __init__(self, url, dbname, replica_uid=None, full_commit=True, - session=None, ensure_ddocs=True): - old_class.__init__(self, url, dbname, replica_uid, full_commit, - session, ensure_ddocs=ensure_ddocs) + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): + old_class.__init__(self, url, dbname, replica_uid=replica_uid, + ensure_ddocs=ensure_ddocs) self._indexes = {} def _put_doc(self, old_doc, doc): @@ -447,19 +463,22 @@ class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase): self.db1 = None self.db2 = None self.db3 = None + self.db1_copy = None + self.db2_copy = None test_sync.DatabaseSyncTests.setUp(self) def tearDown(self): - self.db and self.db.delete_database() - self.db1 and self.db1.delete_database() - self.db2 and self.db2.delete_database() - self.db3 and self.db3.delete_database() - db = self.create_database('test1_copy', 'source') - db.delete_database() - db = self.create_database('test2_copy', 'target') - db.delete_database() - db = self.create_database('test3', 'target') - db.delete_database() + for db in [self.db, self.db1, self.db2, self.db3, self.db1_copy, + self.db2_copy]: + if db is not None: + db.delete_database() + db.close() + for replica_uid, dbname in [('test1_copy', 'source'), + ('test2_copy', 'target'), ('test3', 'target')]: + db = self.create_database(replica_uid, dbname) + db.delete_database() + # cleanup connections to avoid leaking of file descriptors + db.close() test_sync.DatabaseSyncTests.tearDown(self) @@ -467,12 +486,14 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): def setUp(self): CouchDBTestCase.setUp(self) - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=False) # note that we don't enforce ddocs here def tearDown(self): self.db.delete_database() + self.db.close() def test_missing_design_doc_raises(self): """ @@ -509,8 +530,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents list functions will raise if the functions are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # erase views from _design/transactions transactions = self.db._database['_design/transactions'] @@ -538,8 +560,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents list functions will raise if the functions are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # erase views from _design/transactions transactions = self.db._database['_design/transactions'] @@ -567,8 +590,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents' named views will raise if the views are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # erase views from _design/docs docs = self.db._database['_design/docs'] @@ -608,8 +632,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents will raise if the design docs are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # delete _design/docs del self.db._database['_design/docs'] diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 3c457cc5..b03f79e7 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -24,6 +24,10 @@ import mock import tempfile import threading + +from urlparse import urljoin + + from leap.soledad.client import Soledad from leap.soledad.common.couch import CouchDatabase, CouchServerState from leap.soledad.common.tests.test_couch import CouchDBTestCase @@ -101,12 +105,16 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): TestCaseWithServer.setUp(self) CouchDBTestCase.setUp(self) self._couch_url = 'http://localhost:' + str(self.wrapper.port) - self.db = CouchDatabase( - self._couch_url, 'user-user-uuid', replica_uid='replica') + self.db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + replica_uid='replica', + ensure_ddocs=True) self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") def tearDown(self): self.db.delete_database() + self.db.close() CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) @@ -211,6 +219,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): _create_docs_and_sync(sol, 0) _create_docs_and_sync(sol, 1) + sol.close() # # Concurrency tests @@ -344,6 +353,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): self.assertEqual( 1, len(filter(lambda t: t[0] == doc_id, transaction_log))) + sol.close() def test_concurrent_syncs_do_not_fail(self): """ @@ -387,3 +397,4 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): self.assertEqual( 1, len(filter(lambda t: t[0] == doc_id, transaction_log))) + sol.close() diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index af11bc76..4b2470ba 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -44,7 +44,6 @@ from leap.soledad.common.crypto import WrongMac, UnknownMacMethod from leap.soledad.common.tests.u1db_tests import ( simple_doc, nested_doc, - TestCaseWithServer, ) @@ -95,6 +94,7 @@ class RecoveryDocumentTestCase(BaseSoledadTest): self.assertEqual(self._soledad._get_storage_secret(), s._get_storage_secret(), 'Failed settinng secret for symmetric encryption.') + s.close() class SoledadSecretsTestCase(BaseSoledadTest): @@ -110,6 +110,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # generate new secret secret_id_2 = sol._gen_secret() self.assertTrue(secret_id_1 != secret_id_2) + sol.close() # re-instantiate sol = self._soledad_instance( user='user@leap.se', @@ -130,6 +131,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # assert id is hash of new secret self.assertTrue( secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) + sol.close() def test__has_secret(self): sol = self._soledad_instance( @@ -144,6 +146,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # but not being able to decrypt correctly should sol._secrets[sol.secret_id][sol.SECRET_KEY] = None self.assertFalse(sol._has_secret()) + sol.close() class MacAuthTestCase(BaseSoledadTest): diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index f8d2a64f..1c5a7407 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -27,6 +27,7 @@ import mock import time import binascii +from urlparse import urljoin from leap.common.testing.basetest import BaseLeapTest from leap.soledad.common.couch import ( @@ -56,7 +57,8 @@ from leap.soledad.server.auth import URLToAuthorization def _couch_ensure_database(self, dbname): db = CouchDatabase.open_database( self._couch_url + '/' + dbname, - create=True) + create=True, + ensure_ddocs=True) return db, db._replica_uid CouchServerState.ensure_database = _couch_ensure_database @@ -352,11 +354,10 @@ class EncryptedSyncTestCase( self.assertEqual([], doclist) doc1 = sol1.create_doc(json.loads(simple_doc)) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-<uuid>". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -390,6 +391,9 @@ class EncryptedSyncTestCase( # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) db.delete_database() + db.close() + sol1.close() + sol2.close() def test_encrypted_sym_sync_with_unicode_passphrase(self): """ @@ -408,11 +412,10 @@ class EncryptedSyncTestCase( self.assertEqual([], doclist) doc1 = sol1.create_doc(json.loads(simple_doc)) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-<uuid>". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -450,6 +453,9 @@ class EncryptedSyncTestCase( # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) db.delete_database() + db.close() + sol1.close() + sol2.close() def test_sync_very_large_files(self): """ @@ -468,11 +474,10 @@ class EncryptedSyncTestCase( content = binascii.hexlify(os.urandom(length/2)) # len() == length doc1 = sol1.create_doc({'data': content}) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-<uuid>". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -493,6 +498,9 @@ class EncryptedSyncTestCase( self.assertEqual(doc1, doc2) # delete remote database db.delete_database() + db.close() + sol1.close() + sol2.close() def test_sync_many_small_files(self): @@ -512,11 +520,10 @@ class EncryptedSyncTestCase( for i in range(0, number_of_docs): sol1.create_doc(json.loads(simple_doc)) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-<uuid>". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -537,6 +544,9 @@ class EncryptedSyncTestCase( self.assertEqual(sol1.get_doc(doc.doc_id), doc) # delete remote database db.delete_database() + db.close() + sol1.close() + sol2.close() class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): @@ -558,8 +568,14 @@ class LockResourceTestCase( self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) # create the databases - CouchDatabase(self._couch_url, 'shared') - CouchDatabase(self._couch_url, 'tokens') + CouchDatabase.open_database( + urljoin(self._couch_url, 'shared'), + create=True, + ensure_ddocs=True) + CouchDatabase.open_database( + urljoin(self._couch_url, 'tokens'), + create=True, + ensure_ddocs=True) self._state = CouchServerState( self._couch_url, 'shared', 'tokens') @@ -567,10 +583,10 @@ class LockResourceTestCase( CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) # delete remote database - db = CouchDatabase( - self._couch_url, - 'shared', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'shared'), + create=True, + ensure_ddocs=True) db.delete_database() def test__try_obtain_filesystem_lock(self): diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 035c5ac5..5a3bf2b0 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -49,6 +49,7 @@ class AuxMethodsTestCase(BaseSoledadTest): secrets_path = os.path.dirname(sol.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) + sol.close() def test__init_db(self): sol = self._soledad_instance() @@ -61,6 +62,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol._init_db() from leap.soledad.client.sqlcipher import SQLCipherDatabase self.assertIsInstance(sol._db, SQLCipherDatabase) + sol.close() def test__init_config_defaults(self): """ @@ -103,6 +105,7 @@ class AuxMethodsTestCase(BaseSoledadTest): os.path.join(self.tempdir, 'value_2'), sol.local_db_path) self.assertEqual('value_1', sol.server_url) + sol.close() def test_change_passphrase(self): """ @@ -118,6 +121,7 @@ class AuxMethodsTestCase(BaseSoledadTest): # change the passphrase sol.change_passphrase(u'654321') + sol.close() self.assertRaises( WrongMac, @@ -132,6 +136,7 @@ class AuxMethodsTestCase(BaseSoledadTest): prefix=self.rand_prefix) doc2 = sol2.get_doc(doc_id) self.assertEqual(doc, doc2) + sol2.close() def test_change_passphrase_with_short_passphrase_raises(self): """ @@ -145,6 +150,7 @@ class AuxMethodsTestCase(BaseSoledadTest): self.assertRaises( PassphraseTooShort, sol.change_passphrase, u'54321') + sol.close() def test_get_passphrase(self): """ @@ -152,6 +158,7 @@ class AuxMethodsTestCase(BaseSoledadTest): """ sol = self._soledad_instance() self.assertEqual('123', sol.passphrase) + sol.close() class SoledadSharedDBTestCase(BaseSoledadTest): @@ -165,6 +172,9 @@ class SoledadSharedDBTestCase(BaseSoledadTest): 'https://provider/', ADDRESS, document_factory=SoledadDocument, creds=None) + def tearDown(self): + BaseSoledadTest.tearDown(self) + def test__get_secrets_from_shared_db(self): """ Ensure the shared db is queried with the correct doc_id. @@ -209,7 +219,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): BaseSoledadTest.setUp(self) def tearDown(self): - pass + BaseSoledadTest.tearDown(self) def _pop_mock_call(self, mocked): mocked.call_args_list.pop() @@ -283,6 +293,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # assert db was locked and unlocked sol._shared_db.lock.assert_called_with() sol._shared_db.unlock.assert_called_with('atoken') + sol.close() def test_stage2_bootstrap_signals(self): """ @@ -305,6 +316,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def __call__(self): return self + sol.close() # reset mock soledad.client.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals @@ -328,6 +340,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) + sol.close() def test_stage1_bootstrap_signals(self): """ @@ -337,6 +350,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get an existent instance so it emits only some of bootstrap signals sol = self._soledad_instance() self.assertEqual([], soledad.client.signal.mock_calls) + sol.close() def test_sync_signals(self): """ @@ -355,6 +369,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DATA_SYNC, ADDRESS, ) + sol.close() def test_need_sync_signals(self): """ @@ -375,3 +390,4 @@ class SoledadSignalingTestCase(BaseSoledadTest): ADDRESS, ) SoledadSyncTarget.get_sync_info = old_get_sync_info + sol.close() diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 66a673b6..c79a6045 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -469,11 +469,9 @@ class SQLCipherDatabaseSyncTests( def setUp(self): test_sync.DatabaseSyncTests.setUp(self) - BaseSoledadTest.setUp(self) def tearDown(self): test_sync.DatabaseSyncTests.tearDown(self) - BaseSoledadTest.tearDown(self) def test_sync_autoresolves(self): """ @@ -683,11 +681,9 @@ class SQLCipherSyncTargetTests( def setUp(self): test_sync.DatabaseSyncTargetTests.setUp(self) - #BaseSoledadTest.setUp(self) def tearDown(self): test_sync.DatabaseSyncTargetTests.tearDown(self) - BaseSoledadTest.tearDown(self) def test_sync_exchange(self): """ diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py index 5a541745..c1e00d52 100644 --- a/common/src/leap/soledad/common/tests/test_target.py +++ b/common/src/leap/soledad/common/tests/test_target.py @@ -272,11 +272,9 @@ class TestSoledadParsingSyncStream( def setUp(self): test_remote_sync_target.TestParsingSyncStream.setUp(self) - BaseSoledadTest.setUp(self) def tearDown(self): test_remote_sync_target.TestParsingSyncStream.tearDown(self) - BaseSoledadTest.tearDown(self) def test_extra_comma(self): """ @@ -646,6 +644,7 @@ class SoledadDatabaseSyncTargetTests( self.assertEqual(([], 1, last_trans_id), (self.other_changes, new_gen, last_trans_id)) self.assertEqual(10, self.st.get_sync_info('replica')[3]) + sol.close() def test_sync_exchange_push_many(self): """ diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py index 3bc12487..99ff77b4 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py @@ -208,8 +208,8 @@ class DatabaseBaseTests(TestCase): self.db = self.create_database('test') def tearDown(self): - # TODO: Add close_database parameterization - # self.close_database(self.db) + if hasattr(self, 'db') and self.db is not None: + self.db.close() super(DatabaseBaseTests, self).tearDown() def assertTransactionLog(self, doc_ids, db): @@ -335,6 +335,13 @@ class TestCaseWithServer(TestCase): super(TestCaseWithServer, self).setUp() self.server = self.server_thread = None + def tearDown(self): + if self.server is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + super(TestCaseWithServer, self).tearDown() + @property def url_scheme(self): return self.server_def()[-1] diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index d2a91d11..c0a7e1f7 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -355,6 +355,9 @@ class LocalDatabaseTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS + def setUp(self): + tests.DatabaseBaseTests.setUp(self) + def test_create_doc_different_ids_diff_db(self): doc1 = self.db.create_doc_from_json(simple_doc) db2 = self.create_database('other-uid') diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py index 1f78f912..a37c36db 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py @@ -85,7 +85,6 @@ class DatabaseSyncTargetTests(tests.DatabaseBaseTests, whitebox = True def setUp(self): - tests.DatabaseBaseTests.setUp(self) tests.TestCaseWithServer.setUp(self) self.db, self.st = self.create_db_and_target(self) self.other_changes = [] @@ -94,7 +93,6 @@ class DatabaseSyncTargetTests(tests.DatabaseBaseTests, # We delete them explicitly, so that connections are cleanly closed del self.st self.db.close() - del self.db super(DatabaseSyncTargetTests, self).tearDown() def create_db_and_target(self, *args): @@ -1013,30 +1011,30 @@ class DatabaseSyncTests(tests.DatabaseBaseTests, def test_sync_supersedes_conflicts(self): self.db1 = self.create_database('test1', 'both') self.db2 = self.create_database('test2', 'target') - db3 = self.create_database('test3', 'both') + self.db3 = self.create_database('test3', 'both') doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(db3, self.db1) + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) self.assertEqual( self.db1._get_generation_info(), - db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) self.assertEqual( - db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(db3._replica_uid)) - self.sync(db3, self.db2) + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) self.assertEqual( self.db2._get_generation_info(), - db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) self.assertEqual( - db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(db3._replica_uid)) - self.assertEqual(3, len(db3.get_doc_conflicts('the-doc'))) + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) doc1.set_json('{"a": 2}') self.db1.put_doc(doc1) - self.sync(db3, self.db1) + self.sync(self.db3, self.db1) # original doc1 should have been removed from conflicts - self.assertEqual(3, len(db3.get_doc_conflicts('the-doc'))) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) def test_sync_stops_after_get_sync_info(self): self.db1 = self.create_database('test1', 'source') @@ -1054,70 +1052,70 @@ class DatabaseSyncTests(tests.DatabaseBaseTests, self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') self.sync(self.db1, self.db2) - db1_copy = self.copy_database(self.db1) + self.db1_copy = self.copy_database(self.db1) self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidGeneration, self.sync, db1_copy, self.db2) + errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) def test_sync_detects_rollback_in_target(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) - db2_copy = self.copy_database(self.db2) + self.db2_copy = self.copy_database(self.db2) self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1, db2_copy) + errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) def test_sync_detects_diverged_source(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') - db3 = self.copy_database(self.db1) + self.db3 = self.copy_database(self.db1) self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidTransactionId, self.sync, db3, self.db2) + errors.InvalidTransactionId, self.sync, self.db3, self.db2) def test_sync_detects_diverged_target(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') - db3 = self.copy_database(self.db2) - db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, db3) + errors.InvalidTransactionId, self.sync, self.db1, self.db3) def test_sync_detects_rollback_and_divergence_in_source(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') self.sync(self.db1, self.db2) - db1_copy = self.copy_database(self.db1) + self.db1_copy = self.copy_database(self.db1) self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.sync(self.db1, self.db2) - db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.assertRaises( - errors.InvalidTransactionId, self.sync, db1_copy, self.db2) + errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) def test_sync_detects_rollback_and_divergence_in_target(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) - db2_copy = self.copy_database(self.db2) + self.db2_copy = self.copy_database(self.db2) self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.sync(self.db1, self.db2) - db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, db2_copy) + errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) class TestDbSync(tests.TestCaseWithServer): diff --git a/scripts/update_design_docs.py b/scripts/ddocs/update_design_docs.py index e7b5a29c..e7b5a29c 100644 --- a/scripts/update_design_docs.py +++ b/scripts/ddocs/update_design_docs.py diff --git a/scripts/backends_cpu_usage/log_cpu_usage.py b/scripts/profiling/backends_cpu_usage/log_cpu_usage.py index 2674e1ff..2674e1ff 100755 --- a/scripts/backends_cpu_usage/log_cpu_usage.py +++ b/scripts/profiling/backends_cpu_usage/log_cpu_usage.py diff --git a/scripts/backends_cpu_usage/movingaverage.py b/scripts/profiling/backends_cpu_usage/movingaverage.py index bac1b3e1..bac1b3e1 100644 --- a/scripts/backends_cpu_usage/movingaverage.py +++ b/scripts/profiling/backends_cpu_usage/movingaverage.py diff --git a/scripts/backends_cpu_usage/plot.py b/scripts/profiling/backends_cpu_usage/plot.py index 4e5083ad..4e5083ad 100755 --- a/scripts/backends_cpu_usage/plot.py +++ b/scripts/profiling/backends_cpu_usage/plot.py diff --git a/scripts/backends_cpu_usage/test_u1db_sync.py b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py index 26ef8f9f..26ef8f9f 100755 --- a/scripts/backends_cpu_usage/test_u1db_sync.py +++ b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py diff --git a/scripts/doc_put_memory_usage/find_max_upload_size.py b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py index 02c68015..02c68015 100755 --- a/scripts/doc_put_memory_usage/find_max_upload_size.py +++ b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py diff --git a/scripts/doc_put_memory_usage/get-mem.py b/scripts/profiling/doc_put_memory_usage/get-mem.py index d64875fc..d64875fc 100755 --- a/scripts/doc_put_memory_usage/get-mem.py +++ b/scripts/profiling/doc_put_memory_usage/get-mem.py diff --git a/scripts/doc_put_memory_usage/plot-mem.py b/scripts/profiling/doc_put_memory_usage/plot-mem.py index e24679a2..e24679a2 100755 --- a/scripts/doc_put_memory_usage/plot-mem.py +++ b/scripts/profiling/doc_put_memory_usage/plot-mem.py |