From 3cfe76a694bb19d5da21192e4a9a467a9b4c59e1 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 7 Apr 2014 11:40:40 -0300 Subject: Add script to compile design docs (#5315) --- common/src/leap/soledad/common/couch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'common/src/leap') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 8e8613a1..b836c997 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -1411,7 +1411,7 @@ class CouchDatabase(CommonBackend): # strptime here by evaluating the conversion of an arbitrary date. # This will not be needed when/if we switch from python-couchdb to # paisley. - time.strptime('Mar 4 1917', '%b %d %Y') + time.strptime('Mar 8 1917', '%b %d %Y') # spawn threads to retrieve docs threads = [] for doc_id in doc_ids: -- cgit v1.2.3 From 45ffe1dd22d6e902a7e8eeca786b7ad63ec1d139 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 7 Apr 2014 12:48:25 -0300 Subject: Prevent Couch Server State from making one unneeded GET request (#5386). --- common/src/leap/soledad/common/couch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'common/src/leap') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index b836c997..9fb717c2 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -1489,9 +1489,9 @@ class CouchServerState(ServerState): :return: The CouchDatabase object. :rtype: CouchDatabase """ - return CouchDatabase.open_database( - self._couch_url + '/' + dbname, - create=False, + return CouchDatabase( + self._couch_url, + dbname, ensure_ddocs=False) def ensure_database(self, dbname): -- cgit v1.2.3 From 764fb1e8060cb7169744dd6602d2550ad0c02178 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 7 Apr 2014 14:55:26 -0300 Subject: Fix class doc. --- common/src/leap/soledad/common/couch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'common/src/leap') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 9fb717c2..a7828e98 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -353,7 +353,7 @@ class CouchDatabase(CommonBackend): release_fun): """ :param db: The database from where to get the document. - :type db: u1db.Database + :type db: CouchDatabase :param doc_id: The doc_id of the document to be retrieved. :type doc_id: str :param check_for_conflicts: Whether the get_doc() method should -- cgit v1.2.3 From 11757fb0d071b753819a04d8504a72baed80db2f Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 8 Apr 2014 11:20:19 -0300 Subject: Prevent couch instantiation from creating the db (#5386). --- common/src/leap/soledad/common/couch.py | 36 +++++------ common/src/leap/soledad/common/tests/test_couch.py | 69 ++++++++++++++-------- .../tests/test_couch_operations_atomicity.py | 11 +++- .../src/leap/soledad/common/tests/test_server.py | 58 +++++++++--------- 4 files changed, 99 insertions(+), 75 deletions(-) (limited to 'common/src/leap') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index a7828e98..8ed704ba 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -31,9 +31,10 @@ import threading from StringIO import StringIO from collections import defaultdict +from urlparse import urljoin -from couchdb.client import Server +from couchdb.client import Server, Database from couchdb.http import ( ResourceConflict, ResourceNotFound, @@ -380,7 +381,7 @@ class CouchDatabase(CommonBackend): self._release_fun() @classmethod - def open_database(cls, url, create, ensure_ddocs=False): + def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False): """ Open a U1DB database using CouchDB as backend. @@ -388,6 +389,8 @@ class CouchDatabase(CommonBackend): :type url: str :param create: should the replica be created if it does not exist? :type create: bool + :param replica_uid: an optional unique replica identifier + :type replica_uid: str :param ensure_ddocs: Ensure that the design docs exist on server. :type ensure_ddocs: bool @@ -406,10 +409,11 @@ class CouchDatabase(CommonBackend): except ResourceNotFound: if not create: raise DatabaseDoesNotExist() - return cls(url, dbname, ensure_ddocs=ensure_ddocs) + server.create(dbname) + return cls(url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) - def __init__(self, url, dbname, replica_uid=None, full_commit=True, - session=None, ensure_ddocs=True): + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True, + session=None): """ Create a new Couch data container. @@ -419,31 +423,23 @@ class CouchDatabase(CommonBackend): :type dbname: str :param replica_uid: an optional unique replica identifier :type replica_uid: str - :param full_commit: turn on the X-Couch-Full-Commit header - :type full_commit: bool - :param session: an http.Session instance or None for a default session - :type session: http.Session :param ensure_ddocs: Ensure that the design docs exist on server. :type ensure_ddocs: bool + :param session: an http.Session instance or None for a default session + :type session: http.Session """ # save params self._url = url - self._full_commit = full_commit if session is None: session = Session(timeout=COUCH_TIMEOUT) self._session = session self._factory = CouchDocument self._real_replica_uid = None # configure couch - self._server = Server(url=self._url, - full_commit=self._full_commit, - session=self._session) self._dbname = dbname - try: - self._database = self._server[self._dbname] - except ResourceNotFound: - self._server.create(self._dbname) - self._database = self._server[self._dbname] + self._database = Database( + urljoin(self._url, self._dbname), + self._session) if replica_uid is not None: self._set_replica_uid(replica_uid) if ensure_ddocs: @@ -482,7 +478,8 @@ class CouchDatabase(CommonBackend): """ Delete a U1DB CouchDB database. """ - del(self._server[self._dbname]) + server = Server(url=self._url) + del(server[self._dbname]) def close(self): """ @@ -494,7 +491,6 @@ class CouchDatabase(CommonBackend): self._url = None self._full_commit = None self._session = None - self._server = None self._database = None return True diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 86bb4b93..77c46e61 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -25,6 +25,7 @@ import copy import shutil from base64 import b64decode from mock import Mock +from urlparse import urljoin from couchdb.client import Server from u1db import errors as u1db_errors @@ -151,8 +152,11 @@ class CouchDBTestCase(unittest.TestCase): class TestCouchBackendImpl(CouchDBTestCase): def test__allocate_doc_id(self): - db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port), - 'u1db_tests', ensure_ddocs=True) + db = couch.CouchDatabase.open_database( + urljoin( + 'http://localhost:' + str(self.wrapper.port), 'u1db_tests'), + create=True, + ensure_ddocs=True) doc_id1 = db._allocate_doc_id() self.assertTrue(doc_id1.startswith('D-')) self.assertEqual(34, len(doc_id1)) @@ -166,28 +170,35 @@ class TestCouchBackendImpl(CouchDBTestCase): def make_couch_database_for_test(test, replica_uid): port = str(test.wrapper.port) - return couch.CouchDatabase('http://localhost:' + port, replica_uid, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) + return couch.CouchDatabase.open_database( + urljoin('http://localhost:' + port, replica_uid), + create=True, + replica_uid=replica_uid or 'test', + ensure_ddocs=True) def copy_couch_database_for_test(test, db): port = str(test.wrapper.port) couch_url = 'http://localhost:' + port new_dbname = db._replica_uid + '_copy' - new_db = couch.CouchDatabase(couch_url, - new_dbname, - replica_uid=db._replica_uid or 'test') + new_db = couch.CouchDatabase.open_database( + urljoin(couch_url, new_dbname), + create=True, + replica_uid=db._replica_uid or 'test') # copy all docs old_couch_db = Server(couch_url)[db._replica_uid] new_couch_db = Server(couch_url)[new_dbname] for doc_id in old_couch_db: doc = old_couch_db.get(doc_id) + # bypass u1db_config document + if doc_id == 'u1db_config': + pass # copy design docs - if ('u1db_rev' not in doc): + elif doc_id.startswith('_design'): + del doc['_rev'] new_couch_db.save(doc) # copy u1db docs - else: + elif 'u1db_rev' in doc: new_doc = { '_id': doc['_id'], 'u1db_transactions': doc['u1db_transactions'], @@ -228,7 +239,7 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): def setUp(self): test_backends.AllDatabaseTests.setUp(self) # save db info because of test_close - self._server = self.db._server + self._url = self.db._url self._dbname = self.db._dbname def tearDown(self): @@ -238,7 +249,8 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): if self.id() == \ 'leap.soledad.common.tests.test_couch.CouchTests.' \ 'test_close(couch)': - del(self._server[self._dbname]) + server = Server(url=self._url) + del(server[self._dbname]) else: self.db.delete_database() test_backends.AllDatabaseTests.tearDown(self) @@ -355,10 +367,10 @@ from u1db.backends.inmemory import InMemoryIndex class IndexedCouchDatabase(couch.CouchDatabase): - def __init__(self, url, dbname, replica_uid=None, full_commit=True, - session=None, ensure_ddocs=True): - old_class.__init__(self, url, dbname, replica_uid, full_commit, - session, ensure_ddocs=ensure_ddocs) + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True, + session=None): + old_class.__init__(self, url, dbname, replica_uid=replica_uid, + ensure_ddocs=ensure_ddocs, session=session) self._indexes = {} def _put_doc(self, old_doc, doc): @@ -467,8 +479,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): def setUp(self): CouchDBTestCase.setUp(self) - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=False) # note that we don't enforce ddocs here def tearDown(self): @@ -509,8 +522,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents list functions will raise if the functions are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # erase views from _design/transactions transactions = self.db._database['_design/transactions'] @@ -538,8 +552,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents list functions will raise if the functions are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # erase views from _design/transactions transactions = self.db._database['_design/transactions'] @@ -567,8 +582,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents' named views will raise if the views are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # erase views from _design/docs docs = self.db._database['_design/docs'] @@ -608,8 +624,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): Test that all methods that access design documents will raise if the design docs are not present. """ - self.db = couch.CouchDatabase( - 'http://127.0.0.1:%d' % self.wrapper.port, 'test', + self.db = couch.CouchDatabase.open_database( + urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'), + create=True, ensure_ddocs=True) # delete _design/docs del self.db._database['_design/docs'] diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 3c457cc5..3c219b91 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -24,6 +24,10 @@ import mock import tempfile import threading + +from urlparse import urljoin + + from leap.soledad.client import Soledad from leap.soledad.common.couch import CouchDatabase, CouchServerState from leap.soledad.common.tests.test_couch import CouchDBTestCase @@ -101,8 +105,11 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): TestCaseWithServer.setUp(self) CouchDBTestCase.setUp(self) self._couch_url = 'http://localhost:' + str(self.wrapper.port) - self.db = CouchDatabase( - self._couch_url, 'user-user-uuid', replica_uid='replica') + self.db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + replica_uid='replica', + ensure_ddocs=True) self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") def tearDown(self): diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index f8d2a64f..6fe9211c 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -27,6 +27,7 @@ import mock import time import binascii +from urlparse import urljoin from leap.common.testing.basetest import BaseLeapTest from leap.soledad.common.couch import ( @@ -56,7 +57,8 @@ from leap.soledad.server.auth import URLToAuthorization def _couch_ensure_database(self, dbname): db = CouchDatabase.open_database( self._couch_url + '/' + dbname, - create=True) + create=True, + ensure_ddocs=True) return db, db._replica_uid CouchServerState.ensure_database = _couch_ensure_database @@ -352,11 +354,10 @@ class EncryptedSyncTestCase( self.assertEqual([], doclist) doc1 = sol1.create_doc(json.loads(simple_doc)) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -408,11 +409,10 @@ class EncryptedSyncTestCase( self.assertEqual([], doclist) doc1 = sol1.create_doc(json.loads(simple_doc)) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -468,11 +468,10 @@ class EncryptedSyncTestCase( content = binascii.hexlify(os.urandom(length/2)) # len() == length doc1 = sol1.create_doc({'data': content}) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -512,11 +511,10 @@ class EncryptedSyncTestCase( for i in range(0, number_of_docs): sol1.create_doc(json.loads(simple_doc)) # ensure remote db exists before syncing - db = CouchDatabase( - self._couch_url, - # the name of the user database is "user-". - 'user-user-uuid', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) # sync with server sol1._server_url = self.getURL() sol1.sync() @@ -558,8 +556,14 @@ class LockResourceTestCase( self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) # create the databases - CouchDatabase(self._couch_url, 'shared') - CouchDatabase(self._couch_url, 'tokens') + CouchDatabase.open_database( + urljoin(self._couch_url, 'shared'), + create=True, + ensure_ddocs=True) + CouchDatabase.open_database( + urljoin(self._couch_url, 'tokens'), + create=True, + ensure_ddocs=True) self._state = CouchServerState( self._couch_url, 'shared', 'tokens') @@ -567,10 +571,10 @@ class LockResourceTestCase( CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) # delete remote database - db = CouchDatabase( - self._couch_url, - 'shared', - ) + db = CouchDatabase.open_database( + urljoin(self._couch_url, 'shared'), + create=True, + ensure_ddocs=True) db.delete_database() def test__try_obtain_filesystem_lock(self): -- cgit v1.2.3 From ae5b0b57a14e0df45e2ed708eb5c8a495530ddde Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 10 Apr 2014 13:17:37 -0300 Subject: Properly close connections on couch backend (#5493). --- common/src/leap/soledad/common/couch.py | 87 +++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 27 deletions(-) (limited to 'common/src/leap') diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 8ed704ba..0aa84170 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -32,6 +32,7 @@ import threading from StringIO import StringIO from collections import defaultdict from urlparse import urljoin +from contextlib import contextmanager from couchdb.client import Server, Database @@ -39,7 +40,7 @@ from couchdb.http import ( ResourceConflict, ResourceNotFound, ServerError, - Session, + Session as CouchHTTPSession, ) from u1db import query_parser, vectorclock from u1db.errors import ( @@ -332,6 +333,35 @@ class MultipartWriter(object): self.headers[name] = value +class Session(CouchHTTPSession): + """ + An HTTP session that can be closed. + """ + + def close_connections(self): + for key, conns in list(self.conns.items()): + for conn in conns: + conn.close() + + +@contextmanager +def couch_server(url): + """ + Provide a connection to a couch server and cleanup after use. + + For database creation and deletion we use an ephemeral connection to the + couch server. That connection has to be properly closed, so we provide it + as a context manager. + + :param url: The URL of the Couch server. + :type url: str + """ + session = Session(timeout=COUCH_TIMEOUT) + server = Server(url=url, session=session) + yield server + session.close_connections() + + class CouchDatabase(CommonBackend): """ A U1DB implementation that uses CouchDB as its persistence layer. @@ -403,17 +433,16 @@ class CouchDatabase(CommonBackend): raise InvalidURLError url = m.group(1) dbname = m.group(2) - server = Server(url=url) - try: - server[dbname] - except ResourceNotFound: - if not create: - raise DatabaseDoesNotExist() - server.create(dbname) + with couch_server(url) as server: + try: + server[dbname] + except ResourceNotFound: + if not create: + raise DatabaseDoesNotExist() + server.create(dbname) return cls(url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) - def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True, - session=None): + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): """ Create a new Couch data container. @@ -425,14 +454,10 @@ class CouchDatabase(CommonBackend): :type replica_uid: str :param ensure_ddocs: Ensure that the design docs exist on server. :type ensure_ddocs: bool - :param session: an http.Session instance or None for a default session - :type session: http.Session """ # save params self._url = url - if session is None: - session = Session(timeout=COUCH_TIMEOUT) - self._session = session + self._session = Session(timeout=COUCH_TIMEOUT) self._factory = CouchDocument self._real_replica_uid = None # configure couch @@ -478,8 +503,9 @@ class CouchDatabase(CommonBackend): """ Delete a U1DB CouchDB database. """ - server = Server(url=self._url) - del(server[self._dbname]) + with couch_server(self._url) as server: + del(server[self._dbname]) + self.close_connections() def close(self): """ @@ -488,12 +514,26 @@ class CouchDatabase(CommonBackend): :return: True if db was succesfully closed. :rtype: bool """ + self.close_connections() self._url = None self._full_commit = None self._session = None self._database = None return True + def close_connections(self): + """ + Close all open connections to the couch server. + """ + if self._session is not None: + self._session.close_connections() + + def __del__(self): + """ + Close the database upon garbage collection. + """ + self.close() + def _set_replica_uid(self, replica_uid): """ Force the replica uid to be set. @@ -851,7 +891,9 @@ class CouchDatabase(CommonBackend): try: self._database.resource.put_json( doc.doc_id, body=buf.getvalue(), headers=envelope.headers) - self._renew_couch_session() + # What follows is a workaround for an ugly bug. See: + # https://leap.se/code/issues/5448 + self.close_connections() except ResourceConflict: raise RevisionConflict() @@ -1423,15 +1465,6 @@ class CouchDatabase(CommonBackend): continue yield t._doc - def _renew_couch_session(self): - """ - Create a new couch connection session. - - This is a workaround for #5448. Will not be needed once bigcouch is - merged with couchdb. - """ - self._database.resource.session = Session(timeout=COUCH_TIMEOUT) - class CouchSyncTarget(CommonSyncTarget): """ -- cgit v1.2.3 From f20d32698576f521f2fb53de0e636d318e826729 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 10 Apr 2014 13:18:02 -0300 Subject: Prevent file descriptor leaks on tests. --- common/src/leap/soledad/common/tests/test_couch.py | 42 ++++++++------ .../tests/test_couch_operations_atomicity.py | 4 ++ .../src/leap/soledad/common/tests/test_crypto.py | 5 +- .../src/leap/soledad/common/tests/test_server.py | 12 ++++ .../src/leap/soledad/common/tests/test_soledad.py | 18 +++++- .../leap/soledad/common/tests/test_sqlcipher.py | 4 -- .../src/leap/soledad/common/tests/test_target.py | 3 +- .../soledad/common/tests/u1db_tests/__init__.py | 11 +++- .../common/tests/u1db_tests/test_backends.py | 3 + .../soledad/common/tests/u1db_tests/test_sync.py | 64 +++++++++++----------- 10 files changed, 106 insertions(+), 60 deletions(-) (limited to 'common/src/leap') diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 77c46e61..17d4a519 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -27,8 +27,8 @@ from base64 import b64decode from mock import Mock from urlparse import urljoin -from couchdb.client import Server from u1db import errors as u1db_errors +from couchdb.client import Server from leap.common.files import mkdir_p @@ -186,8 +186,9 @@ def copy_couch_database_for_test(test, db): create=True, replica_uid=db._replica_uid or 'test') # copy all docs - old_couch_db = Server(couch_url)[db._replica_uid] - new_couch_db = Server(couch_url)[new_dbname] + session = couch.Session() + old_couch_db = Server(couch_url, session=session)[db._replica_uid] + new_couch_db = Server(couch_url, session=session)[new_dbname] for doc_id in old_couch_db: doc = old_couch_db.get(doc_id) # bypass u1db_config document @@ -217,6 +218,8 @@ def copy_couch_database_for_test(test, db): if (att is not None): new_couch_db.put_attachment(new_doc, att, filename=att_name) + # cleanup connections to prevent file descriptor leaking + session.close_connections() return new_db @@ -249,8 +252,10 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): if self.id() == \ 'leap.soledad.common.tests.test_couch.CouchTests.' \ 'test_close(couch)': - server = Server(url=self._url) + session = couch.Session() + server = Server(url=self._url, session=session) del(server[self._dbname]) + session.close_connections() else: self.db.delete_database() test_backends.AllDatabaseTests.tearDown(self) @@ -367,10 +372,9 @@ from u1db.backends.inmemory import InMemoryIndex class IndexedCouchDatabase(couch.CouchDatabase): - def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True, - session=None): + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): old_class.__init__(self, url, dbname, replica_uid=replica_uid, - ensure_ddocs=ensure_ddocs, session=session) + ensure_ddocs=ensure_ddocs) self._indexes = {} def _put_doc(self, old_doc, doc): @@ -459,19 +463,22 @@ class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase): self.db1 = None self.db2 = None self.db3 = None + self.db1_copy = None + self.db2_copy = None test_sync.DatabaseSyncTests.setUp(self) def tearDown(self): - self.db and self.db.delete_database() - self.db1 and self.db1.delete_database() - self.db2 and self.db2.delete_database() - self.db3 and self.db3.delete_database() - db = self.create_database('test1_copy', 'source') - db.delete_database() - db = self.create_database('test2_copy', 'target') - db.delete_database() - db = self.create_database('test3', 'target') - db.delete_database() + for db in [self.db, self.db1, self.db2, self.db3, self.db1_copy, + self.db2_copy]: + if db is not None: + db.delete_database() + db.close() + for replica_uid, dbname in [('test1_copy', 'source'), + ('test2_copy', 'target'), ('test3', 'target')]: + db = self.create_database(replica_uid, dbname) + db.delete_database() + # cleanup connections to avoid leaking of file descriptors + db.close() test_sync.DatabaseSyncTests.tearDown(self) @@ -486,6 +493,7 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): def tearDown(self): self.db.delete_database() + self.db.close() def test_missing_design_doc_raises(self): """ diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 3c219b91..b03f79e7 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -114,6 +114,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): def tearDown(self): self.db.delete_database() + self.db.close() CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) @@ -218,6 +219,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): _create_docs_and_sync(sol, 0) _create_docs_and_sync(sol, 1) + sol.close() # # Concurrency tests @@ -351,6 +353,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): self.assertEqual( 1, len(filter(lambda t: t[0] == doc_id, transaction_log))) + sol.close() def test_concurrent_syncs_do_not_fail(self): """ @@ -394,3 +397,4 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): self.assertEqual( 1, len(filter(lambda t: t[0] == doc_id, transaction_log))) + sol.close() diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index af11bc76..4b2470ba 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -44,7 +44,6 @@ from leap.soledad.common.crypto import WrongMac, UnknownMacMethod from leap.soledad.common.tests.u1db_tests import ( simple_doc, nested_doc, - TestCaseWithServer, ) @@ -95,6 +94,7 @@ class RecoveryDocumentTestCase(BaseSoledadTest): self.assertEqual(self._soledad._get_storage_secret(), s._get_storage_secret(), 'Failed settinng secret for symmetric encryption.') + s.close() class SoledadSecretsTestCase(BaseSoledadTest): @@ -110,6 +110,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # generate new secret secret_id_2 = sol._gen_secret() self.assertTrue(secret_id_1 != secret_id_2) + sol.close() # re-instantiate sol = self._soledad_instance( user='user@leap.se', @@ -130,6 +131,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # assert id is hash of new secret self.assertTrue( secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) + sol.close() def test__has_secret(self): sol = self._soledad_instance( @@ -144,6 +146,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # but not being able to decrypt correctly should sol._secrets[sol.secret_id][sol.SECRET_KEY] = None self.assertFalse(sol._has_secret()) + sol.close() class MacAuthTestCase(BaseSoledadTest): diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 6fe9211c..1c5a7407 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -391,6 +391,9 @@ class EncryptedSyncTestCase( # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) db.delete_database() + db.close() + sol1.close() + sol2.close() def test_encrypted_sym_sync_with_unicode_passphrase(self): """ @@ -450,6 +453,9 @@ class EncryptedSyncTestCase( # assert incoming doc is equal to the first sent doc self.assertEqual(doc1, doc2) db.delete_database() + db.close() + sol1.close() + sol2.close() def test_sync_very_large_files(self): """ @@ -492,6 +498,9 @@ class EncryptedSyncTestCase( self.assertEqual(doc1, doc2) # delete remote database db.delete_database() + db.close() + sol1.close() + sol2.close() def test_sync_many_small_files(self): @@ -535,6 +544,9 @@ class EncryptedSyncTestCase( self.assertEqual(sol1.get_doc(doc.doc_id), doc) # delete remote database db.delete_database() + db.close() + sol1.close() + sol2.close() class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 035c5ac5..5a3bf2b0 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -49,6 +49,7 @@ class AuxMethodsTestCase(BaseSoledadTest): secrets_path = os.path.dirname(sol.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) + sol.close() def test__init_db(self): sol = self._soledad_instance() @@ -61,6 +62,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol._init_db() from leap.soledad.client.sqlcipher import SQLCipherDatabase self.assertIsInstance(sol._db, SQLCipherDatabase) + sol.close() def test__init_config_defaults(self): """ @@ -103,6 +105,7 @@ class AuxMethodsTestCase(BaseSoledadTest): os.path.join(self.tempdir, 'value_2'), sol.local_db_path) self.assertEqual('value_1', sol.server_url) + sol.close() def test_change_passphrase(self): """ @@ -118,6 +121,7 @@ class AuxMethodsTestCase(BaseSoledadTest): # change the passphrase sol.change_passphrase(u'654321') + sol.close() self.assertRaises( WrongMac, @@ -132,6 +136,7 @@ class AuxMethodsTestCase(BaseSoledadTest): prefix=self.rand_prefix) doc2 = sol2.get_doc(doc_id) self.assertEqual(doc, doc2) + sol2.close() def test_change_passphrase_with_short_passphrase_raises(self): """ @@ -145,6 +150,7 @@ class AuxMethodsTestCase(BaseSoledadTest): self.assertRaises( PassphraseTooShort, sol.change_passphrase, u'54321') + sol.close() def test_get_passphrase(self): """ @@ -152,6 +158,7 @@ class AuxMethodsTestCase(BaseSoledadTest): """ sol = self._soledad_instance() self.assertEqual('123', sol.passphrase) + sol.close() class SoledadSharedDBTestCase(BaseSoledadTest): @@ -165,6 +172,9 @@ class SoledadSharedDBTestCase(BaseSoledadTest): 'https://provider/', ADDRESS, document_factory=SoledadDocument, creds=None) + def tearDown(self): + BaseSoledadTest.tearDown(self) + def test__get_secrets_from_shared_db(self): """ Ensure the shared db is queried with the correct doc_id. @@ -209,7 +219,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): BaseSoledadTest.setUp(self) def tearDown(self): - pass + BaseSoledadTest.tearDown(self) def _pop_mock_call(self, mocked): mocked.call_args_list.pop() @@ -283,6 +293,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # assert db was locked and unlocked sol._shared_db.lock.assert_called_with() sol._shared_db.unlock.assert_called_with('atoken') + sol.close() def test_stage2_bootstrap_signals(self): """ @@ -305,6 +316,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def __call__(self): return self + sol.close() # reset mock soledad.client.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals @@ -328,6 +340,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) + sol.close() def test_stage1_bootstrap_signals(self): """ @@ -337,6 +350,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get an existent instance so it emits only some of bootstrap signals sol = self._soledad_instance() self.assertEqual([], soledad.client.signal.mock_calls) + sol.close() def test_sync_signals(self): """ @@ -355,6 +369,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): proto.SOLEDAD_DONE_DATA_SYNC, ADDRESS, ) + sol.close() def test_need_sync_signals(self): """ @@ -375,3 +390,4 @@ class SoledadSignalingTestCase(BaseSoledadTest): ADDRESS, ) SoledadSyncTarget.get_sync_info = old_get_sync_info + sol.close() diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 66a673b6..c79a6045 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -469,11 +469,9 @@ class SQLCipherDatabaseSyncTests( def setUp(self): test_sync.DatabaseSyncTests.setUp(self) - BaseSoledadTest.setUp(self) def tearDown(self): test_sync.DatabaseSyncTests.tearDown(self) - BaseSoledadTest.tearDown(self) def test_sync_autoresolves(self): """ @@ -683,11 +681,9 @@ class SQLCipherSyncTargetTests( def setUp(self): test_sync.DatabaseSyncTargetTests.setUp(self) - #BaseSoledadTest.setUp(self) def tearDown(self): test_sync.DatabaseSyncTargetTests.tearDown(self) - BaseSoledadTest.tearDown(self) def test_sync_exchange(self): """ diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py index 5a541745..c1e00d52 100644 --- a/common/src/leap/soledad/common/tests/test_target.py +++ b/common/src/leap/soledad/common/tests/test_target.py @@ -272,11 +272,9 @@ class TestSoledadParsingSyncStream( def setUp(self): test_remote_sync_target.TestParsingSyncStream.setUp(self) - BaseSoledadTest.setUp(self) def tearDown(self): test_remote_sync_target.TestParsingSyncStream.tearDown(self) - BaseSoledadTest.tearDown(self) def test_extra_comma(self): """ @@ -646,6 +644,7 @@ class SoledadDatabaseSyncTargetTests( self.assertEqual(([], 1, last_trans_id), (self.other_changes, new_gen, last_trans_id)) self.assertEqual(10, self.st.get_sync_info('replica')[3]) + sol.close() def test_sync_exchange_push_many(self): """ diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py index 3bc12487..99ff77b4 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py @@ -208,8 +208,8 @@ class DatabaseBaseTests(TestCase): self.db = self.create_database('test') def tearDown(self): - # TODO: Add close_database parameterization - # self.close_database(self.db) + if hasattr(self, 'db') and self.db is not None: + self.db.close() super(DatabaseBaseTests, self).tearDown() def assertTransactionLog(self, doc_ids, db): @@ -335,6 +335,13 @@ class TestCaseWithServer(TestCase): super(TestCaseWithServer, self).setUp() self.server = self.server_thread = None + def tearDown(self): + if self.server is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + super(TestCaseWithServer, self).tearDown() + @property def url_scheme(self): return self.server_def()[-1] diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index d2a91d11..c0a7e1f7 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -355,6 +355,9 @@ class LocalDatabaseTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS + def setUp(self): + tests.DatabaseBaseTests.setUp(self) + def test_create_doc_different_ids_diff_db(self): doc1 = self.db.create_doc_from_json(simple_doc) db2 = self.create_database('other-uid') diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py index 1f78f912..a37c36db 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py @@ -85,7 +85,6 @@ class DatabaseSyncTargetTests(tests.DatabaseBaseTests, whitebox = True def setUp(self): - tests.DatabaseBaseTests.setUp(self) tests.TestCaseWithServer.setUp(self) self.db, self.st = self.create_db_and_target(self) self.other_changes = [] @@ -94,7 +93,6 @@ class DatabaseSyncTargetTests(tests.DatabaseBaseTests, # We delete them explicitly, so that connections are cleanly closed del self.st self.db.close() - del self.db super(DatabaseSyncTargetTests, self).tearDown() def create_db_and_target(self, *args): @@ -1013,30 +1011,30 @@ class DatabaseSyncTests(tests.DatabaseBaseTests, def test_sync_supersedes_conflicts(self): self.db1 = self.create_database('test1', 'both') self.db2 = self.create_database('test2', 'target') - db3 = self.create_database('test3', 'both') + self.db3 = self.create_database('test3', 'both') doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(db3, self.db1) + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) self.assertEqual( self.db1._get_generation_info(), - db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) self.assertEqual( - db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(db3._replica_uid)) - self.sync(db3, self.db2) + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) self.assertEqual( self.db2._get_generation_info(), - db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) self.assertEqual( - db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(db3._replica_uid)) - self.assertEqual(3, len(db3.get_doc_conflicts('the-doc'))) + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) doc1.set_json('{"a": 2}') self.db1.put_doc(doc1) - self.sync(db3, self.db1) + self.sync(self.db3, self.db1) # original doc1 should have been removed from conflicts - self.assertEqual(3, len(db3.get_doc_conflicts('the-doc'))) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) def test_sync_stops_after_get_sync_info(self): self.db1 = self.create_database('test1', 'source') @@ -1054,70 +1052,70 @@ class DatabaseSyncTests(tests.DatabaseBaseTests, self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') self.sync(self.db1, self.db2) - db1_copy = self.copy_database(self.db1) + self.db1_copy = self.copy_database(self.db1) self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidGeneration, self.sync, db1_copy, self.db2) + errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) def test_sync_detects_rollback_in_target(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) - db2_copy = self.copy_database(self.db2) + self.db2_copy = self.copy_database(self.db2) self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1, db2_copy) + errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) def test_sync_detects_diverged_source(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') - db3 = self.copy_database(self.db1) + self.db3 = self.copy_database(self.db1) self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidTransactionId, self.sync, db3, self.db2) + errors.InvalidTransactionId, self.sync, self.db3, self.db2) def test_sync_detects_diverged_target(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') - db3 = self.copy_database(self.db2) - db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, db3) + errors.InvalidTransactionId, self.sync, self.db1, self.db3) def test_sync_detects_rollback_and_divergence_in_source(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') self.sync(self.db1, self.db2) - db1_copy = self.copy_database(self.db1) + self.db1_copy = self.copy_database(self.db1) self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.sync(self.db1, self.db2) - db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.assertRaises( - errors.InvalidTransactionId, self.sync, db1_copy, self.db2) + errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) def test_sync_detects_rollback_and_divergence_in_target(self): self.db1 = self.create_database('test1', 'source') self.db2 = self.create_database('test2', 'target') self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") self.sync(self.db1, self.db2) - db2_copy = self.copy_database(self.db2) + self.db2_copy = self.copy_database(self.db2) self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.sync(self.db1, self.db2) - db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, db2_copy) + errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) class TestDbSync(tests.TestCaseWithServer): -- cgit v1.2.3