From bf18c2bc6e3f533187281a3b31febd37ef22f8c0 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 27 May 2015 17:19:13 -0300 Subject: [feat] Make it optional to have a dedicated pool As @meskio pointed out, some cases could need a dedicated pool with different parameters. This is a suggested implementation where the pool is reused by default, creating a dedicated one just if needed/asked. This way we ensure that resources are under control and special cases are still handled. --- src/leap/common/tests/test_http.py | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/leap/common/tests/test_http.py (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_http.py b/src/leap/common/tests/test_http.py new file mode 100644 index 0000000..e240ca3 --- /dev/null +++ b/src/leap/common/tests/test_http.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# test_http.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for: + * leap/common/http.py +""" +import os +try: + import unittest2 as unittest +except ImportError: + import unittest + +from leap.common import http +from leap.common.testing.basetest import BaseLeapTest + +TEST_CERT_PEM = os.path.join( + os.path.split(__file__)[0], + '..', 'testing', "leaptest_combined_keycert.pem") + + +class HTTPClientTest(BaseLeapTest): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_agents_sharing_pool_by_default(self): + client = http.HTTPClient() + client2 = http.HTTPClient(TEST_CERT_PEM) + self.assertNotEquals(client._agent, client2._agent, "Expected dedicated agents") + self.assertEquals(client._agent._pool, client2._agent._pool, "Pool was not reused by default") + + def test_agent_can_have_dedicated_custom_pool(self): + custom_pool = http.createPool(maxPersistentPerHost=42, persistent=False) + self.assertEquals(custom_pool.maxPersistentPerHost, 42, + "Custom persistent connections limit is not being respected") + self.assertFalse(custom_pool.persistent, + "Custom persistence is not being respected") + default_client = http.HTTPClient() + custom_client = http.HTTPClient(pool=custom_pool) + + self.assertNotEquals(default_client._agent, custom_client._agent, "No agent reuse is expected") + self.assertEquals(custom_pool, custom_client._agent._pool, "Custom pool usage was not respected") + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From 07d421a32f3bb45932668f4951233166ada4e770 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Jul 2015 17:03:50 -0300 Subject: [bug] fix events ioloop client tests The events ioloop client is not completelly thread safe, so we have to use reactor.callFromThread whenever we need it to be thread-safe. Examples of this behaviour are the events ioloop client tests that depend on the firing of deferreds, which are not thread safe. This commit fixes tese tests. --- src/leap/common/tests/test_events.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index 7ef3e1b..611781c 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -20,6 +20,7 @@ import os import logging import time +from twisted.internet.reactor import callFromThread from twisted.trial import unittest from twisted.internet import defer @@ -80,8 +81,8 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d = defer.Deferred() - cbk_fail = lambda event, _: d.errback(event) - cbk_succeed = lambda event, _: d.callback(event) + cbk_fail = lambda event, _: callFromThread(d.errback, event) + cbk_succeed = lambda event, _: callFromThread(d.callback, event) self._client.register(event, cbk_fail, uid=1) self._client.register(event, cbk_succeed, uid=1, replace=True) self._client.emit(event, None) @@ -105,9 +106,9 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d1 = defer.Deferred() - cbk1 = lambda event, _: d1.callback(event) + cbk1 = lambda event, _: callFromThread(d1.callback, event) d2 = defer.Deferred() - cbk2 = lambda event, _: d2.callback(event) + cbk2 = lambda event, _: callFromThread(d2.callback, event) self._client.register(event, cbk1) self._client.register(event, cbk2) self._client.emit(event, None) @@ -121,7 +122,7 @@ class EventsGenericClientTestCase(object): event = catalog.CLIENT_UID d = defer.Deferred() def cbk(events, _): - d.callback(event) + callFromThread(d.callback, event) self._client.register(event, cbk) self._client.emit(event, None) return d @@ -133,14 +134,17 @@ class EventsGenericClientTestCase(object): event1 = catalog.CLIENT_UID d = defer.Deferred() # register more than one callback for the same event - self._client.register(event1, lambda ev, _: d.errback(None)) - self._client.register(event1, lambda ev, _: d.errback(None)) + self._client.register( + event1, lambda ev, _: callFromThread(d.errback, None)) + self._client.register( + event1, lambda ev, _: callFromThread(d.errback, None)) # unregister and emit the event self._client.unregister(event1) self._client.emit(event1, None) # register and emit another event so the deferred can succeed event2 = catalog.CLIENT_SESSION_ID - self._client.register(event2, lambda ev, _: d.callback(None)) + self._client.register( + event2, lambda ev, _: callFromThread(d.callback, None)) self._client.emit(event2, None) return d @@ -151,9 +155,11 @@ class EventsGenericClientTestCase(object): event = catalog.CLIENT_UID d = defer.Deferred() # register one callback that would fail - uid = self._client.register(event, lambda ev, _: d.errback(None)) + uid = self._client.register( + event, lambda ev, _: callFromThread(d.errback, None)) # register one callback that will succeed - self._client.register(event, lambda ev, _: d.callback(None)) + self._client.register( + event, lambda ev, _: callFromThread(d.callback, None)) # unregister by uid and emit the event self._client.unregister(event, uid=uid) self._client.emit(event, None) -- cgit v1.2.3 From 25d3b7c80f85cd94159b574274108061a94f1bc9 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 22 Jul 2015 15:38:56 -0300 Subject: [style] Fixed pep8 warnings --- src/leap/common/tests/test_certs.py | 6 ++++-- src/leap/common/tests/test_events.py | 33 ++++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_certs.py b/src/leap/common/tests/test_certs.py index 999071f..209e051 100644 --- a/src/leap/common/tests/test_certs.py +++ b/src/leap/common/tests/test_certs.py @@ -60,11 +60,13 @@ class CertsTest(BaseLeapTest): self.assertTrue(certs.should_redownload(cert_path)) def test_should_redownload_if_before(self): - new_now = lambda: time.struct_time(CERT_NOT_BEFORE) + def new_now(): + time.struct_time(CERT_NOT_BEFORE) self.assertTrue(certs.should_redownload(TEST_CERT_PEM, now=new_now)) def test_should_redownload_if_after(self): - new_now = lambda: time.struct_time(CERT_NOT_AFTER) + def new_now(): + time.struct_time(CERT_NOT_AFTER) self.assertTrue(certs.should_redownload(TEST_CERT_PEM, now=new_now)) def test_not_should_redownload(self): diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index 611781c..e2a918f 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -1,4 +1,4 @@ -## -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # test_events.py # Copyright (C) 2013 LEAP # @@ -60,7 +60,10 @@ class EventsGenericClientTestCase(object): 'There should be no callback for this event.') # register one event event1 = catalog.CLIENT_UID - cbk1 = lambda event, _: True + + def cbk1(event, _): + return True + uid1 = self._client.register(event1, cbk1) # assert for correct registration self.assertTrue(len(callbacks) == 1) @@ -68,7 +71,10 @@ class EventsGenericClientTestCase(object): 'Could not register event in local client.') # register another event event2 = catalog.CLIENT_SESSION_ID - cbk2 = lambda event, _: True + + def cbk2(event, _): + return True + uid2 = self._client.register(event2, cbk2) # assert for correct registration self.assertTrue(len(callbacks) == 2) @@ -81,8 +87,13 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d = defer.Deferred() - cbk_fail = lambda event, _: callFromThread(d.errback, event) - cbk_succeed = lambda event, _: callFromThread(d.callback, event) + + def cbk_fail(event, _): + return callFromThread(d.errback, event) + + def cbk_succeed(event, _): + return callFromThread(d.callback, event) + self._client.register(event, cbk_fail, uid=1) self._client.register(event, cbk_succeed, uid=1, replace=True) self._client.emit(event, None) @@ -106,9 +117,15 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d1 = defer.Deferred() - cbk1 = lambda event, _: callFromThread(d1.callback, event) + + def cbk1(event, _): + return callFromThread(d1.callback, event) + d2 = defer.Deferred() - cbk2 = lambda event, _: callFromThread(d2.callback, event) + + def cbk2(event, _): + return d2.callback(event) + self._client.register(event, cbk1) self._client.register(event, cbk2) self._client.emit(event, None) @@ -121,8 +138,10 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d = defer.Deferred() + def cbk(events, _): callFromThread(d.callback, event) + self._client.register(event, cbk) self._client.emit(event, None) return d -- cgit v1.2.3 From 301be892a67f88b8b9479531003403a77b36a5f2 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 22 Jul 2015 19:17:03 -0300 Subject: [tests] set environment for certs test --- src/leap/common/tests/test_certs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_certs.py b/src/leap/common/tests/test_certs.py index 209e051..8ebc0f4 100644 --- a/src/leap/common/tests/test_certs.py +++ b/src/leap/common/tests/test_certs.py @@ -43,10 +43,10 @@ CERT_NOT_AFTER = (2023, 9, 1, 17, 52, 16, 4, 244, 0) class CertsTest(BaseLeapTest): def setUp(self): - pass + self.setUpEnv() def tearDown(self): - pass + self.tearDownEnv() def test_should_redownload_if_no_cert(self): self.assertTrue(certs.should_redownload(certfile="")) -- cgit v1.2.3 From 5502318835626527d9818c360c7fd2b1a4b01145 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 22 Jul 2015 23:55:39 -0300 Subject: [tests] implemented http feature according to test Two test cases were broken and were implemented here: The first was that HTTPClient should share the connection between clients if a pool was not passed explicitly. If you initialize an HTTPClient without a pool, it will reuse a pool created on the class. The second was that you should be able to pass to the HTTPCLient a pool on initialization. Added that possibility and fixed the tests accordingly --- src/leap/common/tests/test_http.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_http.py b/src/leap/common/tests/test_http.py index e240ca3..a586fd1 100644 --- a/src/leap/common/tests/test_http.py +++ b/src/leap/common/tests/test_http.py @@ -47,7 +47,12 @@ class HTTPClientTest(BaseLeapTest): self.assertEquals(client._agent._pool, client2._agent._pool, "Pool was not reused by default") def test_agent_can_have_dedicated_custom_pool(self): - custom_pool = http.createPool(maxPersistentPerHost=42, persistent=False) + custom_pool = http._HTTPConnectionPool( + None, + timeout=10, + maxPersistentPerHost=42, + persistent=False + ) self.assertEquals(custom_pool.maxPersistentPerHost, 42, "Custom persistent connections limit is not being respected") self.assertFalse(custom_pool.persistent, -- cgit v1.2.3 From 486da2654c63262f0dbc2d603125f0c8c5c4ea74 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Thu, 23 Jul 2015 00:00:39 -0300 Subject: [tests] fixed events tests The events tests check for register and emit signals, but because the flag set_events_enabled was False by default in the tests, no signals were being emitted. I added the flag to the setUp and tearDown of the tests, they are still very slow but at least they are passing now --- src/leap/common/tests/test_events.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index e2a918f..8f08eeb 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -26,6 +26,7 @@ from twisted.internet import defer from leap.common.events import server from leap.common.events import client +from leap.common.events import flags from leap.common.events import txclient from leap.common.events import catalog from leap.common.events.errors import CallbackAlreadyRegisteredError @@ -44,10 +45,12 @@ class EventsGenericClientTestCase(object): self._client.configure_client( emit_addr="tcp://127.0.0.1:%d" % self._server.pull_port, reg_addr="tcp://127.0.0.1:%d" % self._server.pub_port) + flags.set_events_enabled(True) def tearDown(self): self._client.shutdown() self._server.shutdown() + flags.set_events_enabled(False) # wait a bit for sockets to close properly time.sleep(0.1) -- cgit v1.2.3 From a119dd4fa2fc4a14577fd2d6e32dff950d934193 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 28 Jul 2015 11:41:32 -0400 Subject: [style] more pep8 cleanup --- src/leap/common/tests/test_http.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_http.py b/src/leap/common/tests/test_http.py index a586fd1..f44550f 100644 --- a/src/leap/common/tests/test_http.py +++ b/src/leap/common/tests/test_http.py @@ -43,8 +43,11 @@ class HTTPClientTest(BaseLeapTest): def test_agents_sharing_pool_by_default(self): client = http.HTTPClient() client2 = http.HTTPClient(TEST_CERT_PEM) - self.assertNotEquals(client._agent, client2._agent, "Expected dedicated agents") - self.assertEquals(client._agent._pool, client2._agent._pool, "Pool was not reused by default") + self.assertNotEquals( + client._agent, client2._agent, "Expected dedicated agents") + self.assertEquals( + client._agent._pool, client2._agent._pool, + "Pool was not reused by default") def test_agent_can_have_dedicated_custom_pool(self): custom_pool = http._HTTPConnectionPool( @@ -54,14 +57,19 @@ class HTTPClientTest(BaseLeapTest): persistent=False ) self.assertEquals(custom_pool.maxPersistentPerHost, 42, - "Custom persistent connections limit is not being respected") + "Custom persistent connections " + "limit is not being respected") self.assertFalse(custom_pool.persistent, "Custom persistence is not being respected") default_client = http.HTTPClient() custom_client = http.HTTPClient(pool=custom_pool) - self.assertNotEquals(default_client._agent, custom_client._agent, "No agent reuse is expected") - self.assertEquals(custom_pool, custom_client._agent._pool, "Custom pool usage was not respected") + self.assertNotEquals( + default_client._agent, custom_client._agent, + "No agent reuse is expected") + self.assertEquals( + custom_pool, custom_client._agent._pool, + "Custom pool usage was not respected") if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From e47f4bcc0eb9c3d08c649adcb62b0325f439113e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 28 Jul 2015 16:48:04 -0300 Subject: [test] set flag before starting client Test client will only start with flag set to True. Change EventsGenericClientTestCase to set the flag on the first line of setUp. --- src/leap/common/tests/test_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common/tests') diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index 8f08eeb..2ad097e 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -39,13 +39,13 @@ if 'DEBUG' in os.environ: class EventsGenericClientTestCase(object): def setUp(self): + flags.set_events_enabled(True) self._server = server.ensure_server( emit_addr="tcp://127.0.0.1:0", reg_addr="tcp://127.0.0.1:0") self._client.configure_client( emit_addr="tcp://127.0.0.1:%d" % self._server.pull_port, reg_addr="tcp://127.0.0.1:%d" % self._server.pub_port) - flags.set_events_enabled(True) def tearDown(self): self._client.shutdown() -- cgit v1.2.3