summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2017-07-07 18:17:38 -0300
committerdrebs <drebs@leap.se>2017-07-08 07:48:47 -0300
commitf21d9e1a4e6d0a2ed6d7af83191e39a2a883241b (patch)
treea3d917c93c4ed7f11889182094172924d2759a9c
parent16d8677ad67ceb0cdf78f3b0a7d6b4e4dae4b624 (diff)
[benchmarks] run benchmarks twice, for time and resources
We noticed that instrumentation added for watching resources has an impact in time statistics (i.e. it increases average and stddev). This commit makes the benchmark tests run twice: once for measuring time and a second time for measuring resources.
-rw-r--r--testing/tests/benchmarks/conftest.py8
-rw-r--r--testing/tests/conftest.py8
-rw-r--r--testing/tox.ini2
3 files changed, 10 insertions, 8 deletions
diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py
index 0d171ef4..ac29f17f 100644
--- a/testing/tests/benchmarks/conftest.py
+++ b/testing/tests/benchmarks/conftest.py
@@ -22,14 +22,6 @@ server.ensure_server()
# pytest customizations
#
-def pytest_addoption(parser):
- parser.addoption(
- "--watch-resources", default=False, action="store_true",
- help="whether to monitor CPU and memory percentages during test run. "
- "**Warning**: enabling this will impact the time taken by the "
- "benchmarked code, so use with caution!")
-
-
# mark benchmark tests using their group names (thanks ionelmc! :)
def pytest_collection_modifyitems(items):
for item in items:
diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py
index 4d88072e..994b1610 100644
--- a/testing/tests/conftest.py
+++ b/testing/tests/conftest.py
@@ -41,6 +41,14 @@ def pytest_addoption(parser):
"--couch-url", type="string", default="http://127.0.0.1:5984",
help="the url for the couch server to be used during tests")
+ # the following option is only used in benchmarks, but has to be defined
+ # here due to how pytest discovers plugins during startup.
+ parser.addoption(
+ "--watch-resources", default=False, action="store_true",
+ help="whether to monitor CPU and memory percentages during test run. "
+ "**Warning**: enabling this will impact the time taken by the "
+ "benchmarked code, so use with caution!")
+
@pytest.fixture
def couch_url(request):
diff --git a/testing/tox.ini b/testing/tox.ini
index 9aaa1ae3..88b27d48 100644
--- a/testing/tox.ini
+++ b/testing/tox.ini
@@ -76,7 +76,9 @@ commands =
# we must make sure that installed pysqlcipher was built with the HAVE_USLEEP
# flag, or we might have problems with concurrent db access.
./ensure-pysqlcipher-has-usleep.sh
+# run benchmarks twice: once for time and a second time for resources
py.test --benchmark-only {posargs}
+ py.test --benchmark-only --watch-resources {posargs}
passenv = HOST_HOSTNAME
[testenv:code-check]