From 16d8677ad67ceb0cdf78f3b0a7d6b4e4dae4b624 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 7 Jul 2017 18:06:30 -0300 Subject: [benchmarks] add --watch-resources option This commit adds the --watch-resources command line option for benchmarks tests, and allows to running the benchmark test suite with and without resource monitoring instrumentation code. This is needed because resource consumption monitoring impacts the mean time and standard deviation of time taken to run benchmarked tests. --- testing/tests/benchmarks/conftest.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'testing') diff --git a/testing/tests/benchmarks/conftest.py b/testing/tests/benchmarks/conftest.py index 4dbc4377..0d171ef4 100644 --- a/testing/tests/benchmarks/conftest.py +++ b/testing/tests/benchmarks/conftest.py @@ -24,8 +24,10 @@ server.ensure_server() def pytest_addoption(parser): parser.addoption( - "--num-docs", type="int", default=100, - help="the number of documents to use in performance tests") + "--watch-resources", default=False, action="store_true", + help="whether to monitor CPU and memory percentages during test run. " + "**Warning**: enabling this will impact the time taken by the " + "benchmarked code, so use with caution!") # mark benchmark tests using their group names (thanks ionelmc! :) @@ -139,12 +141,21 @@ def _monitored_benchmark(benchmark_fixture, benchmark_function, }) +def _watch_resources(request): + return request.config.getoption('--watch-resources') + + @pytest.fixture -def monitored_benchmark(benchmark): - return functools.partial(_monitored_benchmark, benchmark, benchmark) +def monitored_benchmark(benchmark, request): + if not _watch_resources(request): + return benchmark + return functools.partial( + _monitored_benchmark, benchmark, benchmark) @pytest.fixture -def monitored_benchmark_with_setup(benchmark): +def monitored_benchmark_with_setup(benchmark, request): + if not _watch_resources(request): + return benchmark.pedantic return functools.partial( _monitored_benchmark, benchmark, benchmark.pedantic) -- cgit v1.2.3