summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xscripts/benchmark/check-for-outliers.py (renamed from scripts/benchmark/compare-results-with-history.py)31
-rwxr-xr-xscripts/benchmark/run-benchmarks-ci-job.sh14
2 files changed, 24 insertions, 21 deletions
diff --git a/scripts/benchmark/compare-results-with-history.py b/scripts/benchmark/check-for-outliers.py
index ed609552..6037ef00 100755
--- a/scripts/benchmark/compare-results-with-history.py
+++ b/scripts/benchmark/check-for-outliers.py
@@ -9,6 +9,8 @@
# master branch.
#
# - compare the result in the file with the results in elastic.
+#
+# - if there are bad outliers, exit with status code given in command line.
import argparse
import copy
@@ -27,6 +29,9 @@ def parse_args():
parser.add_argument(
'file',
help='The file with JSON results of pytest-benchmark')
+ parser.add_argument(
+ '--status-code', type=int, default=0,
+ help='The status code to exit with in case bad outliers are detected.')
return parser.parse_args()
@@ -127,31 +132,31 @@ def detect_bad_outlier(test, mean, extra):
if 'memory_percent' in extra:
mem = get_mem_stats(test)
value = extra['memory_percent']['stats']['max']
- bad |= _detect_outlier('mem', value, mem) > 0
+ bad |= _detect_outlier(test, 'mem', value, mem) > 0
else:
time, cpu = get_time_cpu_stats(test)
value = mean
- bad |= _detect_outlier('time', value, time) > 0
+ bad |= _detect_outlier(test, 'time', value, time) > 0
value = extra['cpu_percent']
- bad |= _detect_outlier('cpu', value, cpu) > 0
+ bad |= _detect_outlier(test, 'cpu', value, cpu) > 0
return bad
-def _detect_outlier(name, value, list):
+def _detect_outlier(test, name, value, list):
mean = _mean(list)
std = _std(list)
result = 0
- print "%s: %f ? %f +- %f * %f" \
- % (name, value, mean, MULTIPLIER, std)
+ print "Checking %s (%s):" % (test, name)
+ print " value: %f" % (value,)
+ print " lower limit: %f" % (mean - (MULTIPLIER * std))
+ print " upper limit: %f" % (mean + (MULTIPLIER * std))
if value < mean - MULTIPLIER * std:
- print "%s: %f < %f - %f * %f" \
- % (name, value, mean, MULTIPLIER, std)
+ print " => good outlier detected!"
result = -1
elif value > mean + MULTIPLIER * std:
- print "%s: %f > %f - %f * %f" \
- % (name, value, mean, MULTIPLIER, std)
+ print " => bad outlier detected!"
result = 1
return result
@@ -159,8 +164,12 @@ def _detect_outlier(name, value, list):
if __name__ == '__main__':
args = parse_args()
tests = parse_file(args.file)
+ print "Checking %d test results for outliers..." % len(tests)
failed = False
for test, mean, extra in tests:
failed |= detect_bad_outlier(test, mean, extra)
if failed:
- sys.exit(1)
+ print "Tests have bad outliers! o_O"
+ sys.exit(args.status_code)
+ else:
+ print "All good, no outliers were detected. :-)"
diff --git a/scripts/benchmark/run-benchmarks-ci-job.sh b/scripts/benchmark/run-benchmarks-ci-job.sh
index 30c6ecf5..adf37b7a 100755
--- a/scripts/benchmark/run-benchmarks-ci-job.sh
+++ b/scripts/benchmark/run-benchmarks-ci-job.sh
@@ -16,8 +16,8 @@
# Environment Variables
# ---------------------
#
-# RUN_BENCHMARKS: If not set, skip this run.
-# CHECK_FOR_OUTLIERS: If set, check if results are outliers.
+# RUN_BENCHMARKS - If not set, skip this run.
+# STATUS_CODE_IF_OUTLIERS - Exit with this status code if outliers are detected.
set -eu
set -o xtrace
@@ -55,12 +55,6 @@ if [ -z "$(echo ${ENVIRONMENT} | grep ^benchmark-)" ]; then
exit 0
fi
-# stop here unless the CHECK_FOR_OUTLIERS environment variable is set
-if [ -z "${CHECK_FOR_OUTLIERS:-}" ]; then
- exit 0
-fi
-
-# fail test for bad outliers
-echo "Comparing current test results with history..."
+# check for bad outliers
basedir=$(dirname "${0}")
-${basedir}/compare-results-with-history.py ${tempfile}
+${basedir}/check-for-outliers.py --status-code ${STATUS_CODE_IF_OUTLIERS:-0} ${tempfile}