diff options
| -rwxr-xr-x | scripts/benchmark/check-for-outliers.py (renamed from scripts/benchmark/compare-results-with-history.py) | 31 | ||||
| -rwxr-xr-x | scripts/benchmark/run-benchmarks-ci-job.sh | 14 |
2 files changed, 24 insertions, 21 deletions
diff --git a/scripts/benchmark/compare-results-with-history.py b/scripts/benchmark/check-for-outliers.py index ed609552..6037ef00 100755 --- a/scripts/benchmark/compare-results-with-history.py +++ b/scripts/benchmark/check-for-outliers.py @@ -9,6 +9,8 @@ # master branch. # # - compare the result in the file with the results in elastic. +# +# - if there are bad outliers, exit with status code given in command line. import argparse import copy @@ -27,6 +29,9 @@ def parse_args(): parser.add_argument( 'file', help='The file with JSON results of pytest-benchmark') + parser.add_argument( + '--status-code', type=int, default=0, + help='The status code to exit with in case bad outliers are detected.') return parser.parse_args() @@ -127,31 +132,31 @@ def detect_bad_outlier(test, mean, extra): if 'memory_percent' in extra: mem = get_mem_stats(test) value = extra['memory_percent']['stats']['max'] - bad |= _detect_outlier('mem', value, mem) > 0 + bad |= _detect_outlier(test, 'mem', value, mem) > 0 else: time, cpu = get_time_cpu_stats(test) value = mean - bad |= _detect_outlier('time', value, time) > 0 + bad |= _detect_outlier(test, 'time', value, time) > 0 value = extra['cpu_percent'] - bad |= _detect_outlier('cpu', value, cpu) > 0 + bad |= _detect_outlier(test, 'cpu', value, cpu) > 0 return bad -def _detect_outlier(name, value, list): +def _detect_outlier(test, name, value, list): mean = _mean(list) std = _std(list) result = 0 - print "%s: %f ? %f +- %f * %f" \ - % (name, value, mean, MULTIPLIER, std) + print "Checking %s (%s):" % (test, name) + print " value: %f" % (value,) + print " lower limit: %f" % (mean - (MULTIPLIER * std)) + print " upper limit: %f" % (mean + (MULTIPLIER * std)) if value < mean - MULTIPLIER * std: - print "%s: %f < %f - %f * %f" \ - % (name, value, mean, MULTIPLIER, std) + print " => good outlier detected!" result = -1 elif value > mean + MULTIPLIER * std: - print "%s: %f > %f - %f * %f" \ - % (name, value, mean, MULTIPLIER, std) + print " => bad outlier detected!" result = 1 return result @@ -159,8 +164,12 @@ def _detect_outlier(name, value, list): if __name__ == '__main__': args = parse_args() tests = parse_file(args.file) + print "Checking %d test results for outliers..." % len(tests) failed = False for test, mean, extra in tests: failed |= detect_bad_outlier(test, mean, extra) if failed: - sys.exit(1) + print "Tests have bad outliers! o_O" + sys.exit(args.status_code) + else: + print "All good, no outliers were detected. :-)" diff --git a/scripts/benchmark/run-benchmarks-ci-job.sh b/scripts/benchmark/run-benchmarks-ci-job.sh index 30c6ecf5..adf37b7a 100755 --- a/scripts/benchmark/run-benchmarks-ci-job.sh +++ b/scripts/benchmark/run-benchmarks-ci-job.sh @@ -16,8 +16,8 @@ # Environment Variables # --------------------- # -# RUN_BENCHMARKS: If not set, skip this run. -# CHECK_FOR_OUTLIERS: If set, check if results are outliers. +# RUN_BENCHMARKS - If not set, skip this run. +# STATUS_CODE_IF_OUTLIERS - Exit with this status code if outliers are detected. set -eu set -o xtrace @@ -55,12 +55,6 @@ if [ -z "$(echo ${ENVIRONMENT} | grep ^benchmark-)" ]; then exit 0 fi -# stop here unless the CHECK_FOR_OUTLIERS environment variable is set -if [ -z "${CHECK_FOR_OUTLIERS:-}" ]; then - exit 0 -fi - -# fail test for bad outliers -echo "Comparing current test results with history..." +# check for bad outliers basedir=$(dirname "${0}") -${basedir}/compare-results-with-history.py ${tempfile} +${basedir}/check-for-outliers.py --status-code ${STATUS_CODE_IF_OUTLIERS:-0} ${tempfile} |