1 files changed, 40 insertions, 7 deletions
diff --git a/scripts/benchmark/run-benchmarks-ci-job.sh b/scripts/benchmark/run-benchmarks-ci-job.sh
index b2a8c417..835f8c7f 100755
--- a/scripts/benchmark/run-benchmarks-ci-job.sh
+++ b/scripts/benchmark/run-benchmarks-ci-job.sh
@@ -1,16 +1,23 @@
 #!/bin/sh
 
+# Run benchmakr tests for CI jobs, and optionally compare results with historic
+# series.
+#
+# Usage Example
+# -------------
+#
+# Run this script with the environment name as the only argument:
+#
+#   ./run-benchmarks-ci-job.sh environment-name
+#
 # This script is used in .gitlab-ci.yml to run benchmark jobs. It has been
 # factored out from that file to avoid bloating it with too much information.
 #
-# The benchmark job will be skiped if the RUN_BENCHMARKS variable is not set,
-# thus allowing for opting in to benchmarking.
+# Environment Variables
+# ---------------------
 #
-# This is an attempt to make life of developers easier, by diminishing the time
-# of the pipeline by not running benchmarks by default. The canonical repo
-# (i.e. https://0xacab.org/leap/soledad) should have the RUN_BENCHMARKS
-# variable set to ensure that these jobs will run. Developers can add or remove
-# the variable from their environments as they see fit.
+#   RUN_BENCHMARKS:     If not set, skip this run.
+#   CHECK_FOR_OUTLIERS: If set, check if results are outliers.
 
 set -eu
 set -o xtrace
@@ -25,10 +32,36 @@ fi
 
 echo "Running tox in environment ${ENVIRONMENT}..."
 
+#
+# run benchmark tests with tox
+#
+
+tempfile=$(mktemp)
 /usr/bin/unbuffer \
   tox \
     --recreate \
     -e ${ENVIRONMENT} \
     -- \
     --couch-url http://couchdb:5984 \
+    --benchmark-json=${tempfile} \
+    -m runme \
   | /usr/bin/ts -s
+
+#
+# check results for bad outlier detecion
+#
+
+# stop here unless environment starts with "benchmark-"
+if [ -z "$(echo ${ENVIRONMENT} | grep ^benchmark-)" ]; then
+  exit 0
+fi
+
+# stop here unless the CHECK_FOR_OUTLIERS environment variable is set
+if [ -z "${CHECK_FOR_OUTLIERS:-}" ]; then
+  exit 0
+fi
+
+# fail test for bad outliers
+echo "Comparing current test results with history..."
+basedir=$(dirname "${0}")
+${basedir}/compare-results-with-history.py ${tempfile}