4 files changed, 48 insertions, 26 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 731c5da6..0b7510c5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,33 +66,10 @@ benchmark:
     - benchmark
   services:
     - couchdb
-  allow_failure: true
   script:
-    # By default, gitlab-runner will checkout in a detached HEAD
-    # (see https://gitlab.com/gitlab-org/gitlab-ce/issues/19421)
-    # We want pytest-benchmark to report the proper branch name,
-    # so we make sure we stay at the current branch.
-    # Also, variable names have changed in latest version of gitlab,
-    # but the doc seems to be outdated (see also
-    # https://docs.gitlab.com/ce/ci/variables/README.html).
-    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_REF"
-    # Gitlab will checkout current revision as master / origin/master
-    # We need some files from the "furure" in order to post the right
-    # benchmarking data to elasticsearch.
-    - git remote add leap https://leap.se/git/soledad
-    - git fetch leap
-    - git checkout leap/master
-        testing/tox.ini
-        testing/tests/benchmarks/conftest.py
-        testing/tests/conftest.py
-        testing/check-pysqlcipher.py
-    - git checkout leap/benchmark-all-commits scripts/benchmark/tune-adbapi-parameters.sh
-    # ensure larger timeout and more retries for async/concurrent sqlcipher access
-    - ./scripts/benchmark/tune-adbapi-parameters.sh
-    - git status
-    - git show -s --pretty=fuller
     - cd testing
     - curl -s couchdb:5984
+    - '[ -n "${BENCHMARK_ALL_COMMITS}" ] && ./scripts/benchmark/setup-all-commits-env.sh'
     # You can provide a $NETRC variable containing the creds for your
     # elasticsearch instance so it's protected from being leaked in the
     # CI console
diff --git a/scripts/benchmark/README.md b/scripts/benchmark/README.md
index 0910219a..4df43b3d 100644
--- a/scripts/benchmark/README.md
+++ b/scripts/benchmark/README.md
@@ -16,6 +16,8 @@ So what this script does is the following:
 
   - check out .gitlab-ci.yml from current master.
   - use gitlab-runner to exec a build locally.
+  - set an environment variable (BENCHMARK_ALL_COMMITS=1)so the build script
+    knows that it has to setup the environment.
   - store timestamp and exit status of each run.
   - checkout previous commit and iterate.
 
diff --git a/scripts/benchmark/benchmark-all-commits.sh b/scripts/benchmark/benchmark-all-commits.sh
index 7de63e17..57b2adc2 100755
--- a/scripts/benchmark/benchmark-all-commits.sh
+++ b/scripts/benchmark/benchmark-all-commits.sh
@@ -45,8 +45,8 @@ do
   #git checkout testing/tox.ini testing/tests/benchmarks/conftest.py testing/tests/conftest.py
 
   # Option 2: Run couchdb + tox in docker container using gitlab-runner
-  git checkout origin/benchmark-all-commits .gitlab-ci.yml
-  time gitlab-runner exec docker \
+  git checkout origin/master .gitlab-ci.yml
+  BENCHMARK_ALL_COMMITS=1 time gitlab-runner exec docker \
     benchmark \
     --timeout 7200 \
     --env PYTEST_OPTS="$PYTEST_OPTS" \
diff --git a/scripts/benchmark/setup-all-commits-env.sh b/scripts/benchmark/setup-all-commits-env.sh
new file mode 100755
index 00000000..79a9324b
--- /dev/null
+++ b/scripts/benchmark/setup-all-commits-env.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# This script is supposed to be called while running the benchmark job defined
+# in `/.gitlab-ci.yml`, to setup the environment when benchmarking all commits
+# back in time. It will checkout some needed files and do whatever is needed to
+# run tests from older versions of the repository with the newest benchmarking
+# facilities.
+
+set -e
+
+# By default, gitlab-runner will checkout in a detached HEAD (see
+# https://gitlab.com/gitlab-org/gitlab-ce/issues/19421) We want
+# pytest-benchmark to report the proper branch name, so we make sure we stay at
+# the current branch.
+
+# Variable names have changed in latest version of gitlab, but the doc seems to
+# be outdated (see also https://docs.gitlab.com/ce/ci/variables/README.html).
+
+# Old variable names:
+#git checkout -B "$CI_BUILD_RF_NAME" "$CI_BUILD_REF"
+
+# New (expected) variable names:
+#git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+
+# Working variable names:
+git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_REF"
+
+# Gitlab will checkout current revision as master / origin/master
+# We need some files from the "furure" in order to post the right
+# benchmarking data to elasticsearch.
+git remote add leap https://leap.se/git/soledad
+git fetch leap
+git checkout leap/master \
+    testing/tox.ini \
+    testing/tests/benchmarks/conftest.py \
+    testing/tests/conftest.py \
+    testing/check-pysqlcipher.py \
+    scripts/benchmark/tune-adbapi-parameters.sh
+
+# ensure larger timeout and more retries for async/concurrent sqlcipher access
+./scripts/benchmark/tune-adbapi-parameters.sh
+git status
+git show -s --pretty=fuller