summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitlab-ci.yml25
-rw-r--r--scripts/benchmark/README.md2
-rwxr-xr-xscripts/benchmark/benchmark-all-commits.sh4
-rwxr-xr-xscripts/benchmark/setup-all-commits-env.sh43
4 files changed, 48 insertions, 26 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 731c5da6..0b7510c5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,33 +66,10 @@ benchmark:
- benchmark
services:
- couchdb
- allow_failure: true
script:
- # By default, gitlab-runner will checkout in a detached HEAD
- # (see https://gitlab.com/gitlab-org/gitlab-ce/issues/19421)
- # We want pytest-benchmark to report the proper branch name,
- # so we make sure we stay at the current branch.
- # Also, variable names have changed in latest version of gitlab,
- # but the doc seems to be outdated (see also
- # https://docs.gitlab.com/ce/ci/variables/README.html).
- - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_REF"
- # Gitlab will checkout current revision as master / origin/master
- # We need some files from the "furure" in order to post the right
- # benchmarking data to elasticsearch.
- - git remote add leap https://leap.se/git/soledad
- - git fetch leap
- - git checkout leap/master
- testing/tox.ini
- testing/tests/benchmarks/conftest.py
- testing/tests/conftest.py
- testing/check-pysqlcipher.py
- - git checkout leap/benchmark-all-commits scripts/benchmark/tune-adbapi-parameters.sh
- # ensure larger timeout and more retries for async/concurrent sqlcipher access
- - ./scripts/benchmark/tune-adbapi-parameters.sh
- - git status
- - git show -s --pretty=fuller
- cd testing
- curl -s couchdb:5984
+ - '[ -n "${BENCHMARK_ALL_COMMITS}" ] && ./scripts/benchmark/setup-all-commits-env.sh'
# You can provide a $NETRC variable containing the creds for your
# elasticsearch instance so it's protected from being leaked in the
# CI console
diff --git a/scripts/benchmark/README.md b/scripts/benchmark/README.md
index 0910219a..4df43b3d 100644
--- a/scripts/benchmark/README.md
+++ b/scripts/benchmark/README.md
@@ -16,6 +16,8 @@ So what this script does is the following:
- check out .gitlab-ci.yml from current master.
- use gitlab-runner to exec a build locally.
+ - set an environment variable (BENCHMARK_ALL_COMMITS=1)so the build script
+ knows that it has to setup the environment.
- store timestamp and exit status of each run.
- checkout previous commit and iterate.
diff --git a/scripts/benchmark/benchmark-all-commits.sh b/scripts/benchmark/benchmark-all-commits.sh
index 7de63e17..57b2adc2 100755
--- a/scripts/benchmark/benchmark-all-commits.sh
+++ b/scripts/benchmark/benchmark-all-commits.sh
@@ -45,8 +45,8 @@ do
#git checkout testing/tox.ini testing/tests/benchmarks/conftest.py testing/tests/conftest.py
# Option 2: Run couchdb + tox in docker container using gitlab-runner
- git checkout origin/benchmark-all-commits .gitlab-ci.yml
- time gitlab-runner exec docker \
+ git checkout origin/master .gitlab-ci.yml
+ BENCHMARK_ALL_COMMITS=1 time gitlab-runner exec docker \
benchmark \
--timeout 7200 \
--env PYTEST_OPTS="$PYTEST_OPTS" \
diff --git a/scripts/benchmark/setup-all-commits-env.sh b/scripts/benchmark/setup-all-commits-env.sh
new file mode 100755
index 00000000..79a9324b
--- /dev/null
+++ b/scripts/benchmark/setup-all-commits-env.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# This script is supposed to be called while running the benchmark job defined
+# in `/.gitlab-ci.yml`, to setup the environment when benchmarking all commits
+# back in time. It will checkout some needed files and do whatever is needed to
+# run tests from older versions of the repository with the newest benchmarking
+# facilities.
+
+set -e
+
+# By default, gitlab-runner will checkout in a detached HEAD (see
+# https://gitlab.com/gitlab-org/gitlab-ce/issues/19421) We want
+# pytest-benchmark to report the proper branch name, so we make sure we stay at
+# the current branch.
+
+# Variable names have changed in latest version of gitlab, but the doc seems to
+# be outdated (see also https://docs.gitlab.com/ce/ci/variables/README.html).
+
+# Old variable names:
+#git checkout -B "$CI_BUILD_RF_NAME" "$CI_BUILD_REF"
+
+# New (expected) variable names:
+#git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+
+# Working variable names:
+git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_REF"
+
+# Gitlab will checkout current revision as master / origin/master
+# We need some files from the "furure" in order to post the right
+# benchmarking data to elasticsearch.
+git remote add leap https://leap.se/git/soledad
+git fetch leap
+git checkout leap/master \
+ testing/tox.ini \
+ testing/tests/benchmarks/conftest.py \
+ testing/tests/conftest.py \
+ testing/check-pysqlcipher.py \
+ scripts/benchmark/tune-adbapi-parameters.sh
+
+# ensure larger timeout and more retries for async/concurrent sqlcipher access
+./scripts/benchmark/tune-adbapi-parameters.sh
+git status
+git show -s --pretty=fuller