summaryrefslogtreecommitdiff
path: root/scripts/benchmark/benchmark-all-commits.sh
blob: 7de63e17c7a1c675d66f7e3d6a5c82d73bdde9ee (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#!/bin/bash -x
#
# Go backwards in git history and run benchmark tests.
#
# Export PYTEST_OPTS so pytest will post metrics to an elasticsearch
# instance, i.e.
#
#     export PYTEST_OPTS='--benchmark-autosave --benchmark-storage="elasticsearch+https://USER:PASSWORD@moose.leap.se:9200"'
#
# or, for testing a single test group:
#
#     export PYTEST_OPTS='--benchmark-autosave --benchmark-storage="elasticsearch+https://USER:PASSWORD@moose.leap.se:9200" -m benchmark_test_instance'
#
# Todo
#
#   - Error handling. This script is dumb and will run forever.

# Too many commits just fail for different reasons,
# so we just benchmark thoses that succeed.
#set -e



#rm -rf testing/.tox

echo "$(date -Iseconds) Starting a new run of ${0}..." >> results.txt

while /bin/true
do

  commit_id=$(git rev-parse HEAD)
  echo -e '\n\n\n\n\n'
  echo -e "\e[34m$(date): Starting benchmarking of commit ${commit_id:0:8}, which is $(git rev-list --count ${commit_id}..origin/master) commits back from master:\e[0m"
  echo
  git show | head -6
  echo

  # Option 1: Run couchdb + tox localy - dirty!
  # curl -s localhost:5984 || exit 1
  # Make sure we kill all leftover soledad server/pytest procs
  #pkill -f -9 tox
  #git checkout origin/master testing/tox.ini testing/tests/benchmarks/conftest.py testing/tests/conftest.py
  #tox -c testing/tox.ini -e benchmark -- -x "$PYTEST_OPTS"
  #git reset HEAD testing/tox.ini
  #git checkout testing/tox.ini testing/tests/benchmarks/conftest.py testing/tests/conftest.py

  # Option 2: Run couchdb + tox in docker container using gitlab-runner
  git checkout origin/benchmark-all-commits .gitlab-ci.yml
  time gitlab-runner exec docker \
    benchmark \
    --timeout 7200 \
    --env PYTEST_OPTS="$PYTEST_OPTS" \
    --env HOST_HOSTNAME="$(hostname)"

  code=$?
  echo "$(date -Iseconds) ${commit_id} ${code}" >> results.txt

  echo
  echo -e "\e[34m$(date): Finished benchmarking of commit ${commit_id:0:8}, which is $(git rev-list --count ${commit_id}..origin/master) commits back from master:\e[0m"
  echo
  git show | head -6
  echo

  # checkout next commit
  git reset --hard HEAD^

  # stop if we reached the first commit in the repo
  if [ $? -ne 0 ]; then
    break
  fi
done

echo "$(date -Iseconds) Finished the run of ${0}..." >> results.txt