[benchmarks] post responsiveness tests to elasticsearch

4 files changed, 77 insertions, 8 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0934891e..d284d65d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -85,6 +85,20 @@ benchmark:
     # Output locally saved benchmarks if they exist
     - 'if [ -d .benchmarks ]; then find .benchmarks -type f -exec cat {} \; ; fi'
 
+responsiveness:
+  stage: benchmark
+  image: 0xacab.org:4567/leap/soledad:latest
+  tags:
+    - benchmark
+  services:
+    - couchdb
+  script:
+    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+    - cd testing
+    - curl -s couchdb:5984
+    - echo "addopts=--elasticsearch-url=\"$ELASTICSEARCH_URL\"" >> pytest.ini && chmod 600 pytest.ini
+    - /usr/bin/unbuffer tox --recreate -e responsiveness -- --couch-url http://couchdb:5984 | /usr/bin/ts -s
+
 build_docker_image:
   stage: build
   image: 0xacab.org:4567/leap/soledad:latest
diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py
index 6ce97625..857934ff 100644
--- a/testing/tests/conftest.py
+++ b/testing/tests/conftest.py
@@ -59,6 +59,12 @@ def pytest_addoption(parser):
         help="Soledad Server URL. A local server will be started if and only "
              "if  no URL is passed.")
 
+    # the following option is only used in responsiveness tests, but has to be
+    # defined here due to how pytest discovers plugins during startup.
+    parser.addoption(
+        "--elasticsearch-url", type="string", default=None,
+        help="the url for posting responsiveness results to elasticsearch")
+
 
 def _request(method, url, data=None, do=True):
     if do:
diff --git a/testing/tests/responsiveness/conftest.py b/testing/tests/responsiveness/conftest.py
index 6b2687b1..0aaaa32b 100644
--- a/testing/tests/responsiveness/conftest.py
+++ b/testing/tests/responsiveness/conftest.py
@@ -1,22 +1,25 @@
 import pytest
 
-from watchdog import Watchdog
+import elastic
+import watchdog as wd
 
 
-def _post_results(dog):
-    print("\n")
-    print("+" * 50)
-    print(dog.seconds_blocked)
-    print("+" * 50)
+def _post_results(dog, request):
+    elastic.post(dog.seconds_blocked, request)
 
 
 @pytest.fixture
 def watchdog(request):
-    dog = Watchdog()
+    dog = wd.Watchdog()
     dog_d = dog.start()
-    request.addfinalizer(lambda: _post_results(dog))
+    request.addfinalizer(lambda: _post_results(dog, request))
 
     def _run(deferred_fun):
         deferred_fun().addCallback(lambda _: dog.stop())
         return dog_d
     return _run
+
+
+def pytest_configure(config):
+    option = config.getoption("elasticsearch_url", elastic.ELASTICSEARCH_URL)
+    elastic.ELASTICSEARCH_URL = option
diff --git a/testing/tests/responsiveness/elastic.py b/testing/tests/responsiveness/elastic.py
new file mode 100644
index 00000000..50bae0b3
--- /dev/null
+++ b/testing/tests/responsiveness/elastic.py
@@ -0,0 +1,46 @@
+import datetime
+import elasticsearch
+
+from pytest_benchmark.plugin import pytest_benchmark_generate_machine_info
+from pytest_benchmark.utils import get_commit_info, get_tag, get_machine_id
+from pytest_benchmark.storage.elasticsearch import BenchmarkJSONSerializer
+
+
+ELASTICSEARCH_URL = 'http://elastic:changeme@127.0.0.1:9200/'
+
+
+def post(seconds_blocked, request):
+    es = elasticsearch.Elasticsearch(
+        hosts=[ELASTICSEARCH_URL],
+        serializer=BenchmarkJSONSerializer())
+    body, doc_id = get_doc(seconds_blocked, request)
+    es.index(
+        index='responsiveness',
+        doc_type='responsiveness',
+        id=doc_id,
+        body=body)
+
+
+def get_doc(seconds_blocked, request):
+    fullname = request.node._nodeid
+    name = request.node.name
+    group = None
+    marker = request.node.get_marker("responsivness")
+    if marker:
+        group = marker.kwargs.get("group")
+
+    doc = {
+        "datetime": datetime.datetime.utcnow().isoformat(),
+        "commit_info": get_commit_info(),
+        "fullname": fullname,
+        "name": name,
+        "group": group,
+        "machine_info": pytest_benchmark_generate_machine_info(),
+    }
+
+    # generate a doc id like the one used by pytest-benchmark
+    machine_id = get_machine_id()
+    tag = get_tag()
+    doc_id = machine_id + "_" + tag + "_" + fullname
+
+    return doc, doc_id