[elastic] use multiple mustache templates
authordrebs <drebs@riseup.net>
Thu, 27 Apr 2017 18:06:09 +0000 (20:06 +0200)
committerdrebs <drebs@riseup.net>
Fri, 5 May 2017 07:17:34 +0000 (09:17 +0200)
elastic/generate-config.py
elastic/templates/dashboard.mustache [moved from elastic/dashboard/template.mustache with 92% similarity]
elastic/templates/visualization-searchSourceJSON.mustache [new file with mode: 0644]
elastic/templates/visualization-visState.mustache [new file with mode: 0644]
elastic/templates/visualization.mustache [new file with mode: 0644]
elastic/visualization/template.mustache [deleted file]

index 0aa998f..2d4c7bc 100755 (executable)
@@ -13,23 +13,39 @@ import sys
 
 URL='https://moose.leap.se:9200'
 
-# configure a logger
-logger = logging.getLogger(__name__)
-ch = logging.StreamHandler()
-ch.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
-logger.addHandler(ch)
-logger.setLevel(logging.INFO)
-
-
-# Generate visualizations
-def generate_visualizations():
-    dir = './visualization/'
-    with open(os.path.join(dir, 'template.mustache'), 'r') as f:
+def _get_logger():
+    logger = logging.getLogger(__name__)
+    ch = logging.StreamHandler()
+    ch.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
+    logger.addHandler(ch)
+    logger.setLevel(logging.INFO)
+    return logger
+
+
+def _delete_jsons(directory):
+    for f in os.listdir(directory):
+        if f.endswith('.json'):
+            os.unlink(os.path.join(directory, f))
+
+
+def generate_visualizations(tests):
+    with open(os.path.join('templates', 'visualization.mustache'), 'r') as f:
         template = f.read()
+    with open(os.path.join('templates', 'visualization-visState.mustache')) as f:
+        template_visState = f.read()
+    with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f:
+        template_searchSourceJSON = f.read()
+
+    out_dir = './visualization'
+    _delete_jsons(out_dir)
+
+    # remove all json files in output directory
+    for f in os.listdir(out_dir):
+        if f.endswith('.json'):
+            os.unlink(os.path.join(out_dir, f))
 
-    # test_names = tests.
     for test_name in tests:
-        out_file = os.path.join(dir, test_name + '.json')
+        out_file = os.path.join(out_dir, test_name + '.json')
         logger.info('Generating ' + out_file)
 
         context = {
@@ -39,25 +55,30 @@ def generate_visualizations():
                     "AND machine_info.host='weasel' "
                     "AND name='" + test_name + "'"
         }
-        rendered = pystache.render(template, context)
 
-        # verify we generated valid JSON visualizations
-        content = json.loads(rendered)
-        visState = content['visState']
-        searchSourceJSON = content['kibanaSavedObjectMeta']['searchSourceJSON']
+        visState = pystache.render(template_visState, context)
         assert json.loads(visState)
+        context['visState'] = json.dumps(visState)
+
+        searchSourceJSON = pystache.render(template_searchSourceJSON, context)
         assert json.loads(searchSourceJSON)
+        context['searchSourceJSON'] = json.dumps(searchSourceJSON)
+
+        rendered = pystache.render(template, context)
+        assert json.loads(rendered)
 
         with open(out_file, 'w') as out:
             out.write(rendered)
 
-# Generate Dashboard
-def generate_dashboards():
-    dir = './dashboard/'
-    with open(os.path.join(dir, 'template.mustache'), 'r') as f:
+
+def generate_dashboards(tests):
+    with open(os.path.join('templates', 'dashboard.mustache'), 'r') as f:
         template = f.read()
 
-    out_file = os.path.join(dir, 'soledad-benchmarks.json')
+    out_dir = './dashboard'
+    _delete_jsons(out_dir)
+
+    out_file = os.path.join(out_dir, 'soledad-benchmarks.json')
     logger.info('Generating ' + out_file)
 
     panels = []
@@ -74,27 +95,27 @@ def generate_dashboards():
             'type': 'visualization',
         })
 
-    panels_json = {'panels_json': json.dumps(panels)}
-    rendered = pystache.render(template, panels_json)
+    context = {'panels_json': json.dumps(panels)}
+    rendered = pystache.render(template, context)
+    assert json.loads(rendered), rendered
 
     with open(out_file, 'w') as f:
         f.write(rendered)
 
 
-# Main
-
-# Get all test names
-with open('./query/test_names.json') as f:
-    response = requests.get(URL + '/_search', data=f)
-    buckets = response.json()['aggregations']['test_names']['buckets']
-
-tests = []
-for d in buckets:
-    name = d['key']
-    tests.append(name)
-tests = sorted(tests)
+def _get_test_names():
+    with open('./query/test_names.json') as f:
+        response = requests.get(URL + '/_search', data=f)
+        buckets = response.json()['aggregations']['test_names']['buckets']
+    tests = []
+    for d in buckets:
+        name = d['key']
+        tests.append(name)
+    return sorted(tests)
 
 
 if __name__ == '__main__':
-    generate_dashboards()
-    generate_visualizations()
+    logger = _get_logger()
+    tests = _get_test_names()
+    generate_dashboards(tests)
+    generate_visualizations(tests)
similarity index 92%
rename from elastic/dashboard/template.mustache
rename to elastic/templates/dashboard.mustache
index 0b31c82..e54a019 100644 (file)
@@ -2,7 +2,7 @@
   "title": "Soledad-Benchmarks",
   "hits": 0,
   "description": "All Soledad benchmak tests run on Weasel",
-  "panelsJSON": "{{{panels_json}}}",
+  "panelsJSON": {{{panels_json}}},
   "optionsJSON": "{\"darkTheme\":true}",
   "uiStateJSON": "{}",
   "version": 1,
diff --git a/elastic/templates/visualization-searchSourceJSON.mustache b/elastic/templates/visualization-searchSourceJSON.mustache
new file mode 100644 (file)
index 0000000..b04be2c
--- /dev/null
@@ -0,0 +1,23 @@
+{
+   "filter" : [],
+   "query" : {
+      "query_string" : {
+         "query" : "{{{query}}}",
+         "analyze_wildcard" : true
+      }
+   },
+   "highlight" : {
+      "post_tags" : [
+         "@/kibana-highlighted-field@"
+      ],
+      "pre_tags" : [
+         "@kibana-highlighted-field@"
+      ],
+      "fragment_size" : 2147483647,
+      "fields" : {
+         "*" : {}
+      },
+      "require_field_match" : false
+   },
+   "index" : "benchmark*"
+}
diff --git a/elastic/templates/visualization-visState.mustache b/elastic/templates/visualization-visState.mustache
new file mode 100644 (file)
index 0000000..9689f68
--- /dev/null
@@ -0,0 +1,78 @@
+{
+   "type" : "line",
+   "listeners" : {},
+   "title" : "Soledad benchmark for {{{title}}}",
+   "params" : {
+      "legendPosition" : "right",
+      "defaultYExtents" : false,
+      "setYExtents" : false,
+      "drawLinesBetweenPoints" : true,
+      "interpolate" : "linear",
+      "addTimeMarker" : false,
+      "radiusRatio" : 9,
+      "showCircles" : true,
+      "scale" : "linear",
+      "addTooltip" : true,
+      "addLegend" : true,
+      "times" : []
+   },
+   "aggs" : [
+      {
+         "type" : "avg",
+         "schema" : "metric",
+         "params" : {
+            "field" : "stats.median"
+         },
+         "enabled" : true,
+         "id" : "1"
+      },
+      {
+         "enabled" : true,
+         "id" : "2",
+         "params" : {
+            "order" : "asc",
+            "orderBy" : "custom",
+            "field" : "commit_id_and_date",
+            "size" : 1000,
+            "orderAgg" : {
+               "params" : {
+                  "field" : "commit_info.time"
+               },
+               "schema" : "orderAgg",
+               "type" : "min",
+               "enabled" : true,
+               "id" : "2-orderAgg"
+            }
+         },
+         "schema" : "segment",
+         "type" : "terms"
+      },
+      {
+         "enabled" : true,
+         "id" : "3",
+         "schema" : "metric",
+         "params" : {
+            "field" : "stats.mean"
+         },
+         "type" : "avg"
+      },
+      {
+         "id" : "4",
+         "enabled" : true,
+         "type" : "avg",
+         "schema" : "metric",
+         "params" : {
+            "field" : "stats.iqr"
+         }
+      },
+      {
+         "type" : "avg",
+         "schema" : "metric",
+         "params" : {
+            "field" : "stats.stddev"
+         },
+         "id" : "5",
+         "enabled" : true
+      }
+   ]
+}
diff --git a/elastic/templates/visualization.mustache b/elastic/templates/visualization.mustache
new file mode 100644 (file)
index 0000000..8a9def1
--- /dev/null
@@ -0,0 +1,10 @@
+{
+    "title" : "{{{title}}}",
+    "visState" : {{{visState}}},
+    "uiStateJSON" : "{}",
+    "description" : "Soledad benchmark for {{{title}}}",
+    "version" : 1,
+    "kibanaSavedObjectMeta" : {
+        "searchSourceJSON": {{{searchSourceJSON}}}
+    }
+}
diff --git a/elastic/visualization/template.mustache b/elastic/visualization/template.mustache
deleted file mode 100644 (file)
index 2467a3b..0000000
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-"title": "{{{title}}}",
-"visState": "{\"title\":\"Soledad benchmark for {{{title}}}\",\"type\":\"line\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.median\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"commit_id_and_date\",\"size\":1000,\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"min\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"commit_info.time\"}},\"order\":\"asc\",\"orderBy\":\"custom\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.mean\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.iqr\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.stddev\"}}],\"listeners\":{}}",
-  "uiStateJSON": "{}",
-  "description": "Soledad benchmark for {{{title}}}",
-  "version": 1,
-  "kibanaSavedObjectMeta": {
-  "searchSourceJSON": "{\"index\":\"benchmark*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"{{{query}}}\",\"analyze_wildcard\":true}}}"
-  }
-}