summaryrefslogtreecommitdiff
path: root/elastic
diff options
context:
space:
mode:
authordrebs <drebs@riseup.net>2017-04-27 20:06:09 +0200
committerdrebs <drebs@riseup.net>2017-05-05 09:17:34 +0200
commitab04840d828aa5216911f711577077561208569a (patch)
tree68871774aa5036dca056cc3fad4ad66315f4c792 /elastic
parentf0190a962e5f6ad81898652978bd5a8cfb3c4c51 (diff)
[elastic] use multiple mustache templates
Diffstat (limited to 'elastic')
-rwxr-xr-xelastic/generate-config.py101
-rw-r--r--elastic/templates/dashboard.mustache (renamed from elastic/dashboard/template.mustache)2
-rw-r--r--elastic/templates/visualization-searchSourceJSON.mustache23
-rw-r--r--elastic/templates/visualization-visState.mustache78
-rw-r--r--elastic/templates/visualization.mustache10
-rw-r--r--elastic/visualization/template.mustache10
6 files changed, 173 insertions, 51 deletions
diff --git a/elastic/generate-config.py b/elastic/generate-config.py
index 0aa998f..2d4c7bc 100755
--- a/elastic/generate-config.py
+++ b/elastic/generate-config.py
@@ -13,23 +13,39 @@ import sys
URL='https://moose.leap.se:9200'
-# configure a logger
-logger = logging.getLogger(__name__)
-ch = logging.StreamHandler()
-ch.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
-logger.addHandler(ch)
-logger.setLevel(logging.INFO)
-
-
-# Generate visualizations
-def generate_visualizations():
- dir = './visualization/'
- with open(os.path.join(dir, 'template.mustache'), 'r') as f:
+def _get_logger():
+ logger = logging.getLogger(__name__)
+ ch = logging.StreamHandler()
+ ch.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
+ logger.addHandler(ch)
+ logger.setLevel(logging.INFO)
+ return logger
+
+
+def _delete_jsons(directory):
+ for f in os.listdir(directory):
+ if f.endswith('.json'):
+ os.unlink(os.path.join(directory, f))
+
+
+def generate_visualizations(tests):
+ with open(os.path.join('templates', 'visualization.mustache'), 'r') as f:
template = f.read()
+ with open(os.path.join('templates', 'visualization-visState.mustache')) as f:
+ template_visState = f.read()
+ with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f:
+ template_searchSourceJSON = f.read()
+
+ out_dir = './visualization'
+ _delete_jsons(out_dir)
+
+ # remove all json files in output directory
+ for f in os.listdir(out_dir):
+ if f.endswith('.json'):
+ os.unlink(os.path.join(out_dir, f))
- # test_names = tests.
for test_name in tests:
- out_file = os.path.join(dir, test_name + '.json')
+ out_file = os.path.join(out_dir, test_name + '.json')
logger.info('Generating ' + out_file)
context = {
@@ -39,25 +55,30 @@ def generate_visualizations():
"AND machine_info.host='weasel' "
"AND name='" + test_name + "'"
}
- rendered = pystache.render(template, context)
- # verify we generated valid JSON visualizations
- content = json.loads(rendered)
- visState = content['visState']
- searchSourceJSON = content['kibanaSavedObjectMeta']['searchSourceJSON']
+ visState = pystache.render(template_visState, context)
assert json.loads(visState)
+ context['visState'] = json.dumps(visState)
+
+ searchSourceJSON = pystache.render(template_searchSourceJSON, context)
assert json.loads(searchSourceJSON)
+ context['searchSourceJSON'] = json.dumps(searchSourceJSON)
+
+ rendered = pystache.render(template, context)
+ assert json.loads(rendered)
with open(out_file, 'w') as out:
out.write(rendered)
-# Generate Dashboard
-def generate_dashboards():
- dir = './dashboard/'
- with open(os.path.join(dir, 'template.mustache'), 'r') as f:
+
+def generate_dashboards(tests):
+ with open(os.path.join('templates', 'dashboard.mustache'), 'r') as f:
template = f.read()
- out_file = os.path.join(dir, 'soledad-benchmarks.json')
+ out_dir = './dashboard'
+ _delete_jsons(out_dir)
+
+ out_file = os.path.join(out_dir, 'soledad-benchmarks.json')
logger.info('Generating ' + out_file)
panels = []
@@ -74,27 +95,27 @@ def generate_dashboards():
'type': 'visualization',
})
- panels_json = {'panels_json': json.dumps(panels)}
- rendered = pystache.render(template, panels_json)
+ context = {'panels_json': json.dumps(panels)}
+ rendered = pystache.render(template, context)
+ assert json.loads(rendered), rendered
with open(out_file, 'w') as f:
f.write(rendered)
-# Main
-
-# Get all test names
-with open('./query/test_names.json') as f:
- response = requests.get(URL + '/_search', data=f)
- buckets = response.json()['aggregations']['test_names']['buckets']
-
-tests = []
-for d in buckets:
- name = d['key']
- tests.append(name)
-tests = sorted(tests)
+def _get_test_names():
+ with open('./query/test_names.json') as f:
+ response = requests.get(URL + '/_search', data=f)
+ buckets = response.json()['aggregations']['test_names']['buckets']
+ tests = []
+ for d in buckets:
+ name = d['key']
+ tests.append(name)
+ return sorted(tests)
if __name__ == '__main__':
- generate_dashboards()
- generate_visualizations()
+ logger = _get_logger()
+ tests = _get_test_names()
+ generate_dashboards(tests)
+ generate_visualizations(tests)
diff --git a/elastic/dashboard/template.mustache b/elastic/templates/dashboard.mustache
index 0b31c82..e54a019 100644
--- a/elastic/dashboard/template.mustache
+++ b/elastic/templates/dashboard.mustache
@@ -2,7 +2,7 @@
"title": "Soledad-Benchmarks",
"hits": 0,
"description": "All Soledad benchmak tests run on Weasel",
- "panelsJSON": "{{{panels_json}}}",
+ "panelsJSON": {{{panels_json}}},
"optionsJSON": "{\"darkTheme\":true}",
"uiStateJSON": "{}",
"version": 1,
diff --git a/elastic/templates/visualization-searchSourceJSON.mustache b/elastic/templates/visualization-searchSourceJSON.mustache
new file mode 100644
index 0000000..b04be2c
--- /dev/null
+++ b/elastic/templates/visualization-searchSourceJSON.mustache
@@ -0,0 +1,23 @@
+{
+ "filter" : [],
+ "query" : {
+ "query_string" : {
+ "query" : "{{{query}}}",
+ "analyze_wildcard" : true
+ }
+ },
+ "highlight" : {
+ "post_tags" : [
+ "@/kibana-highlighted-field@"
+ ],
+ "pre_tags" : [
+ "@kibana-highlighted-field@"
+ ],
+ "fragment_size" : 2147483647,
+ "fields" : {
+ "*" : {}
+ },
+ "require_field_match" : false
+ },
+ "index" : "benchmark*"
+}
diff --git a/elastic/templates/visualization-visState.mustache b/elastic/templates/visualization-visState.mustache
new file mode 100644
index 0000000..9689f68
--- /dev/null
+++ b/elastic/templates/visualization-visState.mustache
@@ -0,0 +1,78 @@
+{
+ "type" : "line",
+ "listeners" : {},
+ "title" : "Soledad benchmark for {{{title}}}",
+ "params" : {
+ "legendPosition" : "right",
+ "defaultYExtents" : false,
+ "setYExtents" : false,
+ "drawLinesBetweenPoints" : true,
+ "interpolate" : "linear",
+ "addTimeMarker" : false,
+ "radiusRatio" : 9,
+ "showCircles" : true,
+ "scale" : "linear",
+ "addTooltip" : true,
+ "addLegend" : true,
+ "times" : []
+ },
+ "aggs" : [
+ {
+ "type" : "avg",
+ "schema" : "metric",
+ "params" : {
+ "field" : "stats.median"
+ },
+ "enabled" : true,
+ "id" : "1"
+ },
+ {
+ "enabled" : true,
+ "id" : "2",
+ "params" : {
+ "order" : "asc",
+ "orderBy" : "custom",
+ "field" : "commit_id_and_date",
+ "size" : 1000,
+ "orderAgg" : {
+ "params" : {
+ "field" : "commit_info.time"
+ },
+ "schema" : "orderAgg",
+ "type" : "min",
+ "enabled" : true,
+ "id" : "2-orderAgg"
+ }
+ },
+ "schema" : "segment",
+ "type" : "terms"
+ },
+ {
+ "enabled" : true,
+ "id" : "3",
+ "schema" : "metric",
+ "params" : {
+ "field" : "stats.mean"
+ },
+ "type" : "avg"
+ },
+ {
+ "id" : "4",
+ "enabled" : true,
+ "type" : "avg",
+ "schema" : "metric",
+ "params" : {
+ "field" : "stats.iqr"
+ }
+ },
+ {
+ "type" : "avg",
+ "schema" : "metric",
+ "params" : {
+ "field" : "stats.stddev"
+ },
+ "id" : "5",
+ "enabled" : true
+ }
+ ]
+}
diff --git a/elastic/templates/visualization.mustache b/elastic/templates/visualization.mustache
new file mode 100644
index 0000000..8a9def1
--- /dev/null
+++ b/elastic/templates/visualization.mustache
@@ -0,0 +1,10 @@
+{
+ "title" : "{{{title}}}",
+ "visState" : {{{visState}}},
+ "uiStateJSON" : "{}",
+ "description" : "Soledad benchmark for {{{title}}}",
+ "version" : 1,
+ "kibanaSavedObjectMeta" : {
+ "searchSourceJSON": {{{searchSourceJSON}}}
+ }
+}
diff --git a/elastic/visualization/template.mustache b/elastic/visualization/template.mustache
deleted file mode 100644
index 2467a3b..0000000
--- a/elastic/visualization/template.mustache
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-"title": "{{{title}}}",
-"visState": "{\"title\":\"Soledad benchmark for {{{title}}}\",\"type\":\"line\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.median\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"commit_id_and_date\",\"size\":1000,\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"min\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"commit_info.time\"}},\"order\":\"asc\",\"orderBy\":\"custom\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.mean\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.iqr\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.stddev\"}}],\"listeners\":{}}",
- "uiStateJSON": "{}",
- "description": "Soledad benchmark for {{{title}}}",
- "version": 1,
- "kibanaSavedObjectMeta": {
- "searchSourceJSON": "{\"index\":\"benchmark*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"{{{query}}}\",\"analyze_wildcard\":true}}}"
- }
-}