From ab04840d828aa5216911f711577077561208569a Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 27 Apr 2017 20:06:09 +0200 Subject: [elastic] use multiple mustache templates --- elastic/dashboard/template.mustache | 20 ---- elastic/generate-config.py | 101 +++++++++++++-------- elastic/templates/dashboard.mustache | 20 ++++ .../visualization-searchSourceJSON.mustache | 23 +++++ elastic/templates/visualization-visState.mustache | 78 ++++++++++++++++ elastic/templates/visualization.mustache | 10 ++ elastic/visualization/template.mustache | 10 -- 7 files changed, 192 insertions(+), 70 deletions(-) delete mode 100644 elastic/dashboard/template.mustache create mode 100644 elastic/templates/dashboard.mustache create mode 100644 elastic/templates/visualization-searchSourceJSON.mustache create mode 100644 elastic/templates/visualization-visState.mustache create mode 100644 elastic/templates/visualization.mustache delete mode 100644 elastic/visualization/template.mustache (limited to 'elastic') diff --git a/elastic/dashboard/template.mustache b/elastic/dashboard/template.mustache deleted file mode 100644 index 0b31c82..0000000 --- a/elastic/dashboard/template.mustache +++ /dev/null @@ -1,20 +0,0 @@ -{ - "title": "Soledad-Benchmarks", - "hits": 0, - "description": "All Soledad benchmak tests run on Weasel", - "panelsJSON": "{{{panels_json}}}", - "optionsJSON": "{\"darkTheme\":true}", - "uiStateJSON": "{}", - "version": 1, - "timeRestore": true, - "timeTo": "now", - "timeFrom": "now/y", - "refreshInterval": { - "display": "Off", - "pause": false, - "value": 0 - }, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" - } -} diff --git a/elastic/generate-config.py b/elastic/generate-config.py index 0aa998f..2d4c7bc 100755 --- a/elastic/generate-config.py +++ b/elastic/generate-config.py @@ -13,23 +13,39 @@ import sys URL='https://moose.leap.se:9200' -# configure a logger -logger = logging.getLogger(__name__) -ch = logging.StreamHandler() -ch.setFormatter(logging.Formatter('%(asctime)s %(message)s')) -logger.addHandler(ch) -logger.setLevel(logging.INFO) - - -# Generate visualizations -def generate_visualizations(): - dir = './visualization/' - with open(os.path.join(dir, 'template.mustache'), 'r') as f: +def _get_logger(): + logger = logging.getLogger(__name__) + ch = logging.StreamHandler() + ch.setFormatter(logging.Formatter('%(asctime)s %(message)s')) + logger.addHandler(ch) + logger.setLevel(logging.INFO) + return logger + + +def _delete_jsons(directory): + for f in os.listdir(directory): + if f.endswith('.json'): + os.unlink(os.path.join(directory, f)) + + +def generate_visualizations(tests): + with open(os.path.join('templates', 'visualization.mustache'), 'r') as f: template = f.read() + with open(os.path.join('templates', 'visualization-visState.mustache')) as f: + template_visState = f.read() + with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f: + template_searchSourceJSON = f.read() + + out_dir = './visualization' + _delete_jsons(out_dir) + + # remove all json files in output directory + for f in os.listdir(out_dir): + if f.endswith('.json'): + os.unlink(os.path.join(out_dir, f)) - # test_names = tests. for test_name in tests: - out_file = os.path.join(dir, test_name + '.json') + out_file = os.path.join(out_dir, test_name + '.json') logger.info('Generating ' + out_file) context = { @@ -39,25 +55,30 @@ def generate_visualizations(): "AND machine_info.host='weasel' " "AND name='" + test_name + "'" } - rendered = pystache.render(template, context) - # verify we generated valid JSON visualizations - content = json.loads(rendered) - visState = content['visState'] - searchSourceJSON = content['kibanaSavedObjectMeta']['searchSourceJSON'] + visState = pystache.render(template_visState, context) assert json.loads(visState) + context['visState'] = json.dumps(visState) + + searchSourceJSON = pystache.render(template_searchSourceJSON, context) assert json.loads(searchSourceJSON) + context['searchSourceJSON'] = json.dumps(searchSourceJSON) + + rendered = pystache.render(template, context) + assert json.loads(rendered) with open(out_file, 'w') as out: out.write(rendered) -# Generate Dashboard -def generate_dashboards(): - dir = './dashboard/' - with open(os.path.join(dir, 'template.mustache'), 'r') as f: + +def generate_dashboards(tests): + with open(os.path.join('templates', 'dashboard.mustache'), 'r') as f: template = f.read() - out_file = os.path.join(dir, 'soledad-benchmarks.json') + out_dir = './dashboard' + _delete_jsons(out_dir) + + out_file = os.path.join(out_dir, 'soledad-benchmarks.json') logger.info('Generating ' + out_file) panels = [] @@ -74,27 +95,27 @@ def generate_dashboards(): 'type': 'visualization', }) - panels_json = {'panels_json': json.dumps(panels)} - rendered = pystache.render(template, panels_json) + context = {'panels_json': json.dumps(panels)} + rendered = pystache.render(template, context) + assert json.loads(rendered), rendered with open(out_file, 'w') as f: f.write(rendered) -# Main - -# Get all test names -with open('./query/test_names.json') as f: - response = requests.get(URL + '/_search', data=f) - buckets = response.json()['aggregations']['test_names']['buckets'] - -tests = [] -for d in buckets: - name = d['key'] - tests.append(name) -tests = sorted(tests) +def _get_test_names(): + with open('./query/test_names.json') as f: + response = requests.get(URL + '/_search', data=f) + buckets = response.json()['aggregations']['test_names']['buckets'] + tests = [] + for d in buckets: + name = d['key'] + tests.append(name) + return sorted(tests) if __name__ == '__main__': - generate_dashboards() - generate_visualizations() + logger = _get_logger() + tests = _get_test_names() + generate_dashboards(tests) + generate_visualizations(tests) diff --git a/elastic/templates/dashboard.mustache b/elastic/templates/dashboard.mustache new file mode 100644 index 0000000..e54a019 --- /dev/null +++ b/elastic/templates/dashboard.mustache @@ -0,0 +1,20 @@ +{ + "title": "Soledad-Benchmarks", + "hits": 0, + "description": "All Soledad benchmak tests run on Weasel", + "panelsJSON": {{{panels_json}}}, + "optionsJSON": "{\"darkTheme\":true}", + "uiStateJSON": "{}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now/y", + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } +} diff --git a/elastic/templates/visualization-searchSourceJSON.mustache b/elastic/templates/visualization-searchSourceJSON.mustache new file mode 100644 index 0000000..b04be2c --- /dev/null +++ b/elastic/templates/visualization-searchSourceJSON.mustache @@ -0,0 +1,23 @@ +{ + "filter" : [], + "query" : { + "query_string" : { + "query" : "{{{query}}}", + "analyze_wildcard" : true + } + }, + "highlight" : { + "post_tags" : [ + "@/kibana-highlighted-field@" + ], + "pre_tags" : [ + "@kibana-highlighted-field@" + ], + "fragment_size" : 2147483647, + "fields" : { + "*" : {} + }, + "require_field_match" : false + }, + "index" : "benchmark*" +} diff --git a/elastic/templates/visualization-visState.mustache b/elastic/templates/visualization-visState.mustache new file mode 100644 index 0000000..9689f68 --- /dev/null +++ b/elastic/templates/visualization-visState.mustache @@ -0,0 +1,78 @@ +{ + "type" : "line", + "listeners" : {}, + "title" : "Soledad benchmark for {{{title}}}", + "params" : { + "legendPosition" : "right", + "defaultYExtents" : false, + "setYExtents" : false, + "drawLinesBetweenPoints" : true, + "interpolate" : "linear", + "addTimeMarker" : false, + "radiusRatio" : 9, + "showCircles" : true, + "scale" : "linear", + "addTooltip" : true, + "addLegend" : true, + "times" : [] + }, + "aggs" : [ + { + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "stats.median" + }, + "enabled" : true, + "id" : "1" + }, + { + "enabled" : true, + "id" : "2", + "params" : { + "order" : "asc", + "orderBy" : "custom", + "field" : "commit_id_and_date", + "size" : 1000, + "orderAgg" : { + "params" : { + "field" : "commit_info.time" + }, + "schema" : "orderAgg", + "type" : "min", + "enabled" : true, + "id" : "2-orderAgg" + } + }, + "schema" : "segment", + "type" : "terms" + }, + { + "enabled" : true, + "id" : "3", + "schema" : "metric", + "params" : { + "field" : "stats.mean" + }, + "type" : "avg" + }, + { + "id" : "4", + "enabled" : true, + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "stats.iqr" + } + }, + { + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "stats.stddev" + }, + "id" : "5", + "enabled" : true + } + ] +} diff --git a/elastic/templates/visualization.mustache b/elastic/templates/visualization.mustache new file mode 100644 index 0000000..8a9def1 --- /dev/null +++ b/elastic/templates/visualization.mustache @@ -0,0 +1,10 @@ +{ + "title" : "{{{title}}}", + "visState" : {{{visState}}}, + "uiStateJSON" : "{}", + "description" : "Soledad benchmark for {{{title}}}", + "version" : 1, + "kibanaSavedObjectMeta" : { + "searchSourceJSON": {{{searchSourceJSON}}} + } +} diff --git a/elastic/visualization/template.mustache b/elastic/visualization/template.mustache deleted file mode 100644 index 2467a3b..0000000 --- a/elastic/visualization/template.mustache +++ /dev/null @@ -1,10 +0,0 @@ -{ -"title": "{{{title}}}", -"visState": "{\"title\":\"Soledad benchmark for {{{title}}}\",\"type\":\"line\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.median\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"commit_id_and_date\",\"size\":1000,\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"min\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"commit_info.time\"}},\"order\":\"asc\",\"orderBy\":\"custom\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.mean\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.iqr\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"stats.stddev\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "Soledad benchmark for {{{title}}}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"benchmark*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"{{{query}}}\",\"analyze_wildcard\":true}}}" - } -} -- cgit v1.2.3