From 231bafb691d60151ba5fdb73258780e3136e845a Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 8 Jun 2017 12:54:19 -0300 Subject: [elastic] create resources visualization --- elastic/generate-config.py | 78 ++++++++++++--------- .../visualization-visState-resources.mustache | 60 ++++++++++++++++ .../templates/visualization-visState-time.mustache | 79 ++++++++++++++++++++++ elastic/templates/visualization-visState.mustache | 78 --------------------- elastic/templates/visualization.mustache | 2 +- 5 files changed, 186 insertions(+), 111 deletions(-) create mode 100644 elastic/templates/visualization-visState-resources.mustache create mode 100644 elastic/templates/visualization-visState-time.mustache delete mode 100644 elastic/templates/visualization-visState.mustache (limited to 'elastic') diff --git a/elastic/generate-config.py b/elastic/generate-config.py index be8afcb..88d6e82 100755 --- a/elastic/generate-config.py +++ b/elastic/generate-config.py @@ -33,8 +33,10 @@ def _delete_jsons(directory): def generate_visualizations(tests): with open(os.path.join('templates', 'visualization.mustache'), 'r') as f: template = f.read() - with open(os.path.join('templates', 'visualization-visState.mustache')) as f: - template_visState = f.read() + with open(os.path.join('templates', 'visualization-visState-time.mustache')) as f: + template_visState_time = f.read() + with open(os.path.join('templates', 'visualization-visState-resources.mustache')) as f: + template_visState_resources = f.read() with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f: template_searchSourceJSON = f.read() @@ -46,31 +48,40 @@ def generate_visualizations(tests): if f.endswith('.json'): os.unlink(os.path.join(out_dir, f)) + graphs = [ + ('time', 'weasel', template_visState_time), + ('resources', 'Beluga', template_visState_resources) + ] + for test_name in tests: - out_file = os.path.join(out_dir, test_name + '.json') - logger.info('Generating ' + out_file) - context = { - 'title': test_name, - 'query': "commit_info.project:soledad " - "AND commit_info.branch='master' " - "AND machine_info.host='weasel' " - "AND name='" + test_name + "'" - } + for type, host, template_visState in graphs: + + out_file = os.path.join(out_dir, test_name + '_' + type + '.json') + logger.info('Generating ' + out_file) + + context = { + 'title': test_name + "_" + type, + 'test_name': test_name, + 'query': "commit_info.project:soledad " +# "AND commit_info.branch='master' " + "AND machine_info.host='" + host + "' " + "AND name='" + test_name + "'" + } - visState = pystache.render(template_visState, context) - assert json.loads(visState) - context['visState'] = json.dumps(visState) + visState = pystache.render(template_visState, context) + assert json.loads(visState) + context['visState'] = json.dumps(visState) - searchSourceJSON = pystache.render(template_searchSourceJSON, context) - assert json.loads(searchSourceJSON) - context['searchSourceJSON'] = json.dumps(searchSourceJSON) + searchSourceJSON = pystache.render(template_searchSourceJSON, context) + assert json.loads(searchSourceJSON) + context['searchSourceJSON'] = json.dumps(searchSourceJSON) - rendered = pystache.render(template, context) - assert json.loads(rendered) + rendered = pystache.render(template, context) + assert json.loads(rendered) - with open(out_file, 'w') as out: - out.write(rendered) + with open(out_file, 'w') as out: + out.write(rendered) def generate_dashboards(tests): @@ -84,18 +95,21 @@ def generate_dashboards(tests): logger.info('Generating ' + out_file) panels = [] - count = 0 + panelIndex = 1 + row = 1 for test_name in tests: - count += 1 - panels.append({ - 'id': test_name, - 'panelIndex': count, - 'col': 1, - 'row': count, - 'size_x': 6, - 'size_y': 3, - 'type': 'visualization', - }) + for type, col in [('time', 1), ('resources', 7)]: + panels.append({ + 'id': test_name + '_' + type, + 'panelIndex': panelIndex, + 'col': col, + 'row': row, + 'size_x': 6, + 'size_y': 3, + 'type': 'visualization', + }) + panelIndex += 1 + row += 3 context = {'panels_json': json.dumps(json.dumps(panels))} rendered = pystache.render(template, context) diff --git a/elastic/templates/visualization-visState-resources.mustache b/elastic/templates/visualization-visState-resources.mustache new file mode 100644 index 0000000..d47db6b --- /dev/null +++ b/elastic/templates/visualization-visState-resources.mustache @@ -0,0 +1,60 @@ +{ + "type" : "line", + "listeners" : {}, + "title" : "Test resources for {{{test_name}}}", + "params" : { + "legendPosition" : "right", + "defaultYExtents" : false, + "setYExtents" : false, + "drawLinesBetweenPoints" : true, + "interpolate" : "linear", + "addTimeMarker" : false, + "radiusRatio" : 9, + "showCircles" : true, + "scale" : "linear", + "addTooltip" : true, + "addLegend" : true, + "times" : [] + }, + "aggs" : [ + { + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "extra_info.cpu_percent" + }, + "enabled" : true, + "id" : "1" + }, + { + "enabled" : true, + "id" : "2", + "params" : { + "order" : "asc", + "orderBy" : "custom", + "field" : "commit_id_and_date", + "size" : 1000, + "orderAgg" : { + "params" : { + "field" : "commit_info.time" + }, + "schema" : "orderAgg", + "type" : "min", + "enabled" : true, + "id" : "2-orderAgg" + } + }, + "schema" : "segment", + "type" : "terms" + }, + { + "enabled" : true, + "id" : "3", + "schema" : "metric", + "params" : { + "field" : "extra_info.memory_percent.stats.max" + }, + "type" : "avg" + } + ] +} diff --git a/elastic/templates/visualization-visState-time.mustache b/elastic/templates/visualization-visState-time.mustache new file mode 100644 index 0000000..93ec6f5 --- /dev/null +++ b/elastic/templates/visualization-visState-time.mustache @@ -0,0 +1,79 @@ +{ + "type" : "line", + "listeners" : {}, + "title" : "Test time for {{{test_name}}}", + "params" : { + "legendPosition" : "right", + "defaultYExtents" : false, + "setYExtents" : false, + "drawLinesBetweenPoints" : true, + "interpolate" : "linear", + "addTimeMarker" : false, + "radiusRatio" : 9, + "showCircles" : true, + "scale" : "linear", + "addTooltip" : true, + "addLegend" : true, + "times" : [] + }, + "aggs" : [ + { + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "stats.median" + }, + "enabled" : true, + "id" : "1" + }, + { + "enabled" : true, + "id" : "2", + "params" : { + "order" : "asc", + "orderBy" : "custom", + "field" : "commit_id_and_date", + "size" : 1000, + "orderAgg" : { + "params" : { + "field" : "commit_info.time" + }, + "schema" : "orderAgg", + "type" : "min", + "enabled" : true, + "id" : "2-orderAgg" + }, + "ranges": [{"from": "*2017-4-1"}] + }, + "schema" : "segment", + "type" : "terms" + }, + { + "enabled" : true, + "id" : "3", + "schema" : "metric", + "params" : { + "field" : "stats.mean" + }, + "type" : "avg" + }, + { + "id" : "4", + "enabled" : true, + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "stats.iqr" + } + }, + { + "type" : "avg", + "schema" : "metric", + "params" : { + "field" : "stats.stddev" + }, + "id" : "5", + "enabled" : true + } + ] +} diff --git a/elastic/templates/visualization-visState.mustache b/elastic/templates/visualization-visState.mustache deleted file mode 100644 index 9689f68..0000000 --- a/elastic/templates/visualization-visState.mustache +++ /dev/null @@ -1,78 +0,0 @@ -{ - "type" : "line", - "listeners" : {}, - "title" : "Soledad benchmark for {{{title}}}", - "params" : { - "legendPosition" : "right", - "defaultYExtents" : false, - "setYExtents" : false, - "drawLinesBetweenPoints" : true, - "interpolate" : "linear", - "addTimeMarker" : false, - "radiusRatio" : 9, - "showCircles" : true, - "scale" : "linear", - "addTooltip" : true, - "addLegend" : true, - "times" : [] - }, - "aggs" : [ - { - "type" : "avg", - "schema" : "metric", - "params" : { - "field" : "stats.median" - }, - "enabled" : true, - "id" : "1" - }, - { - "enabled" : true, - "id" : "2", - "params" : { - "order" : "asc", - "orderBy" : "custom", - "field" : "commit_id_and_date", - "size" : 1000, - "orderAgg" : { - "params" : { - "field" : "commit_info.time" - }, - "schema" : "orderAgg", - "type" : "min", - "enabled" : true, - "id" : "2-orderAgg" - } - }, - "schema" : "segment", - "type" : "terms" - }, - { - "enabled" : true, - "id" : "3", - "schema" : "metric", - "params" : { - "field" : "stats.mean" - }, - "type" : "avg" - }, - { - "id" : "4", - "enabled" : true, - "type" : "avg", - "schema" : "metric", - "params" : { - "field" : "stats.iqr" - } - }, - { - "type" : "avg", - "schema" : "metric", - "params" : { - "field" : "stats.stddev" - }, - "id" : "5", - "enabled" : true - } - ] -} diff --git a/elastic/templates/visualization.mustache b/elastic/templates/visualization.mustache index 8a9def1..1d4eda3 100644 --- a/elastic/templates/visualization.mustache +++ b/elastic/templates/visualization.mustache @@ -2,7 +2,7 @@ "title" : "{{{title}}}", "visState" : {{{visState}}}, "uiStateJSON" : "{}", - "description" : "Soledad benchmark for {{{title}}}", + "description" : "Test {{{type}}} for {{{test_name}}}", "version" : 1, "kibanaSavedObjectMeta" : { "searchSourceJSON": {{{searchSourceJSON}}} -- cgit v1.2.3