From 2cf09ffecd09d7e19d8a8da8b7914ac04d9986af Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 17 Jul 2017 08:53:24 -0300 Subject: [elastic] remove all elastic-related stuff All this was moved to the puppet repository so kibana is autoconfigured and the website is autogenerated. --- elastic/generate-config.py | 189 --------------------------------------------- 1 file changed, 189 deletions(-) delete mode 100755 elastic/generate-config.py (limited to 'elastic/generate-config.py') diff --git a/elastic/generate-config.py b/elastic/generate-config.py deleted file mode 100755 index b1389e1..0000000 --- a/elastic/generate-config.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -# -# Todo: -# -# - Remove all json files from previous runs -import json -import logging -import os -import pystache -import requests - - -URL = 'https://moose.leap.se:9200' - - -def _get_logger(): - logger = logging.getLogger(__name__) - ch = logging.StreamHandler() - ch.setFormatter(logging.Formatter('%(asctime)s %(message)s')) - logger.addHandler(ch) - logger.setLevel(logging.INFO) - return logger - - -def _delete_jsons(directory): - logger.info('removing *.json from %s' % directory) - for f in os.listdir(directory): - if f.endswith('.json'): - os.unlink(os.path.join(directory, f)) - - -def generate_visualizations(tests): - with open(os.path.join('templates', 'visualization.mustache'), 'r') as f: - template = f.read() - - path = os.path.join('templates', 'visualization-visState-time.mustache') - with open(path) as f: - template_visState_time = f.read() - - path = os.path.join('templates', 'visualization-visState-cpu.mustache') - with open(path) as f: - template_visState_cpu = f.read() - - path = os.path.join('templates', 'visualization-visState-memory.mustache') - with open(path) as f: - template_visState_memory = f.read() - - path = os.path.join('templates', 'visualization-searchSourceJSON.mustache') - with open(path) as f: - template_searchSourceJSON = f.read() - - out_dir = './visualization' - _delete_jsons(out_dir) - - # remove all json files in output directory - for f in os.listdir(out_dir): - if f.endswith('.json'): - os.unlink(os.path.join(out_dir, f)) - - graphs = [ - ('time', 'Time taken by ', 'weasel', template_visState_time), - ('cpu', 'CPU percentage used by ', 'weasel', template_visState_cpu), - ('memory', 'Memory used by ', 'weasel', template_visState_memory) - ] - - for test_name in tests: - - for type, title, host, template_visState in graphs: - - query = "commit_info.project:soledad" \ - " AND machine_info.host='" + host + "'" \ - " AND name='" + test_name + "'" -# " AND commit_info.branch='master'" - - # results that are accurate for time are not accurate for memory, - # and vice versa. - if type == 'time' or type == 'cpu': - query += " AND NOT _exists_:extra_info.memory_percent" - else: - query += " AND _exists_:extra_info.memory_percent" - - if type == 'cpu': - query += " AND _exists_:extra_info.cpu_percent" - - # create visualization for many ranges - for r in ['', '-1w', '-1M', '-1y']: - - rmod = "" - tmod = "" - if r: - rmod = " AND commit_info.time:[\\\"now%s\\\" TO \\\"now\\\"]" % r - tmod = ' (' + r + ')' - - context = { - 'title': title + test_name + tmod, - 'test_name': test_name, - 'query': query + rmod, - } - - file_name = test_name + '_' + type + r + '.json' - out_file = os.path.join(out_dir, file_name) - logger.info('Generating ' + out_file) - - visState = pystache.render(template_visState, context) - assert json.loads(visState) - context['visState'] = json.dumps(visState) - - searchSourceJSON = pystache.render( - template_searchSourceJSON, context) - assert json.loads(searchSourceJSON) - context['searchSourceJSON'] = json.dumps(searchSourceJSON) - - rendered = pystache.render(template, context) - assert json.loads(rendered) - - with open(out_file, 'w') as out: - out.write(rendered) - - -def _dashboard_for_all_tests(tests): - with open(os.path.join('templates', 'dashboard.mustache'), 'r') as f: - template = f.read() - - out_dir = './dashboard' - _delete_jsons(out_dir) - - out_file = os.path.join(out_dir, 'soledad-benchmarks.json') - logger.info('Generating ' + out_file) - - panels = [] - panelIndex = 1 - row = 1 - for test_name in tests: - for type, col in [('time', 1), ('resources', 7)]: - panels.append({ - 'id': test_name + '_' + type, - 'panelIndex': panelIndex, - 'col': col, - 'row': row, - 'size_x': 6, - 'size_y': 3, - 'type': 'visualization', - }) - panelIndex += 1 - row += 3 - - context = {'panels_json': json.dumps(json.dumps(panels))} - rendered = pystache.render(template, context) - assert json.loads(rendered), rendered - - with open(out_file, 'w') as f: - f.write(rendered) - - -def _dashboards_for_one_test(test): - path = os.path.join('templates', 'dashboard-per-test.mustache') - with open(path, 'r') as f: - template = f.read() - for r in ['', '-1w', '-1M', '-1y']: - context = {'test_name': test, 'range_modifier': r} - rendered = pystache.render(template, context) - name = test + r - out_file = os.path.join('dashboard', 'dashboard-%s.json' % name) - with open(out_file, 'w') as out: - out.write(rendered) - - -def generate_dashboards(tests): - _dashboard_for_all_tests(tests) - for test in tests: - _dashboards_for_one_test(test) - - -def _get_test_names(): - with open('./query/test_names.json') as f: - response = requests.get(URL + '/_search', data=f) - buckets = response.json()['aggregations']['test_names']['buckets'] - tests = [] - for d in buckets: - name = d['key'] - tests.append(name) - return sorted(tests) - - -if __name__ == '__main__': - logger = _get_logger() - tests = _get_test_names() - generate_dashboards(tests) - generate_visualizations(tests) -- cgit v1.2.3