#!/usr/bin/env python3 # # Todo: # # - Remove all json files from previous runs import json import logging import os import pystache import requests import sys URL='https://moose.leap.se:9200' def _get_logger(): logger = logging.getLogger(__name__) ch = logging.StreamHandler() ch.setFormatter(logging.Formatter('%(asctime)s %(message)s')) logger.addHandler(ch) logger.setLevel(logging.INFO) return logger def _delete_jsons(directory): logger.info('removing *.json from %s' % directory) for f in os.listdir(directory): if f.endswith('.json'): os.unlink(os.path.join(directory, f)) def generate_visualizations(tests): with open(os.path.join('templates', 'visualization.mustache'), 'r') as f: template = f.read() with open(os.path.join('templates', 'visualization-visState.mustache')) as f: template_visState = f.read() with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f: template_searchSourceJSON = f.read() out_dir = './visualization' _delete_jsons(out_dir) # remove all json files in output directory for f in os.listdir(out_dir): if f.endswith('.json'): os.unlink(os.path.join(out_dir, f)) for test_name in tests: out_file = os.path.join(out_dir, test_name + '.json') logger.info('Generating ' + out_file) context = { 'title': test_name, 'query': "commit_info.project:soledad " "AND commit_info.branch='master' " "AND machine_info.host='weasel' " "AND name='" + test_name + "'" } visState = pystache.render(template_visState, context) assert json.loads(visState) context['visState'] = json.dumps(visState) searchSourceJSON = pystache.render(template_searchSourceJSON, context) assert json.loads(searchSourceJSON) context['searchSourceJSON'] = json.dumps(searchSourceJSON) rendered = pystache.render(template, context) assert json.loads(rendered) with open(out_file, 'w') as out: out.write(rendered) def generate_dashboards(tests): with open(os.path.join('templates', 'dashboard.mustache'), 'r') as f: template = f.read() out_dir = './dashboard' _delete_jsons(out_dir) out_file = os.path.join(out_dir, 'soledad-benchmarks.json') logger.info('Generating ' + out_file) panels = [] count = 0 for test_name in tests: count += 1 panels.append({ 'id': test_name, 'panelIndex': count, 'col': 1, 'row': count, 'size_x': 6, 'size_y': 3, 'type': 'visualization', }) context = {'panels_json': json.dumps(json.dumps(panels))} rendered = pystache.render(template, context) assert json.loads(rendered), rendered with open(out_file, 'w') as f: f.write(rendered) def _get_test_names(): with open('./query/test_names.json') as f: response = requests.get(URL + '/_search', data=f) buckets = response.json()['aggregations']['test_names']['buckets'] tests = [] for d in buckets: name = d['key'] tests.append(name) return sorted(tests) if __name__ == '__main__': logger = _get_logger() tests = _get_test_names() generate_dashboards(tests) generate_visualizations(tests)