[elastic] use multiple mustache templates
[scripts.git] / elastic / generate-config.py
1 #!/usr/bin/env python3
2 #
3 # Todo:
4 #
5 # - Remove all json files from previous runs
6 import json
7 import logging
8 import os
9 import pystache
10 import requests
11 import sys
12
13
14 URL='https://moose.leap.se:9200'
15
16 def _get_logger():
17     logger = logging.getLogger(__name__)
18     ch = logging.StreamHandler()
19     ch.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
20     logger.addHandler(ch)
21     logger.setLevel(logging.INFO)
22     return logger
23
24
25 def _delete_jsons(directory):
26     for f in os.listdir(directory):
27         if f.endswith('.json'):
28             os.unlink(os.path.join(directory, f))
29
30
31 def generate_visualizations(tests):
32     with open(os.path.join('templates', 'visualization.mustache'), 'r') as f:
33         template = f.read()
34     with open(os.path.join('templates', 'visualization-visState.mustache')) as f:
35         template_visState = f.read()
36     with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f:
37         template_searchSourceJSON = f.read()
38
39     out_dir = './visualization'
40     _delete_jsons(out_dir)
41
42     # remove all json files in output directory
43     for f in os.listdir(out_dir):
44         if f.endswith('.json'):
45             os.unlink(os.path.join(out_dir, f))
46
47     for test_name in tests:
48         out_file = os.path.join(out_dir, test_name + '.json')
49         logger.info('Generating ' + out_file)
50
51         context = {
52            'title': test_name,
53            'query': "commit_info.project:soledad "
54                     "AND commit_info.branch='master' "
55                     "AND machine_info.host='weasel' "
56                     "AND name='" + test_name + "'"
57         }
58
59         visState = pystache.render(template_visState, context)
60         assert json.loads(visState)
61         context['visState'] = json.dumps(visState)
62
63         searchSourceJSON = pystache.render(template_searchSourceJSON, context)
64         assert json.loads(searchSourceJSON)
65         context['searchSourceJSON'] = json.dumps(searchSourceJSON)
66
67         rendered = pystache.render(template, context)
68         assert json.loads(rendered)
69
70         with open(out_file, 'w') as out:
71             out.write(rendered)
72
73
74 def generate_dashboards(tests):
75     with open(os.path.join('templates', 'dashboard.mustache'), 'r') as f:
76         template = f.read()
77
78     out_dir = './dashboard'
79     _delete_jsons(out_dir)
80
81     out_file = os.path.join(out_dir, 'soledad-benchmarks.json')
82     logger.info('Generating ' + out_file)
83
84     panels = []
85     count = 0
86     for test_name in tests:
87         count += 1
88         panels.append({
89             'id': test_name,
90             'panelIndex': count,
91             'col': 1,
92             'row': count,
93             'size_x': 6,
94             'size_y': 3,
95             'type': 'visualization',
96         })
97
98     context = {'panels_json': json.dumps(panels)}
99     rendered = pystache.render(template, context)
100     assert json.loads(rendered), rendered
101
102     with open(out_file, 'w') as f:
103         f.write(rendered)
104
105
106 def _get_test_names():
107     with open('./query/test_names.json') as f:
108         response = requests.get(URL + '/_search', data=f)
109         buckets = response.json()['aggregations']['test_names']['buckets']
110     tests = []
111     for d in buckets:
112         name = d['key']
113         tests.append(name)
114     return sorted(tests)
115
116
117 if __name__ == '__main__':
118     logger = _get_logger()
119     tests = _get_test_names()
120     generate_dashboards(tests)
121     generate_visualizations(tests)