[elastic] create resources visualization
authordrebs <drebs@riseup.net>
Thu, 8 Jun 2017 15:54:19 +0000 (12:54 -0300)
committerdrebs <drebs@riseup.net>
Thu, 8 Jun 2017 17:41:53 +0000 (14:41 -0300)
elastic/generate-config.py
elastic/templates/visualization-visState-resources.mustache [new file with mode: 0644]
elastic/templates/visualization-visState-time.mustache [moved from elastic/templates/visualization-visState.mustache with 94% similarity]
elastic/templates/visualization.mustache

index be8afcb..88d6e82 100755 (executable)
@@ -33,8 +33,10 @@ def _delete_jsons(directory):
 def generate_visualizations(tests):
     with open(os.path.join('templates', 'visualization.mustache'), 'r') as f:
         template = f.read()
-    with open(os.path.join('templates', 'visualization-visState.mustache')) as f:
-        template_visState = f.read()
+    with open(os.path.join('templates', 'visualization-visState-time.mustache')) as f:
+        template_visState_time = f.read()
+    with open(os.path.join('templates', 'visualization-visState-resources.mustache')) as f:
+        template_visState_resources = f.read()
     with open(os.path.join('templates', 'visualization-searchSourceJSON.mustache')) as f:
         template_searchSourceJSON = f.read()
 
@@ -46,31 +48,40 @@ def generate_visualizations(tests):
         if f.endswith('.json'):
             os.unlink(os.path.join(out_dir, f))
 
+    graphs = [
+        ('time', 'weasel', template_visState_time),
+        ('resources', 'Beluga', template_visState_resources)
+    ]
+
     for test_name in tests:
-        out_file = os.path.join(out_dir, test_name + '.json')
-        logger.info('Generating ' + out_file)
 
-        context = {
-           'title': test_name,
-           'query': "commit_info.project:soledad "
-                    "AND commit_info.branch='master' "
-                    "AND machine_info.host='weasel' "
-                    "AND name='" + test_name + "'"
-        }
+        for type, host, template_visState in graphs:
+
+            out_file = os.path.join(out_dir, test_name + '_' + type + '.json')
+            logger.info('Generating ' + out_file)
+
+            context = {
+               'title': test_name + "_" + type,
+               'test_name': test_name,
+               'query': "commit_info.project:soledad "
+#                        "AND commit_info.branch='master' "
+                        "AND machine_info.host='" + host + "' "
+                        "AND name='" + test_name + "'"
+            }
 
-        visState = pystache.render(template_visState, context)
-        assert json.loads(visState)
-        context['visState'] = json.dumps(visState)
+            visState = pystache.render(template_visState, context)
+            assert json.loads(visState)
+            context['visState'] = json.dumps(visState)
 
-        searchSourceJSON = pystache.render(template_searchSourceJSON, context)
-        assert json.loads(searchSourceJSON)
-        context['searchSourceJSON'] = json.dumps(searchSourceJSON)
+            searchSourceJSON = pystache.render(template_searchSourceJSON, context)
+            assert json.loads(searchSourceJSON)
+            context['searchSourceJSON'] = json.dumps(searchSourceJSON)
 
-        rendered = pystache.render(template, context)
-        assert json.loads(rendered)
+            rendered = pystache.render(template, context)
+            assert json.loads(rendered)
 
-        with open(out_file, 'w') as out:
-            out.write(rendered)
+            with open(out_file, 'w') as out:
+                out.write(rendered)
 
 
 def generate_dashboards(tests):
@@ -84,18 +95,21 @@ def generate_dashboards(tests):
     logger.info('Generating ' + out_file)
 
     panels = []
-    count = 0
+    panelIndex = 1
+    row = 1
     for test_name in tests:
-        count += 1
-        panels.append({
-            'id': test_name,
-            'panelIndex': count,
-            'col': 1,
-            'row': count,
-            'size_x': 6,
-            'size_y': 3,
-            'type': 'visualization',
-        })
+        for type, col in [('time', 1), ('resources', 7)]:
+            panels.append({
+                'id': test_name + '_' + type,
+                'panelIndex': panelIndex,
+                'col': col,
+                'row': row,
+                'size_x': 6,
+                'size_y': 3,
+                'type': 'visualization',
+            })
+            panelIndex += 1
+        row += 3
 
     context = {'panels_json': json.dumps(json.dumps(panels))}
     rendered = pystache.render(template, context)
diff --git a/elastic/templates/visualization-visState-resources.mustache b/elastic/templates/visualization-visState-resources.mustache
new file mode 100644 (file)
index 0000000..d47db6b
--- /dev/null
@@ -0,0 +1,60 @@
+{
+   "type" : "line",
+   "listeners" : {},
+   "title" : "Test resources for {{{test_name}}}",
+   "params" : {
+      "legendPosition" : "right",
+      "defaultYExtents" : false,
+      "setYExtents" : false,
+      "drawLinesBetweenPoints" : true,
+      "interpolate" : "linear",
+      "addTimeMarker" : false,
+      "radiusRatio" : 9,
+      "showCircles" : true,
+      "scale" : "linear",
+      "addTooltip" : true,
+      "addLegend" : true,
+      "times" : []
+   },
+   "aggs" : [
+      {
+         "type" : "avg",
+         "schema" : "metric",
+         "params" : {
+            "field" : "extra_info.cpu_percent"
+         },
+         "enabled" : true,
+         "id" : "1"
+      },
+      {
+         "enabled" : true,
+         "id" : "2",
+         "params" : {
+            "order" : "asc",
+            "orderBy" : "custom",
+            "field" : "commit_id_and_date",
+            "size" : 1000,
+            "orderAgg" : {
+               "params" : {
+                  "field" : "commit_info.time"
+               },
+               "schema" : "orderAgg",
+               "type" : "min",
+               "enabled" : true,
+               "id" : "2-orderAgg"
+            }
+         },
+         "schema" : "segment",
+         "type" : "terms"
+      },
+      {
+         "enabled" : true,
+         "id" : "3",
+         "schema" : "metric",
+         "params" : {
+            "field" : "extra_info.memory_percent.stats.max"
+         },
+         "type" : "avg"
+      }
+   ]
+}
@@ -1,7 +1,7 @@
 {
    "type" : "line",
    "listeners" : {},
-   "title" : "Soledad benchmark for {{{title}}}",
+   "title" : "Test time for {{{test_name}}}",
    "params" : {
       "legendPosition" : "right",
       "defaultYExtents" : false,
@@ -42,7 +42,8 @@
                "type" : "min",
                "enabled" : true,
                "id" : "2-orderAgg"
-            }
+            },
+            "ranges": [{"from": "*2017-4-1"}]
          },
          "schema" : "segment",
          "type" : "terms"
index 8a9def1..1d4eda3 100644 (file)
@@ -2,7 +2,7 @@
     "title" : "{{{title}}}",
     "visState" : {{{visState}}},
     "uiStateJSON" : "{}",
-    "description" : "Soledad benchmark for {{{title}}}",
+    "description" : "Test {{{type}}} for {{{test_name}}}",
     "version" : 1,
     "kibanaSavedObjectMeta" : {
         "searchSourceJSON": {{{searchSourceJSON}}}