[test] move legacy-vs-blobs graphing to benchmark scripts dir

4 files changed, 444 insertions, 0 deletions

diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json
new file mode 100644
index 00000000..a585ad3a
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json
@@ -0,0 +1,111 @@
+{
+    "commit_info": {
+        "dirty": true, 
+        "branch": "8856", 
+        "author_time": "2017-09-01T09:50:11-03:00", 
+        "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", 
+        "time": "2017-09-01T09:50:20-03:00", 
+        "project": "soledad"
+    }, 
+    "machine_info": {
+        "node": "pajeh", 
+        "processor": "", 
+        "machine": "x86_64", 
+        "system": "Linux", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "python_implementation": "CPython", 
+        "python_implementation_version": "2.7.13", 
+        "host": "pajeh", 
+        "cpu": {
+            "hardware": "unknown", 
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", 
+            "vendor_id": "GenuineIntel"
+        }, 
+        "release": "4.9.0-3-amd64", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "python_version": "2.7.13"
+    }, 
+    "datetime": "2017-09-01T16:28:31.243069", 
+    "version": "3.1.1", 
+    "benchmarks": [
+        {
+            "stats": {
+                "ld15iqr": 38.49253582954407, 
+                "outliers": "2;0", 
+                "median": 52.51530361175537, 
+                "iterations": 1, 
+                "q3": 65.07869005203247, 
+                "iqr_outliers": 0, 
+                "rounds": 4, 
+                "ops": 0.018550090252706083, 
+                "total": 215.63237404823303, 
+                "mean": 53.90809351205826, 
+                "max": 72.10923099517822, 
+                "iqr": 22.341193079948425, 
+                "stddev_outliers": 2, 
+                "q1": 42.737496972084045, 
+                "min": 38.49253582954407, 
+                "hd15iqr": 72.10923099517822, 
+                "stddev": 14.537598498896823
+            }, 
+            "name": "test_legacy_1000_10k", 
+            "param": null, 
+            "group": "test_legacy_vs_blobs_1000_10k", 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k", 
+            "params": null, 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 26.9
+            }, 
+            "options": {
+                "warmup": false, 
+                "max_time": 1.0, 
+                "min_time": 5e-06, 
+                "min_rounds": 5, 
+                "timer": "time", 
+                "disable_gc": false
+            }
+        }, 
+        {
+            "stats": {
+                "ld15iqr": 176.17668509483337, 
+                "outliers": "2;0", 
+                "median": 192.90700149536133, 
+                "iterations": 1, 
+                "q3": 206.93408203125, 
+                "iqr_outliers": 0, 
+                "rounds": 4, 
+                "ops": 0.005138941398078599, 
+                "total": 778.3704249858856, 
+                "mean": 194.5926062464714, 
+                "max": 216.3797369003296, 
+                "iqr": 24.68295156955719, 
+                "stddev_outliers": 2, 
+                "q1": 182.2511304616928, 
+                "min": 176.17668509483337, 
+                "hd15iqr": 216.3797369003296, 
+                "stddev": 16.94586147142875
+            }, 
+            "name": "test_blobs_1000_10k", 
+            "param": null, 
+            "group": "test_legacy_vs_blobs_1000_10k", 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", 
+            "params": null, 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 10.9
+            }, 
+            "options": {
+                "warmup": false, 
+                "max_time": 1.0, 
+                "min_time": 5e-06, 
+                "min_rounds": 5, 
+                "timer": "time", 
+                "disable_gc": false
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json
new file mode 100644
index 00000000..2cbbe638
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json
@@ -0,0 +1,111 @@
+{
+    "commit_info": {
+        "time": "2017-09-01T09:50:20-03:00", 
+        "branch": "8856", 
+        "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", 
+        "project": "soledad", 
+        "dirty": true, 
+        "author_time": "2017-09-01T09:50:11-03:00"
+    }, 
+    "version": "3.1.1", 
+    "benchmarks": [
+        {
+            "stats": {
+                "stddev_outliers": 1, 
+                "ops": 0.08172728164268425, 
+                "stddev": 1.0083003646390902, 
+                "hd15iqr": 13.605333805084229, 
+                "total": 48.943264961242676, 
+                "iqr": 1.4744374752044678, 
+                "iqr_outliers": 0, 
+                "q1": 11.498597502708435, 
+                "q3": 12.973034977912903, 
+                "ld15iqr": 11.303098917007446, 
+                "min": 11.303098917007446, 
+                "iterations": 1, 
+                "max": 13.605333805084229, 
+                "outliers": "1;0", 
+                "rounds": 4, 
+                "median": 12.0174161195755, 
+                "mean": 12.235816240310669
+            }, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k", 
+            "options": {
+                "warmup": false, 
+                "min_time": 5e-06, 
+                "max_time": 1.0, 
+                "disable_gc": false, 
+                "min_rounds": 5, 
+                "timer": "time"
+            }, 
+            "param": null, 
+            "name": "test_legacy_100_100k", 
+            "group": "test_legacy_vs_blobs_100_100k", 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 25.2
+            }, 
+            "params": null
+        }, 
+        {
+            "stats": {
+                "stddev_outliers": 2, 
+                "ops": 0.052999047893223836, 
+                "stddev": 0.6994077398028641, 
+                "hd15iqr": 19.686198949813843, 
+                "total": 75.47305393218994, 
+                "iqr": 1.0998969078063965, 
+                "iqr_outliers": 0, 
+                "q1": 18.318315029144287, 
+                "q3": 19.418211936950684, 
+                "ld15iqr": 18.07945704460144, 
+                "min": 18.07945704460144, 
+                "iterations": 1, 
+                "max": 19.686198949813843, 
+                "outliers": "2;0", 
+                "rounds": 4, 
+                "median": 18.85369896888733, 
+                "mean": 18.868263483047485
+            }, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k", 
+            "options": {
+                "warmup": false, 
+                "min_time": 5e-06, 
+                "max_time": 1.0, 
+                "disable_gc": false, 
+                "min_rounds": 5, 
+                "timer": "time"
+            }, 
+            "param": null, 
+            "name": "test_blobs_100_100k", 
+            "group": "test_legacy_vs_blobs_100_100k", 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 14.7
+            }, 
+            "params": null
+        }
+    ], 
+    "datetime": "2017-09-01T15:44:20.745800", 
+    "machine_info": {
+        "host": "pajeh", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "python_implementation": "CPython", 
+        "cpu": {
+            "hardware": "unknown", 
+            "vendor_id": "GenuineIntel", 
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+        }, 
+        "node": "pajeh", 
+        "release": "4.9.0-3-amd64", 
+        "processor": "", 
+        "python_implementation_version": "2.7.13", 
+        "python_version": "2.7.13", 
+        "system": "Linux", 
+        "machine": "x86_64"
+    }
+}
\ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json
new file mode 100644
index 00000000..778e5c63
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json
@@ -0,0 +1,111 @@
+{
+    "version": "3.1.1", 
+    "machine_info": {
+        "python_implementation": "CPython", 
+        "host": "pajeh", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "cpu": {
+            "hardware": "unknown", 
+            "vendor_id": "GenuineIntel", 
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+        }, 
+        "release": "4.9.0-3-amd64", 
+        "python_version": "2.7.13", 
+        "machine": "x86_64", 
+        "python_implementation_version": "2.7.13", 
+        "processor": "", 
+        "node": "pajeh", 
+        "system": "Linux", 
+        "python_compiler": "GCC 6.3.0 20170118"
+    }, 
+    "commit_info": {
+        "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", 
+        "time": "2017-09-01T09:50:20-03:00", 
+        "project": "soledad", 
+        "dirty": true, 
+        "branch": "8856", 
+        "author_time": "2017-09-01T09:50:11-03:00"
+    }, 
+    "datetime": "2017-09-01T15:37:48.278845", 
+    "benchmarks": [
+        {
+            "param": null, 
+            "extra_info": {
+                "cpu_percent": 27.9, 
+                "doc": ""
+            }, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k", 
+            "group": "test_legacy_vs_blobs_10_1000k", 
+            "params": null, 
+            "name": "test_legacy_10_1000k", 
+            "stats": {
+                "mean": 8.937849223613739, 
+                "min": 8.711518049240112, 
+                "outliers": "2;0", 
+                "total": 35.751396894454956, 
+                "iqr_outliers": 0, 
+                "iterations": 1, 
+                "ops": 0.11188374014612, 
+                "rounds": 4, 
+                "stddev": 0.20914882546041347, 
+                "hd15iqr": 9.213748931884766, 
+                "median": 8.913064956665039, 
+                "stddev_outliers": 2, 
+                "max": 9.213748931884766, 
+                "q1": 8.794075012207031, 
+                "ld15iqr": 8.711518049240112, 
+                "q3": 9.081623435020447, 
+                "iqr": 0.2875484228134155
+            }, 
+            "options": {
+                "disable_gc": false, 
+                "min_time": 5e-06, 
+                "min_rounds": 5, 
+                "timer": "time", 
+                "max_time": 1.0, 
+                "warmup": false
+            }
+        }, 
+        {
+            "param": null, 
+            "extra_info": {
+                "cpu_percent": 31.7, 
+                "doc": ""
+            }, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", 
+            "group": "test_legacy_vs_blobs_10_1000k", 
+            "params": null, 
+            "name": "test_blobs_10_1000k", 
+            "stats": {
+                "mean": 3.1771076917648315, 
+                "min": 3.077280044555664, 
+                "outliers": "1;0", 
+                "total": 12.708430767059326, 
+                "iqr_outliers": 0, 
+                "iterations": 1, 
+                "ops": 0.3147516851858793, 
+                "rounds": 4, 
+                "stddev": 0.0741566234351459, 
+                "hd15iqr": 3.2503108978271484, 
+                "median": 3.190419912338257, 
+                "stddev_outliers": 1, 
+                "max": 3.2503108978271484, 
+                "q1": 3.1237324476242065, 
+                "ld15iqr": 3.077280044555664, 
+                "q3": 3.2304829359054565, 
+                "iqr": 0.10675048828125
+            }, 
+            "options": {
+                "disable_gc": false, 
+                "min_time": 5e-06, 
+                "min_rounds": 5, 
+                "timer": "time", 
+                "max_time": 1.0, 
+                "warmup": false
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json
new file mode 100644
index 00000000..0697cd6b
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json
@@ -0,0 +1,111 @@
+{
+    "benchmarks": [
+        {
+            "options": {
+                "disable_gc": false, 
+                "timer": "time", 
+                "min_rounds": 5, 
+                "min_time": 5e-06, 
+                "max_time": 1.0, 
+                "warmup": false
+            }, 
+            "params": null, 
+            "extra_info": {
+                "cpu_percent": 25.8, 
+                "doc": ""
+            }, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k", 
+            "group": "test_legacy_vs_blobs_1_10000k", 
+            "name": "test_legacy_1_10000k", 
+            "param": null, 
+            "stats": {
+                "hd15iqr": 9.522547006607056, 
+                "stddev": 0.04062225700829493, 
+                "ops": 0.10556240321945518, 
+                "iterations": 1, 
+                "min": 9.423872947692871, 
+                "total": 37.89227867126465, 
+                "stddev_outliers": 2, 
+                "max": 9.522547006607056, 
+                "q1": 9.445196866989136, 
+                "mean": 9.473069667816162, 
+                "iqr_outliers": 0, 
+                "rounds": 4, 
+                "iqr": 0.055745601654052734, 
+                "median": 9.47292935848236, 
+                "q3": 9.500942468643188, 
+                "ld15iqr": 9.423872947692871, 
+                "outliers": "2;0"
+            }
+        }, 
+        {
+            "options": {
+                "disable_gc": false, 
+                "timer": "time", 
+                "min_rounds": 5, 
+                "min_time": 5e-06, 
+                "max_time": 1.0, 
+                "warmup": false
+            }, 
+            "params": null, 
+            "extra_info": {
+                "cpu_percent": 33.3, 
+                "doc": ""
+            }, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", 
+            "group": "test_legacy_vs_blobs_1_10000k", 
+            "name": "test_blobs_1_10000k", 
+            "param": null, 
+            "stats": {
+                "hd15iqr": 3.758697032928467, 
+                "stddev": 0.619090853876154, 
+                "ops": 0.3292143381447483, 
+                "iterations": 1, 
+                "min": 2.344238042831421, 
+                "total": 12.150139093399048, 
+                "stddev_outliers": 2, 
+                "max": 3.758697032928467, 
+                "q1": 2.5475854873657227, 
+                "mean": 3.037534773349762, 
+                "iqr_outliers": 0, 
+                "rounds": 4, 
+                "iqr": 0.9798985719680786, 
+                "median": 3.02360200881958, 
+                "q3": 3.5274840593338013, 
+                "ld15iqr": 2.344238042831421, 
+                "outliers": "2;0"
+            }
+        }
+    ], 
+    "version": "3.1.1", 
+    "commit_info": {
+        "author_time": "2017-09-01T14:16:21-03:00", 
+        "project": "soledad", 
+        "id": "7a5698ad8c1457460d95cbd6dd237fb96e218d43", 
+        "time": "2017-09-01T14:16:26-03:00", 
+        "dirty": true, 
+        "branch": "8856"
+    }, 
+    "machine_info": {
+        "host": "pajeh", 
+        "python_version": "2.7.13", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "system": "Linux", 
+        "python_implementation": "CPython", 
+        "python_implementation_version": "2.7.13", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "machine": "x86_64", 
+        "release": "4.9.0-3-amd64", 
+        "processor": "", 
+        "node": "pajeh", 
+        "cpu": {
+            "hardware": "unknown", 
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", 
+            "vendor_id": "GenuineIntel"
+        }
+    }, 
+    "datetime": "2017-09-01T22:20:37.591520"
+}
\ No newline at end of file