[test] move legacy-vs-blobs graphing to benchmark scripts dir

4 files changed, 444 insertions, 0 deletions

diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json
new file mode 100644
index 00000000..5be17758
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json
@@ -0,0 +1,111 @@
+{
+    "datetime": "2017-09-02T04:22:20.074434", 
+    "machine_info": {
+        "python_implementation": "CPython", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "machine": "x86_64", 
+        "cpu": {
+            "hardware": "unknown", 
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", 
+            "vendor_id": "GenuineIntel"
+        }, 
+        "host": "pajeh", 
+        "system": "Linux", 
+        "processor": "", 
+        "python_implementation_version": "2.7.13", 
+        "release": "4.9.0-3-amd64", 
+        "python_version": "2.7.13", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "node": "pajeh"
+    }, 
+    "commit_info": {
+        "author_time": "2017-09-02T00:27:58-03:00", 
+        "time": "2017-09-02T00:39:13-03:00", 
+        "dirty": true, 
+        "project": "soledad", 
+        "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", 
+        "branch": "8856"
+    }, 
+    "benchmarks": [
+        {
+            "name": "test_legacy_1000_10k", 
+            "stats": {
+                "median": 48.28695607185364, 
+                "rounds": 4, 
+                "ld15iqr": 36.51543307304382, 
+                "iqr_outliers": 0, 
+                "stddev": 10.431966763848191, 
+                "q1": 40.382596015930176, 
+                "min": 36.51543307304382, 
+                "q3": 56.539175629615784, 
+                "total": 193.84354329109192, 
+                "iterations": 1, 
+                "max": 60.75419807434082, 
+                "stddev_outliers": 2, 
+                "mean": 48.46088582277298, 
+                "ops": 0.020635198532216573, 
+                "hd15iqr": 60.75419807434082, 
+                "outliers": "2;0", 
+                "iqr": 16.156579613685608
+            }, 
+            "extra_info": {
+                "cpu_percent": 26.6, 
+                "doc": ""
+            }, 
+            "params": null, 
+            "param": null, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k", 
+            "group": "test_legacy_vs_blobs_1000_10k", 
+            "options": {
+                "timer": "time", 
+                "min_rounds": 5, 
+                "warmup": false, 
+                "disable_gc": false, 
+                "min_time": 5e-06, 
+                "max_time": 1.0
+            }
+        }, 
+        {
+            "name": "test_blobs_1000_10k", 
+            "stats": {
+                "median": 87.17049705982208, 
+                "rounds": 4, 
+                "ld15iqr": 65.08280301094055, 
+                "iqr_outliers": 0, 
+                "stddev": 19.00300388729206, 
+                "q1": 72.67573952674866, 
+                "min": 65.08280301094055, 
+                "q3": 101.80426609516144, 
+                "total": 348.9600112438202, 
+                "iterations": 1, 
+                "max": 109.53621411323547, 
+                "stddev_outliers": 2, 
+                "mean": 87.24000281095505, 
+                "ops": 0.01146263145092914, 
+                "hd15iqr": 109.53621411323547, 
+                "outliers": "2;0", 
+                "iqr": 29.12852656841278
+            }, 
+            "extra_info": {
+                "cpu_percent": 19.4, 
+                "doc": ""
+            }, 
+            "params": null, 
+            "param": null, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", 
+            "group": "test_legacy_vs_blobs_1000_10k", 
+            "options": {
+                "timer": "time", 
+                "min_rounds": 5, 
+                "warmup": false, 
+                "disable_gc": false, 
+                "min_time": 5e-06, 
+                "max_time": 1.0
+            }
+        }
+    ], 
+    "version": "3.1.1"
+}
\ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json
new file mode 100644
index 00000000..17fc598b
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json
@@ -0,0 +1,111 @@
+{
+    "benchmarks": [
+        {
+            "params": null, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k", 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 25.1
+            }, 
+            "group": "test_legacy_vs_blobs_100_100k", 
+            "stats": {
+                "q1": 11.39308750629425, 
+                "rounds": 4, 
+                "q3": 12.677581906318665, 
+                "hd15iqr": 12.946987867355347, 
+                "median": 12.090978026390076, 
+                "stddev_outliers": 2, 
+                "mean": 12.035334706306458, 
+                "iqr_outliers": 0, 
+                "iterations": 1, 
+                "total": 48.14133882522583, 
+                "min": 11.012394905090332, 
+                "outliers": "2;0", 
+                "ops": 0.08308867384269794, 
+                "ld15iqr": 11.012394905090332, 
+                "stddev": 0.833654348956266, 
+                "iqr": 1.284494400024414, 
+                "max": 12.946987867355347
+            }, 
+            "name": "test_legacy_100_100k", 
+            "param": null, 
+            "options": {
+                "min_rounds": 5, 
+                "timer": "time", 
+                "max_time": 1.0, 
+                "min_time": 5e-06, 
+                "warmup": false, 
+                "disable_gc": false
+            }
+        }, 
+        {
+            "params": null, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k", 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 24.9
+            }, 
+            "group": "test_legacy_vs_blobs_100_100k", 
+            "stats": {
+                "q1": 5.015861988067627, 
+                "rounds": 4, 
+                "q3": 6.166973114013672, 
+                "hd15iqr": 6.362173080444336, 
+                "median": 5.64124858379364, 
+                "stddev_outliers": 2, 
+                "mean": 5.591417551040649, 
+                "iqr_outliers": 0, 
+                "iterations": 1, 
+                "total": 22.365670204162598, 
+                "min": 4.7209999561309814, 
+                "outliers": "2;0", 
+                "ops": 0.1788455236747405, 
+                "ld15iqr": 4.7209999561309814, 
+                "stddev": 0.7246033659461454, 
+                "iqr": 1.151111125946045, 
+                "max": 6.362173080444336
+            }, 
+            "name": "test_blobs_100_100k", 
+            "param": null, 
+            "options": {
+                "min_rounds": 5, 
+                "timer": "time", 
+                "max_time": 1.0, 
+                "min_time": 5e-06, 
+                "warmup": false, 
+                "disable_gc": false
+            }
+        }
+    ], 
+    "datetime": "2017-09-02T03:53:29.055256", 
+    "version": "3.1.1", 
+    "machine_info": {
+        "node": "pajeh", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "python_version": "2.7.13", 
+        "machine": "x86_64", 
+        "python_implementation": "CPython", 
+        "cpu": {
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", 
+            "hardware": "unknown", 
+            "vendor_id": "GenuineIntel"
+        }, 
+        "python_implementation_version": "2.7.13", 
+        "system": "Linux", 
+        "host": "pajeh", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "release": "4.9.0-3-amd64", 
+        "processor": ""
+    }, 
+    "commit_info": {
+        "project": "soledad", 
+        "time": "2017-09-02T00:39:13-03:00", 
+        "author_time": "2017-09-02T00:27:58-03:00", 
+        "branch": "8856", 
+        "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", 
+        "dirty": true
+    }
+}
\ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json
new file mode 100644
index 00000000..1a733f2d
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json
@@ -0,0 +1,111 @@
+{
+    "benchmarks": [
+        {
+            "params": null, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k", 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 27.7
+            }, 
+            "group": "test_legacy_vs_blobs_10_1000k", 
+            "stats": {
+                "q1": 8.741152048110962, 
+                "rounds": 4, 
+                "q3": 8.863584876060486, 
+                "median": 8.83247196674347, 
+                "stddev_outliers": 1, 
+                "iqr": 0.12243282794952393, 
+                "mean": 8.802368462085724, 
+                "iqr_outliers": 0, 
+                "iterations": 1, 
+                "total": 35.209473848342896, 
+                "min": 8.67733097076416, 
+                "outliers": "1;0", 
+                "ops": 0.11360578738634734, 
+                "ld15iqr": 8.67733097076416, 
+                "stddev": 0.08786766595777334, 
+                "hd15iqr": 8.867198944091797, 
+                "max": 8.867198944091797
+            }, 
+            "name": "test_legacy_10_1000k", 
+            "param": null, 
+            "options": {
+                "min_rounds": 5, 
+                "timer": "time", 
+                "max_time": 1.0, 
+                "min_time": 5e-06, 
+                "warmup": false, 
+                "disable_gc": false
+            }
+        }, 
+        {
+            "params": null, 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", 
+            "extra_info": {
+                "doc": "", 
+                "cpu_percent": 37.0
+            }, 
+            "group": "test_legacy_vs_blobs_10_1000k", 
+            "stats": {
+                "q1": 1.8672034740447998, 
+                "rounds": 4, 
+                "q3": 1.9366559982299805, 
+                "median": 1.8831886053085327, 
+                "stddev_outliers": 1, 
+                "iqr": 0.06945252418518066, 
+                "mean": 1.9019297361373901, 
+                "iqr_outliers": 0, 
+                "iterations": 1, 
+                "total": 7.6077189445495605, 
+                "min": 1.8517439365386963, 
+                "outliers": "1;0", 
+                "ops": 0.5257817788952025, 
+                "ld15iqr": 1.8517439365386963, 
+                "stddev": 0.06029736288568101, 
+                "hd15iqr": 1.9895977973937988, 
+                "max": 1.9895977973937988
+            }, 
+            "name": "test_blobs_10_1000k", 
+            "param": null, 
+            "options": {
+                "min_rounds": 5, 
+                "timer": "time", 
+                "max_time": 1.0, 
+                "min_time": 5e-06, 
+                "warmup": false, 
+                "disable_gc": false
+            }
+        }
+    ], 
+    "datetime": "2017-09-02T03:48:24.650044", 
+    "version": "3.1.1", 
+    "machine_info": {
+        "python_implementation_version": "2.7.13", 
+        "system": "Linux", 
+        "host": "pajeh", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "release": "4.9.0-3-amd64", 
+        "python_implementation": "CPython", 
+        "processor": "", 
+        "node": "pajeh", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "python_version": "2.7.13", 
+        "machine": "x86_64", 
+        "cpu": {
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", 
+            "hardware": "unknown", 
+            "vendor_id": "GenuineIntel"
+        }
+    }, 
+    "commit_info": {
+        "author_time": "2017-09-02T00:27:58-03:00", 
+        "time": "2017-09-02T00:39:13-03:00", 
+        "project": "soledad", 
+        "branch": "8856", 
+        "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", 
+        "dirty": true
+    }
+}
\ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json
new file mode 100644
index 00000000..2d970c6e
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json
@@ -0,0 +1,111 @@
+{
+    "datetime": "2017-09-02T03:46:17.875002", 
+    "machine_info": {
+        "processor": "", 
+        "python_implementation_version": "2.7.13", 
+        "python_version": "2.7.13", 
+        "machine": "x86_64", 
+        "python_build": [
+            "default", 
+            "Jan 19 2017 14:48:08"
+        ], 
+        "python_implementation": "CPython", 
+        "python_compiler": "GCC 6.3.0 20170118", 
+        "cpu": {
+            "vendor_id": "GenuineIntel", 
+            "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", 
+            "hardware": "unknown"
+        }, 
+        "release": "4.9.0-3-amd64", 
+        "system": "Linux", 
+        "host": "pajeh", 
+        "node": "pajeh"
+    }, 
+    "version": "3.1.1", 
+    "commit_info": {
+        "time": "2017-09-02T00:39:13-03:00", 
+        "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", 
+        "project": "soledad", 
+        "author_time": "2017-09-02T00:27:58-03:00", 
+        "branch": "8856", 
+        "dirty": true
+    }, 
+    "benchmarks": [
+        {
+            "extra_info": {
+                "cpu_percent": 25.8, 
+                "doc": ""
+            }, 
+            "name": "test_legacy_1_10000k", 
+            "params": null, 
+            "options": {
+                "max_time": 1.0, 
+                "min_rounds": 5, 
+                "min_time": 5e-06, 
+                "warmup": false, 
+                "timer": "time", 
+                "disable_gc": false
+            }, 
+            "group": "test_legacy_vs_blobs_1_10000k", 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k", 
+            "param": null, 
+            "stats": {
+                "mean": 9.607597470283508, 
+                "ld15iqr": 9.551655054092407, 
+                "iqr_outliers": 0, 
+                "outliers": "1;0", 
+                "q3": 9.663439989089966, 
+                "q1": 9.55175495147705, 
+                "iqr": 0.11168503761291504, 
+                "total": 38.43038988113403, 
+                "rounds": 4, 
+                "median": 9.563774466514587, 
+                "stddev_outliers": 1, 
+                "min": 9.551655054092407, 
+                "hd15iqr": 9.751185894012451, 
+                "ops": 0.10408429402803564, 
+                "stddev": 0.09638854573531173, 
+                "iterations": 1, 
+                "max": 9.751185894012451
+            }
+        }, 
+        {
+            "extra_info": {
+                "cpu_percent": 40.2, 
+                "doc": ""
+            }, 
+            "name": "test_blobs_1_10000k", 
+            "params": null, 
+            "options": {
+                "max_time": 1.0, 
+                "min_rounds": 5, 
+                "min_time": 5e-06, 
+                "warmup": false, 
+                "timer": "time", 
+                "disable_gc": false
+            }, 
+            "group": "test_legacy_vs_blobs_1_10000k", 
+            "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", 
+            "param": null, 
+            "stats": {
+                "mean": 2.179791212081909, 
+                "ld15iqr": 2.1572468280792236, 
+                "iqr_outliers": 0, 
+                "outliers": "2;0", 
+                "q3": 2.193374514579773, 
+                "q1": 2.1662079095840454, 
+                "iqr": 0.02716660499572754, 
+                "total": 8.719164848327637, 
+                "rounds": 4, 
+                "median": 2.1782270669937134, 
+                "stddev_outliers": 2, 
+                "min": 2.1572468280792236, 
+                "hd15iqr": 2.2054638862609863, 
+                "ops": 0.45875953369171735, 
+                "stddev": 0.01992429175453132, 
+                "iterations": 1, 
+                "max": 2.2054638862609863
+            }
+        }
+    ]
+}
\ No newline at end of file