From dd7b2e414adf2c84873debcd7b7d526859036bc2 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 13 Sep 2017 11:39:36 -0300 Subject: [test] move legacy-vs-blobs graphing to benchmark scripts dir --- scripts/benchmark/legacy-vs-blobs/README.rst | 43 +++++++ .../legacy-vs-blobs/data/cache/1000_10k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/cache/100_100k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/cache/10_1000k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/cache/1_10000k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/no-cache/1000_10k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/no-cache/100_100k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/no-cache/10_1000k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/no-cache/1_10000k.json | 111 ++++++++++++++++++ .../legacy-vs-blobs/data/persistent/1000_10k.json | 73 ++++++++++++ .../legacy-vs-blobs/data/persistent/100_100k.json | 73 ++++++++++++ .../legacy-vs-blobs/data/persistent/10_1000k.json | 73 ++++++++++++ .../legacy-vs-blobs/data/persistent/1_10000k.json | 73 ++++++++++++ .../benchmark/legacy-vs-blobs/legacy-vs-blobs.png | Bin 0 -> 38517 bytes .../benchmark/legacy-vs-blobs/legacy-vs-blobs.py | 126 +++++++++++++++++++++ scripts/benchmark/legacy-vs-blobs/makefile | 14 +++ scripts/benchmark/legacy-vs-blobs/requirements.pip | 2 + scripts/legacy-vs-blobs/README.rst | 43 ------- scripts/legacy-vs-blobs/data/cache/1000_10k.json | 111 ------------------ scripts/legacy-vs-blobs/data/cache/100_100k.json | 111 ------------------ scripts/legacy-vs-blobs/data/cache/10_1000k.json | 111 ------------------ scripts/legacy-vs-blobs/data/cache/1_10000k.json | 111 ------------------ .../legacy-vs-blobs/data/no-cache/1000_10k.json | 111 ------------------ .../legacy-vs-blobs/data/no-cache/100_100k.json | 111 ------------------ .../legacy-vs-blobs/data/no-cache/10_1000k.json | 111 ------------------ .../legacy-vs-blobs/data/no-cache/1_10000k.json | 111 ------------------ .../legacy-vs-blobs/data/persistent/1000_10k.json | 73 ------------ .../legacy-vs-blobs/data/persistent/100_100k.json | 73 ------------ .../legacy-vs-blobs/data/persistent/10_1000k.json | 73 ------------ .../legacy-vs-blobs/data/persistent/1_10000k.json | 73 ------------ scripts/legacy-vs-blobs/legacy-vs-blobs.png | Bin 38517 -> 0 bytes scripts/legacy-vs-blobs/legacy-vs-blobs.py | 126 --------------------- scripts/legacy-vs-blobs/makefile | 14 --- scripts/legacy-vs-blobs/requirements.pip | 2 - 34 files changed, 1365 insertions(+), 1365 deletions(-) create mode 100644 scripts/benchmark/legacy-vs-blobs/README.rst create mode 100644 scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json create mode 100644 scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png create mode 100755 scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py create mode 100644 scripts/benchmark/legacy-vs-blobs/makefile create mode 100644 scripts/benchmark/legacy-vs-blobs/requirements.pip delete mode 100644 scripts/legacy-vs-blobs/README.rst delete mode 100644 scripts/legacy-vs-blobs/data/cache/1000_10k.json delete mode 100644 scripts/legacy-vs-blobs/data/cache/100_100k.json delete mode 100644 scripts/legacy-vs-blobs/data/cache/10_1000k.json delete mode 100644 scripts/legacy-vs-blobs/data/cache/1_10000k.json delete mode 100644 scripts/legacy-vs-blobs/data/no-cache/1000_10k.json delete mode 100644 scripts/legacy-vs-blobs/data/no-cache/100_100k.json delete mode 100644 scripts/legacy-vs-blobs/data/no-cache/10_1000k.json delete mode 100644 scripts/legacy-vs-blobs/data/no-cache/1_10000k.json delete mode 100644 scripts/legacy-vs-blobs/data/persistent/1000_10k.json delete mode 100644 scripts/legacy-vs-blobs/data/persistent/100_100k.json delete mode 100644 scripts/legacy-vs-blobs/data/persistent/10_1000k.json delete mode 100644 scripts/legacy-vs-blobs/data/persistent/1_10000k.json delete mode 100644 scripts/legacy-vs-blobs/legacy-vs-blobs.png delete mode 100755 scripts/legacy-vs-blobs/legacy-vs-blobs.py delete mode 100644 scripts/legacy-vs-blobs/makefile delete mode 100644 scripts/legacy-vs-blobs/requirements.pip diff --git a/scripts/benchmark/legacy-vs-blobs/README.rst b/scripts/benchmark/legacy-vs-blobs/README.rst new file mode 100644 index 00000000..3eedb3a4 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/README.rst @@ -0,0 +1,43 @@ +Ploggin Legacy vs Blobs results +=============================== + +This script produces a bar graph comparing different implementations of the +mail pipeline (legacy, blobs, blobs with session cache, blobs with session +cache and persistent http connections) for different amount/sizes of incoming +data (1 x 10M, 10 x 1M, 100 x 100K, 1000 x 10K). + +Data acquisition +---------------- + +Data was gathered through different runs of the code in +`soledad/tests/benchmarks/test_legacy_vs_blobs.py`. As these tests are skipped +by default, you have to (1) remove the @pytest.mark.skip decorator and (2) +remove the second call to pytest in `tox.ini` that would cause a second run of +the tests for measuring memory. This script only plots time, for now. + +Once you done the above, chdir into `soledad/tests` and do something like this, +for each test group: + + tox -e benchmarks -- -m benchmark_test_legacy_vs_blobs_10_1000k --benchmark-autosave + +Each run of tox as above produces a JSON file in `soledad/tests/.benchmarks` +with stats. Each file has to be copied in a subdirectory of the `data/` +directory of the folder where this script lies. Code has to be adapted for each +extra set of results added to the `data/` directory. + +Currently, we are comparing 4 implementation variations: + + - data/no-cache: contains legacy as baseline and blobs as the current master + implementation of blobs. + - data/cache: adds twisted session cache to blobs connections. + - data/persistent: adds session cache and persistent HTTP connections. + +Plotting +-------- + +Once you have all your data in subdirectories of the `data/` directory and the +script is correctly tuned for your data, running `make` should be enough to +create a virtualenvironment with dependencies and plotting data. + +By default, the script will generate a file called `./legacy-vs-blobs.png` in +the current directory with the plot of the data. diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json new file mode 100644 index 00000000..5be17758 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json @@ -0,0 +1,111 @@ +{ + "datetime": "2017-09-02T04:22:20.074434", + "machine_info": { + "python_implementation": "CPython", + "python_compiler": "GCC 6.3.0 20170118", + "machine": "x86_64", + "cpu": { + "hardware": "unknown", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", + "vendor_id": "GenuineIntel" + }, + "host": "pajeh", + "system": "Linux", + "processor": "", + "python_implementation_version": "2.7.13", + "release": "4.9.0-3-amd64", + "python_version": "2.7.13", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "node": "pajeh" + }, + "commit_info": { + "author_time": "2017-09-02T00:27:58-03:00", + "time": "2017-09-02T00:39:13-03:00", + "dirty": true, + "project": "soledad", + "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", + "branch": "8856" + }, + "benchmarks": [ + { + "name": "test_legacy_1000_10k", + "stats": { + "median": 48.28695607185364, + "rounds": 4, + "ld15iqr": 36.51543307304382, + "iqr_outliers": 0, + "stddev": 10.431966763848191, + "q1": 40.382596015930176, + "min": 36.51543307304382, + "q3": 56.539175629615784, + "total": 193.84354329109192, + "iterations": 1, + "max": 60.75419807434082, + "stddev_outliers": 2, + "mean": 48.46088582277298, + "ops": 0.020635198532216573, + "hd15iqr": 60.75419807434082, + "outliers": "2;0", + "iqr": 16.156579613685608 + }, + "extra_info": { + "cpu_percent": 26.6, + "doc": "" + }, + "params": null, + "param": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k", + "group": "test_legacy_vs_blobs_1000_10k", + "options": { + "timer": "time", + "min_rounds": 5, + "warmup": false, + "disable_gc": false, + "min_time": 5e-06, + "max_time": 1.0 + } + }, + { + "name": "test_blobs_1000_10k", + "stats": { + "median": 87.17049705982208, + "rounds": 4, + "ld15iqr": 65.08280301094055, + "iqr_outliers": 0, + "stddev": 19.00300388729206, + "q1": 72.67573952674866, + "min": 65.08280301094055, + "q3": 101.80426609516144, + "total": 348.9600112438202, + "iterations": 1, + "max": 109.53621411323547, + "stddev_outliers": 2, + "mean": 87.24000281095505, + "ops": 0.01146263145092914, + "hd15iqr": 109.53621411323547, + "outliers": "2;0", + "iqr": 29.12852656841278 + }, + "extra_info": { + "cpu_percent": 19.4, + "doc": "" + }, + "params": null, + "param": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", + "group": "test_legacy_vs_blobs_1000_10k", + "options": { + "timer": "time", + "min_rounds": 5, + "warmup": false, + "disable_gc": false, + "min_time": 5e-06, + "max_time": 1.0 + } + } + ], + "version": "3.1.1" +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json new file mode 100644 index 00000000..17fc598b --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json @@ -0,0 +1,111 @@ +{ + "benchmarks": [ + { + "params": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k", + "extra_info": { + "doc": "", + "cpu_percent": 25.1 + }, + "group": "test_legacy_vs_blobs_100_100k", + "stats": { + "q1": 11.39308750629425, + "rounds": 4, + "q3": 12.677581906318665, + "hd15iqr": 12.946987867355347, + "median": 12.090978026390076, + "stddev_outliers": 2, + "mean": 12.035334706306458, + "iqr_outliers": 0, + "iterations": 1, + "total": 48.14133882522583, + "min": 11.012394905090332, + "outliers": "2;0", + "ops": 0.08308867384269794, + "ld15iqr": 11.012394905090332, + "stddev": 0.833654348956266, + "iqr": 1.284494400024414, + "max": 12.946987867355347 + }, + "name": "test_legacy_100_100k", + "param": null, + "options": { + "min_rounds": 5, + "timer": "time", + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false, + "disable_gc": false + } + }, + { + "params": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k", + "extra_info": { + "doc": "", + "cpu_percent": 24.9 + }, + "group": "test_legacy_vs_blobs_100_100k", + "stats": { + "q1": 5.015861988067627, + "rounds": 4, + "q3": 6.166973114013672, + "hd15iqr": 6.362173080444336, + "median": 5.64124858379364, + "stddev_outliers": 2, + "mean": 5.591417551040649, + "iqr_outliers": 0, + "iterations": 1, + "total": 22.365670204162598, + "min": 4.7209999561309814, + "outliers": "2;0", + "ops": 0.1788455236747405, + "ld15iqr": 4.7209999561309814, + "stddev": 0.7246033659461454, + "iqr": 1.151111125946045, + "max": 6.362173080444336 + }, + "name": "test_blobs_100_100k", + "param": null, + "options": { + "min_rounds": 5, + "timer": "time", + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false, + "disable_gc": false + } + } + ], + "datetime": "2017-09-02T03:53:29.055256", + "version": "3.1.1", + "machine_info": { + "node": "pajeh", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "python_version": "2.7.13", + "machine": "x86_64", + "python_implementation": "CPython", + "cpu": { + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", + "hardware": "unknown", + "vendor_id": "GenuineIntel" + }, + "python_implementation_version": "2.7.13", + "system": "Linux", + "host": "pajeh", + "python_compiler": "GCC 6.3.0 20170118", + "release": "4.9.0-3-amd64", + "processor": "" + }, + "commit_info": { + "project": "soledad", + "time": "2017-09-02T00:39:13-03:00", + "author_time": "2017-09-02T00:27:58-03:00", + "branch": "8856", + "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", + "dirty": true + } +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json new file mode 100644 index 00000000..1a733f2d --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json @@ -0,0 +1,111 @@ +{ + "benchmarks": [ + { + "params": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k", + "extra_info": { + "doc": "", + "cpu_percent": 27.7 + }, + "group": "test_legacy_vs_blobs_10_1000k", + "stats": { + "q1": 8.741152048110962, + "rounds": 4, + "q3": 8.863584876060486, + "median": 8.83247196674347, + "stddev_outliers": 1, + "iqr": 0.12243282794952393, + "mean": 8.802368462085724, + "iqr_outliers": 0, + "iterations": 1, + "total": 35.209473848342896, + "min": 8.67733097076416, + "outliers": "1;0", + "ops": 0.11360578738634734, + "ld15iqr": 8.67733097076416, + "stddev": 0.08786766595777334, + "hd15iqr": 8.867198944091797, + "max": 8.867198944091797 + }, + "name": "test_legacy_10_1000k", + "param": null, + "options": { + "min_rounds": 5, + "timer": "time", + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false, + "disable_gc": false + } + }, + { + "params": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", + "extra_info": { + "doc": "", + "cpu_percent": 37.0 + }, + "group": "test_legacy_vs_blobs_10_1000k", + "stats": { + "q1": 1.8672034740447998, + "rounds": 4, + "q3": 1.9366559982299805, + "median": 1.8831886053085327, + "stddev_outliers": 1, + "iqr": 0.06945252418518066, + "mean": 1.9019297361373901, + "iqr_outliers": 0, + "iterations": 1, + "total": 7.6077189445495605, + "min": 1.8517439365386963, + "outliers": "1;0", + "ops": 0.5257817788952025, + "ld15iqr": 1.8517439365386963, + "stddev": 0.06029736288568101, + "hd15iqr": 1.9895977973937988, + "max": 1.9895977973937988 + }, + "name": "test_blobs_10_1000k", + "param": null, + "options": { + "min_rounds": 5, + "timer": "time", + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false, + "disable_gc": false + } + } + ], + "datetime": "2017-09-02T03:48:24.650044", + "version": "3.1.1", + "machine_info": { + "python_implementation_version": "2.7.13", + "system": "Linux", + "host": "pajeh", + "python_compiler": "GCC 6.3.0 20170118", + "release": "4.9.0-3-amd64", + "python_implementation": "CPython", + "processor": "", + "node": "pajeh", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "python_version": "2.7.13", + "machine": "x86_64", + "cpu": { + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", + "hardware": "unknown", + "vendor_id": "GenuineIntel" + } + }, + "commit_info": { + "author_time": "2017-09-02T00:27:58-03:00", + "time": "2017-09-02T00:39:13-03:00", + "project": "soledad", + "branch": "8856", + "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", + "dirty": true + } +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json new file mode 100644 index 00000000..2d970c6e --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json @@ -0,0 +1,111 @@ +{ + "datetime": "2017-09-02T03:46:17.875002", + "machine_info": { + "processor": "", + "python_implementation_version": "2.7.13", + "python_version": "2.7.13", + "machine": "x86_64", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "python_implementation": "CPython", + "python_compiler": "GCC 6.3.0 20170118", + "cpu": { + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", + "hardware": "unknown" + }, + "release": "4.9.0-3-amd64", + "system": "Linux", + "host": "pajeh", + "node": "pajeh" + }, + "version": "3.1.1", + "commit_info": { + "time": "2017-09-02T00:39:13-03:00", + "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", + "project": "soledad", + "author_time": "2017-09-02T00:27:58-03:00", + "branch": "8856", + "dirty": true + }, + "benchmarks": [ + { + "extra_info": { + "cpu_percent": 25.8, + "doc": "" + }, + "name": "test_legacy_1_10000k", + "params": null, + "options": { + "max_time": 1.0, + "min_rounds": 5, + "min_time": 5e-06, + "warmup": false, + "timer": "time", + "disable_gc": false + }, + "group": "test_legacy_vs_blobs_1_10000k", + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k", + "param": null, + "stats": { + "mean": 9.607597470283508, + "ld15iqr": 9.551655054092407, + "iqr_outliers": 0, + "outliers": "1;0", + "q3": 9.663439989089966, + "q1": 9.55175495147705, + "iqr": 0.11168503761291504, + "total": 38.43038988113403, + "rounds": 4, + "median": 9.563774466514587, + "stddev_outliers": 1, + "min": 9.551655054092407, + "hd15iqr": 9.751185894012451, + "ops": 0.10408429402803564, + "stddev": 0.09638854573531173, + "iterations": 1, + "max": 9.751185894012451 + } + }, + { + "extra_info": { + "cpu_percent": 40.2, + "doc": "" + }, + "name": "test_blobs_1_10000k", + "params": null, + "options": { + "max_time": 1.0, + "min_rounds": 5, + "min_time": 5e-06, + "warmup": false, + "timer": "time", + "disable_gc": false + }, + "group": "test_legacy_vs_blobs_1_10000k", + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", + "param": null, + "stats": { + "mean": 2.179791212081909, + "ld15iqr": 2.1572468280792236, + "iqr_outliers": 0, + "outliers": "2;0", + "q3": 2.193374514579773, + "q1": 2.1662079095840454, + "iqr": 0.02716660499572754, + "total": 8.719164848327637, + "rounds": 4, + "median": 2.1782270669937134, + "stddev_outliers": 2, + "min": 2.1572468280792236, + "hd15iqr": 2.2054638862609863, + "ops": 0.45875953369171735, + "stddev": 0.01992429175453132, + "iterations": 1, + "max": 2.2054638862609863 + } + } + ] +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json new file mode 100644 index 00000000..a585ad3a --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json @@ -0,0 +1,111 @@ +{ + "commit_info": { + "dirty": true, + "branch": "8856", + "author_time": "2017-09-01T09:50:11-03:00", + "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", + "time": "2017-09-01T09:50:20-03:00", + "project": "soledad" + }, + "machine_info": { + "node": "pajeh", + "processor": "", + "machine": "x86_64", + "system": "Linux", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "python_implementation": "CPython", + "python_implementation_version": "2.7.13", + "host": "pajeh", + "cpu": { + "hardware": "unknown", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", + "vendor_id": "GenuineIntel" + }, + "release": "4.9.0-3-amd64", + "python_compiler": "GCC 6.3.0 20170118", + "python_version": "2.7.13" + }, + "datetime": "2017-09-01T16:28:31.243069", + "version": "3.1.1", + "benchmarks": [ + { + "stats": { + "ld15iqr": 38.49253582954407, + "outliers": "2;0", + "median": 52.51530361175537, + "iterations": 1, + "q3": 65.07869005203247, + "iqr_outliers": 0, + "rounds": 4, + "ops": 0.018550090252706083, + "total": 215.63237404823303, + "mean": 53.90809351205826, + "max": 72.10923099517822, + "iqr": 22.341193079948425, + "stddev_outliers": 2, + "q1": 42.737496972084045, + "min": 38.49253582954407, + "hd15iqr": 72.10923099517822, + "stddev": 14.537598498896823 + }, + "name": "test_legacy_1000_10k", + "param": null, + "group": "test_legacy_vs_blobs_1000_10k", + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k", + "params": null, + "extra_info": { + "doc": "", + "cpu_percent": 26.9 + }, + "options": { + "warmup": false, + "max_time": 1.0, + "min_time": 5e-06, + "min_rounds": 5, + "timer": "time", + "disable_gc": false + } + }, + { + "stats": { + "ld15iqr": 176.17668509483337, + "outliers": "2;0", + "median": 192.90700149536133, + "iterations": 1, + "q3": 206.93408203125, + "iqr_outliers": 0, + "rounds": 4, + "ops": 0.005138941398078599, + "total": 778.3704249858856, + "mean": 194.5926062464714, + "max": 216.3797369003296, + "iqr": 24.68295156955719, + "stddev_outliers": 2, + "q1": 182.2511304616928, + "min": 176.17668509483337, + "hd15iqr": 216.3797369003296, + "stddev": 16.94586147142875 + }, + "name": "test_blobs_1000_10k", + "param": null, + "group": "test_legacy_vs_blobs_1000_10k", + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", + "params": null, + "extra_info": { + "doc": "", + "cpu_percent": 10.9 + }, + "options": { + "warmup": false, + "max_time": 1.0, + "min_time": 5e-06, + "min_rounds": 5, + "timer": "time", + "disable_gc": false + } + } + ] +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json new file mode 100644 index 00000000..2cbbe638 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json @@ -0,0 +1,111 @@ +{ + "commit_info": { + "time": "2017-09-01T09:50:20-03:00", + "branch": "8856", + "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", + "project": "soledad", + "dirty": true, + "author_time": "2017-09-01T09:50:11-03:00" + }, + "version": "3.1.1", + "benchmarks": [ + { + "stats": { + "stddev_outliers": 1, + "ops": 0.08172728164268425, + "stddev": 1.0083003646390902, + "hd15iqr": 13.605333805084229, + "total": 48.943264961242676, + "iqr": 1.4744374752044678, + "iqr_outliers": 0, + "q1": 11.498597502708435, + "q3": 12.973034977912903, + "ld15iqr": 11.303098917007446, + "min": 11.303098917007446, + "iterations": 1, + "max": 13.605333805084229, + "outliers": "1;0", + "rounds": 4, + "median": 12.0174161195755, + "mean": 12.235816240310669 + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k", + "options": { + "warmup": false, + "min_time": 5e-06, + "max_time": 1.0, + "disable_gc": false, + "min_rounds": 5, + "timer": "time" + }, + "param": null, + "name": "test_legacy_100_100k", + "group": "test_legacy_vs_blobs_100_100k", + "extra_info": { + "doc": "", + "cpu_percent": 25.2 + }, + "params": null + }, + { + "stats": { + "stddev_outliers": 2, + "ops": 0.052999047893223836, + "stddev": 0.6994077398028641, + "hd15iqr": 19.686198949813843, + "total": 75.47305393218994, + "iqr": 1.0998969078063965, + "iqr_outliers": 0, + "q1": 18.318315029144287, + "q3": 19.418211936950684, + "ld15iqr": 18.07945704460144, + "min": 18.07945704460144, + "iterations": 1, + "max": 19.686198949813843, + "outliers": "2;0", + "rounds": 4, + "median": 18.85369896888733, + "mean": 18.868263483047485 + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k", + "options": { + "warmup": false, + "min_time": 5e-06, + "max_time": 1.0, + "disable_gc": false, + "min_rounds": 5, + "timer": "time" + }, + "param": null, + "name": "test_blobs_100_100k", + "group": "test_legacy_vs_blobs_100_100k", + "extra_info": { + "doc": "", + "cpu_percent": 14.7 + }, + "params": null + } + ], + "datetime": "2017-09-01T15:44:20.745800", + "machine_info": { + "host": "pajeh", + "python_compiler": "GCC 6.3.0 20170118", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "python_implementation": "CPython", + "cpu": { + "hardware": "unknown", + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" + }, + "node": "pajeh", + "release": "4.9.0-3-amd64", + "processor": "", + "python_implementation_version": "2.7.13", + "python_version": "2.7.13", + "system": "Linux", + "machine": "x86_64" + } +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json new file mode 100644 index 00000000..778e5c63 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json @@ -0,0 +1,111 @@ +{ + "version": "3.1.1", + "machine_info": { + "python_implementation": "CPython", + "host": "pajeh", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "cpu": { + "hardware": "unknown", + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" + }, + "release": "4.9.0-3-amd64", + "python_version": "2.7.13", + "machine": "x86_64", + "python_implementation_version": "2.7.13", + "processor": "", + "node": "pajeh", + "system": "Linux", + "python_compiler": "GCC 6.3.0 20170118" + }, + "commit_info": { + "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", + "time": "2017-09-01T09:50:20-03:00", + "project": "soledad", + "dirty": true, + "branch": "8856", + "author_time": "2017-09-01T09:50:11-03:00" + }, + "datetime": "2017-09-01T15:37:48.278845", + "benchmarks": [ + { + "param": null, + "extra_info": { + "cpu_percent": 27.9, + "doc": "" + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k", + "group": "test_legacy_vs_blobs_10_1000k", + "params": null, + "name": "test_legacy_10_1000k", + "stats": { + "mean": 8.937849223613739, + "min": 8.711518049240112, + "outliers": "2;0", + "total": 35.751396894454956, + "iqr_outliers": 0, + "iterations": 1, + "ops": 0.11188374014612, + "rounds": 4, + "stddev": 0.20914882546041347, + "hd15iqr": 9.213748931884766, + "median": 8.913064956665039, + "stddev_outliers": 2, + "max": 9.213748931884766, + "q1": 8.794075012207031, + "ld15iqr": 8.711518049240112, + "q3": 9.081623435020447, + "iqr": 0.2875484228134155 + }, + "options": { + "disable_gc": false, + "min_time": 5e-06, + "min_rounds": 5, + "timer": "time", + "max_time": 1.0, + "warmup": false + } + }, + { + "param": null, + "extra_info": { + "cpu_percent": 31.7, + "doc": "" + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", + "group": "test_legacy_vs_blobs_10_1000k", + "params": null, + "name": "test_blobs_10_1000k", + "stats": { + "mean": 3.1771076917648315, + "min": 3.077280044555664, + "outliers": "1;0", + "total": 12.708430767059326, + "iqr_outliers": 0, + "iterations": 1, + "ops": 0.3147516851858793, + "rounds": 4, + "stddev": 0.0741566234351459, + "hd15iqr": 3.2503108978271484, + "median": 3.190419912338257, + "stddev_outliers": 1, + "max": 3.2503108978271484, + "q1": 3.1237324476242065, + "ld15iqr": 3.077280044555664, + "q3": 3.2304829359054565, + "iqr": 0.10675048828125 + }, + "options": { + "disable_gc": false, + "min_time": 5e-06, + "min_rounds": 5, + "timer": "time", + "max_time": 1.0, + "warmup": false + } + } + ] +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json new file mode 100644 index 00000000..0697cd6b --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json @@ -0,0 +1,111 @@ +{ + "benchmarks": [ + { + "options": { + "disable_gc": false, + "timer": "time", + "min_rounds": 5, + "min_time": 5e-06, + "max_time": 1.0, + "warmup": false + }, + "params": null, + "extra_info": { + "cpu_percent": 25.8, + "doc": "" + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k", + "group": "test_legacy_vs_blobs_1_10000k", + "name": "test_legacy_1_10000k", + "param": null, + "stats": { + "hd15iqr": 9.522547006607056, + "stddev": 0.04062225700829493, + "ops": 0.10556240321945518, + "iterations": 1, + "min": 9.423872947692871, + "total": 37.89227867126465, + "stddev_outliers": 2, + "max": 9.522547006607056, + "q1": 9.445196866989136, + "mean": 9.473069667816162, + "iqr_outliers": 0, + "rounds": 4, + "iqr": 0.055745601654052734, + "median": 9.47292935848236, + "q3": 9.500942468643188, + "ld15iqr": 9.423872947692871, + "outliers": "2;0" + } + }, + { + "options": { + "disable_gc": false, + "timer": "time", + "min_rounds": 5, + "min_time": 5e-06, + "max_time": 1.0, + "warmup": false + }, + "params": null, + "extra_info": { + "cpu_percent": 33.3, + "doc": "" + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", + "group": "test_legacy_vs_blobs_1_10000k", + "name": "test_blobs_1_10000k", + "param": null, + "stats": { + "hd15iqr": 3.758697032928467, + "stddev": 0.619090853876154, + "ops": 0.3292143381447483, + "iterations": 1, + "min": 2.344238042831421, + "total": 12.150139093399048, + "stddev_outliers": 2, + "max": 3.758697032928467, + "q1": 2.5475854873657227, + "mean": 3.037534773349762, + "iqr_outliers": 0, + "rounds": 4, + "iqr": 0.9798985719680786, + "median": 3.02360200881958, + "q3": 3.5274840593338013, + "ld15iqr": 2.344238042831421, + "outliers": "2;0" + } + } + ], + "version": "3.1.1", + "commit_info": { + "author_time": "2017-09-01T14:16:21-03:00", + "project": "soledad", + "id": "7a5698ad8c1457460d95cbd6dd237fb96e218d43", + "time": "2017-09-01T14:16:26-03:00", + "dirty": true, + "branch": "8856" + }, + "machine_info": { + "host": "pajeh", + "python_version": "2.7.13", + "python_compiler": "GCC 6.3.0 20170118", + "system": "Linux", + "python_implementation": "CPython", + "python_implementation_version": "2.7.13", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "machine": "x86_64", + "release": "4.9.0-3-amd64", + "processor": "", + "node": "pajeh", + "cpu": { + "hardware": "unknown", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", + "vendor_id": "GenuineIntel" + } + }, + "datetime": "2017-09-01T22:20:37.591520" +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json new file mode 100644 index 00000000..909e6f21 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json @@ -0,0 +1,73 @@ +{ + "version": "3.1.1", + "machine_info": { + "python_version": "2.7.13", + "processor": "", + "node": "pajeh", + "python_compiler": "GCC 6.3.0 20170118", + "python_implementation_version": "2.7.13", + "python_implementation": "CPython", + "machine": "x86_64", + "system": "Linux", + "host": "pajeh", + "cpu": { + "hardware": "unknown", + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" + }, + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "release": "4.9.0-3-amd64" + }, + "commit_info": { + "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", + "time": "2017-09-02T02:38:57-03:00", + "dirty": true, + "branch": "8856", + "author_time": "2017-09-02T02:38:57-03:00", + "project": "soledad" + }, + "benchmarks": [ + { + "param": null, + "stats": { + "ld15iqr": 50.158271074295044, + "total": 290.0146920681, + "iterations": 1, + "stddev": 19.375312299153435, + "stddev_outliers": 2, + "mean": 72.503673017025, + "q1": 56.99843108654022, + "rounds": 4, + "q3": 88.00891494750977, + "iqr": 31.010483860969543, + "max": 93.9663679599762, + "iqr_outliers": 0, + "hd15iqr": 93.9663679599762, + "median": 72.94502651691437, + "ops": 0.013792404693279255, + "min": 50.158271074295044, + "outliers": "2;0" + }, + "params": null, + "name": "test_blobs_1000_10k", + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", + "group": "test_legacy_vs_blobs_1000_10k", + "options": { + "timer": "time", + "disable_gc": false, + "min_rounds": 5, + "min_time": 5e-06, + "warmup": false, + "max_time": 1.0 + }, + "extra_info": { + "cpu_percent": 17.4, + "doc": "" + } + } + ], + "datetime": "2017-09-02T05:51:50.400601" +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json new file mode 100644 index 00000000..48f0e077 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json @@ -0,0 +1,73 @@ +{ + "machine_info": { + "host": "pajeh", + "python_compiler": "GCC 6.3.0 20170118", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "python_implementation": "CPython", + "processor": "", + "cpu": { + "hardware": "unknown", + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" + }, + "node": "pajeh", + "system": "Linux", + "release": "4.9.0-3-amd64", + "python_version": "2.7.13", + "machine": "x86_64", + "python_implementation_version": "2.7.13" + }, + "commit_info": { + "branch": "8856", + "author_time": "2017-09-02T02:38:57-03:00", + "time": "2017-09-02T02:38:57-03:00", + "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", + "project": "soledad", + "dirty": true + }, + "version": "3.1.1", + "benchmarks": [ + { + "group": "test_legacy_vs_blobs_100_100k", + "params": null, + "name": "test_blobs_100_100k", + "stats": { + "total": 20.738121271133423, + "stddev_outliers": 2, + "stddev": 0.9161744731162039, + "hd15iqr": 6.378293991088867, + "rounds": 4, + "max": 6.378293991088867, + "median": 5.054657578468323, + "iqr": 1.3696125745773315, + "q3": 5.8693366050720215, + "q1": 4.49972403049469, + "mean": 5.184530317783356, + "ops": 0.1928815029916827, + "iqr_outliers": 0, + "min": 4.25051212310791, + "iterations": 1, + "ld15iqr": 4.25051212310791, + "outliers": "2;0" + }, + "options": { + "timer": "time", + "disable_gc": false, + "warmup": false, + "max_time": 1.0, + "min_rounds": 5, + "min_time": 5e-06 + }, + "param": null, + "extra_info": { + "cpu_percent": 24.7, + "doc": "" + }, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k" + } + ], + "datetime": "2017-09-02T05:56:28.497384" +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json new file mode 100644 index 00000000..529cca8c --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json @@ -0,0 +1,73 @@ +{ + "machine_info": { + "python_compiler": "GCC 6.3.0 20170118", + "python_implementation_version": "2.7.13", + "node": "pajeh", + "machine": "x86_64", + "host": "pajeh", + "cpu": { + "hardware": "unknown", + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" + }, + "system": "Linux", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "release": "4.9.0-3-amd64", + "python_version": "2.7.13", + "processor": "", + "python_implementation": "CPython" + }, + "commit_info": { + "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", + "time": "2017-09-02T02:38:57-03:00", + "dirty": true, + "branch": "8856", + "author_time": "2017-09-02T02:38:57-03:00", + "project": "soledad" + }, + "version": "3.1.1", + "benchmarks": [ + { + "stats": { + "ops": 0.4919825789326136, + "stddev": 0.1042862860397157, + "mean": 2.032592296600342, + "stddev_outliers": 2, + "q3": 2.112483024597168, + "q1": 1.9527015686035156, + "rounds": 4, + "max": 2.1607320308685303, + "hd15iqr": 2.1607320308685303, + "iqr_outliers": 0, + "ld15iqr": 1.91776704788208, + "iterations": 1, + "median": 2.0259350538253784, + "iqr": 0.15978145599365234, + "total": 8.130369186401367, + "outliers": "2;0", + "min": 1.91776704788208 + }, + "param": null, + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", + "params": null, + "name": "test_blobs_10_1000k", + "group": "test_legacy_vs_blobs_10_1000k", + "options": { + "disable_gc": false, + "min_rounds": 5, + "timer": "time", + "warmup": false, + "min_time": 5e-06, + "max_time": 1.0 + }, + "extra_info": { + "cpu_percent": 36.3, + "doc": "" + } + } + ], + "datetime": "2017-09-02T05:57:48.679847" +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json new file mode 100644 index 00000000..30d362ea --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json @@ -0,0 +1,73 @@ +{ + "machine_info": { + "python_compiler": "GCC 6.3.0 20170118", + "host": "pajeh", + "python_build": [ + "default", + "Jan 19 2017 14:48:08" + ], + "cpu": { + "hardware": "unknown", + "vendor_id": "GenuineIntel", + "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" + }, + "node": "pajeh", + "system": "Linux", + "release": "4.9.0-3-amd64", + "python_implementation": "CPython", + "processor": "", + "python_implementation_version": "2.7.13", + "python_version": "2.7.13", + "machine": "x86_64" + }, + "commit_info": { + "branch": "8856", + "author_time": "2017-09-02T02:38:57-03:00", + "dirty": true, + "time": "2017-09-02T02:38:57-03:00", + "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", + "project": "soledad" + }, + "version": "3.1.1", + "benchmarks": [ + { + "stats": { + "max": 2.1960339546203613, + "median": 2.1957188844680786, + "stddev_outliers": 1, + "q3": 2.19601047039032, + "iqr_outliers": 0, + "q1": 2.194662928581238, + "ld15iqr": 2.1938750743865967, + "ops": 0.4555109930218146, + "stddev": 0.001009686487541645, + "hd15iqr": 2.1960339546203613, + "rounds": 4, + "iterations": 1, + "total": 8.781346797943115, + "mean": 2.195336699485779, + "iqr": 0.0013475418090820312, + "outliers": "1;0", + "min": 2.1938750743865967 + }, + "options": { + "warmup": false, + "max_time": 1.0, + "timer": "time", + "min_rounds": 5, + "min_time": 5e-06, + "disable_gc": false + }, + "group": "test_legacy_vs_blobs_1_10000k", + "params": null, + "name": "test_blobs_1_10000k", + "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", + "param": null, + "extra_info": { + "cpu_percent": 40.2, + "doc": "" + } + } + ], + "datetime": "2017-09-02T06:00:45.837579" +} \ No newline at end of file diff --git a/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png new file mode 100644 index 00000000..99080238 Binary files /dev/null and b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png differ diff --git a/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py new file mode 100755 index 00000000..57a42376 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +# Plot bars comparing different implementations of mail pipeline. +# +# This script can be improved to account for arbitrary number of data sets, but +# it is not doing it right now. + +import json +import numpy as np +import matplotlib.pyplot as plt + +# make a prettier graph +from mpltools import style +style.use('ggplot') + +OUTPUT_FILENAME = 'legacy-vs-blobs.png' + +# each value below will generate one bar for each for each (amount, size) pair. +# The script expects to find files in ./data/SET/ for each set of +# implementations. +# +# The baseline values will be the legacy results in ./data/no-cache/. + +graphs = [ + '1000_10k', + '100_100k', + '10_1000k', + '1_10000k', +] + + +# the JSON structure returned by the following function is ugly, but the +# original JSONs are even uglier, so this is here just to make the life of the +# script easier. +# +# We want to have something like: +# +# data[variation][graph][implementation] = +# +# Where: +# +# - variation is one data set under ./data (i.e. no-cache, cache, persistent, +# etc). +# - graph is one of the values in graphs variable above. +# - implementation is either legacy or blobs (we just need legacy for the +# no-cache variation, as that is the one we are using as baseline. + +def get_data(): + folders = ['cache', 'no-cache', 'persistent'] + data = {} + for folder in folders: + data[folder] = {} + for graph in graphs: + with open('data/%s/%s.json' % (folder, graph)) as f: + d = json.loads(f.read()) + benchmarks = d['benchmarks'] + data[folder][graph] = {} + for t in ['blobs', 'legacy']: + result = filter(lambda b: t in b['name'], benchmarks) + if result: + result = result.pop() + data[folder][graph][t] = result['stats'] + return data + + +def plot_data(data): + + N = 4 + + # this is our baseline (i.e. legacy / legacy) + absolutes = (1, 1, 1, 1) + + ind = np.arange(N) # the x locations for the groups + width = 0.20 # the width of the bars + + fig, ax = plt.subplots() + rects1 = ax.bar(ind, absolutes, width) + + # for each graph, calculate the ratios + ratios = {'no-cache': [], 'cache': [], 'persistent': []} + for graph in graphs: + legacy = data['no-cache'][graph]['legacy']['mean'] + + # calculate ratios for no-cache / legacy + ratio = data['no-cache'][graph]['blobs']['mean'] / legacy + ratios['no-cache'].append(ratio) + + # calculate ratios for cache / legacy + ratio = data['cache'][graph]['blobs']['mean'] / legacy + ratios['cache'].append(ratio) + + # calculate ratios for persistent / legacy + ratio = data['persistent'][graph]['blobs']['mean'] / legacy + ratios['persistent'].append(ratio) + + # create the boxes with the ratios + nocache = tuple(ratios['no-cache']) + rects2 = ax.bar(ind + width, nocache, width) + + cache = tuple(ratios['cache']) + rects3 = ax.bar(ind + (2 * width), cache, width) + + persistent = tuple(ratios['persistent']) + rects4 = ax.bar(ind + (3 * width), persistent, width) + + # add some text for labels, title and axes ticks + ax.set_ylabel('Ratio of time (legacy is baseline)') + ax.set_xlabel('Amount and size of email messages') + ax.set_title('Inbox loading time: legacy vs blobs mail pipeline') + ax.set_xticks(ind + (1.5 * width)) + ax.set_xticklabels( + tuple(map(lambda name: name.replace('_', ' x '), graphs))) + + ax.legend( + (rects1[0], rects2[0], rects3[0], rects4[0]), + ('legacy', 'blobs', 'blobs + session cache', + 'blobs + session cache + persistent http')) + # ax.grid() + + plt.savefig(OUTPUT_FILENAME) + # plt.show() + + +if __name__ == '__main__': + data = get_data() + plot_data(data) diff --git a/scripts/benchmark/legacy-vs-blobs/makefile b/scripts/benchmark/legacy-vs-blobs/makefile new file mode 100644 index 00000000..649e181c --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/makefile @@ -0,0 +1,14 @@ +VIRTUALENV_NAME = plot +VIRTUALENV_ROOT = ~/.virtualenvs/$(VIRTUALENV_NAME) + +all: graph + +env: + if [ ! -d $(VIRTUALENV_ROOT) ]; then \ + mkdir -p $$(dirname $(VIRTUALENV_ROOT)); \ + virtualenv $(VIRTUALENV_ROOT); \ + $(VIRTUALENV_ROOT)/bin/pip install requirements.pip; \ + fi + +graph: env + $(VIRTUALENV_ROOT)/bin/python ./legacy-vs-blobs.py diff --git a/scripts/benchmark/legacy-vs-blobs/requirements.pip b/scripts/benchmark/legacy-vs-blobs/requirements.pip new file mode 100644 index 00000000..db5d81e0 --- /dev/null +++ b/scripts/benchmark/legacy-vs-blobs/requirements.pip @@ -0,0 +1,2 @@ +matplotlib +numpy diff --git a/scripts/legacy-vs-blobs/README.rst b/scripts/legacy-vs-blobs/README.rst deleted file mode 100644 index 3eedb3a4..00000000 --- a/scripts/legacy-vs-blobs/README.rst +++ /dev/null @@ -1,43 +0,0 @@ -Ploggin Legacy vs Blobs results -=============================== - -This script produces a bar graph comparing different implementations of the -mail pipeline (legacy, blobs, blobs with session cache, blobs with session -cache and persistent http connections) for different amount/sizes of incoming -data (1 x 10M, 10 x 1M, 100 x 100K, 1000 x 10K). - -Data acquisition ----------------- - -Data was gathered through different runs of the code in -`soledad/tests/benchmarks/test_legacy_vs_blobs.py`. As these tests are skipped -by default, you have to (1) remove the @pytest.mark.skip decorator and (2) -remove the second call to pytest in `tox.ini` that would cause a second run of -the tests for measuring memory. This script only plots time, for now. - -Once you done the above, chdir into `soledad/tests` and do something like this, -for each test group: - - tox -e benchmarks -- -m benchmark_test_legacy_vs_blobs_10_1000k --benchmark-autosave - -Each run of tox as above produces a JSON file in `soledad/tests/.benchmarks` -with stats. Each file has to be copied in a subdirectory of the `data/` -directory of the folder where this script lies. Code has to be adapted for each -extra set of results added to the `data/` directory. - -Currently, we are comparing 4 implementation variations: - - - data/no-cache: contains legacy as baseline and blobs as the current master - implementation of blobs. - - data/cache: adds twisted session cache to blobs connections. - - data/persistent: adds session cache and persistent HTTP connections. - -Plotting --------- - -Once you have all your data in subdirectories of the `data/` directory and the -script is correctly tuned for your data, running `make` should be enough to -create a virtualenvironment with dependencies and plotting data. - -By default, the script will generate a file called `./legacy-vs-blobs.png` in -the current directory with the plot of the data. diff --git a/scripts/legacy-vs-blobs/data/cache/1000_10k.json b/scripts/legacy-vs-blobs/data/cache/1000_10k.json deleted file mode 100644 index 5be17758..00000000 --- a/scripts/legacy-vs-blobs/data/cache/1000_10k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "datetime": "2017-09-02T04:22:20.074434", - "machine_info": { - "python_implementation": "CPython", - "python_compiler": "GCC 6.3.0 20170118", - "machine": "x86_64", - "cpu": { - "hardware": "unknown", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", - "vendor_id": "GenuineIntel" - }, - "host": "pajeh", - "system": "Linux", - "processor": "", - "python_implementation_version": "2.7.13", - "release": "4.9.0-3-amd64", - "python_version": "2.7.13", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "node": "pajeh" - }, - "commit_info": { - "author_time": "2017-09-02T00:27:58-03:00", - "time": "2017-09-02T00:39:13-03:00", - "dirty": true, - "project": "soledad", - "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", - "branch": "8856" - }, - "benchmarks": [ - { - "name": "test_legacy_1000_10k", - "stats": { - "median": 48.28695607185364, - "rounds": 4, - "ld15iqr": 36.51543307304382, - "iqr_outliers": 0, - "stddev": 10.431966763848191, - "q1": 40.382596015930176, - "min": 36.51543307304382, - "q3": 56.539175629615784, - "total": 193.84354329109192, - "iterations": 1, - "max": 60.75419807434082, - "stddev_outliers": 2, - "mean": 48.46088582277298, - "ops": 0.020635198532216573, - "hd15iqr": 60.75419807434082, - "outliers": "2;0", - "iqr": 16.156579613685608 - }, - "extra_info": { - "cpu_percent": 26.6, - "doc": "" - }, - "params": null, - "param": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k", - "group": "test_legacy_vs_blobs_1000_10k", - "options": { - "timer": "time", - "min_rounds": 5, - "warmup": false, - "disable_gc": false, - "min_time": 5e-06, - "max_time": 1.0 - } - }, - { - "name": "test_blobs_1000_10k", - "stats": { - "median": 87.17049705982208, - "rounds": 4, - "ld15iqr": 65.08280301094055, - "iqr_outliers": 0, - "stddev": 19.00300388729206, - "q1": 72.67573952674866, - "min": 65.08280301094055, - "q3": 101.80426609516144, - "total": 348.9600112438202, - "iterations": 1, - "max": 109.53621411323547, - "stddev_outliers": 2, - "mean": 87.24000281095505, - "ops": 0.01146263145092914, - "hd15iqr": 109.53621411323547, - "outliers": "2;0", - "iqr": 29.12852656841278 - }, - "extra_info": { - "cpu_percent": 19.4, - "doc": "" - }, - "params": null, - "param": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", - "group": "test_legacy_vs_blobs_1000_10k", - "options": { - "timer": "time", - "min_rounds": 5, - "warmup": false, - "disable_gc": false, - "min_time": 5e-06, - "max_time": 1.0 - } - } - ], - "version": "3.1.1" -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/cache/100_100k.json b/scripts/legacy-vs-blobs/data/cache/100_100k.json deleted file mode 100644 index 17fc598b..00000000 --- a/scripts/legacy-vs-blobs/data/cache/100_100k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "benchmarks": [ - { - "params": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k", - "extra_info": { - "doc": "", - "cpu_percent": 25.1 - }, - "group": "test_legacy_vs_blobs_100_100k", - "stats": { - "q1": 11.39308750629425, - "rounds": 4, - "q3": 12.677581906318665, - "hd15iqr": 12.946987867355347, - "median": 12.090978026390076, - "stddev_outliers": 2, - "mean": 12.035334706306458, - "iqr_outliers": 0, - "iterations": 1, - "total": 48.14133882522583, - "min": 11.012394905090332, - "outliers": "2;0", - "ops": 0.08308867384269794, - "ld15iqr": 11.012394905090332, - "stddev": 0.833654348956266, - "iqr": 1.284494400024414, - "max": 12.946987867355347 - }, - "name": "test_legacy_100_100k", - "param": null, - "options": { - "min_rounds": 5, - "timer": "time", - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false, - "disable_gc": false - } - }, - { - "params": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k", - "extra_info": { - "doc": "", - "cpu_percent": 24.9 - }, - "group": "test_legacy_vs_blobs_100_100k", - "stats": { - "q1": 5.015861988067627, - "rounds": 4, - "q3": 6.166973114013672, - "hd15iqr": 6.362173080444336, - "median": 5.64124858379364, - "stddev_outliers": 2, - "mean": 5.591417551040649, - "iqr_outliers": 0, - "iterations": 1, - "total": 22.365670204162598, - "min": 4.7209999561309814, - "outliers": "2;0", - "ops": 0.1788455236747405, - "ld15iqr": 4.7209999561309814, - "stddev": 0.7246033659461454, - "iqr": 1.151111125946045, - "max": 6.362173080444336 - }, - "name": "test_blobs_100_100k", - "param": null, - "options": { - "min_rounds": 5, - "timer": "time", - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false, - "disable_gc": false - } - } - ], - "datetime": "2017-09-02T03:53:29.055256", - "version": "3.1.1", - "machine_info": { - "node": "pajeh", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "python_version": "2.7.13", - "machine": "x86_64", - "python_implementation": "CPython", - "cpu": { - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", - "hardware": "unknown", - "vendor_id": "GenuineIntel" - }, - "python_implementation_version": "2.7.13", - "system": "Linux", - "host": "pajeh", - "python_compiler": "GCC 6.3.0 20170118", - "release": "4.9.0-3-amd64", - "processor": "" - }, - "commit_info": { - "project": "soledad", - "time": "2017-09-02T00:39:13-03:00", - "author_time": "2017-09-02T00:27:58-03:00", - "branch": "8856", - "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", - "dirty": true - } -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/cache/10_1000k.json b/scripts/legacy-vs-blobs/data/cache/10_1000k.json deleted file mode 100644 index 1a733f2d..00000000 --- a/scripts/legacy-vs-blobs/data/cache/10_1000k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "benchmarks": [ - { - "params": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k", - "extra_info": { - "doc": "", - "cpu_percent": 27.7 - }, - "group": "test_legacy_vs_blobs_10_1000k", - "stats": { - "q1": 8.741152048110962, - "rounds": 4, - "q3": 8.863584876060486, - "median": 8.83247196674347, - "stddev_outliers": 1, - "iqr": 0.12243282794952393, - "mean": 8.802368462085724, - "iqr_outliers": 0, - "iterations": 1, - "total": 35.209473848342896, - "min": 8.67733097076416, - "outliers": "1;0", - "ops": 0.11360578738634734, - "ld15iqr": 8.67733097076416, - "stddev": 0.08786766595777334, - "hd15iqr": 8.867198944091797, - "max": 8.867198944091797 - }, - "name": "test_legacy_10_1000k", - "param": null, - "options": { - "min_rounds": 5, - "timer": "time", - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false, - "disable_gc": false - } - }, - { - "params": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", - "extra_info": { - "doc": "", - "cpu_percent": 37.0 - }, - "group": "test_legacy_vs_blobs_10_1000k", - "stats": { - "q1": 1.8672034740447998, - "rounds": 4, - "q3": 1.9366559982299805, - "median": 1.8831886053085327, - "stddev_outliers": 1, - "iqr": 0.06945252418518066, - "mean": 1.9019297361373901, - "iqr_outliers": 0, - "iterations": 1, - "total": 7.6077189445495605, - "min": 1.8517439365386963, - "outliers": "1;0", - "ops": 0.5257817788952025, - "ld15iqr": 1.8517439365386963, - "stddev": 0.06029736288568101, - "hd15iqr": 1.9895977973937988, - "max": 1.9895977973937988 - }, - "name": "test_blobs_10_1000k", - "param": null, - "options": { - "min_rounds": 5, - "timer": "time", - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false, - "disable_gc": false - } - } - ], - "datetime": "2017-09-02T03:48:24.650044", - "version": "3.1.1", - "machine_info": { - "python_implementation_version": "2.7.13", - "system": "Linux", - "host": "pajeh", - "python_compiler": "GCC 6.3.0 20170118", - "release": "4.9.0-3-amd64", - "python_implementation": "CPython", - "processor": "", - "node": "pajeh", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "python_version": "2.7.13", - "machine": "x86_64", - "cpu": { - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", - "hardware": "unknown", - "vendor_id": "GenuineIntel" - } - }, - "commit_info": { - "author_time": "2017-09-02T00:27:58-03:00", - "time": "2017-09-02T00:39:13-03:00", - "project": "soledad", - "branch": "8856", - "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", - "dirty": true - } -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/cache/1_10000k.json b/scripts/legacy-vs-blobs/data/cache/1_10000k.json deleted file mode 100644 index 2d970c6e..00000000 --- a/scripts/legacy-vs-blobs/data/cache/1_10000k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "datetime": "2017-09-02T03:46:17.875002", - "machine_info": { - "processor": "", - "python_implementation_version": "2.7.13", - "python_version": "2.7.13", - "machine": "x86_64", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "python_implementation": "CPython", - "python_compiler": "GCC 6.3.0 20170118", - "cpu": { - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", - "hardware": "unknown" - }, - "release": "4.9.0-3-amd64", - "system": "Linux", - "host": "pajeh", - "node": "pajeh" - }, - "version": "3.1.1", - "commit_info": { - "time": "2017-09-02T00:39:13-03:00", - "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8", - "project": "soledad", - "author_time": "2017-09-02T00:27:58-03:00", - "branch": "8856", - "dirty": true - }, - "benchmarks": [ - { - "extra_info": { - "cpu_percent": 25.8, - "doc": "" - }, - "name": "test_legacy_1_10000k", - "params": null, - "options": { - "max_time": 1.0, - "min_rounds": 5, - "min_time": 5e-06, - "warmup": false, - "timer": "time", - "disable_gc": false - }, - "group": "test_legacy_vs_blobs_1_10000k", - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k", - "param": null, - "stats": { - "mean": 9.607597470283508, - "ld15iqr": 9.551655054092407, - "iqr_outliers": 0, - "outliers": "1;0", - "q3": 9.663439989089966, - "q1": 9.55175495147705, - "iqr": 0.11168503761291504, - "total": 38.43038988113403, - "rounds": 4, - "median": 9.563774466514587, - "stddev_outliers": 1, - "min": 9.551655054092407, - "hd15iqr": 9.751185894012451, - "ops": 0.10408429402803564, - "stddev": 0.09638854573531173, - "iterations": 1, - "max": 9.751185894012451 - } - }, - { - "extra_info": { - "cpu_percent": 40.2, - "doc": "" - }, - "name": "test_blobs_1_10000k", - "params": null, - "options": { - "max_time": 1.0, - "min_rounds": 5, - "min_time": 5e-06, - "warmup": false, - "timer": "time", - "disable_gc": false - }, - "group": "test_legacy_vs_blobs_1_10000k", - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", - "param": null, - "stats": { - "mean": 2.179791212081909, - "ld15iqr": 2.1572468280792236, - "iqr_outliers": 0, - "outliers": "2;0", - "q3": 2.193374514579773, - "q1": 2.1662079095840454, - "iqr": 0.02716660499572754, - "total": 8.719164848327637, - "rounds": 4, - "median": 2.1782270669937134, - "stddev_outliers": 2, - "min": 2.1572468280792236, - "hd15iqr": 2.2054638862609863, - "ops": 0.45875953369171735, - "stddev": 0.01992429175453132, - "iterations": 1, - "max": 2.2054638862609863 - } - } - ] -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/no-cache/1000_10k.json b/scripts/legacy-vs-blobs/data/no-cache/1000_10k.json deleted file mode 100644 index a585ad3a..00000000 --- a/scripts/legacy-vs-blobs/data/no-cache/1000_10k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "commit_info": { - "dirty": true, - "branch": "8856", - "author_time": "2017-09-01T09:50:11-03:00", - "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", - "time": "2017-09-01T09:50:20-03:00", - "project": "soledad" - }, - "machine_info": { - "node": "pajeh", - "processor": "", - "machine": "x86_64", - "system": "Linux", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "python_implementation": "CPython", - "python_implementation_version": "2.7.13", - "host": "pajeh", - "cpu": { - "hardware": "unknown", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", - "vendor_id": "GenuineIntel" - }, - "release": "4.9.0-3-amd64", - "python_compiler": "GCC 6.3.0 20170118", - "python_version": "2.7.13" - }, - "datetime": "2017-09-01T16:28:31.243069", - "version": "3.1.1", - "benchmarks": [ - { - "stats": { - "ld15iqr": 38.49253582954407, - "outliers": "2;0", - "median": 52.51530361175537, - "iterations": 1, - "q3": 65.07869005203247, - "iqr_outliers": 0, - "rounds": 4, - "ops": 0.018550090252706083, - "total": 215.63237404823303, - "mean": 53.90809351205826, - "max": 72.10923099517822, - "iqr": 22.341193079948425, - "stddev_outliers": 2, - "q1": 42.737496972084045, - "min": 38.49253582954407, - "hd15iqr": 72.10923099517822, - "stddev": 14.537598498896823 - }, - "name": "test_legacy_1000_10k", - "param": null, - "group": "test_legacy_vs_blobs_1000_10k", - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k", - "params": null, - "extra_info": { - "doc": "", - "cpu_percent": 26.9 - }, - "options": { - "warmup": false, - "max_time": 1.0, - "min_time": 5e-06, - "min_rounds": 5, - "timer": "time", - "disable_gc": false - } - }, - { - "stats": { - "ld15iqr": 176.17668509483337, - "outliers": "2;0", - "median": 192.90700149536133, - "iterations": 1, - "q3": 206.93408203125, - "iqr_outliers": 0, - "rounds": 4, - "ops": 0.005138941398078599, - "total": 778.3704249858856, - "mean": 194.5926062464714, - "max": 216.3797369003296, - "iqr": 24.68295156955719, - "stddev_outliers": 2, - "q1": 182.2511304616928, - "min": 176.17668509483337, - "hd15iqr": 216.3797369003296, - "stddev": 16.94586147142875 - }, - "name": "test_blobs_1000_10k", - "param": null, - "group": "test_legacy_vs_blobs_1000_10k", - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", - "params": null, - "extra_info": { - "doc": "", - "cpu_percent": 10.9 - }, - "options": { - "warmup": false, - "max_time": 1.0, - "min_time": 5e-06, - "min_rounds": 5, - "timer": "time", - "disable_gc": false - } - } - ] -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/no-cache/100_100k.json b/scripts/legacy-vs-blobs/data/no-cache/100_100k.json deleted file mode 100644 index 2cbbe638..00000000 --- a/scripts/legacy-vs-blobs/data/no-cache/100_100k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "commit_info": { - "time": "2017-09-01T09:50:20-03:00", - "branch": "8856", - "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", - "project": "soledad", - "dirty": true, - "author_time": "2017-09-01T09:50:11-03:00" - }, - "version": "3.1.1", - "benchmarks": [ - { - "stats": { - "stddev_outliers": 1, - "ops": 0.08172728164268425, - "stddev": 1.0083003646390902, - "hd15iqr": 13.605333805084229, - "total": 48.943264961242676, - "iqr": 1.4744374752044678, - "iqr_outliers": 0, - "q1": 11.498597502708435, - "q3": 12.973034977912903, - "ld15iqr": 11.303098917007446, - "min": 11.303098917007446, - "iterations": 1, - "max": 13.605333805084229, - "outliers": "1;0", - "rounds": 4, - "median": 12.0174161195755, - "mean": 12.235816240310669 - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k", - "options": { - "warmup": false, - "min_time": 5e-06, - "max_time": 1.0, - "disable_gc": false, - "min_rounds": 5, - "timer": "time" - }, - "param": null, - "name": "test_legacy_100_100k", - "group": "test_legacy_vs_blobs_100_100k", - "extra_info": { - "doc": "", - "cpu_percent": 25.2 - }, - "params": null - }, - { - "stats": { - "stddev_outliers": 2, - "ops": 0.052999047893223836, - "stddev": 0.6994077398028641, - "hd15iqr": 19.686198949813843, - "total": 75.47305393218994, - "iqr": 1.0998969078063965, - "iqr_outliers": 0, - "q1": 18.318315029144287, - "q3": 19.418211936950684, - "ld15iqr": 18.07945704460144, - "min": 18.07945704460144, - "iterations": 1, - "max": 19.686198949813843, - "outliers": "2;0", - "rounds": 4, - "median": 18.85369896888733, - "mean": 18.868263483047485 - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k", - "options": { - "warmup": false, - "min_time": 5e-06, - "max_time": 1.0, - "disable_gc": false, - "min_rounds": 5, - "timer": "time" - }, - "param": null, - "name": "test_blobs_100_100k", - "group": "test_legacy_vs_blobs_100_100k", - "extra_info": { - "doc": "", - "cpu_percent": 14.7 - }, - "params": null - } - ], - "datetime": "2017-09-01T15:44:20.745800", - "machine_info": { - "host": "pajeh", - "python_compiler": "GCC 6.3.0 20170118", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "python_implementation": "CPython", - "cpu": { - "hardware": "unknown", - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" - }, - "node": "pajeh", - "release": "4.9.0-3-amd64", - "processor": "", - "python_implementation_version": "2.7.13", - "python_version": "2.7.13", - "system": "Linux", - "machine": "x86_64" - } -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/no-cache/10_1000k.json b/scripts/legacy-vs-blobs/data/no-cache/10_1000k.json deleted file mode 100644 index 778e5c63..00000000 --- a/scripts/legacy-vs-blobs/data/no-cache/10_1000k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "version": "3.1.1", - "machine_info": { - "python_implementation": "CPython", - "host": "pajeh", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "cpu": { - "hardware": "unknown", - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" - }, - "release": "4.9.0-3-amd64", - "python_version": "2.7.13", - "machine": "x86_64", - "python_implementation_version": "2.7.13", - "processor": "", - "node": "pajeh", - "system": "Linux", - "python_compiler": "GCC 6.3.0 20170118" - }, - "commit_info": { - "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8", - "time": "2017-09-01T09:50:20-03:00", - "project": "soledad", - "dirty": true, - "branch": "8856", - "author_time": "2017-09-01T09:50:11-03:00" - }, - "datetime": "2017-09-01T15:37:48.278845", - "benchmarks": [ - { - "param": null, - "extra_info": { - "cpu_percent": 27.9, - "doc": "" - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k", - "group": "test_legacy_vs_blobs_10_1000k", - "params": null, - "name": "test_legacy_10_1000k", - "stats": { - "mean": 8.937849223613739, - "min": 8.711518049240112, - "outliers": "2;0", - "total": 35.751396894454956, - "iqr_outliers": 0, - "iterations": 1, - "ops": 0.11188374014612, - "rounds": 4, - "stddev": 0.20914882546041347, - "hd15iqr": 9.213748931884766, - "median": 8.913064956665039, - "stddev_outliers": 2, - "max": 9.213748931884766, - "q1": 8.794075012207031, - "ld15iqr": 8.711518049240112, - "q3": 9.081623435020447, - "iqr": 0.2875484228134155 - }, - "options": { - "disable_gc": false, - "min_time": 5e-06, - "min_rounds": 5, - "timer": "time", - "max_time": 1.0, - "warmup": false - } - }, - { - "param": null, - "extra_info": { - "cpu_percent": 31.7, - "doc": "" - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", - "group": "test_legacy_vs_blobs_10_1000k", - "params": null, - "name": "test_blobs_10_1000k", - "stats": { - "mean": 3.1771076917648315, - "min": 3.077280044555664, - "outliers": "1;0", - "total": 12.708430767059326, - "iqr_outliers": 0, - "iterations": 1, - "ops": 0.3147516851858793, - "rounds": 4, - "stddev": 0.0741566234351459, - "hd15iqr": 3.2503108978271484, - "median": 3.190419912338257, - "stddev_outliers": 1, - "max": 3.2503108978271484, - "q1": 3.1237324476242065, - "ld15iqr": 3.077280044555664, - "q3": 3.2304829359054565, - "iqr": 0.10675048828125 - }, - "options": { - "disable_gc": false, - "min_time": 5e-06, - "min_rounds": 5, - "timer": "time", - "max_time": 1.0, - "warmup": false - } - } - ] -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/no-cache/1_10000k.json b/scripts/legacy-vs-blobs/data/no-cache/1_10000k.json deleted file mode 100644 index 0697cd6b..00000000 --- a/scripts/legacy-vs-blobs/data/no-cache/1_10000k.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "benchmarks": [ - { - "options": { - "disable_gc": false, - "timer": "time", - "min_rounds": 5, - "min_time": 5e-06, - "max_time": 1.0, - "warmup": false - }, - "params": null, - "extra_info": { - "cpu_percent": 25.8, - "doc": "" - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k", - "group": "test_legacy_vs_blobs_1_10000k", - "name": "test_legacy_1_10000k", - "param": null, - "stats": { - "hd15iqr": 9.522547006607056, - "stddev": 0.04062225700829493, - "ops": 0.10556240321945518, - "iterations": 1, - "min": 9.423872947692871, - "total": 37.89227867126465, - "stddev_outliers": 2, - "max": 9.522547006607056, - "q1": 9.445196866989136, - "mean": 9.473069667816162, - "iqr_outliers": 0, - "rounds": 4, - "iqr": 0.055745601654052734, - "median": 9.47292935848236, - "q3": 9.500942468643188, - "ld15iqr": 9.423872947692871, - "outliers": "2;0" - } - }, - { - "options": { - "disable_gc": false, - "timer": "time", - "min_rounds": 5, - "min_time": 5e-06, - "max_time": 1.0, - "warmup": false - }, - "params": null, - "extra_info": { - "cpu_percent": 33.3, - "doc": "" - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", - "group": "test_legacy_vs_blobs_1_10000k", - "name": "test_blobs_1_10000k", - "param": null, - "stats": { - "hd15iqr": 3.758697032928467, - "stddev": 0.619090853876154, - "ops": 0.3292143381447483, - "iterations": 1, - "min": 2.344238042831421, - "total": 12.150139093399048, - "stddev_outliers": 2, - "max": 3.758697032928467, - "q1": 2.5475854873657227, - "mean": 3.037534773349762, - "iqr_outliers": 0, - "rounds": 4, - "iqr": 0.9798985719680786, - "median": 3.02360200881958, - "q3": 3.5274840593338013, - "ld15iqr": 2.344238042831421, - "outliers": "2;0" - } - } - ], - "version": "3.1.1", - "commit_info": { - "author_time": "2017-09-01T14:16:21-03:00", - "project": "soledad", - "id": "7a5698ad8c1457460d95cbd6dd237fb96e218d43", - "time": "2017-09-01T14:16:26-03:00", - "dirty": true, - "branch": "8856" - }, - "machine_info": { - "host": "pajeh", - "python_version": "2.7.13", - "python_compiler": "GCC 6.3.0 20170118", - "system": "Linux", - "python_implementation": "CPython", - "python_implementation_version": "2.7.13", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "machine": "x86_64", - "release": "4.9.0-3-amd64", - "processor": "", - "node": "pajeh", - "cpu": { - "hardware": "unknown", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", - "vendor_id": "GenuineIntel" - } - }, - "datetime": "2017-09-01T22:20:37.591520" -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/persistent/1000_10k.json b/scripts/legacy-vs-blobs/data/persistent/1000_10k.json deleted file mode 100644 index 909e6f21..00000000 --- a/scripts/legacy-vs-blobs/data/persistent/1000_10k.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "version": "3.1.1", - "machine_info": { - "python_version": "2.7.13", - "processor": "", - "node": "pajeh", - "python_compiler": "GCC 6.3.0 20170118", - "python_implementation_version": "2.7.13", - "python_implementation": "CPython", - "machine": "x86_64", - "system": "Linux", - "host": "pajeh", - "cpu": { - "hardware": "unknown", - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" - }, - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "release": "4.9.0-3-amd64" - }, - "commit_info": { - "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", - "time": "2017-09-02T02:38:57-03:00", - "dirty": true, - "branch": "8856", - "author_time": "2017-09-02T02:38:57-03:00", - "project": "soledad" - }, - "benchmarks": [ - { - "param": null, - "stats": { - "ld15iqr": 50.158271074295044, - "total": 290.0146920681, - "iterations": 1, - "stddev": 19.375312299153435, - "stddev_outliers": 2, - "mean": 72.503673017025, - "q1": 56.99843108654022, - "rounds": 4, - "q3": 88.00891494750977, - "iqr": 31.010483860969543, - "max": 93.9663679599762, - "iqr_outliers": 0, - "hd15iqr": 93.9663679599762, - "median": 72.94502651691437, - "ops": 0.013792404693279255, - "min": 50.158271074295044, - "outliers": "2;0" - }, - "params": null, - "name": "test_blobs_1000_10k", - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k", - "group": "test_legacy_vs_blobs_1000_10k", - "options": { - "timer": "time", - "disable_gc": false, - "min_rounds": 5, - "min_time": 5e-06, - "warmup": false, - "max_time": 1.0 - }, - "extra_info": { - "cpu_percent": 17.4, - "doc": "" - } - } - ], - "datetime": "2017-09-02T05:51:50.400601" -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/persistent/100_100k.json b/scripts/legacy-vs-blobs/data/persistent/100_100k.json deleted file mode 100644 index 48f0e077..00000000 --- a/scripts/legacy-vs-blobs/data/persistent/100_100k.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "machine_info": { - "host": "pajeh", - "python_compiler": "GCC 6.3.0 20170118", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "python_implementation": "CPython", - "processor": "", - "cpu": { - "hardware": "unknown", - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" - }, - "node": "pajeh", - "system": "Linux", - "release": "4.9.0-3-amd64", - "python_version": "2.7.13", - "machine": "x86_64", - "python_implementation_version": "2.7.13" - }, - "commit_info": { - "branch": "8856", - "author_time": "2017-09-02T02:38:57-03:00", - "time": "2017-09-02T02:38:57-03:00", - "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", - "project": "soledad", - "dirty": true - }, - "version": "3.1.1", - "benchmarks": [ - { - "group": "test_legacy_vs_blobs_100_100k", - "params": null, - "name": "test_blobs_100_100k", - "stats": { - "total": 20.738121271133423, - "stddev_outliers": 2, - "stddev": 0.9161744731162039, - "hd15iqr": 6.378293991088867, - "rounds": 4, - "max": 6.378293991088867, - "median": 5.054657578468323, - "iqr": 1.3696125745773315, - "q3": 5.8693366050720215, - "q1": 4.49972403049469, - "mean": 5.184530317783356, - "ops": 0.1928815029916827, - "iqr_outliers": 0, - "min": 4.25051212310791, - "iterations": 1, - "ld15iqr": 4.25051212310791, - "outliers": "2;0" - }, - "options": { - "timer": "time", - "disable_gc": false, - "warmup": false, - "max_time": 1.0, - "min_rounds": 5, - "min_time": 5e-06 - }, - "param": null, - "extra_info": { - "cpu_percent": 24.7, - "doc": "" - }, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k" - } - ], - "datetime": "2017-09-02T05:56:28.497384" -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/persistent/10_1000k.json b/scripts/legacy-vs-blobs/data/persistent/10_1000k.json deleted file mode 100644 index 529cca8c..00000000 --- a/scripts/legacy-vs-blobs/data/persistent/10_1000k.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "machine_info": { - "python_compiler": "GCC 6.3.0 20170118", - "python_implementation_version": "2.7.13", - "node": "pajeh", - "machine": "x86_64", - "host": "pajeh", - "cpu": { - "hardware": "unknown", - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" - }, - "system": "Linux", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "release": "4.9.0-3-amd64", - "python_version": "2.7.13", - "processor": "", - "python_implementation": "CPython" - }, - "commit_info": { - "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", - "time": "2017-09-02T02:38:57-03:00", - "dirty": true, - "branch": "8856", - "author_time": "2017-09-02T02:38:57-03:00", - "project": "soledad" - }, - "version": "3.1.1", - "benchmarks": [ - { - "stats": { - "ops": 0.4919825789326136, - "stddev": 0.1042862860397157, - "mean": 2.032592296600342, - "stddev_outliers": 2, - "q3": 2.112483024597168, - "q1": 1.9527015686035156, - "rounds": 4, - "max": 2.1607320308685303, - "hd15iqr": 2.1607320308685303, - "iqr_outliers": 0, - "ld15iqr": 1.91776704788208, - "iterations": 1, - "median": 2.0259350538253784, - "iqr": 0.15978145599365234, - "total": 8.130369186401367, - "outliers": "2;0", - "min": 1.91776704788208 - }, - "param": null, - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k", - "params": null, - "name": "test_blobs_10_1000k", - "group": "test_legacy_vs_blobs_10_1000k", - "options": { - "disable_gc": false, - "min_rounds": 5, - "timer": "time", - "warmup": false, - "min_time": 5e-06, - "max_time": 1.0 - }, - "extra_info": { - "cpu_percent": 36.3, - "doc": "" - } - } - ], - "datetime": "2017-09-02T05:57:48.679847" -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/data/persistent/1_10000k.json b/scripts/legacy-vs-blobs/data/persistent/1_10000k.json deleted file mode 100644 index 30d362ea..00000000 --- a/scripts/legacy-vs-blobs/data/persistent/1_10000k.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "machine_info": { - "python_compiler": "GCC 6.3.0 20170118", - "host": "pajeh", - "python_build": [ - "default", - "Jan 19 2017 14:48:08" - ], - "cpu": { - "hardware": "unknown", - "vendor_id": "GenuineIntel", - "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz" - }, - "node": "pajeh", - "system": "Linux", - "release": "4.9.0-3-amd64", - "python_implementation": "CPython", - "processor": "", - "python_implementation_version": "2.7.13", - "python_version": "2.7.13", - "machine": "x86_64" - }, - "commit_info": { - "branch": "8856", - "author_time": "2017-09-02T02:38:57-03:00", - "dirty": true, - "time": "2017-09-02T02:38:57-03:00", - "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e", - "project": "soledad" - }, - "version": "3.1.1", - "benchmarks": [ - { - "stats": { - "max": 2.1960339546203613, - "median": 2.1957188844680786, - "stddev_outliers": 1, - "q3": 2.19601047039032, - "iqr_outliers": 0, - "q1": 2.194662928581238, - "ld15iqr": 2.1938750743865967, - "ops": 0.4555109930218146, - "stddev": 0.001009686487541645, - "hd15iqr": 2.1960339546203613, - "rounds": 4, - "iterations": 1, - "total": 8.781346797943115, - "mean": 2.195336699485779, - "iqr": 0.0013475418090820312, - "outliers": "1;0", - "min": 2.1938750743865967 - }, - "options": { - "warmup": false, - "max_time": 1.0, - "timer": "time", - "min_rounds": 5, - "min_time": 5e-06, - "disable_gc": false - }, - "group": "test_legacy_vs_blobs_1_10000k", - "params": null, - "name": "test_blobs_1_10000k", - "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k", - "param": null, - "extra_info": { - "cpu_percent": 40.2, - "doc": "" - } - } - ], - "datetime": "2017-09-02T06:00:45.837579" -} \ No newline at end of file diff --git a/scripts/legacy-vs-blobs/legacy-vs-blobs.png b/scripts/legacy-vs-blobs/legacy-vs-blobs.png deleted file mode 100644 index 99080238..00000000 Binary files a/scripts/legacy-vs-blobs/legacy-vs-blobs.png and /dev/null differ diff --git a/scripts/legacy-vs-blobs/legacy-vs-blobs.py b/scripts/legacy-vs-blobs/legacy-vs-blobs.py deleted file mode 100755 index 57a42376..00000000 --- a/scripts/legacy-vs-blobs/legacy-vs-blobs.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python - -# Plot bars comparing different implementations of mail pipeline. -# -# This script can be improved to account for arbitrary number of data sets, but -# it is not doing it right now. - -import json -import numpy as np -import matplotlib.pyplot as plt - -# make a prettier graph -from mpltools import style -style.use('ggplot') - -OUTPUT_FILENAME = 'legacy-vs-blobs.png' - -# each value below will generate one bar for each for each (amount, size) pair. -# The script expects to find files in ./data/SET/ for each set of -# implementations. -# -# The baseline values will be the legacy results in ./data/no-cache/. - -graphs = [ - '1000_10k', - '100_100k', - '10_1000k', - '1_10000k', -] - - -# the JSON structure returned by the following function is ugly, but the -# original JSONs are even uglier, so this is here just to make the life of the -# script easier. -# -# We want to have something like: -# -# data[variation][graph][implementation] = -# -# Where: -# -# - variation is one data set under ./data (i.e. no-cache, cache, persistent, -# etc). -# - graph is one of the values in graphs variable above. -# - implementation is either legacy or blobs (we just need legacy for the -# no-cache variation, as that is the one we are using as baseline. - -def get_data(): - folders = ['cache', 'no-cache', 'persistent'] - data = {} - for folder in folders: - data[folder] = {} - for graph in graphs: - with open('data/%s/%s.json' % (folder, graph)) as f: - d = json.loads(f.read()) - benchmarks = d['benchmarks'] - data[folder][graph] = {} - for t in ['blobs', 'legacy']: - result = filter(lambda b: t in b['name'], benchmarks) - if result: - result = result.pop() - data[folder][graph][t] = result['stats'] - return data - - -def plot_data(data): - - N = 4 - - # this is our baseline (i.e. legacy / legacy) - absolutes = (1, 1, 1, 1) - - ind = np.arange(N) # the x locations for the groups - width = 0.20 # the width of the bars - - fig, ax = plt.subplots() - rects1 = ax.bar(ind, absolutes, width) - - # for each graph, calculate the ratios - ratios = {'no-cache': [], 'cache': [], 'persistent': []} - for graph in graphs: - legacy = data['no-cache'][graph]['legacy']['mean'] - - # calculate ratios for no-cache / legacy - ratio = data['no-cache'][graph]['blobs']['mean'] / legacy - ratios['no-cache'].append(ratio) - - # calculate ratios for cache / legacy - ratio = data['cache'][graph]['blobs']['mean'] / legacy - ratios['cache'].append(ratio) - - # calculate ratios for persistent / legacy - ratio = data['persistent'][graph]['blobs']['mean'] / legacy - ratios['persistent'].append(ratio) - - # create the boxes with the ratios - nocache = tuple(ratios['no-cache']) - rects2 = ax.bar(ind + width, nocache, width) - - cache = tuple(ratios['cache']) - rects3 = ax.bar(ind + (2 * width), cache, width) - - persistent = tuple(ratios['persistent']) - rects4 = ax.bar(ind + (3 * width), persistent, width) - - # add some text for labels, title and axes ticks - ax.set_ylabel('Ratio of time (legacy is baseline)') - ax.set_xlabel('Amount and size of email messages') - ax.set_title('Inbox loading time: legacy vs blobs mail pipeline') - ax.set_xticks(ind + (1.5 * width)) - ax.set_xticklabels( - tuple(map(lambda name: name.replace('_', ' x '), graphs))) - - ax.legend( - (rects1[0], rects2[0], rects3[0], rects4[0]), - ('legacy', 'blobs', 'blobs + session cache', - 'blobs + session cache + persistent http')) - # ax.grid() - - plt.savefig(OUTPUT_FILENAME) - # plt.show() - - -if __name__ == '__main__': - data = get_data() - plot_data(data) diff --git a/scripts/legacy-vs-blobs/makefile b/scripts/legacy-vs-blobs/makefile deleted file mode 100644 index 649e181c..00000000 --- a/scripts/legacy-vs-blobs/makefile +++ /dev/null @@ -1,14 +0,0 @@ -VIRTUALENV_NAME = plot -VIRTUALENV_ROOT = ~/.virtualenvs/$(VIRTUALENV_NAME) - -all: graph - -env: - if [ ! -d $(VIRTUALENV_ROOT) ]; then \ - mkdir -p $$(dirname $(VIRTUALENV_ROOT)); \ - virtualenv $(VIRTUALENV_ROOT); \ - $(VIRTUALENV_ROOT)/bin/pip install requirements.pip; \ - fi - -graph: env - $(VIRTUALENV_ROOT)/bin/python ./legacy-vs-blobs.py diff --git a/scripts/legacy-vs-blobs/requirements.pip b/scripts/legacy-vs-blobs/requirements.pip deleted file mode 100644 index db5d81e0..00000000 --- a/scripts/legacy-vs-blobs/requirements.pip +++ /dev/null @@ -1,2 +0,0 @@ -matplotlib -numpy -- cgit v1.2.3