summaryrefslogtreecommitdiff
path: root/scripts/benchmark/legacy-vs-blobs
diff options
context:
space:
mode:
authordrebs <drebs@riseup.net>2017-09-13 11:39:36 -0300
committerdrebs <drebs@riseup.net>2017-09-14 12:14:44 -0300
commitdd7b2e414adf2c84873debcd7b7d526859036bc2 (patch)
tree57f44bd454e9f4612a1c4fc808fbb944d0716c18 /scripts/benchmark/legacy-vs-blobs
parentaffc09e18a243b215dd1430f53c74a46eab8c098 (diff)
[test] move legacy-vs-blobs graphing to benchmark scripts dir
Diffstat (limited to 'scripts/benchmark/legacy-vs-blobs')
-rw-r--r--scripts/benchmark/legacy-vs-blobs/README.rst43
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json111
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json73
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json73
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json73
-rw-r--r--scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json73
-rw-r--r--scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.pngbin0 -> 38517 bytes
-rwxr-xr-xscripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py126
-rw-r--r--scripts/benchmark/legacy-vs-blobs/makefile14
-rw-r--r--scripts/benchmark/legacy-vs-blobs/requirements.pip2
17 files changed, 1365 insertions, 0 deletions
diff --git a/scripts/benchmark/legacy-vs-blobs/README.rst b/scripts/benchmark/legacy-vs-blobs/README.rst
new file mode 100644
index 00000000..3eedb3a4
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/README.rst
@@ -0,0 +1,43 @@
+Ploggin Legacy vs Blobs results
+===============================
+
+This script produces a bar graph comparing different implementations of the
+mail pipeline (legacy, blobs, blobs with session cache, blobs with session
+cache and persistent http connections) for different amount/sizes of incoming
+data (1 x 10M, 10 x 1M, 100 x 100K, 1000 x 10K).
+
+Data acquisition
+----------------
+
+Data was gathered through different runs of the code in
+`soledad/tests/benchmarks/test_legacy_vs_blobs.py`. As these tests are skipped
+by default, you have to (1) remove the @pytest.mark.skip decorator and (2)
+remove the second call to pytest in `tox.ini` that would cause a second run of
+the tests for measuring memory. This script only plots time, for now.
+
+Once you done the above, chdir into `soledad/tests` and do something like this,
+for each test group:
+
+ tox -e benchmarks -- -m benchmark_test_legacy_vs_blobs_10_1000k --benchmark-autosave
+
+Each run of tox as above produces a JSON file in `soledad/tests/.benchmarks`
+with stats. Each file has to be copied in a subdirectory of the `data/`
+directory of the folder where this script lies. Code has to be adapted for each
+extra set of results added to the `data/` directory.
+
+Currently, we are comparing 4 implementation variations:
+
+ - data/no-cache: contains legacy as baseline and blobs as the current master
+ implementation of blobs.
+ - data/cache: adds twisted session cache to blobs connections.
+ - data/persistent: adds session cache and persistent HTTP connections.
+
+Plotting
+--------
+
+Once you have all your data in subdirectories of the `data/` directory and the
+script is correctly tuned for your data, running `make` should be enough to
+create a virtualenvironment with dependencies and plotting data.
+
+By default, the script will generate a file called `./legacy-vs-blobs.png` in
+the current directory with the plot of the data.
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json
new file mode 100644
index 00000000..5be17758
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/1000_10k.json
@@ -0,0 +1,111 @@
+{
+ "datetime": "2017-09-02T04:22:20.074434",
+ "machine_info": {
+ "python_implementation": "CPython",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "machine": "x86_64",
+ "cpu": {
+ "hardware": "unknown",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz",
+ "vendor_id": "GenuineIntel"
+ },
+ "host": "pajeh",
+ "system": "Linux",
+ "processor": "",
+ "python_implementation_version": "2.7.13",
+ "release": "4.9.0-3-amd64",
+ "python_version": "2.7.13",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "node": "pajeh"
+ },
+ "commit_info": {
+ "author_time": "2017-09-02T00:27:58-03:00",
+ "time": "2017-09-02T00:39:13-03:00",
+ "dirty": true,
+ "project": "soledad",
+ "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8",
+ "branch": "8856"
+ },
+ "benchmarks": [
+ {
+ "name": "test_legacy_1000_10k",
+ "stats": {
+ "median": 48.28695607185364,
+ "rounds": 4,
+ "ld15iqr": 36.51543307304382,
+ "iqr_outliers": 0,
+ "stddev": 10.431966763848191,
+ "q1": 40.382596015930176,
+ "min": 36.51543307304382,
+ "q3": 56.539175629615784,
+ "total": 193.84354329109192,
+ "iterations": 1,
+ "max": 60.75419807434082,
+ "stddev_outliers": 2,
+ "mean": 48.46088582277298,
+ "ops": 0.020635198532216573,
+ "hd15iqr": 60.75419807434082,
+ "outliers": "2;0",
+ "iqr": 16.156579613685608
+ },
+ "extra_info": {
+ "cpu_percent": 26.6,
+ "doc": ""
+ },
+ "params": null,
+ "param": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k",
+ "group": "test_legacy_vs_blobs_1000_10k",
+ "options": {
+ "timer": "time",
+ "min_rounds": 5,
+ "warmup": false,
+ "disable_gc": false,
+ "min_time": 5e-06,
+ "max_time": 1.0
+ }
+ },
+ {
+ "name": "test_blobs_1000_10k",
+ "stats": {
+ "median": 87.17049705982208,
+ "rounds": 4,
+ "ld15iqr": 65.08280301094055,
+ "iqr_outliers": 0,
+ "stddev": 19.00300388729206,
+ "q1": 72.67573952674866,
+ "min": 65.08280301094055,
+ "q3": 101.80426609516144,
+ "total": 348.9600112438202,
+ "iterations": 1,
+ "max": 109.53621411323547,
+ "stddev_outliers": 2,
+ "mean": 87.24000281095505,
+ "ops": 0.01146263145092914,
+ "hd15iqr": 109.53621411323547,
+ "outliers": "2;0",
+ "iqr": 29.12852656841278
+ },
+ "extra_info": {
+ "cpu_percent": 19.4,
+ "doc": ""
+ },
+ "params": null,
+ "param": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k",
+ "group": "test_legacy_vs_blobs_1000_10k",
+ "options": {
+ "timer": "time",
+ "min_rounds": 5,
+ "warmup": false,
+ "disable_gc": false,
+ "min_time": 5e-06,
+ "max_time": 1.0
+ }
+ }
+ ],
+ "version": "3.1.1"
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json
new file mode 100644
index 00000000..17fc598b
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/100_100k.json
@@ -0,0 +1,111 @@
+{
+ "benchmarks": [
+ {
+ "params": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k",
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 25.1
+ },
+ "group": "test_legacy_vs_blobs_100_100k",
+ "stats": {
+ "q1": 11.39308750629425,
+ "rounds": 4,
+ "q3": 12.677581906318665,
+ "hd15iqr": 12.946987867355347,
+ "median": 12.090978026390076,
+ "stddev_outliers": 2,
+ "mean": 12.035334706306458,
+ "iqr_outliers": 0,
+ "iterations": 1,
+ "total": 48.14133882522583,
+ "min": 11.012394905090332,
+ "outliers": "2;0",
+ "ops": 0.08308867384269794,
+ "ld15iqr": 11.012394905090332,
+ "stddev": 0.833654348956266,
+ "iqr": 1.284494400024414,
+ "max": 12.946987867355347
+ },
+ "name": "test_legacy_100_100k",
+ "param": null,
+ "options": {
+ "min_rounds": 5,
+ "timer": "time",
+ "max_time": 1.0,
+ "min_time": 5e-06,
+ "warmup": false,
+ "disable_gc": false
+ }
+ },
+ {
+ "params": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k",
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 24.9
+ },
+ "group": "test_legacy_vs_blobs_100_100k",
+ "stats": {
+ "q1": 5.015861988067627,
+ "rounds": 4,
+ "q3": 6.166973114013672,
+ "hd15iqr": 6.362173080444336,
+ "median": 5.64124858379364,
+ "stddev_outliers": 2,
+ "mean": 5.591417551040649,
+ "iqr_outliers": 0,
+ "iterations": 1,
+ "total": 22.365670204162598,
+ "min": 4.7209999561309814,
+ "outliers": "2;0",
+ "ops": 0.1788455236747405,
+ "ld15iqr": 4.7209999561309814,
+ "stddev": 0.7246033659461454,
+ "iqr": 1.151111125946045,
+ "max": 6.362173080444336
+ },
+ "name": "test_blobs_100_100k",
+ "param": null,
+ "options": {
+ "min_rounds": 5,
+ "timer": "time",
+ "max_time": 1.0,
+ "min_time": 5e-06,
+ "warmup": false,
+ "disable_gc": false
+ }
+ }
+ ],
+ "datetime": "2017-09-02T03:53:29.055256",
+ "version": "3.1.1",
+ "machine_info": {
+ "node": "pajeh",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "python_version": "2.7.13",
+ "machine": "x86_64",
+ "python_implementation": "CPython",
+ "cpu": {
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz",
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel"
+ },
+ "python_implementation_version": "2.7.13",
+ "system": "Linux",
+ "host": "pajeh",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "release": "4.9.0-3-amd64",
+ "processor": ""
+ },
+ "commit_info": {
+ "project": "soledad",
+ "time": "2017-09-02T00:39:13-03:00",
+ "author_time": "2017-09-02T00:27:58-03:00",
+ "branch": "8856",
+ "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8",
+ "dirty": true
+ }
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json
new file mode 100644
index 00000000..1a733f2d
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/10_1000k.json
@@ -0,0 +1,111 @@
+{
+ "benchmarks": [
+ {
+ "params": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k",
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 27.7
+ },
+ "group": "test_legacy_vs_blobs_10_1000k",
+ "stats": {
+ "q1": 8.741152048110962,
+ "rounds": 4,
+ "q3": 8.863584876060486,
+ "median": 8.83247196674347,
+ "stddev_outliers": 1,
+ "iqr": 0.12243282794952393,
+ "mean": 8.802368462085724,
+ "iqr_outliers": 0,
+ "iterations": 1,
+ "total": 35.209473848342896,
+ "min": 8.67733097076416,
+ "outliers": "1;0",
+ "ops": 0.11360578738634734,
+ "ld15iqr": 8.67733097076416,
+ "stddev": 0.08786766595777334,
+ "hd15iqr": 8.867198944091797,
+ "max": 8.867198944091797
+ },
+ "name": "test_legacy_10_1000k",
+ "param": null,
+ "options": {
+ "min_rounds": 5,
+ "timer": "time",
+ "max_time": 1.0,
+ "min_time": 5e-06,
+ "warmup": false,
+ "disable_gc": false
+ }
+ },
+ {
+ "params": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k",
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 37.0
+ },
+ "group": "test_legacy_vs_blobs_10_1000k",
+ "stats": {
+ "q1": 1.8672034740447998,
+ "rounds": 4,
+ "q3": 1.9366559982299805,
+ "median": 1.8831886053085327,
+ "stddev_outliers": 1,
+ "iqr": 0.06945252418518066,
+ "mean": 1.9019297361373901,
+ "iqr_outliers": 0,
+ "iterations": 1,
+ "total": 7.6077189445495605,
+ "min": 1.8517439365386963,
+ "outliers": "1;0",
+ "ops": 0.5257817788952025,
+ "ld15iqr": 1.8517439365386963,
+ "stddev": 0.06029736288568101,
+ "hd15iqr": 1.9895977973937988,
+ "max": 1.9895977973937988
+ },
+ "name": "test_blobs_10_1000k",
+ "param": null,
+ "options": {
+ "min_rounds": 5,
+ "timer": "time",
+ "max_time": 1.0,
+ "min_time": 5e-06,
+ "warmup": false,
+ "disable_gc": false
+ }
+ }
+ ],
+ "datetime": "2017-09-02T03:48:24.650044",
+ "version": "3.1.1",
+ "machine_info": {
+ "python_implementation_version": "2.7.13",
+ "system": "Linux",
+ "host": "pajeh",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "release": "4.9.0-3-amd64",
+ "python_implementation": "CPython",
+ "processor": "",
+ "node": "pajeh",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "python_version": "2.7.13",
+ "machine": "x86_64",
+ "cpu": {
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz",
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel"
+ }
+ },
+ "commit_info": {
+ "author_time": "2017-09-02T00:27:58-03:00",
+ "time": "2017-09-02T00:39:13-03:00",
+ "project": "soledad",
+ "branch": "8856",
+ "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8",
+ "dirty": true
+ }
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json
new file mode 100644
index 00000000..2d970c6e
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/cache/1_10000k.json
@@ -0,0 +1,111 @@
+{
+ "datetime": "2017-09-02T03:46:17.875002",
+ "machine_info": {
+ "processor": "",
+ "python_implementation_version": "2.7.13",
+ "python_version": "2.7.13",
+ "machine": "x86_64",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "python_implementation": "CPython",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "cpu": {
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz",
+ "hardware": "unknown"
+ },
+ "release": "4.9.0-3-amd64",
+ "system": "Linux",
+ "host": "pajeh",
+ "node": "pajeh"
+ },
+ "version": "3.1.1",
+ "commit_info": {
+ "time": "2017-09-02T00:39:13-03:00",
+ "id": "2dea7ebc513eaa506704fd7ae68bae94b220eea8",
+ "project": "soledad",
+ "author_time": "2017-09-02T00:27:58-03:00",
+ "branch": "8856",
+ "dirty": true
+ },
+ "benchmarks": [
+ {
+ "extra_info": {
+ "cpu_percent": 25.8,
+ "doc": ""
+ },
+ "name": "test_legacy_1_10000k",
+ "params": null,
+ "options": {
+ "max_time": 1.0,
+ "min_rounds": 5,
+ "min_time": 5e-06,
+ "warmup": false,
+ "timer": "time",
+ "disable_gc": false
+ },
+ "group": "test_legacy_vs_blobs_1_10000k",
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k",
+ "param": null,
+ "stats": {
+ "mean": 9.607597470283508,
+ "ld15iqr": 9.551655054092407,
+ "iqr_outliers": 0,
+ "outliers": "1;0",
+ "q3": 9.663439989089966,
+ "q1": 9.55175495147705,
+ "iqr": 0.11168503761291504,
+ "total": 38.43038988113403,
+ "rounds": 4,
+ "median": 9.563774466514587,
+ "stddev_outliers": 1,
+ "min": 9.551655054092407,
+ "hd15iqr": 9.751185894012451,
+ "ops": 0.10408429402803564,
+ "stddev": 0.09638854573531173,
+ "iterations": 1,
+ "max": 9.751185894012451
+ }
+ },
+ {
+ "extra_info": {
+ "cpu_percent": 40.2,
+ "doc": ""
+ },
+ "name": "test_blobs_1_10000k",
+ "params": null,
+ "options": {
+ "max_time": 1.0,
+ "min_rounds": 5,
+ "min_time": 5e-06,
+ "warmup": false,
+ "timer": "time",
+ "disable_gc": false
+ },
+ "group": "test_legacy_vs_blobs_1_10000k",
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k",
+ "param": null,
+ "stats": {
+ "mean": 2.179791212081909,
+ "ld15iqr": 2.1572468280792236,
+ "iqr_outliers": 0,
+ "outliers": "2;0",
+ "q3": 2.193374514579773,
+ "q1": 2.1662079095840454,
+ "iqr": 0.02716660499572754,
+ "total": 8.719164848327637,
+ "rounds": 4,
+ "median": 2.1782270669937134,
+ "stddev_outliers": 2,
+ "min": 2.1572468280792236,
+ "hd15iqr": 2.2054638862609863,
+ "ops": 0.45875953369171735,
+ "stddev": 0.01992429175453132,
+ "iterations": 1,
+ "max": 2.2054638862609863
+ }
+ }
+ ]
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json
new file mode 100644
index 00000000..a585ad3a
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1000_10k.json
@@ -0,0 +1,111 @@
+{
+ "commit_info": {
+ "dirty": true,
+ "branch": "8856",
+ "author_time": "2017-09-01T09:50:11-03:00",
+ "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8",
+ "time": "2017-09-01T09:50:20-03:00",
+ "project": "soledad"
+ },
+ "machine_info": {
+ "node": "pajeh",
+ "processor": "",
+ "machine": "x86_64",
+ "system": "Linux",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "python_implementation": "CPython",
+ "python_implementation_version": "2.7.13",
+ "host": "pajeh",
+ "cpu": {
+ "hardware": "unknown",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz",
+ "vendor_id": "GenuineIntel"
+ },
+ "release": "4.9.0-3-amd64",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "python_version": "2.7.13"
+ },
+ "datetime": "2017-09-01T16:28:31.243069",
+ "version": "3.1.1",
+ "benchmarks": [
+ {
+ "stats": {
+ "ld15iqr": 38.49253582954407,
+ "outliers": "2;0",
+ "median": 52.51530361175537,
+ "iterations": 1,
+ "q3": 65.07869005203247,
+ "iqr_outliers": 0,
+ "rounds": 4,
+ "ops": 0.018550090252706083,
+ "total": 215.63237404823303,
+ "mean": 53.90809351205826,
+ "max": 72.10923099517822,
+ "iqr": 22.341193079948425,
+ "stddev_outliers": 2,
+ "q1": 42.737496972084045,
+ "min": 38.49253582954407,
+ "hd15iqr": 72.10923099517822,
+ "stddev": 14.537598498896823
+ },
+ "name": "test_legacy_1000_10k",
+ "param": null,
+ "group": "test_legacy_vs_blobs_1000_10k",
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1000_10k",
+ "params": null,
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 26.9
+ },
+ "options": {
+ "warmup": false,
+ "max_time": 1.0,
+ "min_time": 5e-06,
+ "min_rounds": 5,
+ "timer": "time",
+ "disable_gc": false
+ }
+ },
+ {
+ "stats": {
+ "ld15iqr": 176.17668509483337,
+ "outliers": "2;0",
+ "median": 192.90700149536133,
+ "iterations": 1,
+ "q3": 206.93408203125,
+ "iqr_outliers": 0,
+ "rounds": 4,
+ "ops": 0.005138941398078599,
+ "total": 778.3704249858856,
+ "mean": 194.5926062464714,
+ "max": 216.3797369003296,
+ "iqr": 24.68295156955719,
+ "stddev_outliers": 2,
+ "q1": 182.2511304616928,
+ "min": 176.17668509483337,
+ "hd15iqr": 216.3797369003296,
+ "stddev": 16.94586147142875
+ },
+ "name": "test_blobs_1000_10k",
+ "param": null,
+ "group": "test_legacy_vs_blobs_1000_10k",
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k",
+ "params": null,
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 10.9
+ },
+ "options": {
+ "warmup": false,
+ "max_time": 1.0,
+ "min_time": 5e-06,
+ "min_rounds": 5,
+ "timer": "time",
+ "disable_gc": false
+ }
+ }
+ ]
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json
new file mode 100644
index 00000000..2cbbe638
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/100_100k.json
@@ -0,0 +1,111 @@
+{
+ "commit_info": {
+ "time": "2017-09-01T09:50:20-03:00",
+ "branch": "8856",
+ "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8",
+ "project": "soledad",
+ "dirty": true,
+ "author_time": "2017-09-01T09:50:11-03:00"
+ },
+ "version": "3.1.1",
+ "benchmarks": [
+ {
+ "stats": {
+ "stddev_outliers": 1,
+ "ops": 0.08172728164268425,
+ "stddev": 1.0083003646390902,
+ "hd15iqr": 13.605333805084229,
+ "total": 48.943264961242676,
+ "iqr": 1.4744374752044678,
+ "iqr_outliers": 0,
+ "q1": 11.498597502708435,
+ "q3": 12.973034977912903,
+ "ld15iqr": 11.303098917007446,
+ "min": 11.303098917007446,
+ "iterations": 1,
+ "max": 13.605333805084229,
+ "outliers": "1;0",
+ "rounds": 4,
+ "median": 12.0174161195755,
+ "mean": 12.235816240310669
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_100_100k",
+ "options": {
+ "warmup": false,
+ "min_time": 5e-06,
+ "max_time": 1.0,
+ "disable_gc": false,
+ "min_rounds": 5,
+ "timer": "time"
+ },
+ "param": null,
+ "name": "test_legacy_100_100k",
+ "group": "test_legacy_vs_blobs_100_100k",
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 25.2
+ },
+ "params": null
+ },
+ {
+ "stats": {
+ "stddev_outliers": 2,
+ "ops": 0.052999047893223836,
+ "stddev": 0.6994077398028641,
+ "hd15iqr": 19.686198949813843,
+ "total": 75.47305393218994,
+ "iqr": 1.0998969078063965,
+ "iqr_outliers": 0,
+ "q1": 18.318315029144287,
+ "q3": 19.418211936950684,
+ "ld15iqr": 18.07945704460144,
+ "min": 18.07945704460144,
+ "iterations": 1,
+ "max": 19.686198949813843,
+ "outliers": "2;0",
+ "rounds": 4,
+ "median": 18.85369896888733,
+ "mean": 18.868263483047485
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k",
+ "options": {
+ "warmup": false,
+ "min_time": 5e-06,
+ "max_time": 1.0,
+ "disable_gc": false,
+ "min_rounds": 5,
+ "timer": "time"
+ },
+ "param": null,
+ "name": "test_blobs_100_100k",
+ "group": "test_legacy_vs_blobs_100_100k",
+ "extra_info": {
+ "doc": "",
+ "cpu_percent": 14.7
+ },
+ "params": null
+ }
+ ],
+ "datetime": "2017-09-01T15:44:20.745800",
+ "machine_info": {
+ "host": "pajeh",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "python_implementation": "CPython",
+ "cpu": {
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+ },
+ "node": "pajeh",
+ "release": "4.9.0-3-amd64",
+ "processor": "",
+ "python_implementation_version": "2.7.13",
+ "python_version": "2.7.13",
+ "system": "Linux",
+ "machine": "x86_64"
+ }
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json
new file mode 100644
index 00000000..778e5c63
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/10_1000k.json
@@ -0,0 +1,111 @@
+{
+ "version": "3.1.1",
+ "machine_info": {
+ "python_implementation": "CPython",
+ "host": "pajeh",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "cpu": {
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+ },
+ "release": "4.9.0-3-amd64",
+ "python_version": "2.7.13",
+ "machine": "x86_64",
+ "python_implementation_version": "2.7.13",
+ "processor": "",
+ "node": "pajeh",
+ "system": "Linux",
+ "python_compiler": "GCC 6.3.0 20170118"
+ },
+ "commit_info": {
+ "id": "365d9663cbbd524366b7dfe73ac5cccefeb357c8",
+ "time": "2017-09-01T09:50:20-03:00",
+ "project": "soledad",
+ "dirty": true,
+ "branch": "8856",
+ "author_time": "2017-09-01T09:50:11-03:00"
+ },
+ "datetime": "2017-09-01T15:37:48.278845",
+ "benchmarks": [
+ {
+ "param": null,
+ "extra_info": {
+ "cpu_percent": 27.9,
+ "doc": ""
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_10_1000k",
+ "group": "test_legacy_vs_blobs_10_1000k",
+ "params": null,
+ "name": "test_legacy_10_1000k",
+ "stats": {
+ "mean": 8.937849223613739,
+ "min": 8.711518049240112,
+ "outliers": "2;0",
+ "total": 35.751396894454956,
+ "iqr_outliers": 0,
+ "iterations": 1,
+ "ops": 0.11188374014612,
+ "rounds": 4,
+ "stddev": 0.20914882546041347,
+ "hd15iqr": 9.213748931884766,
+ "median": 8.913064956665039,
+ "stddev_outliers": 2,
+ "max": 9.213748931884766,
+ "q1": 8.794075012207031,
+ "ld15iqr": 8.711518049240112,
+ "q3": 9.081623435020447,
+ "iqr": 0.2875484228134155
+ },
+ "options": {
+ "disable_gc": false,
+ "min_time": 5e-06,
+ "min_rounds": 5,
+ "timer": "time",
+ "max_time": 1.0,
+ "warmup": false
+ }
+ },
+ {
+ "param": null,
+ "extra_info": {
+ "cpu_percent": 31.7,
+ "doc": ""
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k",
+ "group": "test_legacy_vs_blobs_10_1000k",
+ "params": null,
+ "name": "test_blobs_10_1000k",
+ "stats": {
+ "mean": 3.1771076917648315,
+ "min": 3.077280044555664,
+ "outliers": "1;0",
+ "total": 12.708430767059326,
+ "iqr_outliers": 0,
+ "iterations": 1,
+ "ops": 0.3147516851858793,
+ "rounds": 4,
+ "stddev": 0.0741566234351459,
+ "hd15iqr": 3.2503108978271484,
+ "median": 3.190419912338257,
+ "stddev_outliers": 1,
+ "max": 3.2503108978271484,
+ "q1": 3.1237324476242065,
+ "ld15iqr": 3.077280044555664,
+ "q3": 3.2304829359054565,
+ "iqr": 0.10675048828125
+ },
+ "options": {
+ "disable_gc": false,
+ "min_time": 5e-06,
+ "min_rounds": 5,
+ "timer": "time",
+ "max_time": 1.0,
+ "warmup": false
+ }
+ }
+ ]
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json
new file mode 100644
index 00000000..0697cd6b
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/no-cache/1_10000k.json
@@ -0,0 +1,111 @@
+{
+ "benchmarks": [
+ {
+ "options": {
+ "disable_gc": false,
+ "timer": "time",
+ "min_rounds": 5,
+ "min_time": 5e-06,
+ "max_time": 1.0,
+ "warmup": false
+ },
+ "params": null,
+ "extra_info": {
+ "cpu_percent": 25.8,
+ "doc": ""
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_legacy_1_10000k",
+ "group": "test_legacy_vs_blobs_1_10000k",
+ "name": "test_legacy_1_10000k",
+ "param": null,
+ "stats": {
+ "hd15iqr": 9.522547006607056,
+ "stddev": 0.04062225700829493,
+ "ops": 0.10556240321945518,
+ "iterations": 1,
+ "min": 9.423872947692871,
+ "total": 37.89227867126465,
+ "stddev_outliers": 2,
+ "max": 9.522547006607056,
+ "q1": 9.445196866989136,
+ "mean": 9.473069667816162,
+ "iqr_outliers": 0,
+ "rounds": 4,
+ "iqr": 0.055745601654052734,
+ "median": 9.47292935848236,
+ "q3": 9.500942468643188,
+ "ld15iqr": 9.423872947692871,
+ "outliers": "2;0"
+ }
+ },
+ {
+ "options": {
+ "disable_gc": false,
+ "timer": "time",
+ "min_rounds": 5,
+ "min_time": 5e-06,
+ "max_time": 1.0,
+ "warmup": false
+ },
+ "params": null,
+ "extra_info": {
+ "cpu_percent": 33.3,
+ "doc": ""
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k",
+ "group": "test_legacy_vs_blobs_1_10000k",
+ "name": "test_blobs_1_10000k",
+ "param": null,
+ "stats": {
+ "hd15iqr": 3.758697032928467,
+ "stddev": 0.619090853876154,
+ "ops": 0.3292143381447483,
+ "iterations": 1,
+ "min": 2.344238042831421,
+ "total": 12.150139093399048,
+ "stddev_outliers": 2,
+ "max": 3.758697032928467,
+ "q1": 2.5475854873657227,
+ "mean": 3.037534773349762,
+ "iqr_outliers": 0,
+ "rounds": 4,
+ "iqr": 0.9798985719680786,
+ "median": 3.02360200881958,
+ "q3": 3.5274840593338013,
+ "ld15iqr": 2.344238042831421,
+ "outliers": "2;0"
+ }
+ }
+ ],
+ "version": "3.1.1",
+ "commit_info": {
+ "author_time": "2017-09-01T14:16:21-03:00",
+ "project": "soledad",
+ "id": "7a5698ad8c1457460d95cbd6dd237fb96e218d43",
+ "time": "2017-09-01T14:16:26-03:00",
+ "dirty": true,
+ "branch": "8856"
+ },
+ "machine_info": {
+ "host": "pajeh",
+ "python_version": "2.7.13",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "system": "Linux",
+ "python_implementation": "CPython",
+ "python_implementation_version": "2.7.13",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "machine": "x86_64",
+ "release": "4.9.0-3-amd64",
+ "processor": "",
+ "node": "pajeh",
+ "cpu": {
+ "hardware": "unknown",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz",
+ "vendor_id": "GenuineIntel"
+ }
+ },
+ "datetime": "2017-09-01T22:20:37.591520"
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json
new file mode 100644
index 00000000..909e6f21
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/1000_10k.json
@@ -0,0 +1,73 @@
+{
+ "version": "3.1.1",
+ "machine_info": {
+ "python_version": "2.7.13",
+ "processor": "",
+ "node": "pajeh",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "python_implementation_version": "2.7.13",
+ "python_implementation": "CPython",
+ "machine": "x86_64",
+ "system": "Linux",
+ "host": "pajeh",
+ "cpu": {
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+ },
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "release": "4.9.0-3-amd64"
+ },
+ "commit_info": {
+ "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e",
+ "time": "2017-09-02T02:38:57-03:00",
+ "dirty": true,
+ "branch": "8856",
+ "author_time": "2017-09-02T02:38:57-03:00",
+ "project": "soledad"
+ },
+ "benchmarks": [
+ {
+ "param": null,
+ "stats": {
+ "ld15iqr": 50.158271074295044,
+ "total": 290.0146920681,
+ "iterations": 1,
+ "stddev": 19.375312299153435,
+ "stddev_outliers": 2,
+ "mean": 72.503673017025,
+ "q1": 56.99843108654022,
+ "rounds": 4,
+ "q3": 88.00891494750977,
+ "iqr": 31.010483860969543,
+ "max": 93.9663679599762,
+ "iqr_outliers": 0,
+ "hd15iqr": 93.9663679599762,
+ "median": 72.94502651691437,
+ "ops": 0.013792404693279255,
+ "min": 50.158271074295044,
+ "outliers": "2;0"
+ },
+ "params": null,
+ "name": "test_blobs_1000_10k",
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1000_10k",
+ "group": "test_legacy_vs_blobs_1000_10k",
+ "options": {
+ "timer": "time",
+ "disable_gc": false,
+ "min_rounds": 5,
+ "min_time": 5e-06,
+ "warmup": false,
+ "max_time": 1.0
+ },
+ "extra_info": {
+ "cpu_percent": 17.4,
+ "doc": ""
+ }
+ }
+ ],
+ "datetime": "2017-09-02T05:51:50.400601"
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json
new file mode 100644
index 00000000..48f0e077
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/100_100k.json
@@ -0,0 +1,73 @@
+{
+ "machine_info": {
+ "host": "pajeh",
+ "python_compiler": "GCC 6.3.0 20170118",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "python_implementation": "CPython",
+ "processor": "",
+ "cpu": {
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+ },
+ "node": "pajeh",
+ "system": "Linux",
+ "release": "4.9.0-3-amd64",
+ "python_version": "2.7.13",
+ "machine": "x86_64",
+ "python_implementation_version": "2.7.13"
+ },
+ "commit_info": {
+ "branch": "8856",
+ "author_time": "2017-09-02T02:38:57-03:00",
+ "time": "2017-09-02T02:38:57-03:00",
+ "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e",
+ "project": "soledad",
+ "dirty": true
+ },
+ "version": "3.1.1",
+ "benchmarks": [
+ {
+ "group": "test_legacy_vs_blobs_100_100k",
+ "params": null,
+ "name": "test_blobs_100_100k",
+ "stats": {
+ "total": 20.738121271133423,
+ "stddev_outliers": 2,
+ "stddev": 0.9161744731162039,
+ "hd15iqr": 6.378293991088867,
+ "rounds": 4,
+ "max": 6.378293991088867,
+ "median": 5.054657578468323,
+ "iqr": 1.3696125745773315,
+ "q3": 5.8693366050720215,
+ "q1": 4.49972403049469,
+ "mean": 5.184530317783356,
+ "ops": 0.1928815029916827,
+ "iqr_outliers": 0,
+ "min": 4.25051212310791,
+ "iterations": 1,
+ "ld15iqr": 4.25051212310791,
+ "outliers": "2;0"
+ },
+ "options": {
+ "timer": "time",
+ "disable_gc": false,
+ "warmup": false,
+ "max_time": 1.0,
+ "min_rounds": 5,
+ "min_time": 5e-06
+ },
+ "param": null,
+ "extra_info": {
+ "cpu_percent": 24.7,
+ "doc": ""
+ },
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_100_100k"
+ }
+ ],
+ "datetime": "2017-09-02T05:56:28.497384"
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json
new file mode 100644
index 00000000..529cca8c
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/10_1000k.json
@@ -0,0 +1,73 @@
+{
+ "machine_info": {
+ "python_compiler": "GCC 6.3.0 20170118",
+ "python_implementation_version": "2.7.13",
+ "node": "pajeh",
+ "machine": "x86_64",
+ "host": "pajeh",
+ "cpu": {
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+ },
+ "system": "Linux",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "release": "4.9.0-3-amd64",
+ "python_version": "2.7.13",
+ "processor": "",
+ "python_implementation": "CPython"
+ },
+ "commit_info": {
+ "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e",
+ "time": "2017-09-02T02:38:57-03:00",
+ "dirty": true,
+ "branch": "8856",
+ "author_time": "2017-09-02T02:38:57-03:00",
+ "project": "soledad"
+ },
+ "version": "3.1.1",
+ "benchmarks": [
+ {
+ "stats": {
+ "ops": 0.4919825789326136,
+ "stddev": 0.1042862860397157,
+ "mean": 2.032592296600342,
+ "stddev_outliers": 2,
+ "q3": 2.112483024597168,
+ "q1": 1.9527015686035156,
+ "rounds": 4,
+ "max": 2.1607320308685303,
+ "hd15iqr": 2.1607320308685303,
+ "iqr_outliers": 0,
+ "ld15iqr": 1.91776704788208,
+ "iterations": 1,
+ "median": 2.0259350538253784,
+ "iqr": 0.15978145599365234,
+ "total": 8.130369186401367,
+ "outliers": "2;0",
+ "min": 1.91776704788208
+ },
+ "param": null,
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_10_1000k",
+ "params": null,
+ "name": "test_blobs_10_1000k",
+ "group": "test_legacy_vs_blobs_10_1000k",
+ "options": {
+ "disable_gc": false,
+ "min_rounds": 5,
+ "timer": "time",
+ "warmup": false,
+ "min_time": 5e-06,
+ "max_time": 1.0
+ },
+ "extra_info": {
+ "cpu_percent": 36.3,
+ "doc": ""
+ }
+ }
+ ],
+ "datetime": "2017-09-02T05:57:48.679847"
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json b/scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json
new file mode 100644
index 00000000..30d362ea
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/data/persistent/1_10000k.json
@@ -0,0 +1,73 @@
+{
+ "machine_info": {
+ "python_compiler": "GCC 6.3.0 20170118",
+ "host": "pajeh",
+ "python_build": [
+ "default",
+ "Jan 19 2017 14:48:08"
+ ],
+ "cpu": {
+ "hardware": "unknown",
+ "vendor_id": "GenuineIntel",
+ "brand": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz"
+ },
+ "node": "pajeh",
+ "system": "Linux",
+ "release": "4.9.0-3-amd64",
+ "python_implementation": "CPython",
+ "processor": "",
+ "python_implementation_version": "2.7.13",
+ "python_version": "2.7.13",
+ "machine": "x86_64"
+ },
+ "commit_info": {
+ "branch": "8856",
+ "author_time": "2017-09-02T02:38:57-03:00",
+ "dirty": true,
+ "time": "2017-09-02T02:38:57-03:00",
+ "id": "1e132568857e6c2d4fbbbf886d3b7ab2009c3c9e",
+ "project": "soledad"
+ },
+ "version": "3.1.1",
+ "benchmarks": [
+ {
+ "stats": {
+ "max": 2.1960339546203613,
+ "median": 2.1957188844680786,
+ "stddev_outliers": 1,
+ "q3": 2.19601047039032,
+ "iqr_outliers": 0,
+ "q1": 2.194662928581238,
+ "ld15iqr": 2.1938750743865967,
+ "ops": 0.4555109930218146,
+ "stddev": 0.001009686487541645,
+ "hd15iqr": 2.1960339546203613,
+ "rounds": 4,
+ "iterations": 1,
+ "total": 8.781346797943115,
+ "mean": 2.195336699485779,
+ "iqr": 0.0013475418090820312,
+ "outliers": "1;0",
+ "min": 2.1938750743865967
+ },
+ "options": {
+ "warmup": false,
+ "max_time": 1.0,
+ "timer": "time",
+ "min_rounds": 5,
+ "min_time": 5e-06,
+ "disable_gc": false
+ },
+ "group": "test_legacy_vs_blobs_1_10000k",
+ "params": null,
+ "name": "test_blobs_1_10000k",
+ "fullname": "tests/benchmarks/test_legacy_vs_blobs.py::test_blobs_1_10000k",
+ "param": null,
+ "extra_info": {
+ "cpu_percent": 40.2,
+ "doc": ""
+ }
+ }
+ ],
+ "datetime": "2017-09-02T06:00:45.837579"
+} \ No newline at end of file
diff --git a/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png
new file mode 100644
index 00000000..99080238
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.png
Binary files differ
diff --git a/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py
new file mode 100755
index 00000000..57a42376
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/legacy-vs-blobs.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+
+# Plot bars comparing different implementations of mail pipeline.
+#
+# This script can be improved to account for arbitrary number of data sets, but
+# it is not doing it right now.
+
+import json
+import numpy as np
+import matplotlib.pyplot as plt
+
+# make a prettier graph
+from mpltools import style
+style.use('ggplot')
+
+OUTPUT_FILENAME = 'legacy-vs-blobs.png'
+
+# each value below will generate one bar for each for each (amount, size) pair.
+# The script expects to find files in ./data/SET/ for each set of
+# implementations.
+#
+# The baseline values will be the legacy results in ./data/no-cache/.
+
+graphs = [
+ '1000_10k',
+ '100_100k',
+ '10_1000k',
+ '1_10000k',
+]
+
+
+# the JSON structure returned by the following function is ugly, but the
+# original JSONs are even uglier, so this is here just to make the life of the
+# script easier.
+#
+# We want to have something like:
+#
+# data[variation][graph][implementation] = <stats>
+#
+# Where:
+#
+# - variation is one data set under ./data (i.e. no-cache, cache, persistent,
+# etc).
+# - graph is one of the values in graphs variable above.
+# - implementation is either legacy or blobs (we just need legacy for the
+# no-cache variation, as that is the one we are using as baseline.
+
+def get_data():
+ folders = ['cache', 'no-cache', 'persistent']
+ data = {}
+ for folder in folders:
+ data[folder] = {}
+ for graph in graphs:
+ with open('data/%s/%s.json' % (folder, graph)) as f:
+ d = json.loads(f.read())
+ benchmarks = d['benchmarks']
+ data[folder][graph] = {}
+ for t in ['blobs', 'legacy']:
+ result = filter(lambda b: t in b['name'], benchmarks)
+ if result:
+ result = result.pop()
+ data[folder][graph][t] = result['stats']
+ return data
+
+
+def plot_data(data):
+
+ N = 4
+
+ # this is our baseline (i.e. legacy / legacy)
+ absolutes = (1, 1, 1, 1)
+
+ ind = np.arange(N) # the x locations for the groups
+ width = 0.20 # the width of the bars
+
+ fig, ax = plt.subplots()
+ rects1 = ax.bar(ind, absolutes, width)
+
+ # for each graph, calculate the ratios
+ ratios = {'no-cache': [], 'cache': [], 'persistent': []}
+ for graph in graphs:
+ legacy = data['no-cache'][graph]['legacy']['mean']
+
+ # calculate ratios for no-cache / legacy
+ ratio = data['no-cache'][graph]['blobs']['mean'] / legacy
+ ratios['no-cache'].append(ratio)
+
+ # calculate ratios for cache / legacy
+ ratio = data['cache'][graph]['blobs']['mean'] / legacy
+ ratios['cache'].append(ratio)
+
+ # calculate ratios for persistent / legacy
+ ratio = data['persistent'][graph]['blobs']['mean'] / legacy
+ ratios['persistent'].append(ratio)
+
+ # create the boxes with the ratios
+ nocache = tuple(ratios['no-cache'])
+ rects2 = ax.bar(ind + width, nocache, width)
+
+ cache = tuple(ratios['cache'])
+ rects3 = ax.bar(ind + (2 * width), cache, width)
+
+ persistent = tuple(ratios['persistent'])
+ rects4 = ax.bar(ind + (3 * width), persistent, width)
+
+ # add some text for labels, title and axes ticks
+ ax.set_ylabel('Ratio of time (legacy is baseline)')
+ ax.set_xlabel('Amount and size of email messages')
+ ax.set_title('Inbox loading time: legacy vs blobs mail pipeline')
+ ax.set_xticks(ind + (1.5 * width))
+ ax.set_xticklabels(
+ tuple(map(lambda name: name.replace('_', ' x '), graphs)))
+
+ ax.legend(
+ (rects1[0], rects2[0], rects3[0], rects4[0]),
+ ('legacy', 'blobs', 'blobs + session cache',
+ 'blobs + session cache + persistent http'))
+ # ax.grid()
+
+ plt.savefig(OUTPUT_FILENAME)
+ # plt.show()
+
+
+if __name__ == '__main__':
+ data = get_data()
+ plot_data(data)
diff --git a/scripts/benchmark/legacy-vs-blobs/makefile b/scripts/benchmark/legacy-vs-blobs/makefile
new file mode 100644
index 00000000..649e181c
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/makefile
@@ -0,0 +1,14 @@
+VIRTUALENV_NAME = plot
+VIRTUALENV_ROOT = ~/.virtualenvs/$(VIRTUALENV_NAME)
+
+all: graph
+
+env:
+ if [ ! -d $(VIRTUALENV_ROOT) ]; then \
+ mkdir -p $$(dirname $(VIRTUALENV_ROOT)); \
+ virtualenv $(VIRTUALENV_ROOT); \
+ $(VIRTUALENV_ROOT)/bin/pip install requirements.pip; \
+ fi
+
+graph: env
+ $(VIRTUALENV_ROOT)/bin/python ./legacy-vs-blobs.py
diff --git a/scripts/benchmark/legacy-vs-blobs/requirements.pip b/scripts/benchmark/legacy-vs-blobs/requirements.pip
new file mode 100644
index 00000000..db5d81e0
--- /dev/null
+++ b/scripts/benchmark/legacy-vs-blobs/requirements.pip
@@ -0,0 +1,2 @@
+matplotlib
+numpy