From 31b527a9acc8607e5e03927b3b646b7c832e7058 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 2 May 2016 20:25:45 -0300 Subject: refactor and make complete test run with script --- Makefile | 28 +++++++--- defaults.conf.example | 15 ++++-- get_client_cpu_mem.py | 11 ---- get_ping_rate.py | 12 ----- graphit | 9 ---- measure-perf-series.py | 20 -------- out/.empty | 0 run-test.sh | 51 +++++++++++++++---- scripts/__init__.py | 0 scripts/create_payload.py | 29 +++++++++++ scripts/get_client_stats.py | 20 ++++++++ scripts/get_ping_rate.py | 19 +++++++ scripts/measure_perf_series.py | 31 +++++++++++ scripts/preload_server_database.py | 39 ++++++++++++++ scripts/server_with_soledad_syncer.py | 47 +++++++++++++++++ scripts/soledad_sync.py | 96 +++++++++++++++++++++++++++++++++++ scripts/sync_stats.gnuplot | 23 +++++++++ server_with_soledad_syncer.py | 41 --------------- soledad_sync.py | 95 ---------------------------------- 19 files changed, 377 insertions(+), 209 deletions(-) delete mode 100644 get_client_cpu_mem.py delete mode 100644 get_ping_rate.py delete mode 100755 graphit delete mode 100644 measure-perf-series.py create mode 100644 out/.empty create mode 100644 scripts/__init__.py create mode 100755 scripts/create_payload.py create mode 100755 scripts/get_client_stats.py create mode 100755 scripts/get_ping_rate.py create mode 100755 scripts/measure_perf_series.py create mode 100755 scripts/preload_server_database.py create mode 100644 scripts/server_with_soledad_syncer.py create mode 100644 scripts/soledad_sync.py create mode 100755 scripts/sync_stats.gnuplot delete mode 100644 server_with_soledad_syncer.py delete mode 100644 soledad_sync.py diff --git a/Makefile b/Makefile index f9da7d2..53228a5 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,18 @@ # Actual soledad sync soledad-sync-server: - twistd -n web --port 8080 --class=server_with_soledad_syncer.resource + twistd -n web --port 8080 --class=scripts.server_with_soledad_syncer.resource soledad-sync-server-lineprof: - kernprof -l server_with_soledad_syncer.py + kernprof -l ./scripts/server_with_soledad_syncer.py soledad-sync-server-debug: #twistd --profile=stats_obj --profiler=cProfile -n web --port 8080 --class=server_with_soledad_syncer.resource - python -m cProfile -o sync.cprofile server_with_soledad_syncer.py + python -m cProfile -o sync.cprofile ./scripts/server_with_soledad_syncer.py view-lineprofile: - python -m line_profiler server_with_soledad_syncer.py.lprof + python -m line_profiler ./scripts/server_with_soledad_syncer.py.lprof view-profile: @@ -21,9 +21,14 @@ view-profile: measure-ping: httperf --server localhost --port 8080 --num-calls 5 --num-conns 20 --uri /ping +trigger-create-docs: + curl localhost:8080/create-docs + trigger-sync: #time curl localhost:8080/start-sync curl localhost:8080/start-sync + +trigger-stop: curl localhost:8080/stop measure-series: @@ -31,9 +36,16 @@ measure-series: # TODO rm series.log, name it with a timestamp # TODO measure, first of all, the number of seconds from the beginning!!! (right now it's biased) # TODO add cpu/ram usage (ping command COULD RETURN THAT!) - rm -f /tmp/soledadsync/* + #rm -f /tmp/soledadsync/* rm -f ./out/series.log - python measure-perf-series.py + ./scripts/measure_perf_series.py + +graph-image: + gnuplot -e 'call "./scripts/sync_stats.gnuplot" "./out/series.log" "./out/sync-stats.png"' + +graph-view: + gnuplot -e 'call "./scripts/sync_stats.gnuplot" "./out/series.log" ""' -graph-series: - data=./out/series.log ./graphit +kill: + killall -9 twistd + killall -9 python diff --git a/defaults.conf.example b/defaults.conf.example index e4d836d..e997c53 100644 --- a/defaults.conf.example +++ b/defaults.conf.example @@ -1,9 +1,16 @@ [server] -host: http://localhost:2424 +host = http://localhost:2424 [client] -uuid: 1234567890abcdef +uuid = 1234567890abcdef +basedir = /tmp/soledad_client_test +passphrase = 12345678 [sync] -num_docs: 100 -payload: /tmp/payload +num_docs = 100 +payload = /tmp/payload +payload_size = 500 +auth_token = an-auth-token + +[test] +stats_file = ./out/stats.json diff --git a/get_client_cpu_mem.py b/get_client_cpu_mem.py deleted file mode 100644 index fe1aeec..0000000 --- a/get_client_cpu_mem.py +++ /dev/null @@ -1,11 +0,0 @@ -import commands -import urllib -import psutil - -pid, phase = urllib.urlopen('http://localhost:8080/stats').read().split() -res = commands.getoutput("ps -p " + pid + " -o \%cpu,\%mem") -splitted = res.split() -cpu = splitted[2] -mem = splitted[3] - -print cpu, mem, phase diff --git a/get_ping_rate.py b/get_ping_rate.py deleted file mode 100644 index 442f4dc..0000000 --- a/get_ping_rate.py +++ /dev/null @@ -1,12 +0,0 @@ -import commands -import re - -# ORIGINAL, STANDARD MEASURE (TOTALLY RANDOM) -#res = commands.getoutput( - #'httperf --server localhost --port 8080 --num-calls 5 --num-conns 20 ' - #'--uri /ping | grep "Connection rate"') - -res = commands.getoutput( - 'httperf --server localhost --port 8080 --num-calls 5 --num-conns 10 ' - '--uri /ping | grep "Connection rate"') -print re.findall('[-+]?([0-9]*\.?[0-9]+)', res)[0] diff --git a/graphit b/graphit deleted file mode 100755 index 0c809c0..0000000 --- a/graphit +++ /dev/null @@ -1,9 +0,0 @@ -#! /usr/bin/gnuplot - -filename=system("echo $data") -set title filename -set key outside -#plot name with linespoints notitle -#plot filename using 1:2 with linespoints title columnheader -plot for [col=2:5] filename using 1:col with linespoints title columnheader -pause -1 diff --git a/measure-perf-series.py b/measure-perf-series.py deleted file mode 100644 index e87fcac..0000000 --- a/measure-perf-series.py +++ /dev/null @@ -1,20 +0,0 @@ -import commands -import datetime - -# How many points to measure. Make sure that you leave the baseline -# running for a significative amount before triggering the sync, it's possible -# you have significant variability in there. -POINTS = 200 - -commands.getoutput('echo time req/s cpu mem phase > ./out/series.log') -start = datetime.datetime.now() - -for i in range(POINTS): - value = commands.getoutput('python get_ping_rate.py') - print "Step", i, "...", value - stats = commands.getoutput('python get_client_cpu_mem.py') - cpu, mem, phase = stats.split() - now = datetime.datetime.now() - secs = (now - start).total_seconds() - commands.getoutput( - 'echo %s\t%s\t%s\t%s\t%s >> series.log' % (secs, value, cpu, mem, phase)) diff --git a/out/.empty b/out/.empty new file mode 100644 index 0000000..e69de29 diff --git a/run-test.sh b/run-test.sh index 7cc84d8..66bf985 100755 --- a/run-test.sh +++ b/run-test.sh @@ -1,16 +1,49 @@ #!/bin/sh -CLI="python /home/drebs/projetos/leap/repos/soledad/scripts/soledad_test_env/soledad_test_env.py" -${CLI} couch start -${CLI} user-db delete -${CLI} user-db create -${CLI} shared-db create -${CLI} token-db create -${CLI} token-db insert-token -${CLI} soledad-server start +export SOLEDAD_STATS=1 +# assume the cli is in the environment +CLI="soledad_test_env.py" + +_server_setup() { + ${CLI} couch start + ${CLI} shared-db create + ${CLI} token-db create + ${CLI} token-db insert-token #--uuid 1234567890abcdefa --auth-token an-auth-token2 + ${CLI} soledad-server start +} + +function _server_reset() { + ${CLI} user-db delete #--uuid 1234567890abcdefa + ${CLI} user-db create #--uuid 1234567890abcdefa + ./scripts/create_payload.py + ./scripts/preload_server_database.py +} + +function _client_reset() { + rm -rf /tmp/soledad_client_test +} + +_server_setup + +if [ ! "${SKIP_SERVER_RESET}" ]; then + _server_reset +fi +if [ ! "${SKIP_CLIENT_RESET}" ]; then + _client_reset +fi + +# start local test server make soledad-sync-server | grep -v stats | grep -v ping & sleep 5 + +# create documents +make trigger-create-docs + +# launch background series measurement make measure-series & -sleep 5 +sleep 5 # wait a bit for some data points + +# trigger sync and stop afterwards make trigger-sync +make trigger-stop diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/create_payload.py b/scripts/create_payload.py new file mode 100755 index 0000000..d051661 --- /dev/null +++ b/scripts/create_payload.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +""" +Create a payload file in disk to use during tests. Reads the name of the file +and the intended size of the payload from the "defaults.conf" file: + + PAYLOAD = /path/to/payload/file + PAYLOAD_SIZE = 500 # in Kb +""" + +import os +from ConfigParser import ConfigParser + +parser = ConfigParser() +parser.read('defaults.conf') + +PAYLOAD = parser.get('sync', 'payload') +PAYLOAD_SIZE = int(parser.get('sync', 'payload_size')) * 1024 + +if os.path.isfile(PAYLOAD): + os.unlink(PAYLOAD) + +content = 'a' * 1024 + +with open(PAYLOAD, 'w') as f: + length = 0 + while length < PAYLOAD_SIZE: + f.write(content) + length += len(content) diff --git a/scripts/get_client_stats.py b/scripts/get_client_stats.py new file mode 100755 index 0000000..59f9aa6 --- /dev/null +++ b/scripts/get_client_stats.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +""" +Get stats from dummy server running Soledad Client and spit them out. +""" + +import commands +import urllib +import psutil + +stats = urllib.urlopen('http://localhost:8080/stats').read().split() + +pid, sync_phase, sync_exchange_phase = stats + +res = commands.getoutput("ps -p " + pid + " -o \%cpu,\%mem") +splitted = res.split() +cpu = splitted[2] +mem = splitted[3] + +print cpu, mem, sync_phase, sync_exchange_phase diff --git a/scripts/get_ping_rate.py b/scripts/get_ping_rate.py new file mode 100755 index 0000000..19d63d1 --- /dev/null +++ b/scripts/get_ping_rate.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +""" +Measure the connection rate to the dummy server running Soledad Client using +httperf and spit it out. +""" + +import commands +import re + +# ORIGINAL, STANDARD MEASURE (TOTALLY RANDOM) +#res = commands.getoutput( + #'httperf --server localhost --port 8080 --num-calls 5 --num-conns 20 ' + #'--uri /ping | grep "Connection rate"') + +res = commands.getoutput( + 'httperf --server localhost --port 8080 --num-calls 5 --num-conns 10 ' + '--uri /ping | grep "Connection rate"') +print re.findall('[-+]?([0-9]*\.?[0-9]+)', res)[0] diff --git a/scripts/measure_perf_series.py b/scripts/measure_perf_series.py new file mode 100755 index 0000000..587ff7a --- /dev/null +++ b/scripts/measure_perf_series.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +import commands +import datetime + +# How many points to measure. Make sure that you leave the baseline +# running for a significative amount before triggering the sync, it's possible +# you have significant variability in there. +POINTS = 400 + +SCALE = 50 + +commands.getoutput('echo time req/s cpu mem sync_phase sync_exchange_phase > ./out/series.log') +start = datetime.datetime.now() + +for i in range(POINTS): + value = commands.getoutput('./scripts/get_ping_rate.py') + print "Step", i, "...", value + stats = commands.getoutput('./scripts/get_client_stats.py') + cpu, mem, sync_phase, sync_exchange_phase = stats.split() + sync_phase = str(int(sync_phase)*SCALE) + sync_exchange_phase = str(int(sync_exchange_phase)*SCALE) + now = datetime.datetime.now() + secs = (now - start).total_seconds() + try: + # make it so the script exits succesfully when the server is dead + commands.getoutput( + 'echo %s\t%s\t%s\t%s\t%s\t%s >> ./out/series.log' \ + % (secs, value, cpu, mem, sync_phase, sync_exchange_phase)) + except ValueError: + break diff --git a/scripts/preload_server_database.py b/scripts/preload_server_database.py new file mode 100755 index 0000000..95e8cfd --- /dev/null +++ b/scripts/preload_server_database.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +""" +Preload the server database with a certain amount of documents so we can +receive them during the sync process and gather meaningful statistics for that +specific phase of sync. + +Gets uuid, payload file path and amount of documents to preload from +"defaults.conf" config file: + + UUID = some-uuid + PAYLOAD = /path/to/payload/file + NUM_DOCS = 100 +""" + +import os +from ConfigParser import ConfigParser +from leap.soledad.common.couch import CouchDatabase + +parser = ConfigParser() +parser.read('defaults.conf') + +UUID = parser.get('client', 'uuid') +PAYLOAD = parser.get('sync', 'payload') +NUM_DOCS = int(parser.get('sync', 'num_docs')) + +db = CouchDatabase.open_database( + 'http://127.0.0.1:5984/user-%s' % UUID, + False) # should create database? + +payload = None +if os.path.isfile(PAYLOAD): + with open(PAYLOAD, 'r') as f: + payload = f.read() + +for i in xrange(NUM_DOCS): + db.create_doc({'payload': payload}) + +db.close() diff --git a/scripts/server_with_soledad_syncer.py b/scripts/server_with_soledad_syncer.py new file mode 100644 index 0000000..2890e04 --- /dev/null +++ b/scripts/server_with_soledad_syncer.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + + +import os +from klein import run, route, resource +import soledad_sync as sync +from twisted.internet import reactor +import datetime + + +@route('/create-docs') +def create_docs(request): + d = sync.create_docs() + return d + + +@route('/start-sync') +def start_sync(request): + d = sync.start_sync() + return d + + +@route('/ping') +def ping(request): + return 'easy!' + + +@route('/pid') +def pid(request): + return str(os.getpid()) + + +@route('/stop') +def stop(request): + reactor.callLater(1, reactor.stop) + return '' + +@route('/stats') +def stats(request): + pid = os.getpid() + sync_phase, sync_exchange_phase = sync.stats() + return "%d %d %d" % (pid, sync_phase, sync_exchange_phase) + + +if __name__ == "__main__": + run("localhost", 8080) diff --git a/scripts/soledad_sync.py b/scripts/soledad_sync.py new file mode 100644 index 0000000..8c6442f --- /dev/null +++ b/scripts/soledad_sync.py @@ -0,0 +1,96 @@ +import os +import json +from ConfigParser import ConfigParser +from leap.soledad.client.api import Soledad +from twisted.internet import defer, reactor +from twisted.internet.task import deferLater + + +# get configs from file +parser = ConfigParser() +parser.read('defaults.conf') + +HOST = parser.get('server', 'host') + +UUID = parser.get('client', 'uuid') +CLIENT_BASEDIR = parser.get('client', 'basedir') +PASSPHRASE = parser.get('client', 'passphrase') + +NUM_DOCS = int(parser.get('sync', 'num_docs')) +PAYLOAD = parser.get('sync', 'payload') +AUTH_TOKEN = parser.get('sync', 'auth_token') + +STATS_FILE = parser.get('test', 'stats_file') + + +DO_THESEUS = os.environ.get('THESEUS', False) + + +def _get_soledad_instance_from_uuid(uuid, passphrase, basedir, server_url, + cert_file, token): + secrets_path = os.path.join(basedir, '%s.secret' % uuid) + local_db_path = os.path.join(basedir, '%s.db' % uuid) + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True, + syncable=True) + + +def _get_soledad_instance(): + return _get_soledad_instance_from_uuid( + UUID, PASSPHRASE, CLIENT_BASEDIR, HOST, '', AUTH_TOKEN) + +s = _get_soledad_instance() + + +def create_docs(): + global s + # get content for docs + payload = 'a' * 10 + if os.path.isfile(PAYLOAD): + with open(PAYLOAD, 'r') as f: + payload = f.read() + + # create docs + cd = [] + for i in range(NUM_DOCS): + cd.append(s.create_doc({'payload': payload})) + d = defer.gatherResults(cd) + + d.addCallback(lambda _: s.get_all_docs()) + d.addCallback(lambda result: "%d docs created, %d docs on db" % (NUM_DOCS, result[0])) + #d.addCallback(lambda _: s.close()) + + return d + + +def start_sync(): + global s + + if DO_THESEUS: + from theseus import Tracer + t = Tracer() + t.install() + + def stop_tracing(_): + if DO_THESEUS: + with open('callgrind.theseus', 'wb') as outfile: + t.write_data(outfile) + print "STOPPED TRACING, DUMPED IN CALLGRIND.THESEUS<<<<" + + cd = [] + + d = s.sync() + d.addCallback(stop_tracing) + + return d + +def stats(): + global s + return s.sync_stats() diff --git a/scripts/sync_stats.gnuplot b/scripts/sync_stats.gnuplot new file mode 100755 index 0000000..7fdf29a --- /dev/null +++ b/scripts/sync_stats.gnuplot @@ -0,0 +1,23 @@ +#!/usr/bin/gnuplot + +infile="$0" +outfile="$1" + +# maybe save image file +if (outfile ne '') \ + set term png size 1000,400; \ + set output "./out/sync-stats.png" + +# make the graph beautiful +set title 'Soledad Sync Phases' +set xtics 10 +set ytics 50 +set grid +set key outside + +# plot! +plot for [col=2:6] infile using 1:col with linespoints title columnheader + +# pause when not saving image file +if (outfile eq '') \ + pause -1 diff --git a/server_with_soledad_syncer.py b/server_with_soledad_syncer.py deleted file mode 100644 index 456e3dc..0000000 --- a/server_with_soledad_syncer.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/env python - - -import os -from klein import run, route, resource -import soledad_sync as sync -from twisted.internet import reactor -import datetime - - -@route('/start-sync') -def home(request): - print "GOT REQUEST FOR STARTING SYNC..." - d = sync.upload_soledad_stuff() - return d - - -@route('/ping') -def ping(request): - return 'easy!' - - -@route('/pid') -def pid(request): - return str(os.getpid()) - - -@route('/stop') -def stop(request): - reactor.callLater(1, reactor.stop) - return '' - -@route('/stats') -def stats(request): - pid = os.getpid() - return "%d %d" % (pid, sync.phase * 50) - - -if __name__ == "__main__": - run("localhost", 8080) diff --git a/soledad_sync.py b/soledad_sync.py deleted file mode 100644 index e53f49c..0000000 --- a/soledad_sync.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import json -from ConfigParser import ConfigParser -from leap.soledad.client.api import Soledad -from twisted.internet import defer - - -# get configs from file -parser = ConfigParser() -parser.read('defaults.conf') -HOST = parser.get('server', 'host') -UUID = parser.get('client', 'uuid') -NUM_DOCS = int(parser.get('sync', 'num_docs')) -PAYLOAD = parser.get('sync', 'payload') -AUTH_TOKEN = parser.get('sync', 'auth-token') -STATS_FILE = parser.get('test', 'stats-file') - - - - -DO_THESEUS = os.environ.get('THESEUS', False) - - -phase = 0 - - -def _get_soledad_instance_from_uuid(uuid, passphrase, basedir, server_url, - cert_file, token): - secrets_path = os.path.join(basedir, '%s.secret' % uuid) - local_db_path = os.path.join(basedir, '%s.db' % uuid) - return Soledad( - uuid, - unicode(passphrase), - secrets_path=secrets_path, - local_db_path=local_db_path, - server_url=server_url, - cert_file=cert_file, - auth_token=token, - defer_encryption=True, - syncable=True) - - -def onSyncDone(result): - #-------- PHASE 3: sync done. - global phase - phase += 1 - print "SYNC DONE!", result - - -def upload_soledad_stuff(): - global phase - - with open(PAYLOAD, 'r') as f: - payload = f.read() - - if DO_THESEUS: - from theseus import Tracer - t = Tracer() - t.install() - - s = _get_soledad_instance_from_uuid( - UUID, 'pass', '/tmp/soledadsync', HOST, '', AUTH_TOKEN) - - def dumpStats(_): - stats = s.sync_stats() - with open(STATS_FILE, 'w') as f: - f.write(json.dumps(stats)) - - def do_sync(_): - global phase - #-------- PHASE 2: docs created, defer sync - phase += 1 - d = s.sync() - d.addCallback(onSyncDone) - d.addCallback(dumpStats) - return d - - def stop_tracing(_): - if DO_THESEUS: - with open('callgrind.theseus', 'wb') as outfile: - t.write_data(outfile) - print "STOPPED TRACING, DUMPED IN CALLGRIND.THESEUS<<<<" - - cd = [] - #-------- PHASE 1: deferring doc creation - phase += 1 - for i in range(NUM_DOCS): - cd.append(s.create_doc({'payload': payload})) - d1 = defer.gatherResults(cd) - - # XXX comment out to nuke out the actual sync - d1.addCallback(do_sync) - d1.addCallback(stop_tracing) - - return d1 -- cgit v1.2.3