Merge remote-tracking branch 'kali-github/pr/2'

17 files changed, 279 insertions, 100 deletions

diff --git a/Makefile b/Makefile
index c710c68..53228a5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,18 +1,18 @@
 # Actual soledad sync
 
 soledad-sync-server:
-	twistd -n web --port 8080 --class=server_with_soledad_syncer.resource
+	twistd -n web --port 8080 --class=scripts.server_with_soledad_syncer.resource
 
 soledad-sync-server-lineprof:
-	kernprof -l server_with_soledad_syncer.py
+	kernprof -l ./scripts/server_with_soledad_syncer.py
 
 
 soledad-sync-server-debug:
 	#twistd --profile=stats_obj --profiler=cProfile -n web --port 8080 --class=server_with_soledad_syncer.resource
-	python -m cProfile -o sync.cprofile server_with_soledad_syncer.py 
+	python -m cProfile -o sync.cprofile ./scripts/server_with_soledad_syncer.py 
 
 view-lineprofile:
-	python -m line_profiler server_with_soledad_syncer.py.lprof
+	python -m line_profiler ./scripts/server_with_soledad_syncer.py.lprof
 
 
 view-profile:
@@ -21,9 +21,14 @@ view-profile:
 measure-ping:
 	httperf --server localhost --port 8080 --num-calls 5 --num-conns 20 --uri /ping
 
+trigger-create-docs:
+	curl localhost:8080/create-docs
+
 trigger-sync:
 	#time curl localhost:8080/start-sync
 	curl localhost:8080/start-sync
+
+trigger-stop:
 	curl localhost:8080/stop
 
 measure-series:
@@ -31,9 +36,16 @@ measure-series:
 	# TODO rm series.log, name it with a timestamp
 	# TODO measure, first of all, the number of seconds from the beginning!!! (right now it's biased)
 	# TODO add cpu/ram usage (ping command COULD RETURN THAT!)
-	rm -f /tmp/soledadsync/*
-	rm -f series.log
-	python measure-perf-series.py
+	#rm -f /tmp/soledadsync/*
+	rm -f ./out/series.log
+	./scripts/measure_perf_series.py
+
+graph-image:
+	gnuplot -e 'call "./scripts/sync_stats.gnuplot" "./out/series.log" "./out/sync-stats.png"'
+
+graph-view:
+	gnuplot -e 'call "./scripts/sync_stats.gnuplot" "./out/series.log" ""'
 
-graph-series:
-	data=series.log ./graphit
+kill:
+	killall -9 twistd
+	killall -9 python
diff --git a/data/sync-stats-drebs-develop-without-unicode-block.png b/data/sync-stats-drebs-develop-without-unicode-block.png
new file mode 100644
index 0000000..80846e7
--- /dev/null
+++ b/data/sync-stats-drebs-develop-without-unicode-block.png
diff --git a/defaults.conf.example b/defaults.conf.example
index e4d836d..e997c53 100644
--- a/defaults.conf.example
+++ b/defaults.conf.example
@@ -1,9 +1,16 @@
 [server]
-host: http://localhost:2424
+host = http://localhost:2424
 
 [client]
-uuid: 1234567890abcdef
+uuid = 1234567890abcdef
+basedir = /tmp/soledad_client_test
+passphrase = 12345678
 
 [sync]
-num_docs: 100
-payload: /tmp/payload
+num_docs = 100
+payload = /tmp/payload
+payload_size = 500
+auth_token = an-auth-token
+
+[test]
+stats_file = ./out/stats.json
diff --git a/get_client_cpu_mem.py b/get_client_cpu_mem.py
deleted file mode 100644
index fe1aeec..0000000
--- a/get_client_cpu_mem.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import commands
-import urllib
-import psutil
-
-pid, phase = urllib.urlopen('http://localhost:8080/stats').read().split()
-res = commands.getoutput("ps -p " + pid + " -o \%cpu,\%mem")
-splitted = res.split()
-cpu = splitted[2]
-mem = splitted[3]
-
-print cpu, mem, phase
diff --git a/graphit b/graphit
deleted file mode 100755
index 0c809c0..0000000
--- a/graphit
+++ /dev/null
@@ -1,9 +0,0 @@
-#! /usr/bin/gnuplot
-
-filename=system("echo $data")
-set title filename
-set key outside
-#plot name  with linespoints notitle
-#plot filename using 1:2 with linespoints title columnheader
-plot for [col=2:5] filename using 1:col with linespoints title columnheader
-pause -1 
diff --git a/measure-perf-series.py b/measure-perf-series.py
deleted file mode 100644
index 5694bc8..0000000
--- a/measure-perf-series.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import commands
-import datetime
-
-# How many points to measure. Make sure that you leave the baseline
-# running for a significative amount before triggering the sync, it's possible 
-# you have significant variability in there.
-POINTS = 200
-
-commands.getoutput('echo time req/s cpu mem phase > series.log')
-start = datetime.datetime.now()
-
-for i in range(POINTS):
-    value = commands.getoutput('python get_ping_rate.py')
-    print "Step", i, "...", value
-    stats = commands.getoutput('python get_client_cpu_mem.py')
-    cpu, mem, phase = stats.split()
-    now = datetime.datetime.now()
-    secs = (now - start).total_seconds()
-    commands.getoutput(
-        'echo %s\t%s\t%s\t%s\t%s >> series.log' % (secs, value, cpu, mem, phase))
diff --git a/out/.empty b/out/.empty
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/out/.empty
diff --git a/run-test.sh b/run-test.sh
index 7cc84d8..66bf985 100755
--- a/run-test.sh
+++ b/run-test.sh
@@ -1,16 +1,49 @@
 #!/bin/sh
 
-CLI="python /home/drebs/projetos/leap/repos/soledad/scripts/soledad_test_env/soledad_test_env.py"
-${CLI} couch start
-${CLI} user-db delete
-${CLI} user-db create
-${CLI} shared-db create
-${CLI} token-db create
-${CLI} token-db insert-token
-${CLI} soledad-server start
+export SOLEDAD_STATS=1
 
+# assume the cli is in the environment
+CLI="soledad_test_env.py"
+
+_server_setup() {
+  ${CLI} couch start
+  ${CLI} shared-db create
+  ${CLI} token-db create
+  ${CLI} token-db insert-token #--uuid 1234567890abcdefa --auth-token an-auth-token2
+  ${CLI} soledad-server start
+}
+
+function _server_reset() {
+  ${CLI} user-db delete #--uuid 1234567890abcdefa
+  ${CLI} user-db create #--uuid 1234567890abcdefa
+  ./scripts/create_payload.py
+  ./scripts/preload_server_database.py
+}
+
+function _client_reset() {
+  rm -rf /tmp/soledad_client_test
+}
+
+_server_setup
+
+if [ ! "${SKIP_SERVER_RESET}" ]; then
+  _server_reset
+fi
+if [ ! "${SKIP_CLIENT_RESET}" ]; then
+  _client_reset
+fi
+
+# start local test server
 make soledad-sync-server | grep -v stats | grep -v ping  &
 sleep 5
+
+# create documents
+make trigger-create-docs
+
+# launch background series measurement
 make measure-series &
-sleep 5
+sleep 5  # wait a bit for some data points
+
+# trigger sync and stop afterwards
 make trigger-sync
+make trigger-stop
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/scripts/__init__.py
diff --git a/scripts/create_payload.py b/scripts/create_payload.py
new file mode 100755
index 0000000..d051661
--- /dev/null
+++ b/scripts/create_payload.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+"""
+Create a payload file in disk to use during tests. Reads the name of the file
+and the intended size of the payload from the "defaults.conf" file:
+
+  PAYLOAD = /path/to/payload/file
+  PAYLOAD_SIZE = 500  # in Kb
+"""
+
+import os
+from ConfigParser import ConfigParser
+
+parser = ConfigParser()
+parser.read('defaults.conf')
+
+PAYLOAD = parser.get('sync', 'payload')
+PAYLOAD_SIZE = int(parser.get('sync', 'payload_size')) * 1024
+
+if os.path.isfile(PAYLOAD):
+    os.unlink(PAYLOAD)
+
+content = 'a' * 1024
+
+with open(PAYLOAD, 'w') as f:
+    length = 0
+    while length < PAYLOAD_SIZE:
+        f.write(content)
+        length += len(content)
diff --git a/scripts/get_client_stats.py b/scripts/get_client_stats.py
new file mode 100755
index 0000000..59f9aa6
--- /dev/null
+++ b/scripts/get_client_stats.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+"""
+Get stats from dummy server running Soledad Client and spit them out.
+"""
+
+import commands
+import urllib
+import psutil
+
+stats = urllib.urlopen('http://localhost:8080/stats').read().split()
+
+pid, sync_phase, sync_exchange_phase = stats
+
+res = commands.getoutput("ps -p " + pid + " -o \%cpu,\%mem")
+splitted = res.split()
+cpu = splitted[2]
+mem = splitted[3]
+
+print cpu, mem, sync_phase, sync_exchange_phase
diff --git a/get_ping_rate.py b/scripts/get_ping_rate.py
index 442f4dc..19d63d1 100644..100755
--- a/get_ping_rate.py
+++ b/scripts/get_ping_rate.py
@@ -1,3 +1,10 @@
+#!/usr/bin/env python
+
+"""
+Measure the connection rate to the dummy server running Soledad Client using
+httperf and spit it out.
+"""
+
 import commands
 import re
 
diff --git a/scripts/measure_perf_series.py b/scripts/measure_perf_series.py
new file mode 100755
index 0000000..587ff7a
--- /dev/null
+++ b/scripts/measure_perf_series.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+import commands
+import datetime
+
+# How many points to measure. Make sure that you leave the baseline
+# running for a significative amount before triggering the sync, it's possible 
+# you have significant variability in there.
+POINTS = 400
+
+SCALE = 50
+
+commands.getoutput('echo time req/s cpu mem sync_phase sync_exchange_phase > ./out/series.log')
+start = datetime.datetime.now()
+
+for i in range(POINTS):
+    value = commands.getoutput('./scripts/get_ping_rate.py')
+    print "Step", i, "...", value
+    stats = commands.getoutput('./scripts/get_client_stats.py')
+    cpu, mem, sync_phase, sync_exchange_phase = stats.split()
+    sync_phase = str(int(sync_phase)*SCALE)
+    sync_exchange_phase = str(int(sync_exchange_phase)*SCALE)
+    now = datetime.datetime.now()
+    secs = (now - start).total_seconds()
+    try:
+        # make it so the script exits succesfully when the server is dead
+        commands.getoutput(
+            'echo %s\t%s\t%s\t%s\t%s\t%s >> ./out/series.log' \
+            % (secs, value, cpu, mem, sync_phase, sync_exchange_phase))
+    except ValueError:
+        break
diff --git a/scripts/preload_server_database.py b/scripts/preload_server_database.py
new file mode 100755
index 0000000..95e8cfd
--- /dev/null
+++ b/scripts/preload_server_database.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+"""
+Preload the server database with a certain amount of documents so we can
+receive them during the sync process and gather meaningful statistics for that
+specific phase of sync.
+
+Gets uuid, payload file path and amount of documents to preload from
+"defaults.conf" config file:
+
+  UUID = some-uuid
+  PAYLOAD = /path/to/payload/file
+  NUM_DOCS = 100
+"""
+
+import os
+from ConfigParser import ConfigParser
+from leap.soledad.common.couch import CouchDatabase
+
+parser = ConfigParser()
+parser.read('defaults.conf')
+
+UUID = parser.get('client', 'uuid')
+PAYLOAD = parser.get('sync', 'payload')
+NUM_DOCS = int(parser.get('sync', 'num_docs'))
+
+db = CouchDatabase.open_database(
+    'http://127.0.0.1:5984/user-%s' % UUID,
+    False)  # should create database?
+
+payload = None
+if os.path.isfile(PAYLOAD):
+    with open(PAYLOAD, 'r') as f:
+        payload = f.read()
+
+for i in xrange(NUM_DOCS):
+    db.create_doc({'payload': payload})
+
+db.close()
diff --git a/server_with_soledad_syncer.py b/scripts/server_with_soledad_syncer.py
index 3d495b6..2890e04 100644
--- a/server_with_soledad_syncer.py
+++ b/scripts/server_with_soledad_syncer.py
@@ -3,17 +3,21 @@
 
 
 import os
-import datetime
 from klein import run, route, resource
+import soledad_sync as sync
 from twisted.internet import reactor
+import datetime
 
-import soledad_sync as sync
+
+@route('/create-docs')
+def create_docs(request):
+    d = sync.create_docs()
+    return d
 
 
 @route('/start-sync')
-def home(request):
-    print "GOT REQUEST FOR STARTING SYNC..."
-    d = sync.upload_soledad_stuff()
+def start_sync(request):
+    d = sync.start_sync()
     return d
 
 
@@ -35,7 +39,8 @@ def stop(request):
 @route('/stats')
 def stats(request):
     pid = os.getpid()
-    return "%d %d" % (pid, sync.phase * 50)
+    sync_phase, sync_exchange_phase = sync.stats()
+    return "%d %d %d" % (pid, sync_phase, sync_exchange_phase)
 
 
 if __name__ == "__main__":
diff --git a/soledad_sync.py b/scripts/soledad_sync.py
index 525b988..8c6442f 100644
--- a/soledad_sync.py
+++ b/scripts/soledad_sync.py
@@ -1,22 +1,30 @@
 import os
+import json
 from ConfigParser import ConfigParser
 from leap.soledad.client.api import Soledad
-from twisted.internet import defer
+from twisted.internet import defer, reactor
+from twisted.internet.task import deferLater
 
 
 # get configs from file
 parser = ConfigParser()
 parser.read('defaults.conf')
+
 HOST = parser.get('server', 'host')
+
 UUID = parser.get('client', 'uuid')
+CLIENT_BASEDIR = parser.get('client', 'basedir')
+PASSPHRASE = parser.get('client', 'passphrase')
+
 NUM_DOCS = int(parser.get('sync', 'num_docs'))
 PAYLOAD = parser.get('sync', 'payload')
+AUTH_TOKEN = parser.get('sync', 'auth_token')
 
+STATS_FILE = parser.get('test', 'stats_file')
 
-DO_THESEUS = os.environ.get('THESEUS', False)
 
+DO_THESEUS = os.environ.get('THESEUS', False)
 
-phase = 0
 
 def _get_soledad_instance_from_uuid(uuid, passphrase, basedir, server_url,
                                     cert_file, token):
@@ -34,35 +42,42 @@ def _get_soledad_instance_from_uuid(uuid, passphrase, basedir, server_url,
         syncable=True)
 
 
-def onSyncDone(result):
-    #-------- PHASE 3: sync done.
-    global phase
-    phase += 1
-    print "SYNC DONE!", result
+def _get_soledad_instance():
+    return _get_soledad_instance_from_uuid(
+        UUID, PASSPHRASE, CLIENT_BASEDIR, HOST, '', AUTH_TOKEN)
+
+s = _get_soledad_instance()
+
+
+def create_docs():
+    global s
+    # get content for docs
+    payload = 'a' * 10
+    if os.path.isfile(PAYLOAD):
+        with open(PAYLOAD, 'r') as f:
+            payload = f.read()
+
+    # create docs
+    cd = []
+    for i in range(NUM_DOCS):
+        cd.append(s.create_doc({'payload': payload}))
+    d = defer.gatherResults(cd)
 
+    d.addCallback(lambda _: s.get_all_docs())
+    d.addCallback(lambda result: "%d docs created, %d docs on db" % (NUM_DOCS, result[0]))
+    #d.addCallback(lambda _: s.close())
 
-def upload_soledad_stuff():
-    global phase
+    return d
 
-    with open(PAYLOAD, 'r') as f:
-        payload = f.read()
+
+def start_sync():
+    global s
 
     if DO_THESEUS:
         from theseus import Tracer
         t = Tracer()
         t.install()
 
-    s = _get_soledad_instance_from_uuid(
-        UUID, 'pass', '/tmp/soledadsync', HOST, '', 'an-auth-token')
-
-    def do_sync(_):
-        global phase
-        #-------- PHASE 2: docs created, defer sync
-        phase += 1
-        d = s.sync()
-        d.addCallback(onSyncDone)
-        return d
-
     def stop_tracing(_):
         if DO_THESEUS:
             with open('callgrind.theseus', 'wb') as outfile:
@@ -70,14 +85,12 @@ def upload_soledad_stuff():
             print "STOPPED TRACING, DUMPED IN CALLGRIND.THESEUS<<<<"
 
     cd = []
-    #-------- PHASE 1: deferring doc creation
-    phase += 1
-    for i in range(NUM_DOCS):
-        cd.append(s.create_doc({'payload': payload}))
-    d1 = defer.gatherResults(cd)
 
-    # XXX comment out to nuke out the actual sync
-    d1.addCallback(do_sync)
-    d1.addCallback(stop_tracing)
+    d = s.sync()
+    d.addCallback(stop_tracing)
+
+    return d
 
-    return d1
+def stats():
+    global s
+    return s.sync_stats()
diff --git a/scripts/sync_stats.gnuplot b/scripts/sync_stats.gnuplot
new file mode 100755
index 0000000..7fdf29a
--- /dev/null
+++ b/scripts/sync_stats.gnuplot
@@ -0,0 +1,23 @@
+#!/usr/bin/gnuplot
+
+infile="$0"
+outfile="$1"
+
+# maybe save image file
+if (outfile ne '') \
+  set term png size 1000,400; \
+  set output "./out/sync-stats.png"
+
+# make the graph beautiful
+set title 'Soledad Sync Phases'
+set xtics 10
+set ytics 50
+set grid
+set key outside
+
+# plot!
+plot for [col=2:6] infile using 1:col with linespoints title columnheader
+
+# pause when not saving image file
+if (outfile eq '') \
+  pause -1