diff options
Diffstat (limited to 'scripts/profiling/sync')
| l--------- | scripts/profiling/sync/movingaverage.py | 1 | ||||
| -rw-r--r-- | scripts/profiling/sync/profile-decoupled.py | 24 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/profiling/sync/profile-sync.py | 161 | ||||
| -rwxr-xr-x | scripts/profiling/sync/run_plop_server.sh | 2 |
4 files changed, 172 insertions, 16 deletions
diff --git a/scripts/profiling/sync/movingaverage.py b/scripts/profiling/sync/movingaverage.py new file mode 120000 index 00000000..098b0a01 --- /dev/null +++ b/scripts/profiling/sync/movingaverage.py @@ -0,0 +1 @@ +../movingaverage.py
\ No newline at end of file diff --git a/scripts/profiling/sync/profile-decoupled.py b/scripts/profiling/sync/profile-decoupled.py new file mode 100644 index 00000000..a844c3c6 --- /dev/null +++ b/scripts/profiling/sync/profile-decoupled.py @@ -0,0 +1,24 @@ +# test_name: soledad-sync +# start_time: 2014-06-12 20:09:11.232317+00:00 +# elapsed_time total_cpu total_memory proc_cpu proc_memory +0.000225 68.400000 46.100000 105.300000 0.527224 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.250167 0.000000 0.255160 +0.707006 76.200000 46.200000 90.000000 0.562369 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +1.413140 63.200000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +2.123962 0.000000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +2.833941 31.600000 46.100000 0.000000 0.360248 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +3.541532 5.300000 46.100000 0.000000 0.360298 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +4.253390 14.300000 46.100000 11.100000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +4.967365 5.000000 46.100000 0.000000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +5.680172 5.600000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +6.390501 10.500000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +7.101711 23.800000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +7.810529 30.000000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +8.517835 25.000000 46.100000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +9.227455 5.300000 46.000000 9.500000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +9.936479 9.500000 46.000000 10.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +10.645015 52.400000 46.200000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +11.355179 21.100000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +12.066252 36.800000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +12.777689 28.600000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +13.489886 0.000000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +# end_time: 2014-06-12 20:09:25.434677+00:00
\ No newline at end of file diff --git a/scripts/profiling/sync/profile-sync.py b/scripts/profiling/sync/profile-sync.py index fdd5b5a6..9ef2ea92 100644..100755 --- a/scripts/profiling/sync/profile-sync.py +++ b/scripts/profiling/sync/profile-sync.py @@ -1,13 +1,21 @@ -#!/usr/bin/python - +#!/usr/bin/env python import argparse +import commands +import getpass import logging +import mmap +import os +import tempfile +from datetime import datetime +from twisted.internet import reactor from util import StatsLogger, ValidateUserHandle -from client_side_db import get_soledad_instance -#from plot import plot +from client_side_db import _get_soledad_instance, _get_soledad_info +from leap.common.events import flags + +flags.set_events_enabled(False) # create a logger @@ -15,6 +23,50 @@ logger = logging.getLogger(__name__) LOG_FORMAT = '%(asctime)s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) +GITVER = commands.getoutput('git describe') + + +def get_and_run_plop_collector(): + from plop.collector import Collector + collector = Collector() + collector.start() + return collector + + +def get_and_run_theseus_tracer(): + from theseus import Tracer + t = Tracer() + t.install() + return t + + +def bail(msg): + print "[!] %s" % msg + + +def create_docs(soledad, args): + """ + Populates the soledad database with dummy messages, so we can exercise + sending payloads during the sync. + """ + sample_path = args.payload_f + if not sample_path: + bail('Need to pass a --payload-file') + return + if not os.path.isfile(sample_path): + bail('--payload-file does not exist!') + return + + numdocs = args.send_num + docsize = args.send_size + + # XXX this will FAIL if the payload source is smaller to size * num + # XXX could use a cycle iterator + with open(sample_path, "r+b") as sample_f: + fmap = mmap.mmap(sample_f.fileno(), 0, prot=mmap.PROT_READ) + for index in xrange(numdocs): + payload = fmap.read(docsize * 1024) + s.create_doc({payload: payload}) # main program @@ -33,6 +85,34 @@ if __name__ == '__main__': parser.add_argument( '-l', dest='logfile', required=False, default='/tmp/profile.log', help='the file to which write the log') + parser.add_argument( + '--no-send', dest='do_send', action='store_false', + help='skip sending messages') + parser.add_argument( + '--send-size', dest='send_size', default=10, + help='size of doc to send, in KB (default: 10)') + parser.add_argument( + '--send-num', dest='send_num', default=10, + help='number of docs to send (default: 10)') + parser.add_argument( + '--payload-file', dest="payload_f", default=None, + help='path to a sample file to use for the payloads') + parser.add_argument( + '--no-stats', dest='do_stats', action='store_false', + help='skip system stats') + parser.add_argument( + '--plot', dest='do_plot', action='store_true', + help='do a graphical plot') + parser.add_argument( + '--plop', dest='do_plop', action='store_true', + help='run sync script under plop profiler') + parser.add_argument( + '--theseus', dest='do_theseus', action='store_true', + help='run sync script under theseus profiler') + parser.set_defaults( + do_send=True, do_stats=True, do_plot=False, do_plop=False, + do_theseus=False, + ) args = parser.parse_args() # get the password @@ -47,16 +127,65 @@ if __name__ == '__main__': basedir = tempfile.mkdtemp() logger.info('Using %s as base directory.' % basedir) + uuid, server_url, cert_file, token = \ + _get_soledad_info( + args.username, args.provider, passphrase, basedir) # get the soledad instance - s = get_soledad_instance( - args.username, args.provider, passphrase, basedir) - for i in xrange(10): - s.create_doc({}) - - sl = StatsLogger( - "soledad-sync", args.logfile, procs=["python"], interval=0.001) - sl.start() - s.sync() - sl.stop() - - #plot(args.logfile) + s = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + + if args.do_send: + create_docs(s, args) + + def start_sync(): + if args.do_stats: + sl = StatsLogger( + "soledad-sync", args.logfile, procs=["python"], interval=0.001) + sl.start() + else: + sl = None + + if args.do_plop: + plop_collector = get_and_run_plop_collector() + else: + plop_collector = None + + if args.do_theseus: + theseus = get_and_run_theseus_tracer() + else: + theseus = None + + t0 = datetime.now() + d = s.sync() + d.addCallback(onSyncDone, sl, t0, plop_collector, theseus) + + def onSyncDone(sync_result, sl, t0, plop_collector, theseus): + # TODO should write this to a result file + print "GOT SYNC RESULT: ", sync_result + t1 = datetime.now() + if sl: + sl.stop() + if plop_collector: + from plop.collector import PlopFormatter + formatter = PlopFormatter() + plop_collector.stop() + if not os.path.isdir('profiles'): + os.mkdir('profiles') + with open('profiles/plop-sync-%s' % GITVER, 'w') as f: + f.write(formatter.format(plop_collector)) + if theseus: + with open('callgrind.theseus', 'wb') as outfile: + theseus.write_data(outfile) + theseus.uninstall() + + delta = (t1 - t0).total_seconds() + # TODO should write this to a result file + print "[+] Sync took %s seconds." % delta + reactor.stop() + + if args.do_plot: + from plot import plot + plot(args.logfile) + + reactor.callWhenRunning(start_sync) + reactor.run() diff --git a/scripts/profiling/sync/run_plop_server.sh b/scripts/profiling/sync/run_plop_server.sh new file mode 100755 index 00000000..7b905c3d --- /dev/null +++ b/scripts/profiling/sync/run_plop_server.sh @@ -0,0 +1,2 @@ +#!/bin/sh +python -m plop.viewer --datadir=profiles |