diff options
| -rwxr-xr-x | scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py | 46 | ||||
| -rwxr-xr-x | scripts/profiling/doc_put_memory_usage/profile-procs.py | 54 | ||||
| l--------- | scripts/profiling/sync/client_side_db.py | 1 | ||||
| -rwxr-xr-x | scripts/profiling/sync/plot.py | 93 | ||||
| -rw-r--r-- | scripts/profiling/sync/profile-sync.py | 62 | ||||
| l--------- | scripts/profiling/sync/util.py | 1 | ||||
| -rw-r--r-- | scripts/profiling/util.py | 75 |
7 files changed, 332 insertions, 0 deletions
diff --git a/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py b/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py new file mode 100755 index 00000000..b2b35d30 --- /dev/null +++ b/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py @@ -0,0 +1,46 @@ +#!/usr/bin/python + + +import logging +import argparse +import psutil +import time + + +def find_procs(procs): + result = [] + for name, executable in procs: + found = filter( + lambda p: executable == p.name, + psutil.process_iter()) + if len(found) == 1: + result.append(found[0]) + return result + + +def log_memory(soledad, bigcouch): + while True: + print "%f %f" % \ + (soledad.get_memory_percent(), bigcouch.get_memory_percent()) + time.sleep(1) + + +if __name__ == '__main__': + + # configure logger + logger = logging.getLogger(__name__) + LOG_FORMAT = '%(asctime)s %(message)s' + logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + '-l', dest='logfile', + help='log output to file') + args = parser.parse_args() + + log_memory(*find_procs([ + ('Soledad', 'twistd'), + ('Bigcouch', 'beam.smp')])) + diff --git a/scripts/profiling/doc_put_memory_usage/profile-procs.py b/scripts/profiling/doc_put_memory_usage/profile-procs.py new file mode 100755 index 00000000..53f5977b --- /dev/null +++ b/scripts/profiling/doc_put_memory_usage/profile-procs.py @@ -0,0 +1,54 @@ +#!/usr/bin/python + + +import logging +import argparse +import psutil +import time + + +def find_procs(procs): + result = [] + for name, executable in procs: + found = filter( + lambda p: executable == p.name, + psutil.process_iter()) + if len(found) == 1: + result.append(found[0]) + return result + + +def log_usage(procs, logger): + names = [proc.name for proc in procs] + logger.info("Logging cpu and memory for: %s" % names) + while True: + s = '%f %f' %\ + (psutil.cpu_percent(), psutil.phymem_usage().percent) + for proc in procs: + s += ' %f %f' % \ + (proc.get_cpu_percent(), proc.get_memory_percent()) + logger.info(s) + time.sleep(1) + + +if __name__ == '__main__': + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + '-l', dest='logfile', + help='log output to file') + args = parser.parse_args() + + # configure logger + logger = logging.getLogger(__name__) + LOG_FORMAT = '%(asctime)s %(message)s' + logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + if args.logfile is not None: + handler = logging.FileHandler(args.logfile, mode='a') + handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT)) + logger.addHandler(handler) + + log_usage(find_procs([ + ('Soledad', 'twistd'), + ('Bigcouch', 'beam.smp')]), logger) diff --git a/scripts/profiling/sync/client_side_db.py b/scripts/profiling/sync/client_side_db.py new file mode 120000 index 00000000..9e49a7f0 --- /dev/null +++ b/scripts/profiling/sync/client_side_db.py @@ -0,0 +1 @@ +../../db_access/client_side_db.py
\ No newline at end of file diff --git a/scripts/profiling/sync/plot.py b/scripts/profiling/sync/plot.py new file mode 100755 index 00000000..8e3e1c15 --- /dev/null +++ b/scripts/profiling/sync/plot.py @@ -0,0 +1,93 @@ +#!/usr/bin/python + + +import argparse +from matplotlib import pyplot as plt +from movingaverage import movingaverage +from scipy.interpolate import interp1d +from numpy import linspace + + +def smooth(l): + return movingaverage(l, 3, data_is_list=True, avoid_fp_drift=False) + + +def plot(filename, subtitle=''): + + # config the plot + plt.xlabel('time (s)') + plt.ylabel('usage (%)') + title = 'soledad sync' + if subtitle != '': + title += '- %s' % subtitle + plt.title(title) + + x = [] + ycpu = [] + ymem = [] + ypcpu = [] + ypmem = [] + + ys = [ + (ycpu, 'total cpu', 'r'), + (ymem, 'total mem', 'b'), + (ypcpu, 'proc cpu', 'm'), + (ypmem, 'proc mem', 'g'), + ] + + # read data from file + with open(filename, 'r') as f: + line = f.readline() + while True: + line = f.readline() + if line.startswith('#'): + continue + if line == '' or line is None: + break + time, cpu, mem, pcpu, pmem = tuple(line.strip().split(' ')) + x.append(float(time)) + ycpu.append(float(cpu)) + ymem.append(float(mem)) + ypcpu.append(float(pcpu)) + ypmem.append(float(pmem)) + + smoothx = [n for n in smooth(x)] + #xnew = linspace(0.01, 19, 100) + + for y in ys: + kwargs = { + 'linewidth': 1.0, + 'linestyle': '-', + # 'marker': '.', + 'color': y[2], + } + #f = interp1d(x, y[0], kind='cubic') + plt.plot( + smoothx, + [n for n in smooth(y[0])], + #xnew, + #f(xnew), + label=y[1], **kwargs) + + #plt.axes().get_xaxis().set_ticks(x) + #plt.axes().get_xaxis().set_ticklabels(x) + + # annotate max and min values + plt.xlim(0, 20) + plt.ylim(0, 100) + plt.grid() + plt.legend() + plt.show() + + +if __name__ == '__main__': + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + '-d', dest='datafile', required=False, default='/tmp/profile.log', + help='the data file to plot') + parser.add_argument( + '-s', dest='subtitle', required=False, default='', + help='a subtitle for the plot') + args = parser.parse_args() + plot(args.datafile, args.subtitle) diff --git a/scripts/profiling/sync/profile-sync.py b/scripts/profiling/sync/profile-sync.py new file mode 100644 index 00000000..fdd5b5a6 --- /dev/null +++ b/scripts/profiling/sync/profile-sync.py @@ -0,0 +1,62 @@ +#!/usr/bin/python + + +import argparse +import logging + + +from util import StatsLogger, ValidateUserHandle +from client_side_db import get_soledad_instance +#from plot import plot + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +# main program + +if __name__ == '__main__': + + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-b', dest='basedir', required=False, default=None, + help='soledad base directory') + parser.add_argument( + '-p', dest='passphrase', required=False, default=None, + help='the user passphrase') + parser.add_argument( + '-l', dest='logfile', required=False, default='/tmp/profile.log', + help='the file to which write the log') + args = parser.parse_args() + + # get the password + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + + # get the basedir + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + logger.info('Using %s as base directory.' % basedir) + + # get the soledad instance + s = get_soledad_instance( + args.username, args.provider, passphrase, basedir) + for i in xrange(10): + s.create_doc({}) + + sl = StatsLogger( + "soledad-sync", args.logfile, procs=["python"], interval=0.001) + sl.start() + s.sync() + sl.stop() + + #plot(args.logfile) diff --git a/scripts/profiling/sync/util.py b/scripts/profiling/sync/util.py new file mode 120000 index 00000000..7f16d684 --- /dev/null +++ b/scripts/profiling/sync/util.py @@ -0,0 +1 @@ +../util.py
\ No newline at end of file diff --git a/scripts/profiling/util.py b/scripts/profiling/util.py new file mode 100644 index 00000000..adf1de8c --- /dev/null +++ b/scripts/profiling/util.py @@ -0,0 +1,75 @@ +import re +import psutil +import time +import threading +import argparse +import pytz +import datetime + + +class ValidateUserHandle(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + m = re.compile('^([^@]+)@([^@]+\.[^@]+)$') + res = m.match(values) + if res == None: + parser.error('User handle should have the form user@provider.') + setattr(namespace, 'username', res.groups()[0]) + setattr(namespace, 'provider', res.groups()[1]) + + +class StatsLogger(threading.Thread): + + def __init__(self, name, fname, procs=[], interval=0.01): + threading.Thread.__init__(self) + self._stopped = True + self._name = name + self._fname = fname + self._procs = self._find_procs(procs) + self._interval = interval + + def _find_procs(self, procs): + return filter(lambda p: p.name in procs, psutil.process_iter()) + + def run(self): + self._stopped = False + with open(self._fname, 'w') as f: + self._start = time.time() + f.write(self._make_header()) + while self._stopped is False: + f.write('%s %s\n' % + (self._make_general_stats(), self._make_proc_stats())) + time.sleep(self._interval) + f.write(self._make_footer()) + + def _make_general_stats(self): + now = time.time() + stats = [] + stats.append("%f" % (now - self._start)) # elapsed time + stats.append("%f" % psutil.cpu_percent()) # total cpu + stats.append("%f" % psutil.phymem_usage().percent) # total memory + return ' '.join(stats) + + def _make_proc_stats(self): + stats = [] + for p in self._procs: + stats.append('%f' % p.get_cpu_percent()) # proc cpu + stats.append('%f' % p.get_memory_percent()) # proc memory + return ' '.join(stats) + + def _make_header(self): + header = [] + header.append('# test_name: %s' % self._name) + header.append('# start_time: %s' % datetime.datetime.now(pytz.utc)) + header.append( + '# elapsed_time total_cpu total_memory proc_cpu proc_memory ') + return '\n'.join(header) + '\n' + + def _make_footer(self): + footer = [] + footer.append('# end_time: %s' % datetime.datetime.now(pytz.utc)) + return '\n'.join(footer) + + def stop(self): + self._stopped = True + + |