summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2014-06-09 11:52:54 -0300
committerdrebs <drebs@leap.se>2014-06-09 11:52:54 -0300
commitb57a493e5992ed9e225901762782b36701074a5e (patch)
tree489ba9fc5fe3dc1aecc8ca439b4a0c037ec084fc
parentee53e05fbff3509a7cdb9bc510c891922f9a8bca (diff)
Add profiling scripts.
-rwxr-xr-xscripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py46
-rwxr-xr-xscripts/profiling/doc_put_memory_usage/profile-procs.py54
l---------scripts/profiling/sync/client_side_db.py1
-rwxr-xr-xscripts/profiling/sync/plot.py93
-rw-r--r--scripts/profiling/sync/profile-sync.py62
l---------scripts/profiling/sync/util.py1
-rw-r--r--scripts/profiling/util.py75
7 files changed, 332 insertions, 0 deletions
diff --git a/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py b/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py
new file mode 100755
index 00000000..b2b35d30
--- /dev/null
+++ b/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+
+
+import logging
+import argparse
+import psutil
+import time
+
+
+def find_procs(procs):
+ result = []
+ for name, executable in procs:
+ found = filter(
+ lambda p: executable == p.name,
+ psutil.process_iter())
+ if len(found) == 1:
+ result.append(found[0])
+ return result
+
+
+def log_memory(soledad, bigcouch):
+ while True:
+ print "%f %f" % \
+ (soledad.get_memory_percent(), bigcouch.get_memory_percent())
+ time.sleep(1)
+
+
+if __name__ == '__main__':
+
+ # configure logger
+ logger = logging.getLogger(__name__)
+ LOG_FORMAT = '%(asctime)s %(message)s'
+ logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '-l', dest='logfile',
+ help='log output to file')
+ args = parser.parse_args()
+
+ log_memory(*find_procs([
+ ('Soledad', 'twistd'),
+ ('Bigcouch', 'beam.smp')]))
+
diff --git a/scripts/profiling/doc_put_memory_usage/profile-procs.py b/scripts/profiling/doc_put_memory_usage/profile-procs.py
new file mode 100755
index 00000000..53f5977b
--- /dev/null
+++ b/scripts/profiling/doc_put_memory_usage/profile-procs.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+
+import logging
+import argparse
+import psutil
+import time
+
+
+def find_procs(procs):
+ result = []
+ for name, executable in procs:
+ found = filter(
+ lambda p: executable == p.name,
+ psutil.process_iter())
+ if len(found) == 1:
+ result.append(found[0])
+ return result
+
+
+def log_usage(procs, logger):
+ names = [proc.name for proc in procs]
+ logger.info("Logging cpu and memory for: %s" % names)
+ while True:
+ s = '%f %f' %\
+ (psutil.cpu_percent(), psutil.phymem_usage().percent)
+ for proc in procs:
+ s += ' %f %f' % \
+ (proc.get_cpu_percent(), proc.get_memory_percent())
+ logger.info(s)
+ time.sleep(1)
+
+
+if __name__ == '__main__':
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '-l', dest='logfile',
+ help='log output to file')
+ args = parser.parse_args()
+
+ # configure logger
+ logger = logging.getLogger(__name__)
+ LOG_FORMAT = '%(asctime)s %(message)s'
+ logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+ if args.logfile is not None:
+ handler = logging.FileHandler(args.logfile, mode='a')
+ handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT))
+ logger.addHandler(handler)
+
+ log_usage(find_procs([
+ ('Soledad', 'twistd'),
+ ('Bigcouch', 'beam.smp')]), logger)
diff --git a/scripts/profiling/sync/client_side_db.py b/scripts/profiling/sync/client_side_db.py
new file mode 120000
index 00000000..9e49a7f0
--- /dev/null
+++ b/scripts/profiling/sync/client_side_db.py
@@ -0,0 +1 @@
+../../db_access/client_side_db.py \ No newline at end of file
diff --git a/scripts/profiling/sync/plot.py b/scripts/profiling/sync/plot.py
new file mode 100755
index 00000000..8e3e1c15
--- /dev/null
+++ b/scripts/profiling/sync/plot.py
@@ -0,0 +1,93 @@
+#!/usr/bin/python
+
+
+import argparse
+from matplotlib import pyplot as plt
+from movingaverage import movingaverage
+from scipy.interpolate import interp1d
+from numpy import linspace
+
+
+def smooth(l):
+ return movingaverage(l, 3, data_is_list=True, avoid_fp_drift=False)
+
+
+def plot(filename, subtitle=''):
+
+ # config the plot
+ plt.xlabel('time (s)')
+ plt.ylabel('usage (%)')
+ title = 'soledad sync'
+ if subtitle != '':
+ title += '- %s' % subtitle
+ plt.title(title)
+
+ x = []
+ ycpu = []
+ ymem = []
+ ypcpu = []
+ ypmem = []
+
+ ys = [
+ (ycpu, 'total cpu', 'r'),
+ (ymem, 'total mem', 'b'),
+ (ypcpu, 'proc cpu', 'm'),
+ (ypmem, 'proc mem', 'g'),
+ ]
+
+ # read data from file
+ with open(filename, 'r') as f:
+ line = f.readline()
+ while True:
+ line = f.readline()
+ if line.startswith('#'):
+ continue
+ if line == '' or line is None:
+ break
+ time, cpu, mem, pcpu, pmem = tuple(line.strip().split(' '))
+ x.append(float(time))
+ ycpu.append(float(cpu))
+ ymem.append(float(mem))
+ ypcpu.append(float(pcpu))
+ ypmem.append(float(pmem))
+
+ smoothx = [n for n in smooth(x)]
+ #xnew = linspace(0.01, 19, 100)
+
+ for y in ys:
+ kwargs = {
+ 'linewidth': 1.0,
+ 'linestyle': '-',
+ # 'marker': '.',
+ 'color': y[2],
+ }
+ #f = interp1d(x, y[0], kind='cubic')
+ plt.plot(
+ smoothx,
+ [n for n in smooth(y[0])],
+ #xnew,
+ #f(xnew),
+ label=y[1], **kwargs)
+
+ #plt.axes().get_xaxis().set_ticks(x)
+ #plt.axes().get_xaxis().set_ticklabels(x)
+
+ # annotate max and min values
+ plt.xlim(0, 20)
+ plt.ylim(0, 100)
+ plt.grid()
+ plt.legend()
+ plt.show()
+
+
+if __name__ == '__main__':
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '-d', dest='datafile', required=False, default='/tmp/profile.log',
+ help='the data file to plot')
+ parser.add_argument(
+ '-s', dest='subtitle', required=False, default='',
+ help='a subtitle for the plot')
+ args = parser.parse_args()
+ plot(args.datafile, args.subtitle)
diff --git a/scripts/profiling/sync/profile-sync.py b/scripts/profiling/sync/profile-sync.py
new file mode 100644
index 00000000..fdd5b5a6
--- /dev/null
+++ b/scripts/profiling/sync/profile-sync.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python
+
+
+import argparse
+import logging
+
+
+from util import StatsLogger, ValidateUserHandle
+from client_side_db import get_soledad_instance
+#from plot import plot
+
+
+# create a logger
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+# main program
+
+if __name__ == '__main__':
+
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'user@provider', action=ValidateUserHandle, help='the user handle')
+ parser.add_argument(
+ '-b', dest='basedir', required=False, default=None,
+ help='soledad base directory')
+ parser.add_argument(
+ '-p', dest='passphrase', required=False, default=None,
+ help='the user passphrase')
+ parser.add_argument(
+ '-l', dest='logfile', required=False, default='/tmp/profile.log',
+ help='the file to which write the log')
+ args = parser.parse_args()
+
+ # get the password
+ passphrase = args.passphrase
+ if passphrase is None:
+ passphrase = getpass.getpass(
+ 'Password for %s@%s: ' % (args.username, args.provider))
+
+ # get the basedir
+ basedir = args.basedir
+ if basedir is None:
+ basedir = tempfile.mkdtemp()
+ logger.info('Using %s as base directory.' % basedir)
+
+ # get the soledad instance
+ s = get_soledad_instance(
+ args.username, args.provider, passphrase, basedir)
+ for i in xrange(10):
+ s.create_doc({})
+
+ sl = StatsLogger(
+ "soledad-sync", args.logfile, procs=["python"], interval=0.001)
+ sl.start()
+ s.sync()
+ sl.stop()
+
+ #plot(args.logfile)
diff --git a/scripts/profiling/sync/util.py b/scripts/profiling/sync/util.py
new file mode 120000
index 00000000..7f16d684
--- /dev/null
+++ b/scripts/profiling/sync/util.py
@@ -0,0 +1 @@
+../util.py \ No newline at end of file
diff --git a/scripts/profiling/util.py b/scripts/profiling/util.py
new file mode 100644
index 00000000..adf1de8c
--- /dev/null
+++ b/scripts/profiling/util.py
@@ -0,0 +1,75 @@
+import re
+import psutil
+import time
+import threading
+import argparse
+import pytz
+import datetime
+
+
+class ValidateUserHandle(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ m = re.compile('^([^@]+)@([^@]+\.[^@]+)$')
+ res = m.match(values)
+ if res == None:
+ parser.error('User handle should have the form user@provider.')
+ setattr(namespace, 'username', res.groups()[0])
+ setattr(namespace, 'provider', res.groups()[1])
+
+
+class StatsLogger(threading.Thread):
+
+ def __init__(self, name, fname, procs=[], interval=0.01):
+ threading.Thread.__init__(self)
+ self._stopped = True
+ self._name = name
+ self._fname = fname
+ self._procs = self._find_procs(procs)
+ self._interval = interval
+
+ def _find_procs(self, procs):
+ return filter(lambda p: p.name in procs, psutil.process_iter())
+
+ def run(self):
+ self._stopped = False
+ with open(self._fname, 'w') as f:
+ self._start = time.time()
+ f.write(self._make_header())
+ while self._stopped is False:
+ f.write('%s %s\n' %
+ (self._make_general_stats(), self._make_proc_stats()))
+ time.sleep(self._interval)
+ f.write(self._make_footer())
+
+ def _make_general_stats(self):
+ now = time.time()
+ stats = []
+ stats.append("%f" % (now - self._start)) # elapsed time
+ stats.append("%f" % psutil.cpu_percent()) # total cpu
+ stats.append("%f" % psutil.phymem_usage().percent) # total memory
+ return ' '.join(stats)
+
+ def _make_proc_stats(self):
+ stats = []
+ for p in self._procs:
+ stats.append('%f' % p.get_cpu_percent()) # proc cpu
+ stats.append('%f' % p.get_memory_percent()) # proc memory
+ return ' '.join(stats)
+
+ def _make_header(self):
+ header = []
+ header.append('# test_name: %s' % self._name)
+ header.append('# start_time: %s' % datetime.datetime.now(pytz.utc))
+ header.append(
+ '# elapsed_time total_cpu total_memory proc_cpu proc_memory ')
+ return '\n'.join(header) + '\n'
+
+ def _make_footer(self):
+ footer = []
+ footer.append('# end_time: %s' % datetime.datetime.now(pytz.utc))
+ return '\n'.join(footer)
+
+ def stop(self):
+ self._stopped = True
+
+