Merge remote-tracking branch 'refs/remotes/drebs/feature/add-profiling-scripts' into develop

7 files changed, 332 insertions, 0 deletions

diff --git a/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py b/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py
new file mode 100755
index 00000000..b2b35d30
--- /dev/null
+++ b/scripts/profiling/doc_put_memory_usage/get-soledad-and-couch-mem.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+
+
+import logging
+import argparse
+import psutil
+import time
+
+
+def find_procs(procs):
+    result = []
+    for name, executable in procs:
+        found = filter(
+            lambda p: executable == p.name,
+            psutil.process_iter())
+        if len(found) == 1:
+            result.append(found[0])
+    return result
+
+
+def log_memory(soledad, bigcouch):
+    while True:
+        print "%f %f" % \
+            (soledad.get_memory_percent(), bigcouch.get_memory_percent())
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    
+    # configure logger
+    logger = logging.getLogger(__name__)
+    LOG_FORMAT = '%(asctime)s %(message)s'
+    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+    # parse command line
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-l', dest='logfile',
+        help='log output to file')
+    args = parser.parse_args()
+
+    log_memory(*find_procs([
+        ('Soledad', 'twistd'),
+        ('Bigcouch', 'beam.smp')]))
+    
diff --git a/scripts/profiling/doc_put_memory_usage/profile-procs.py b/scripts/profiling/doc_put_memory_usage/profile-procs.py
new file mode 100755
index 00000000..53f5977b
--- /dev/null
+++ b/scripts/profiling/doc_put_memory_usage/profile-procs.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+
+import logging
+import argparse
+import psutil
+import time
+
+
+def find_procs(procs):
+    result = []
+    for name, executable in procs:
+        found = filter(
+            lambda p: executable == p.name,
+            psutil.process_iter())
+        if len(found) == 1:
+            result.append(found[0])
+    return result
+
+
+def log_usage(procs, logger):
+    names = [proc.name for proc in procs]
+    logger.info("Logging cpu and memory for: %s" % names)
+    while True:
+        s = '%f %f' %\
+            (psutil.cpu_percent(), psutil.phymem_usage().percent)
+        for proc in procs:
+            s += ' %f %f' % \
+                 (proc.get_cpu_percent(), proc.get_memory_percent())
+        logger.info(s)
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    # parse command line
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-l', dest='logfile',
+        help='log output to file')
+    args = parser.parse_args()
+
+    # configure logger
+    logger = logging.getLogger(__name__)
+    LOG_FORMAT = '%(asctime)s %(message)s'
+    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+    if args.logfile is not None:
+        handler = logging.FileHandler(args.logfile, mode='a')
+        handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT))
+        logger.addHandler(handler)
+
+    log_usage(find_procs([
+        ('Soledad', 'twistd'),
+        ('Bigcouch', 'beam.smp')]), logger)
diff --git a/scripts/profiling/sync/client_side_db.py b/scripts/profiling/sync/client_side_db.py
new file mode 120000
index 00000000..9e49a7f0
--- /dev/null
+++ b/scripts/profiling/sync/client_side_db.py
@@ -0,0 +1 @@
+../../db_access/client_side_db.py
\ No newline at end of file
diff --git a/scripts/profiling/sync/plot.py b/scripts/profiling/sync/plot.py
new file mode 100755
index 00000000..8e3e1c15
--- /dev/null
+++ b/scripts/profiling/sync/plot.py
@@ -0,0 +1,93 @@
+#!/usr/bin/python
+
+
+import argparse
+from matplotlib import pyplot as plt
+from movingaverage import movingaverage
+from scipy.interpolate import interp1d
+from numpy import linspace
+
+
+def smooth(l):
+    return movingaverage(l, 3, data_is_list=True, avoid_fp_drift=False)
+
+
+def plot(filename, subtitle=''):
+
+    # config the plot
+    plt.xlabel('time (s)')
+    plt.ylabel('usage (%)')
+    title = 'soledad sync'
+    if subtitle != '':
+        title += '- %s' % subtitle
+    plt.title(title)
+
+    x = []
+    ycpu = []
+    ymem = []
+    ypcpu = []
+    ypmem = []
+
+    ys = [
+        (ycpu, 'total cpu', 'r'),
+        (ymem, 'total mem', 'b'),
+        (ypcpu, 'proc cpu', 'm'),
+        (ypmem, 'proc mem', 'g'),
+    ]
+
+    # read data from file
+    with open(filename, 'r') as f:
+        line = f.readline()
+        while True:
+            line = f.readline()
+            if line.startswith('#'):
+                continue
+            if line == '' or line is None:
+                break
+            time, cpu, mem, pcpu, pmem = tuple(line.strip().split(' '))
+            x.append(float(time))
+            ycpu.append(float(cpu))
+            ymem.append(float(mem))
+            ypcpu.append(float(pcpu))
+            ypmem.append(float(pmem))
+
+    smoothx = [n for n in smooth(x)]
+    #xnew = linspace(0.01, 19, 100)
+
+    for y in ys:
+        kwargs = {
+            'linewidth': 1.0,
+            'linestyle': '-',
+        #    'marker': '.',
+            'color': y[2],
+        }
+        #f = interp1d(x, y[0], kind='cubic')
+        plt.plot(
+            smoothx,
+            [n for n in smooth(y[0])],
+            #xnew,
+            #f(xnew),
+            label=y[1], **kwargs)
+
+    #plt.axes().get_xaxis().set_ticks(x)
+    #plt.axes().get_xaxis().set_ticklabels(x)
+
+    # annotate max and min values
+    plt.xlim(0, 20)
+    plt.ylim(0, 100)
+    plt.grid()
+    plt.legend()
+    plt.show()
+
+
+if __name__ == '__main__':
+    # parse command line
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-d', dest='datafile', required=False, default='/tmp/profile.log',
+        help='the data file to plot')
+    parser.add_argument(
+        '-s', dest='subtitle', required=False, default='',
+        help='a subtitle for the plot')
+    args = parser.parse_args()
+    plot(args.datafile, args.subtitle)
diff --git a/scripts/profiling/sync/profile-sync.py b/scripts/profiling/sync/profile-sync.py
new file mode 100644
index 00000000..fdd5b5a6
--- /dev/null
+++ b/scripts/profiling/sync/profile-sync.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python
+
+
+import argparse
+import logging
+
+
+from util import StatsLogger, ValidateUserHandle
+from client_side_db import get_soledad_instance
+#from plot import plot
+
+
+# create a logger
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+# main program
+
+if __name__ == '__main__':
+
+    # parse command line
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'user@provider', action=ValidateUserHandle, help='the user handle')
+    parser.add_argument(
+        '-b', dest='basedir', required=False, default=None,
+        help='soledad base directory')
+    parser.add_argument(
+        '-p', dest='passphrase', required=False, default=None,
+        help='the user passphrase')
+    parser.add_argument(
+        '-l', dest='logfile', required=False, default='/tmp/profile.log',
+        help='the file to which write the log')
+    args = parser.parse_args()
+
+    # get the password
+    passphrase = args.passphrase
+    if passphrase is None:
+        passphrase = getpass.getpass(
+            'Password for %s@%s: ' % (args.username, args.provider))
+
+    # get the basedir
+    basedir = args.basedir
+    if basedir is None:
+        basedir = tempfile.mkdtemp()
+    logger.info('Using %s as base directory.' % basedir)
+
+    # get the soledad instance
+    s = get_soledad_instance(
+        args.username, args.provider, passphrase, basedir)
+    for i in xrange(10):
+        s.create_doc({})
+
+    sl = StatsLogger(
+        "soledad-sync", args.logfile, procs=["python"], interval=0.001)
+    sl.start()
+    s.sync()
+    sl.stop()
+
+    #plot(args.logfile)
diff --git a/scripts/profiling/sync/util.py b/scripts/profiling/sync/util.py
new file mode 120000
index 00000000..7f16d684
--- /dev/null
+++ b/scripts/profiling/sync/util.py
@@ -0,0 +1 @@
+../util.py
\ No newline at end of file
diff --git a/scripts/profiling/util.py b/scripts/profiling/util.py
new file mode 100644
index 00000000..adf1de8c
--- /dev/null
+++ b/scripts/profiling/util.py
@@ -0,0 +1,75 @@
+import re
+import psutil
+import time
+import threading
+import argparse
+import pytz
+import datetime
+
+
+class ValidateUserHandle(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        m = re.compile('^([^@]+)@([^@]+\.[^@]+)$')
+        res = m.match(values)
+        if res == None:
+            parser.error('User handle should have the form user@provider.')
+        setattr(namespace, 'username', res.groups()[0])
+        setattr(namespace, 'provider', res.groups()[1])
+
+
+class StatsLogger(threading.Thread):
+
+    def __init__(self, name, fname, procs=[], interval=0.01):
+        threading.Thread.__init__(self)
+        self._stopped = True
+        self._name = name
+        self._fname = fname
+        self._procs = self._find_procs(procs)
+        self._interval = interval
+
+    def _find_procs(self, procs):
+        return filter(lambda p: p.name in procs, psutil.process_iter())
+
+    def run(self):
+        self._stopped = False
+        with open(self._fname, 'w') as f:
+            self._start = time.time()
+            f.write(self._make_header())
+            while self._stopped is False:
+                f.write('%s %s\n' %
+                    (self._make_general_stats(), self._make_proc_stats()))
+                time.sleep(self._interval)
+            f.write(self._make_footer())
+
+    def _make_general_stats(self):
+        now = time.time()
+        stats = []
+        stats.append("%f" % (now - self._start))   # elapsed time
+        stats.append("%f" % psutil.cpu_percent())  # total cpu
+        stats.append("%f" % psutil.phymem_usage().percent)  # total memory
+        return ' '.join(stats)
+
+    def _make_proc_stats(self):
+        stats = []
+        for p in self._procs:
+            stats.append('%f' % p.get_cpu_percent())     # proc cpu
+            stats.append('%f' % p.get_memory_percent())  # proc memory
+        return ' '.join(stats)
+
+    def _make_header(self):
+        header = []
+        header.append('# test_name: %s' % self._name)
+        header.append('# start_time: %s' %  datetime.datetime.now(pytz.utc))
+        header.append(
+            '# elapsed_time total_cpu total_memory proc_cpu proc_memory ')
+        return '\n'.join(header) + '\n'
+
+    def _make_footer(self):
+        footer = []
+        footer.append('# end_time: %s' % datetime.datetime.now(pytz.utc))
+        return '\n'.join(footer)
+
+    def stop(self):
+        self._stopped = True
+
+