Add script to measure PUT memory usage.

5 files changed, 147 insertions, 58 deletions

diff --git a/scripts/client_side_db.py b/scripts/db_access/client_side_db.py
index 15980f5d..15980f5d 100644
--- a/scripts/client_side_db.py
+++ b/scripts/db_access/client_side_db.py
diff --git a/scripts/server_side_db.py b/scripts/db_access/server_side_db.py
index 18641a0f..18641a0f 100644
--- a/scripts/server_side_db.py
+++ b/scripts/db_access/server_side_db.py
diff --git a/scripts/find_max_upload_size.py b/scripts/doc_put_memory_usage/find_max_upload_size.py
index 8abeee78..02c68015 100755
--- a/scripts/find_max_upload_size.py
+++ b/scripts/doc_put_memory_usage/find_max_upload_size.py
@@ -15,18 +15,26 @@
 
 import os
 import configparser
-import couchdb
 import logging
 import argparse
 import random
 import string
 import binascii
 import json
+import time
+import uuid
+
+
+from couchdb.client import Database
+from socket import error as socket_error
+from leap.soledad.common.couch import CouchDatabase
 
 
 SOLEDAD_CONFIG_FILE = '/etc/leap/soledad-server.conf'
 PREFIX = '/tmp/soledad_test'
 LOG_FORMAT = '%(asctime)s %(levelname)s %(message)s'
+RETRIES = 3  # number of times to retry uploading a document of a certain
+             # size after a failure
 
 
 # configure logger
@@ -55,27 +63,19 @@ def get_couch_url(config_file=SOLEDAD_CONFIG_FILE):
 
 
 # generate or load an uploadable doc with the given size in mb
-def gen_body(size):
-    if os.path.exists(
-            os.path.join(PREFIX, 'body-%d.json' % size)):
-        logger.debug('Loading body with %d MB...' % size)
-        with open(os.path.join(PREFIX, 'body-%d.json' % size), 'r') as f:
-            return json.loads(f.read())
+def get_content(size):
+    fname = os.path.join(PREFIX, 'content-%d.json' % size)
+    if os.path.exists(fname):
+        logger.debug('Loading content with %d MB...' % size)
+        with open(fname, 'r') as f:
+            return f.read()
     else:
         length = int(size * 1024 ** 2)
-        hexdata = binascii.hexlify(os.urandom(length))[:length]
-        body = {
-            'couch_rev': None,
-            'u1db_rev': '1',
-            'content': hexdata,
-            'trans_id': '1',
-            'conflicts': None,
-            'update_conflicts': False,
-        }
         logger.debug('Generating body with %d MB...' % size)
-        with open(os.path.join(PREFIX, 'body-%d.json' % size), 'w+') as f:
-            f.write(json.dumps(body))
-        return body
+        content = binascii.hexlify(os.urandom(length))[:length]
+        with open(fname, 'w') as f:
+            f.write(content)
+        return content
 
 
 def delete_doc(db):
@@ -83,57 +83,57 @@ def delete_doc(db):
     db.delete(doc)
 
 
-def upload(db, size):
-    ddoc_path = ['_design', 'docs', '_update', 'put', 'largedoc']
-    resource = db.resource(*ddoc_path)
-    body = gen_body(size)
-    try:
-        logger.debug('Uploading %d MB body...' % size)
-        response = resource.put_json(
-            body=body,
-            headers={'content-type': 'application/json'})
-        # the document might have been updated in between, so we check for
-        # the return message
-        msg = response[2].read()
-        if msg == 'ok':
-            delete_doc(db)
+def upload(db, size, couch_db):
+    # try many times to be sure that size is infeasible
+    for i in range(RETRIES):
+        # wait until server is up to upload
+        while True:
+            try:
+                'largedoc' in couch_db
+                break
+            except socket_error:
+                logger.debug('Waiting for server to come up...')
+                time.sleep(1)
+        # attempt to upload
+        try:
+            logger.debug(
+                'Trying to upload %d MB document (attempt %d/%d)...' %
+                (size, (i+1), RETRIES))
+            content = get_content(size)
+            logger.debug('Starting upload of %d bytes.' % len(content))
+            doc = db.create_doc({'data': content}, doc_id='largedoc')
+            delete_doc(couch_db)
             logger.debug('Success uploading %d MB doc.' % size)
             return True
-        else:
-            # should not happen
-            logger.error('Unexpected error uploading %d MB doc: %s' % (size, msg))
-            return False
-    except Exception as e:
-        logger.debug('Failed to upload %d MB doc: %s' % (size, str(e)))
-        return False
-
-
-def find_max_upload_size(dbname):
-    couch_url = get_couch_url()
-    db_url = '%s/%s' % (couch_url, dbname)
-    logger.debug('Couch URL: %s' % db_url)
-    # get a 'raw' couch handler
-    server = couchdb.client.Server(couch_url)
-    db = server[dbname]
+        except Exception as e:
+            logger.debug('Failed to upload %d MB doc: %s' % (size, str(e)))
+    return False
+
+
+def find_max_upload_size(db_uri):
+    db = CouchDatabase.open_database(db_uri, False)
+    couch_db = Database(db_uri)
+    logger.debug('Database URI: %s' % db_uri)
     # delete eventual leftover from last run
-    largedoc = db.get('largedoc')
-    if largedoc is not None:
-        db.delete(largedoc)
+    if 'largedoc' in couch_db:
+        delete_doc(couch_db)
     # phase 1: increase upload size exponentially
     logger.info('Starting phase 1: increasing size exponentially.')
     size = 1
+    #import ipdb; ipdb.set_trace()
     while True:
-        if upload(db, size):
+        if upload(db, size, couch_db):
             size *= 2
         else:
             break
+
     # phase 2: binary search for maximum value
     unable = size
     able = size / 2
     logger.info('Starting phase 2: binary search for maximum value.')
     while unable - able > 1:
         size = able + ((unable - able) / 2)
-        if upload(db, size):
+        if upload(db, size, couch_db):
             able = size
         else:
             unable = size
@@ -150,12 +150,12 @@ if __name__ == '__main__':
         '-l', dest='logfile',
         help='log output to file')
     parser.add_argument(
-        'dbname', help='the name of the database to test in')
+        'db_uri', help='the couch database URI to test')
     args = parser.parse_args()
 
     # log to file
     if args.logfile is not None:
-        add_file_handler(args.logfile)
+        log_to_file(args.logfile)
 
     # set loglevel
     if args.debug is True:
@@ -164,6 +164,6 @@ if __name__ == '__main__':
         config_log(logging.INFO)
 
     # run test and report
-    logger.info('Will test using db %s.' % args.dbname)
-    maxsize = find_max_upload_size(args.dbname)
+    logger.info('Will test using db at %s.' % args.db_uri)
+    maxsize = find_max_upload_size(args.db_uri)
     logger.info('Max upload size is %d MB.' % maxsize)
diff --git a/scripts/doc_put_memory_usage/get-mem.py b/scripts/doc_put_memory_usage/get-mem.py
new file mode 100755
index 00000000..d64875fc
--- /dev/null
+++ b/scripts/doc_put_memory_usage/get-mem.py
@@ -0,0 +1,16 @@
+#!/usr/bin/python
+
+
+import psutil
+import time
+
+
+delta = 50 * 60
+start = time.time()
+
+while True:
+    now = time.time()
+    print "%s %s" % (now - start, psutil.phymem_usage().used)
+    time.sleep(0.1)
+    if now > start + delta:
+        break
diff --git a/scripts/doc_put_memory_usage/plot-mem.py b/scripts/doc_put_memory_usage/plot-mem.py
new file mode 100755
index 00000000..e24679a2
--- /dev/null
+++ b/scripts/doc_put_memory_usage/plot-mem.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python
+
+
+from matplotlib import pyplot as plt
+
+
+files = [
+    ('local', 'couchdb-json', 'b'),
+    ('local', 'bigcouch-json', 'r'),
+    ('local', 'couchdb-multipart', 'g'),
+    ('local', 'bigcouch-multipart', 'm'),
+]
+
+
+# config the plot
+plt.xlabel('time')
+plt.ylabel('memory usage')
+plt.title('bigcouch versus couch memory usage')
+
+
+for fi in files:
+
+    machine = fi[0]
+    database = fi[1]
+    color = fi[2]
+    filename = '%s-%s.txt' % (machine, database)
+
+    x = []
+    y = []
+
+    xmax = None
+    xmin = None
+    ymax = None
+    ymin = None
+
+    # read data from file
+    with open(filename, 'r') as f:
+        line = f.readline()
+        while line is not None:
+            time, mem = tuple(line.strip().split(' '))
+            mem = float(mem) / (10**9)
+            x.append(float(time))
+            y.append(mem)
+            if ymax == None or mem > ymax:
+                ymax = mem
+                xmax = time
+            if ymin == None or mem < ymin:
+                ymin = mem
+                xmin = time
+            line = f.readline()
+            if line == '':
+                break
+
+    kwargs = {
+        'linewidth': 1.0,
+        'linestyle': '-',
+    #    'marker': '.',
+        'color': color,
+    }
+    plt.plot(x, y, label=database, **kwargs)
+
+    #plt.axes().get_xaxis().set_ticks(x)
+    #plt.axes().get_xaxis().set_ticklabels(x)
+
+    # annotate max and min values
+    #plt.axes().annotate("%.2f GB" % ymax, xy=(xmax, ymax))
+    #plt.axes().annotate("%.2f GB" % ymin, xy=(xmin, ymin))
+
+
+plt.grid()
+plt.legend()
+plt.show()
+