From 94ec89384abeba52d660addf3528b11a9aa5a8b0 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Tue, 11 Mar 2014 16:14:26 -0300
Subject: Add script to measure PUT memory usage.
---
scripts/db_access/client_side_db.py | 145 ++++++++++++++++++++++++++++++++++++
scripts/db_access/server_side_db.py | 42 +++++++++++
2 files changed, 187 insertions(+)
create mode 100644 scripts/db_access/client_side_db.py
create mode 100644 scripts/db_access/server_side_db.py
(limited to 'scripts/db_access')
diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py
new file mode 100644
index 00000000..15980f5d
--- /dev/null
+++ b/scripts/db_access/client_side_db.py
@@ -0,0 +1,145 @@
+#!/usr/bin/python
+
+# This script gives client-side access to one Soledad user database.
+
+
+import sys
+import os
+import argparse
+import re
+import tempfile
+import getpass
+import requests
+import json
+import srp._pysrp as srp
+import binascii
+
+from leap.common.config import get_path_prefix
+from leap.soledad.client import Soledad
+
+
+safe_unhexlify = lambda x: binascii.unhexlify(x) if (
+ len(x) % 2 == 0) else binascii.unhexlify('0' + x)
+
+
+def fail(reason):
+ print 'Fail: ' + reason
+ exit(2)
+
+
+def get_api_info(provider):
+ info = requests.get(
+ 'https://'+provider+'/provider.json', verify=False).json()
+ return info['api_uri'], info['api_version']
+
+
+def login(username, passphrase, provider, api_uri, api_version):
+ usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024)
+ auth = None
+ try:
+ auth = authenticate(api_uri, api_version, usr).json()
+ except requests.exceptions.ConnectionError:
+ fail('Could not connect to server.')
+ if 'errors' in auth:
+ fail(str(auth['errors']))
+ return api_uri, api_version, auth
+
+
+def authenticate(api_uri, api_version, usr):
+ api_url = "%s/%s" % (api_uri, api_version)
+ session = requests.session()
+ uname, A = usr.start_authentication()
+ params = {'login': uname, 'A': binascii.hexlify(A)}
+ init = session.post(
+ api_url + '/sessions', data=params, verify=False).json()
+ if 'errors' in init:
+ fail('test user not found')
+ M = usr.process_challenge(
+ safe_unhexlify(init['salt']), safe_unhexlify(init['B']))
+ return session.put(api_url + '/sessions/' + uname, verify=False,
+ data={'client_auth': binascii.hexlify(M)})
+
+
+def get_soledad_info(username, provider, passphrase, basedir):
+ api_uri, api_version = get_api_info(provider)
+ auth = login(username, passphrase, provider, api_uri, api_version)
+ # get soledad server url
+ service_url = '%s/%s/config/soledad-service.json' % \
+ (api_uri, api_version)
+ soledad_hosts = requests.get(service_url, verify=False).json()['hosts']
+ hostnames = soledad_hosts.keys()
+ # allow for choosing the host
+ host = hostnames[0]
+ if len(hostnames) > 1:
+ i = 1
+ print "There are many available hosts:"
+ for h in hostnames:
+ print " (%d) %s.%s" % (i, h, provider)
+ i += 1
+ choice = raw_input("Choose a host to use (default: 1): ")
+ if choice != '':
+ host = hostnames[int(choice) - 1]
+ server_url = 'https://%s:%d/user-%s' % \
+ (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'],
+ auth[2]['id'])
+ # get provider ca certificate
+ ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text
+ cert_file = os.path.join(basedir, 'ca.crt')
+ with open(cert_file, 'w') as f:
+ f.write(ca_cert)
+ return auth[2]['id'], server_url, cert_file, auth[2]['token']
+
+
+def get_soledad_instance(username, provider, passphrase, basedir):
+ # setup soledad info
+ uuid, server_url, cert_file, token = \
+ get_soledad_info(username, provider, passphrase, basedir)
+ secrets_path = os.path.join(
+ basedir, '%s.secret' % uuid)
+ local_db_path = os.path.join(
+ basedir, '%s.db' % uuid)
+ # instantiate soledad
+ return Soledad(
+ uuid,
+ unicode(passphrase),
+ secrets_path=secrets_path,
+ local_db_path=local_db_path,
+ server_url=server_url,
+ cert_file=cert_file,
+ auth_token=token)
+
+
+# main program
+
+if __name__ == '__main__':
+
+ class ValidateUserHandle(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ m = re.compile('^([^@]+)@([^@]+\.[^@]+)$')
+ res = m.match(values)
+ if res == None:
+ parser.error('User handle should have the form user@provider.')
+ setattr(namespace, 'username', res.groups()[0])
+ setattr(namespace, 'provider', res.groups()[1])
+
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'user@provider', action=ValidateUserHandle, help='the user handle')
+ parser.add_argument(
+ '-b', dest='basedir', required=False, default=None, help='the user handle')
+ args = parser.parse_args()
+
+ # get the password
+ passphrase = getpass.getpass(
+ 'Password for %s@%s: ' % (args.username, args.provider))
+
+ # get the basedir
+ basedir = args.basedir
+ if basedir is None:
+ basedir = tempfile.mkdtemp()
+ print 'Using %s as base directory.' % basedir
+
+ # get the soledad instance
+ s = get_soledad_instance(
+ args.username, args.provider, passphrase, basedir)
diff --git a/scripts/db_access/server_side_db.py b/scripts/db_access/server_side_db.py
new file mode 100644
index 00000000..18641a0f
--- /dev/null
+++ b/scripts/db_access/server_side_db.py
@@ -0,0 +1,42 @@
+#!/usr/bin/python
+
+# This script gives server-side access to one Soledad user database by using
+# the configuration stored in /etc/leap/soledad-server.conf.
+#
+# Use it like this:
+#
+# python -i server-side-db.py <uuid>
+
+import sys
+from ConfigParser import ConfigParser
+
+from leap.soledad.common.couch import CouchDatabase
+
+if len(sys.argv) != 2:
+ print 'Usage: %s <uuid>' % sys.argv[0]
+ exit(1)
+
+uuid = sys.argv[1]
+
+# get couch url
+cp = ConfigParser()
+cp.read('/etc/leap/soledad-server.conf')
+url = cp.get('soledad-server', 'couch_url')
+
+# access user db
+dbname = 'user-%s' % uuid
+db = CouchDatabase(url, dbname)
+
+# get replica info
+replica_uid = db._replica_uid
+gen, docs = db.get_all_docs()
+print "dbname: %s" % dbname
+print "replica_uid: %s" % replica_uid
+print "generation: %d" % gen
+
+# get relevant docs
+schemes = map(lambda d: d.content['_enc_scheme'], docs)
+pubenc = filter(lambda d: d.content['_enc_scheme'] == 'pubkey', docs)
+
+print "total number of docs: %d" % len(docs)
+print "pubkey encrypted docs: %d" % len(pubenc)
--
cgit v1.2.3
From 5a2e9ac138faca940e10920be008a229a7a54cca Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Fri, 14 Mar 2014 16:04:28 -0300
Subject: Improve client db access script.
---
scripts/db_access/client_side_db.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
(limited to 'scripts/db_access')
diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py
index 15980f5d..2bf4ab5e 100644
--- a/scripts/db_access/client_side_db.py
+++ b/scripts/db_access/client_side_db.py
@@ -13,17 +13,24 @@ import requests
import json
import srp._pysrp as srp
import binascii
+import logging
from leap.common.config import get_path_prefix
from leap.soledad.client import Soledad
+# create a logger
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
safe_unhexlify = lambda x: binascii.unhexlify(x) if (
len(x) % 2 == 0) else binascii.unhexlify('0' + x)
def fail(reason):
- print 'Fail: ' + reason
+ logger.error('Fail: ' + reason)
exit(2)
@@ -94,6 +101,8 @@ def get_soledad_instance(username, provider, passphrase, basedir):
# setup soledad info
uuid, server_url, cert_file, token = \
get_soledad_info(username, provider, passphrase, basedir)
+ logger.info('UUID is %s' % uuid)
+ logger.info('Server URL is %s' % server_url)
secrets_path = os.path.join(
basedir, '%s.secret' % uuid)
local_db_path = os.path.join(
@@ -138,7 +147,7 @@ if __name__ == '__main__':
basedir = args.basedir
if basedir is None:
basedir = tempfile.mkdtemp()
- print 'Using %s as base directory.' % basedir
+ logger.info('Using %s as base directory.' % basedir)
# get the soledad instance
s = get_soledad_instance(
--
cgit v1.2.3
From 4dd32df24959b3f6062f525a3bba5cd592729107 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 24 Mar 2014 12:38:44 -0300
Subject: Add script for resetting users database.
---
scripts/db_access/reset_db.py | 79 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 79 insertions(+)
create mode 100644 scripts/db_access/reset_db.py
(limited to 'scripts/db_access')
diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py
new file mode 100644
index 00000000..80871856
--- /dev/null
+++ b/scripts/db_access/reset_db.py
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+
+# This script can be run on server side to completelly reset a user database.
+#
+# WARNING: running this script over a database will delete all documents but
+# the one with id u1db_config (which contains db metadata) and design docs
+# needed for couch backend.
+
+
+import sys
+from ConfigParser import ConfigParser
+import threading
+import logging
+from couchdb import Database as CouchDatabase
+
+
+if len(sys.argv) != 2:
+ print 'Usage: %s <uuid>' % sys.argv[0]
+ exit(1)
+
+uuid = sys.argv[1]
+
+
+# create a logger
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+# get couch url
+cp = ConfigParser()
+cp.read('/etc/leap/soledad-server.conf')
+url = cp.get('soledad-server', 'couch_url')
+
+
+# confirm
+yes = raw_input("Are you sure you want to reset the database for user %s "
+ "(type YES)? " % uuid)
+if yes != 'YES':
+ print 'Bailing out...'
+ exit(2)
+
+
+db = CouchDatabase('%s/user-%s' % (url, uuid))
+
+
+class _DeleterThread(threading.Thread):
+
+ def __init__(self, db, doc_id, release_fun):
+ threading.Thread.__init__(self)
+ self._db = db
+ self._doc_id = doc_id
+ self._release_fun = release_fun
+
+ def run(self):
+ logger.info('[%s] deleting doc...' % self._doc_id)
+ del self._db[self._doc_id]
+ logger.info('[%s] done.' % self._doc_id)
+ self._release_fun()
+
+
+semaphore_pool = threading.BoundedSemaphore(value=20)
+
+
+threads = []
+for doc_id in db:
+ if doc_id != 'u1db_config' and not doc_id.startswith('_design'):
+ semaphore_pool.acquire()
+ logger.info('[main] launching thread for doc: %s' % doc_id)
+ t = _DeleterThread(db, doc_id, semaphore_pool.release)
+ t.start()
+ threads.append(t)
+
+
+logger.info('[main] waiting for threads.')
+map(lambda thread: thread.join(), threads)
+
+
+logger.info('[main] done.')
--
cgit v1.2.3