From 94ec89384abeba52d660addf3528b11a9aa5a8b0 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 11 Mar 2014 16:14:26 -0300 Subject: Add script to measure PUT memory usage. --- scripts/db_access/client_side_db.py | 145 ++++++++++++++++++++++++++++++++++++ scripts/db_access/server_side_db.py | 42 +++++++++++ 2 files changed, 187 insertions(+) create mode 100644 scripts/db_access/client_side_db.py create mode 100644 scripts/db_access/server_side_db.py (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py new file mode 100644 index 00000000..15980f5d --- /dev/null +++ b/scripts/db_access/client_side_db.py @@ -0,0 +1,145 @@ +#!/usr/bin/python + +# This script gives client-side access to one Soledad user database. + + +import sys +import os +import argparse +import re +import tempfile +import getpass +import requests +import json +import srp._pysrp as srp +import binascii + +from leap.common.config import get_path_prefix +from leap.soledad.client import Soledad + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def fail(reason): + print 'Fail: ' + reason + exit(2) + + +def get_api_info(provider): + info = requests.get( + 'https://'+provider+'/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + fail('Could not connect to server.') + if 'errors' in auth: + fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = get_api_info(provider) + auth = login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def get_soledad_instance(username, provider, passphrase, basedir): + # setup soledad info + uuid, server_url, cert_file, token = \ + get_soledad_info(username, provider, passphrase, basedir) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token) + + +# main program + +if __name__ == '__main__': + + class ValidateUserHandle(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + m = re.compile('^([^@]+)@([^@]+\.[^@]+)$') + res = m.match(values) + if res == None: + parser.error('User handle should have the form user@provider.') + setattr(namespace, 'username', res.groups()[0]) + setattr(namespace, 'provider', res.groups()[1]) + + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-b', dest='basedir', required=False, default=None, help='the user handle') + args = parser.parse_args() + + # get the password + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + + # get the basedir + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + print 'Using %s as base directory.' % basedir + + # get the soledad instance + s = get_soledad_instance( + args.username, args.provider, passphrase, basedir) diff --git a/scripts/db_access/server_side_db.py b/scripts/db_access/server_side_db.py new file mode 100644 index 00000000..18641a0f --- /dev/null +++ b/scripts/db_access/server_side_db.py @@ -0,0 +1,42 @@ +#!/usr/bin/python + +# This script gives server-side access to one Soledad user database by using +# the configuration stored in /etc/leap/soledad-server.conf. +# +# Use it like this: +# +# python -i server-side-db.py + +import sys +from ConfigParser import ConfigParser + +from leap.soledad.common.couch import CouchDatabase + +if len(sys.argv) != 2: + print 'Usage: %s ' % sys.argv[0] + exit(1) + +uuid = sys.argv[1] + +# get couch url +cp = ConfigParser() +cp.read('/etc/leap/soledad-server.conf') +url = cp.get('soledad-server', 'couch_url') + +# access user db +dbname = 'user-%s' % uuid +db = CouchDatabase(url, dbname) + +# get replica info +replica_uid = db._replica_uid +gen, docs = db.get_all_docs() +print "dbname: %s" % dbname +print "replica_uid: %s" % replica_uid +print "generation: %d" % gen + +# get relevant docs +schemes = map(lambda d: d.content['_enc_scheme'], docs) +pubenc = filter(lambda d: d.content['_enc_scheme'] == 'pubkey', docs) + +print "total number of docs: %d" % len(docs) +print "pubkey encrypted docs: %d" % len(pubenc) -- cgit v1.2.3 From 5a2e9ac138faca940e10920be008a229a7a54cca Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 14 Mar 2014 16:04:28 -0300 Subject: Improve client db access script. --- scripts/db_access/client_side_db.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 15980f5d..2bf4ab5e 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -13,17 +13,24 @@ import requests import json import srp._pysrp as srp import binascii +import logging from leap.common.config import get_path_prefix from leap.soledad.client import Soledad +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + safe_unhexlify = lambda x: binascii.unhexlify(x) if ( len(x) % 2 == 0) else binascii.unhexlify('0' + x) def fail(reason): - print 'Fail: ' + reason + logger.error('Fail: ' + reason) exit(2) @@ -94,6 +101,8 @@ def get_soledad_instance(username, provider, passphrase, basedir): # setup soledad info uuid, server_url, cert_file, token = \ get_soledad_info(username, provider, passphrase, basedir) + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) secrets_path = os.path.join( basedir, '%s.secret' % uuid) local_db_path = os.path.join( @@ -138,7 +147,7 @@ if __name__ == '__main__': basedir = args.basedir if basedir is None: basedir = tempfile.mkdtemp() - print 'Using %s as base directory.' % basedir + logger.info('Using %s as base directory.' % basedir) # get the soledad instance s = get_soledad_instance( -- cgit v1.2.3 From 4dd32df24959b3f6062f525a3bba5cd592729107 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 24 Mar 2014 12:38:44 -0300 Subject: Add script for resetting users database. --- scripts/db_access/reset_db.py | 79 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 scripts/db_access/reset_db.py (limited to 'scripts/db_access') diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py new file mode 100644 index 00000000..80871856 --- /dev/null +++ b/scripts/db_access/reset_db.py @@ -0,0 +1,79 @@ +#!/usr/bin/python + +# This script can be run on server side to completelly reset a user database. +# +# WARNING: running this script over a database will delete all documents but +# the one with id u1db_config (which contains db metadata) and design docs +# needed for couch backend. + + +import sys +from ConfigParser import ConfigParser +import threading +import logging +from couchdb import Database as CouchDatabase + + +if len(sys.argv) != 2: + print 'Usage: %s ' % sys.argv[0] + exit(1) + +uuid = sys.argv[1] + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +# get couch url +cp = ConfigParser() +cp.read('/etc/leap/soledad-server.conf') +url = cp.get('soledad-server', 'couch_url') + + +# confirm +yes = raw_input("Are you sure you want to reset the database for user %s " + "(type YES)? " % uuid) +if yes != 'YES': + print 'Bailing out...' + exit(2) + + +db = CouchDatabase('%s/user-%s' % (url, uuid)) + + +class _DeleterThread(threading.Thread): + + def __init__(self, db, doc_id, release_fun): + threading.Thread.__init__(self) + self._db = db + self._doc_id = doc_id + self._release_fun = release_fun + + def run(self): + logger.info('[%s] deleting doc...' % self._doc_id) + del self._db[self._doc_id] + logger.info('[%s] done.' % self._doc_id) + self._release_fun() + + +semaphore_pool = threading.BoundedSemaphore(value=20) + + +threads = [] +for doc_id in db: + if doc_id != 'u1db_config' and not doc_id.startswith('_design'): + semaphore_pool.acquire() + logger.info('[main] launching thread for doc: %s' % doc_id) + t = _DeleterThread(db, doc_id, semaphore_pool.release) + t.start() + threads.append(t) + + +logger.info('[main] waiting for threads.') +map(lambda thread: thread.join(), threads) + + +logger.info('[main] done.') -- cgit v1.2.3