From 6183ad313298de08e05c31c5f18f133361cd803b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 14 Jul 2014 19:54:43 -0300 Subject: Make client db access script defer decryption. --- scripts/db_access/client_side_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 6c456c41..1c4f3754 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -26,7 +26,7 @@ from util import ValidateUserHandle # create a logger logger = logging.getLogger(__name__) LOG_FORMAT = '%(asctime)s %(message)s' -logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) +logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) safe_unhexlify = lambda x: binascii.unhexlify(x) if ( @@ -119,7 +119,8 @@ def get_soledad_instance(username, provider, passphrase, basedir): local_db_path=local_db_path, server_url=server_url, cert_file=cert_file, - auth_token=token) + auth_token=token, + defer_encryption=True) # main program -- cgit v1.2.3 From 30aa5c040c093aa82be09e94dd403c18597320e5 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 4 Aug 2014 16:42:56 -0300 Subject: Protect sync db with a password. --- scripts/db_access/client_side_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 1c4f3754..67c5dbe1 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -120,7 +120,7 @@ def get_soledad_instance(username, provider, passphrase, basedir): server_url=server_url, cert_file=cert_file, auth_token=token, - defer_encryption=True) + defer_encryption=False) # main program @@ -154,3 +154,4 @@ if __name__ == '__main__': # get the soledad instance s = get_soledad_instance( args.username, args.provider, passphrase, basedir) + s.sync() -- cgit v1.2.3 From eda955ed1f761d8de005a2f2c03fc7d10484ac28 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 5 Dec 2014 14:35:06 -0200 Subject: Add key manager to client db access script. --- scripts/db_access/client_side_db.py | 86 ++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 30 deletions(-) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 67c5dbe1..2b1b7c72 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -2,23 +2,17 @@ # This script gives client-side access to one Soledad user database. - -import sys import os import argparse -import re import tempfile import getpass import requests -import json import srp._pysrp as srp import binascii import logging - -from leap.common.config import get_path_prefix from leap.soledad.client import Soledad - +from leap.keymanager import KeyManager from util import ValidateUserHandle @@ -33,30 +27,30 @@ safe_unhexlify = lambda x: binascii.unhexlify(x) if ( len(x) % 2 == 0) else binascii.unhexlify('0' + x) -def fail(reason): +def _fail(reason): logger.error('Fail: ' + reason) exit(2) -def get_api_info(provider): +def _get_api_info(provider): info = requests.get( 'https://'+provider+'/provider.json', verify=False).json() return info['api_uri'], info['api_version'] -def login(username, passphrase, provider, api_uri, api_version): +def _login(username, passphrase, provider, api_uri, api_version): usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) auth = None try: - auth = authenticate(api_uri, api_version, usr).json() + auth = _authenticate(api_uri, api_version, usr).json() except requests.exceptions.ConnectionError: - fail('Could not connect to server.') + _fail('Could not connect to server.') if 'errors' in auth: - fail(str(auth['errors'])) + _fail(str(auth['errors'])) return api_uri, api_version, auth -def authenticate(api_uri, api_version, usr): +def _authenticate(api_uri, api_version, usr): api_url = "%s/%s" % (api_uri, api_version) session = requests.session() uname, A = usr.start_authentication() @@ -64,16 +58,16 @@ def authenticate(api_uri, api_version, usr): init = session.post( api_url + '/sessions', data=params, verify=False).json() if 'errors' in init: - fail('test user not found') + _fail('test user not found') M = usr.process_challenge( safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) return session.put(api_url + '/sessions/' + uname, verify=False, data={'client_auth': binascii.hexlify(M)}) -def get_soledad_info(username, provider, passphrase, basedir): - api_uri, api_version = get_api_info(provider) - auth = login(username, passphrase, provider, api_uri, api_version) +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) # get soledad server url service_url = '%s/%s/config/soledad-service.json' % \ (api_uri, api_version) @@ -101,10 +95,9 @@ def get_soledad_info(username, provider, passphrase, basedir): return auth[2]['id'], server_url, cert_file, auth[2]['token'] -def get_soledad_instance(username, provider, passphrase, basedir): +def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): # setup soledad info - uuid, server_url, cert_file, token = \ - get_soledad_info(username, provider, passphrase, basedir) logger.info('UUID is %s' % uuid) logger.info('Server URL is %s' % server_url) secrets_path = os.path.join( @@ -123,10 +116,22 @@ def get_soledad_instance(username, provider, passphrase, basedir): defer_encryption=False) -# main program +def _get_keymanager_instance(username, provider, soledad, token, + ca_cert_path=None, api_uri=None, api_version=None, uid=None, + gpgbinary=None): + return KeyManager( + "{username}@{provider}".format(username=username, provider=provider), + "http://uri", + soledad, + token=token, + ca_cert_path=ca_cert_path, + api_uri=api_uri, + api_version=api_version, + uid=uid, + gpgbinary=gpgbinary) -if __name__ == '__main__': +def _parse_args(): # parse command line parser = argparse.ArgumentParser() parser.add_argument( @@ -137,21 +142,42 @@ if __name__ == '__main__': parser.add_argument( '-p', dest='passphrase', required=False, default=None, help='the user passphrase') - args = parser.parse_args() + return parser.parse_args() - # get the password + +def _get_passphrase(args): passphrase = args.passphrase if passphrase is None: passphrase = getpass.getpass( 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + - # get the basedir +def _get_basedir(args): basedir = args.basedir if basedir is None: basedir = tempfile.mkdtemp() logger.info('Using %s as base directory.' % basedir) + return basedir + + +# main program + +if __name__ == '__main__': + args = _parse_args() + passphrase = _get_passphrase(args) + basedir = _get_basedir(args) + uuid, server_url, cert_file, token = \ + _get_soledad_info(args.username, args.provider, passphrase, basedir) + + soledad = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + soledad.sync() + + km = _get_keymanager_instance( + args.username, + args.provider, + soledad, + token, + uid=uuid) - # get the soledad instance - s = get_soledad_instance( - args.username, args.provider, passphrase, basedir) - s.sync() -- cgit v1.2.3 From fa8dacef003d30cd9b56f7e2b07baa3b387c1e20 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 18 Dec 2014 14:42:13 -0200 Subject: Update testing scripts. --- scripts/db_access/reset_db.py | 132 ++++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 37 deletions(-) (limited to 'scripts/db_access') diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py index 80871856..7c6d281b 100644 --- a/scripts/db_access/reset_db.py +++ b/scripts/db_access/reset_db.py @@ -5,20 +5,21 @@ # WARNING: running this script over a database will delete all documents but # the one with id u1db_config (which contains db metadata) and design docs # needed for couch backend. +# +# Run it like this to get some help: +# +# ./reset_db.py --help -import sys -from ConfigParser import ConfigParser import threading import logging -from couchdb import Database as CouchDatabase - +import argparse +import re -if len(sys.argv) != 2: - print 'Usage: %s ' % sys.argv[0] - exit(1) -uuid = sys.argv[1] +from ConfigParser import ConfigParser +from couchdb import Database as CouchDatabase +from couchdb import Server as CouchServer # create a logger @@ -27,23 +28,6 @@ LOG_FORMAT = '%(asctime)s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) -# get couch url -cp = ConfigParser() -cp.read('/etc/leap/soledad-server.conf') -url = cp.get('soledad-server', 'couch_url') - - -# confirm -yes = raw_input("Are you sure you want to reset the database for user %s " - "(type YES)? " % uuid) -if yes != 'YES': - print 'Bailing out...' - exit(2) - - -db = CouchDatabase('%s/user-%s' % (url, uuid)) - - class _DeleterThread(threading.Thread): def __init__(self, db, doc_id, release_fun): @@ -59,21 +43,95 @@ class _DeleterThread(threading.Thread): self._release_fun() -semaphore_pool = threading.BoundedSemaphore(value=20) - - -threads = [] -for doc_id in db: - if doc_id != 'u1db_config' and not doc_id.startswith('_design'): +def get_confirmation(noconfirm, uuid, shared): + msg = "Are you sure you want to reset %s (type YES)? " + if shared: + msg = msg % "the shared database" + elif uuid: + msg = msg % ("the database for user %s" % uuid) + else: + msg = msg % "all databases" + if noconfirm is False: + yes = raw_input(msg) + if yes != 'YES': + print 'Bailing out...' + exit(2) + + +def get_url(empty): + url = None + if empty is False: + # get couch url + cp = ConfigParser() + cp.read('/etc/leap/soledad-server.conf') + url = cp.get('soledad-server', 'couch_url') + else: + with open('/etc/couchdb/couchdb.netrc') as f: + netrc = f.read() + admin_password = re.match('^.* password (.*)$', netrc).groups()[0] + url = 'http://admin:%s@127.0.0.1:5984' % admin_password + return url + + +def reset_all_dbs(url, empty): + server = CouchServer('%s' % (url)) + for dbname in server: + if dbname.startswith('user-') or dbname == 'shared': + reset_db(url, dbname, empty) + + +def reset_db(url, dbname, empty): + db = CouchDatabase('%s/%s' % (url, dbname)) + semaphore_pool = threading.BoundedSemaphore(value=20) + + # launch threads for deleting docs + threads = [] + for doc_id in db: + if empty is False: + if doc_id == 'u1db_config' or doc_id.startswith('_design'): + continue semaphore_pool.acquire() logger.info('[main] launching thread for doc: %s' % doc_id) t = _DeleterThread(db, doc_id, semaphore_pool.release) t.start() threads.append(t) - -logger.info('[main] waiting for threads.') -map(lambda thread: thread.join(), threads) - - -logger.info('[main] done.') + # wait for threads to finish + logger.info('[main] waiting for threads.') + map(lambda thread: thread.join(), threads) + logger.info('[main] done.') + + +def _parse_args(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('-u', dest='uuid', default=False, + help='Reset database of given user.') + group.add_argument('-s', dest='shared', action='store_true', default=False, + help='Reset the shared database.') + group.add_argument('-a', dest='all', action='store_true', default=False, + help='Reset all user databases.') + parser.add_argument( + '-e', dest='empty', action='store_true', required=False, default=False, + help='Empty database (do not preserve minimal set of u1db documents).') + parser.add_argument( + '-y', dest='noconfirm', action='store_true', required=False, + default=False, + help='Do not ask for confirmation.') + return parser.parse_args(), parser + + +if __name__ == '__main__': + args, parser = _parse_args() + if not (args.uuid or args.shared or args.all): + parser.print_help() + exit(1) + + url = get_url(args.empty) + get_confirmation(args.noconfirm, args.uuid, args.shared) + if args.uuid: + reset_db(url, "user-%s" % args.uuid, args.empty) + elif args.shared: + reset_db(url, "shared", args.empty) + elif args.all: + reset_all_dbs(url, args.empty) -- cgit v1.2.3 From 578d986b8eb502cde027c0cd1e05742a59def496 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 24 Mar 2015 13:46:45 -0300 Subject: [fix] turn client side db access script into async With changes in soledad client api to make it async, the supporting scripts also have to be updated. This commit also adds more functionalities as exporting incoming mail and public or private keys. --- scripts/db_access/client_side_db.py | 56 +++++++++++++++++++++++++++++++++---- scripts/db_access/util.py | 1 + 2 files changed, 51 insertions(+), 6 deletions(-) create mode 120000 scripts/db_access/util.py (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 2b1b7c72..a77b0b16 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -11,8 +11,12 @@ import srp._pysrp as srp import binascii import logging +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks + from leap.soledad.client import Soledad from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey from util import ValidateUserHandle @@ -137,11 +141,21 @@ def _parse_args(): parser.add_argument( 'user@provider', action=ValidateUserHandle, help='the user handle') parser.add_argument( - '-b', dest='basedir', required=False, default=None, + '--basedir', '-b', default=None, help='soledad base directory') parser.add_argument( - '-p', dest='passphrase', required=False, default=None, + '--passphrase', '-p', default=None, help='the user passphrase') + parser.add_argument( + '--sync', '-s', action='store_true', + help='synchronize with the server replica') + parser.add_argument( + '--export-public-key', help="export the public key to a file") + parser.add_argument( + '--export-private-key', help="export the private key to a file") + parser.add_argument( + '--export-incoming-messages', + help="export incoming messages to a directory") return parser.parse_args() @@ -161,23 +175,53 @@ def _get_basedir(args): return basedir +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key(address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + # main program +@inlineCallbacks +def _main(soledad, km, args): + if args.sync: + yield soledad.sync() + if args.export_private_key: + yield _export_key(args, km, args.export_private_key, private=True) + if args.export_public_key: + yield _export_key(args, km, args.expoert_public_key, private=False) + if args.export_incoming_messages: + yield _export_incoming_messages(soledad, args.export_incoming_messages) + reactor.stop() + + if __name__ == '__main__': args = _parse_args() passphrase = _get_passphrase(args) basedir = _get_basedir(args) uuid, server_url, cert_file, token = \ _get_soledad_info(args.username, args.provider, passphrase, basedir) - soledad = _get_soledad_instance( uuid, passphrase, basedir, server_url, cert_file, token) - soledad.sync() - km = _get_keymanager_instance( args.username, args.provider, soledad, token, uid=uuid) - + _main(soledad, km, args) + reactor.run() diff --git a/scripts/db_access/util.py b/scripts/db_access/util.py new file mode 120000 index 00000000..368734f7 --- /dev/null +++ b/scripts/db_access/util.py @@ -0,0 +1 @@ +../profiling/util.py \ No newline at end of file -- cgit v1.2.3 From b76f9b6c479d73206a6a29494b83d2b9c66b7a6a Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 25 Mar 2015 17:33:56 -0300 Subject: [bug] create directory in client db access script --- scripts/db_access/client_side_db.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index a77b0b16..d7c54b66 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -171,6 +171,8 @@ def _get_basedir(args): basedir = args.basedir if basedir is None: basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) logger.info('Using %s as base directory.' % basedir) return basedir -- cgit v1.2.3 From eae4468d99029006cc36a021e82350a0f62f7006 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 May 2015 14:49:40 -0300 Subject: [bug] fix order of insertion of decrypted docs This commit actually does some different things: * When doing asynchronous decryption of incoming documents in soledad client during a sync, there was the possibility that a document corresponding to a newer generation would be decrypted and inserted in the local database before a document corresponding to an older generation. When this happened, the metadata about the target database (i.e. its locally-known generation) would be first updated to the newer generation, and then an attempt to insert a document corresponding to an older generation would cause the infamous InvalidGeneration error. To fix that we use the sync-index information that is contained in the sync stream to correctly find the insertable docs to be inserted in the local database, thus avoiding the problem described above. * Refactor the sync encrypt/decrypt pool to its own file. * Fix the use of twisted adbapi with multiprocessing. Closes: #6757. --- scripts/db_access/client_side_db.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index d7c54b66..a047b522 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -10,6 +10,7 @@ import requests import srp._pysrp as srp import binascii import logging +import json from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks @@ -146,6 +147,9 @@ def _parse_args(): parser.add_argument( '--passphrase', '-p', default=None, help='the user passphrase') + parser.add_argument( + '--get-all-docs', '-a', action='store_true', + help='get all documents from the local database') parser.add_argument( '--sync', '-s', action='store_true', help='synchronize with the server replica') @@ -196,12 +200,21 @@ def _export_incoming_messages(soledad, directory): i += 1 +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) + + # main program @inlineCallbacks def _main(soledad, km, args): if args.sync: yield soledad.sync() + if args.get_all_docs: + yield _get_all_docs(soledad) if args.export_private_key: yield _export_key(args, km, args.export_private_key, private=True) if args.export_public_key: -- cgit v1.2.3 From 93717f50c9e8fc6295f74b6117268ba595f13ce9 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 13 May 2015 10:34:22 -0300 Subject: [feature] add --create-doc to client db script --- scripts/db_access/client_side_db.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index a047b522..5dd2bd95 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -150,6 +150,9 @@ def _parse_args(): parser.add_argument( '--get-all-docs', '-a', action='store_true', help='get all documents from the local database') + parser.add_argument( + '--create-doc', '-c', default=None, + help='create a document with give content') parser.add_argument( '--sync', '-s', action='store_true', help='synchronize with the server replica') @@ -211,6 +214,8 @@ def _get_all_docs(soledad): @inlineCallbacks def _main(soledad, km, args): + if args.create_doc: + yield soledad.create_doc({'content': args.create_doc}) if args.sync: yield soledad.sync() if args.get_all_docs: -- cgit v1.2.3 From d59ac3b5ce713787cd7a46e181f2381de3a8fde2 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 20 May 2015 10:58:16 -0300 Subject: [feature] ensure reactor stops on client db script --- scripts/db_access/client_side_db.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'scripts/db_access') diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 5dd2bd95..1d8d32e2 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -214,19 +214,23 @@ def _get_all_docs(soledad): @inlineCallbacks def _main(soledad, km, args): - if args.create_doc: - yield soledad.create_doc({'content': args.create_doc}) - if args.sync: - yield soledad.sync() - if args.get_all_docs: - yield _get_all_docs(soledad) - if args.export_private_key: - yield _export_key(args, km, args.export_private_key, private=True) - if args.export_public_key: - yield _export_key(args, km, args.expoert_public_key, private=False) - if args.export_incoming_messages: - yield _export_incoming_messages(soledad, args.export_incoming_messages) - reactor.stop() + try: + if args.create_doc: + yield soledad.create_doc({'content': args.create_doc}) + if args.sync: + yield soledad.sync() + if args.get_all_docs: + yield _get_all_docs(soledad) + if args.export_private_key: + yield _export_key(args, km, args.export_private_key, private=True) + if args.export_public_key: + yield _export_key(args, km, args.expoert_public_key, private=False) + if args.export_incoming_messages: + yield _export_incoming_messages(soledad, args.export_incoming_messages) + except: + pass + finally: + reactor.stop() if __name__ == '__main__': -- cgit v1.2.3