Merge tag '0.7.2' into debian/experimental

Tag leap.soledad version 0.7.2

3 files changed, 297 insertions, 74 deletions

diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py
index 6c456c41..25eebfbe 100644
--- a/scripts/db_access/client_side_db.py
+++ b/scripts/db_access/client_side_db.py
@@ -1,62 +1,77 @@
 #!/usr/bin/python
 
-# This script gives client-side access to one Soledad user database.
-
-
-import sys
 import os
 import argparse
-import re
 import tempfile
 import getpass
 import requests
-import json
 import srp._pysrp as srp
 import binascii
 import logging
+import json
+import time
 
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
 
-from leap.common.config import get_path_prefix
 from leap.soledad.client import Soledad
-
+from leap.keymanager import KeyManager
+from leap.keymanager.openpgp import OpenPGPKey
 
 from util import ValidateUserHandle
 
 
+"""
+Script to give access to client-side Soledad database.
+
+This is mainly used for tests, but can also be used to recover data from a
+Soledad database (public/private keys, export documents, etc).
+
+To speed up testing/debugging, this script can dump the auth data after
+logging in. Use the --export-auth-data option to export auth data to a file.
+The contents of the file is a json dictionary containing the uuid, server_url,
+cert_file and token, which is enough info to instantiate a soledad client
+without having to interact with the webapp again. Use the --use-auth-data
+option to use the auth data stored in a file.
+
+Use the --help option to see available options.
+"""
+
+
 # create a logger
 logger = logging.getLogger(__name__)
 LOG_FORMAT = '%(asctime)s %(message)s'
-logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG)
 
 
 safe_unhexlify = lambda x: binascii.unhexlify(x) if (
     len(x) % 2 == 0) else binascii.unhexlify('0' + x)
 
 
-def fail(reason):
+def _fail(reason):
     logger.error('Fail: ' + reason)
     exit(2)
 
 
-def get_api_info(provider):
+def _get_api_info(provider):
     info = requests.get(
         'https://'+provider+'/provider.json', verify=False).json()
     return info['api_uri'], info['api_version']
 
 
-def login(username, passphrase, provider, api_uri, api_version):
+def _login(username, passphrase, provider, api_uri, api_version):
     usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024)
     auth = None
     try:
-        auth = authenticate(api_uri, api_version, usr).json()
+        auth = _authenticate(api_uri, api_version, usr).json()
     except requests.exceptions.ConnectionError:
-        fail('Could not connect to server.')
+        _fail('Could not connect to server.')
     if 'errors' in auth:
-        fail(str(auth['errors']))
+        _fail(str(auth['errors']))
     return api_uri, api_version, auth
 
 
-def authenticate(api_uri, api_version, usr):
+def _authenticate(api_uri, api_version, usr):
     api_url = "%s/%s" % (api_uri, api_version)
     session = requests.session()
     uname, A = usr.start_authentication()
@@ -64,16 +79,16 @@ def authenticate(api_uri, api_version, usr):
     init = session.post(
         api_url + '/sessions', data=params, verify=False).json()
     if 'errors' in init:
-        fail('test user not found')
+        _fail('test user not found')
     M = usr.process_challenge(
         safe_unhexlify(init['salt']), safe_unhexlify(init['B']))
     return session.put(api_url + '/sessions/' + uname, verify=False,
                        data={'client_auth': binascii.hexlify(M)})
 
 
-def get_soledad_info(username, provider, passphrase, basedir):
-    api_uri, api_version = get_api_info(provider)
-    auth = login(username, passphrase, provider, api_uri, api_version)
+def _get_soledad_info(username, provider, passphrase, basedir):
+    api_uri, api_version = _get_api_info(provider)
+    auth = _login(username, passphrase, provider, api_uri, api_version)
     # get soledad server url
     service_url = '%s/%s/config/soledad-service.json' % \
                   (api_uri, api_version)
@@ -91,20 +106,19 @@ def get_soledad_info(username, provider, passphrase, basedir):
         if choice != '':
             host = hostnames[int(choice) - 1]
     server_url = 'https://%s:%d/user-%s' % \
-              (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'],
-               auth[2]['id'])
+        (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'],
+         auth[2]['id'])
     # get provider ca certificate
     ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text
     cert_file = os.path.join(basedir, 'ca.crt')
     with open(cert_file, 'w') as f:
-      f.write(ca_cert)
+        f.write(ca_cert)
     return auth[2]['id'], server_url, cert_file, auth[2]['token']
 
 
-def get_soledad_instance(username, provider, passphrase, basedir):
+def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file,
+                          token):
     # setup soledad info
-    uuid, server_url, cert_file, token = \
-        get_soledad_info(username, provider, passphrase, basedir)
     logger.info('UUID is %s' % uuid)
     logger.info('Server URL is %s' % server_url)
     secrets_path = os.path.join(
@@ -119,37 +133,187 @@ def get_soledad_instance(username, provider, passphrase, basedir):
         local_db_path=local_db_path,
         server_url=server_url,
         cert_file=cert_file,
-        auth_token=token)
+        auth_token=token,
+        defer_encryption=True)
 
 
-# main program
+def _get_keymanager_instance(username, provider, soledad, token,
+                             ca_cert_path=None, api_uri=None, api_version=None,
+                             uid=None, gpgbinary=None):
+    return KeyManager(
+        "{username}@{provider}".format(username=username, provider=provider),
+        "http://uri",
+        soledad,
+        token=token,
+        ca_cert_path=ca_cert_path,
+        api_uri=api_uri,
+        api_version=api_version,
+        uid=uid,
+        gpgbinary=gpgbinary)
 
-if __name__ == '__main__':
 
+def _parse_args():
     # parse command line
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'user@provider', action=ValidateUserHandle, help='the user handle')
     parser.add_argument(
-        '-b', dest='basedir', required=False, default=None,
+        '--basedir', '-b', default=None,
         help='soledad base directory')
     parser.add_argument(
-        '-p', dest='passphrase', required=False, default=None,
+        '--passphrase', '-p', default=None,
         help='the user passphrase')
-    args = parser.parse_args()
+    parser.add_argument(
+        '--get-all-docs', '-a', action='store_true',
+        help='get all documents from the local database')
+    parser.add_argument(
+        '--create-docs', '-c', default=0, type=int,
+        help='create a number of documents')
+    parser.add_argument(
+        '--sync', '-s', action='store_true',
+        help='synchronize with the server replica')
+    parser.add_argument(
+        '--repeat-sync', '-r', action='store_true',
+        help='repeat synchronization until no new data is received')
+    parser.add_argument(
+        '--export-public-key', help="export the public key to a file")
+    parser.add_argument(
+        '--export-private-key', help="export the private key to a file")
+    parser.add_argument(
+        '--export-incoming-messages',
+        help="export incoming messages to a directory")
+    parser.add_argument(
+        '--export-auth-data',
+        help="export authentication data to a file")
+    parser.add_argument(
+        '--use-auth-data',
+        help="use authentication data from a file")
+    return parser.parse_args()
+
 
-    # get the password
+def _get_passphrase(args):
     passphrase = args.passphrase
     if passphrase is None:
         passphrase = getpass.getpass(
             'Password for %s@%s: ' % (args.username, args.provider))
+    return passphrase
+
 
-    # get the basedir
+def _get_basedir(args):
     basedir = args.basedir
     if basedir is None:
         basedir = tempfile.mkdtemp()
+    elif not os.path.isdir(basedir):
+        os.mkdir(basedir)
     logger.info('Using %s as base directory.' % basedir)
+    return basedir
+
+
+@inlineCallbacks
+def _export_key(args, km, fname, private=False):
+    address = args.username + "@" + args.provider
+    pkey = yield km.get_key(
+        address, OpenPGPKey, private=private, fetch_remote=False)
+    with open(args.export_private_key, "w") as f:
+        f.write(pkey.key_data)
+
+
+@inlineCallbacks
+def _export_incoming_messages(soledad, directory):
+    yield soledad.create_index("by-incoming", "bool(incoming)")
+    docs = yield soledad.get_from_index("by-incoming", '1')
+    i = 1
+    for doc in docs:
+        with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f:
+            f.write(doc.content["_enc_json"])
+        i += 1
+
+
+@inlineCallbacks
+def _get_all_docs(soledad):
+    _, docs = yield soledad.get_all_docs()
+    for doc in docs:
+        print json.dumps(doc.content, indent=4)
+
+
+# main program
+
+@inlineCallbacks
+def _main(soledad, km, args):
+    try:
+        if args.create_docs:
+            for i in xrange(args.create_docs):
+                t = time.time()
+                logger.debug(
+                    "Creating doc %d/%d..." % (i + 1, args.create_docs))
+                content = {
+                    'datetime': time.strftime(
+                        "%Y-%m-%d %H:%M:%S", time.gmtime(t)),
+                    'timestamp': t,
+                    'index': i,
+                    'total': args.create_docs,
+                }
+                yield soledad.create_doc(content)
+        if args.sync:
+            yield soledad.sync()
+        if args.repeat_sync:
+            old_gen = 0
+            new_gen = yield soledad.sync()
+            while old_gen != new_gen:
+                old_gen = new_gen
+                new_gen = yield soledad.sync()
+        if args.get_all_docs:
+            yield _get_all_docs(soledad)
+        if args.export_private_key:
+            yield _export_key(args, km, args.export_private_key, private=True)
+        if args.export_public_key:
+            yield _export_key(args, km, args.expoert_public_key, private=False)
+        if args.export_incoming_messages:
+            yield _export_incoming_messages(
+                soledad, args.export_incoming_messages)
+    except Exception as e:
+        logger.error(e)
+    finally:
+        soledad.close()
+        reactor.callWhenRunning(reactor.stop)
+
+
+if __name__ == '__main__':
+    args = _parse_args()
+    passphrase = _get_passphrase(args)
+    basedir = _get_basedir(args)
+
+    if not args.use_auth_data:
+        # get auth data from server
+        uuid, server_url, cert_file, token = \
+            _get_soledad_info(
+                args.username, args.provider, passphrase, basedir)
+    else:
+        # load auth data from file
+        with open(args.use_auth_data) as f:
+            auth_data = json.loads(f.read())
+            uuid = auth_data['uuid']
+            server_url = auth_data['server_url']
+            cert_file = auth_data['cert_file']
+            token = auth_data['token']
+
+    # export auth data to a file
+    if args.export_auth_data:
+        with open(args.export_auth_data, "w") as f:
+            f.write(json.dumps({
+                'uuid': uuid,
+                'server_url': server_url,
+                'cert_file': cert_file,
+                'token': token,
+            }))
 
-    # get the soledad instance
-    s = get_soledad_instance(
-        args.username, args.provider, passphrase, basedir)
+    soledad = _get_soledad_instance(
+        uuid, passphrase, basedir, server_url, cert_file, token)
+    km = _get_keymanager_instance(
+        args.username,
+        args.provider,
+        soledad,
+        token,
+        uid=uuid)
+    _main(soledad, km, args)
+    reactor.run()
diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py
index 80871856..7c6d281b 100644
--- a/scripts/db_access/reset_db.py
+++ b/scripts/db_access/reset_db.py
@@ -5,20 +5,21 @@
 # WARNING: running this script over a database will delete all documents but
 # the one with id u1db_config (which contains db metadata) and design docs
 # needed for couch backend.
+#
+# Run it like this to get some help:
+#
+#     ./reset_db.py --help
 
 
-import sys
-from ConfigParser import ConfigParser
 import threading
 import logging
-from couchdb import Database as CouchDatabase
-
+import argparse
+import re
 
-if len(sys.argv) != 2:
-    print 'Usage: %s <uuid>' % sys.argv[0]
-    exit(1)
 
-uuid = sys.argv[1]
+from ConfigParser import ConfigParser
+from couchdb import Database as CouchDatabase
+from couchdb import Server as CouchServer
 
 
 # create a logger
@@ -27,23 +28,6 @@ LOG_FORMAT = '%(asctime)s %(message)s'
 logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
 
 
-# get couch url
-cp = ConfigParser()
-cp.read('/etc/leap/soledad-server.conf')
-url = cp.get('soledad-server', 'couch_url')
-
-
-# confirm
-yes = raw_input("Are you sure you want to reset the database for user %s "
-                "(type YES)? " % uuid)
-if yes != 'YES':
-    print 'Bailing out...'
-    exit(2)
-
-
-db = CouchDatabase('%s/user-%s' % (url, uuid))
-
-
 class _DeleterThread(threading.Thread):
 
     def __init__(self, db, doc_id, release_fun):
@@ -59,21 +43,95 @@ class _DeleterThread(threading.Thread):
         self._release_fun()
 
 
-semaphore_pool = threading.BoundedSemaphore(value=20)
-
-
-threads = []
-for doc_id in db:
-    if doc_id != 'u1db_config' and not doc_id.startswith('_design'):
+def get_confirmation(noconfirm, uuid, shared):
+    msg = "Are you sure you want to reset %s (type YES)? "
+    if shared:
+        msg = msg % "the shared database"
+    elif uuid:
+        msg = msg % ("the database for user %s" % uuid)
+    else:
+        msg = msg % "all databases"
+    if noconfirm is False:
+        yes = raw_input(msg)
+        if yes != 'YES':
+            print 'Bailing out...'
+            exit(2)
+
+
+def get_url(empty):
+    url = None
+    if empty is False:
+        # get couch url
+        cp = ConfigParser()
+        cp.read('/etc/leap/soledad-server.conf')
+        url = cp.get('soledad-server', 'couch_url')
+    else:
+        with open('/etc/couchdb/couchdb.netrc') as f:
+            netrc = f.read()
+            admin_password = re.match('^.* password (.*)$', netrc).groups()[0]
+            url = 'http://admin:%s@127.0.0.1:5984' % admin_password
+    return url
+
+
+def reset_all_dbs(url, empty):
+    server = CouchServer('%s' % (url))
+    for dbname in server:
+        if dbname.startswith('user-') or dbname == 'shared':
+            reset_db(url, dbname, empty)
+
+
+def reset_db(url, dbname, empty):
+    db = CouchDatabase('%s/%s' % (url, dbname))
+    semaphore_pool = threading.BoundedSemaphore(value=20)
+
+    # launch threads for deleting docs
+    threads = []
+    for doc_id in db:
+        if empty is False:
+            if doc_id == 'u1db_config' or doc_id.startswith('_design'):
+                continue
         semaphore_pool.acquire()
         logger.info('[main] launching thread for doc: %s' % doc_id)
         t = _DeleterThread(db, doc_id, semaphore_pool.release)
         t.start()
         threads.append(t)
 
-
-logger.info('[main] waiting for threads.')
-map(lambda thread: thread.join(), threads)
-
-
-logger.info('[main] done.')
+    # wait for threads to finish
+    logger.info('[main] waiting for threads.')
+    map(lambda thread: thread.join(), threads)
+    logger.info('[main] done.')
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser()
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('-u', dest='uuid', default=False,
+        help='Reset database of given user.')
+    group.add_argument('-s', dest='shared', action='store_true', default=False,
+        help='Reset the shared database.')
+    group.add_argument('-a', dest='all', action='store_true', default=False,
+        help='Reset all user databases.')
+    parser.add_argument(
+        '-e', dest='empty', action='store_true', required=False, default=False,
+        help='Empty database (do not preserve minimal set of u1db documents).')
+    parser.add_argument(
+        '-y', dest='noconfirm', action='store_true', required=False,
+        default=False,
+        help='Do not ask for confirmation.')
+    return parser.parse_args(), parser
+
+
+if __name__ == '__main__':
+    args, parser = _parse_args()
+    if not (args.uuid or args.shared or args.all):
+        parser.print_help()
+        exit(1)
+
+    url = get_url(args.empty)
+    get_confirmation(args.noconfirm, args.uuid, args.shared)
+    if args.uuid:
+        reset_db(url, "user-%s" % args.uuid, args.empty)
+    elif args.shared:
+        reset_db(url, "shared", args.empty)
+    elif args.all:
+        reset_all_dbs(url, args.empty)
diff --git a/scripts/db_access/util.py b/scripts/db_access/util.py
new file mode 120000
index 00000000..368734f7
--- /dev/null
+++ b/scripts/db_access/util.py
@@ -0,0 +1 @@
+../profiling/util.py
\ No newline at end of file