summaryrefslogtreecommitdiff
path: root/scripts/db_access
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2015-06-04 11:21:40 -0300
committerdrebs <drebs@leap.se>2015-06-04 11:21:40 -0300
commit293c71080e9a21115d248e46d1a706c53cc8ee37 (patch)
tree3353672f1f770ef5f5c87ea7e3e27debe737a50b /scripts/db_access
parentfa7708e256ba56cd1e9913993d68611b4ae95824 (diff)
parent9fb1c47ca7da06d6feef6846b812aec28128ed78 (diff)
Merge tag '0.7.0'
Tag version 0.7.0. Conflicts: CHANGELOG client/src/leap/soledad/client/__init__.py client/src/leap/soledad/client/sqlcipher.py client/src/leap/soledad/client/target.py server/pkg/soledad-server
Diffstat (limited to 'scripts/db_access')
-rw-r--r--scripts/db_access/client_side_db.py168
-rw-r--r--scripts/db_access/reset_db.py132
l---------scripts/db_access/util.py1
3 files changed, 228 insertions, 73 deletions
diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py
index 6c456c41..1d8d32e2 100644
--- a/scripts/db_access/client_side_db.py
+++ b/scripts/db_access/client_side_db.py
@@ -2,23 +2,22 @@
# This script gives client-side access to one Soledad user database.
-
-import sys
import os
import argparse
-import re
import tempfile
import getpass
import requests
-import json
import srp._pysrp as srp
import binascii
import logging
+import json
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
-from leap.common.config import get_path_prefix
from leap.soledad.client import Soledad
-
+from leap.keymanager import KeyManager
+from leap.keymanager.openpgp import OpenPGPKey
from util import ValidateUserHandle
@@ -26,37 +25,37 @@ from util import ValidateUserHandle
# create a logger
logger = logging.getLogger(__name__)
LOG_FORMAT = '%(asctime)s %(message)s'
-logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG)
safe_unhexlify = lambda x: binascii.unhexlify(x) if (
len(x) % 2 == 0) else binascii.unhexlify('0' + x)
-def fail(reason):
+def _fail(reason):
logger.error('Fail: ' + reason)
exit(2)
-def get_api_info(provider):
+def _get_api_info(provider):
info = requests.get(
'https://'+provider+'/provider.json', verify=False).json()
return info['api_uri'], info['api_version']
-def login(username, passphrase, provider, api_uri, api_version):
+def _login(username, passphrase, provider, api_uri, api_version):
usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024)
auth = None
try:
- auth = authenticate(api_uri, api_version, usr).json()
+ auth = _authenticate(api_uri, api_version, usr).json()
except requests.exceptions.ConnectionError:
- fail('Could not connect to server.')
+ _fail('Could not connect to server.')
if 'errors' in auth:
- fail(str(auth['errors']))
+ _fail(str(auth['errors']))
return api_uri, api_version, auth
-def authenticate(api_uri, api_version, usr):
+def _authenticate(api_uri, api_version, usr):
api_url = "%s/%s" % (api_uri, api_version)
session = requests.session()
uname, A = usr.start_authentication()
@@ -64,16 +63,16 @@ def authenticate(api_uri, api_version, usr):
init = session.post(
api_url + '/sessions', data=params, verify=False).json()
if 'errors' in init:
- fail('test user not found')
+ _fail('test user not found')
M = usr.process_challenge(
safe_unhexlify(init['salt']), safe_unhexlify(init['B']))
return session.put(api_url + '/sessions/' + uname, verify=False,
data={'client_auth': binascii.hexlify(M)})
-def get_soledad_info(username, provider, passphrase, basedir):
- api_uri, api_version = get_api_info(provider)
- auth = login(username, passphrase, provider, api_uri, api_version)
+def _get_soledad_info(username, provider, passphrase, basedir):
+ api_uri, api_version = _get_api_info(provider)
+ auth = _login(username, passphrase, provider, api_uri, api_version)
# get soledad server url
service_url = '%s/%s/config/soledad-service.json' % \
(api_uri, api_version)
@@ -101,10 +100,9 @@ def get_soledad_info(username, provider, passphrase, basedir):
return auth[2]['id'], server_url, cert_file, auth[2]['token']
-def get_soledad_instance(username, provider, passphrase, basedir):
+def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file,
+ token):
# setup soledad info
- uuid, server_url, cert_file, token = \
- get_soledad_info(username, provider, passphrase, basedir)
logger.info('UUID is %s' % uuid)
logger.info('Server URL is %s' % server_url)
secrets_path = os.path.join(
@@ -119,37 +117,135 @@ def get_soledad_instance(username, provider, passphrase, basedir):
local_db_path=local_db_path,
server_url=server_url,
cert_file=cert_file,
- auth_token=token)
-
-
-# main program
-
-if __name__ == '__main__':
-
+ auth_token=token,
+ defer_encryption=False)
+
+
+def _get_keymanager_instance(username, provider, soledad, token,
+ ca_cert_path=None, api_uri=None, api_version=None, uid=None,
+ gpgbinary=None):
+ return KeyManager(
+ "{username}@{provider}".format(username=username, provider=provider),
+ "http://uri",
+ soledad,
+ token=token,
+ ca_cert_path=ca_cert_path,
+ api_uri=api_uri,
+ api_version=api_version,
+ uid=uid,
+ gpgbinary=gpgbinary)
+
+
+def _parse_args():
# parse command line
parser = argparse.ArgumentParser()
parser.add_argument(
'user@provider', action=ValidateUserHandle, help='the user handle')
parser.add_argument(
- '-b', dest='basedir', required=False, default=None,
+ '--basedir', '-b', default=None,
help='soledad base directory')
parser.add_argument(
- '-p', dest='passphrase', required=False, default=None,
+ '--passphrase', '-p', default=None,
help='the user passphrase')
- args = parser.parse_args()
+ parser.add_argument(
+ '--get-all-docs', '-a', action='store_true',
+ help='get all documents from the local database')
+ parser.add_argument(
+ '--create-doc', '-c', default=None,
+ help='create a document with give content')
+ parser.add_argument(
+ '--sync', '-s', action='store_true',
+ help='synchronize with the server replica')
+ parser.add_argument(
+ '--export-public-key', help="export the public key to a file")
+ parser.add_argument(
+ '--export-private-key', help="export the private key to a file")
+ parser.add_argument(
+ '--export-incoming-messages',
+ help="export incoming messages to a directory")
+ return parser.parse_args()
- # get the password
+
+def _get_passphrase(args):
passphrase = args.passphrase
if passphrase is None:
passphrase = getpass.getpass(
'Password for %s@%s: ' % (args.username, args.provider))
+ return passphrase
+
- # get the basedir
+def _get_basedir(args):
basedir = args.basedir
if basedir is None:
basedir = tempfile.mkdtemp()
+ elif not os.path.isdir(basedir):
+ os.mkdir(basedir)
logger.info('Using %s as base directory.' % basedir)
+ return basedir
+
+
+@inlineCallbacks
+def _export_key(args, km, fname, private=False):
+ address = args.username + "@" + args.provider
+ pkey = yield km.get_key(address, OpenPGPKey, private=private, fetch_remote=False)
+ with open(args.export_private_key, "w") as f:
+ f.write(pkey.key_data)
+
+
+@inlineCallbacks
+def _export_incoming_messages(soledad, directory):
+ yield soledad.create_index("by-incoming", "bool(incoming)")
+ docs = yield soledad.get_from_index("by-incoming", '1')
+ i = 1
+ for doc in docs:
+ with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f:
+ f.write(doc.content["_enc_json"])
+ i += 1
+
+
+@inlineCallbacks
+def _get_all_docs(soledad):
+ _, docs = yield soledad.get_all_docs()
+ for doc in docs:
+ print json.dumps(doc.content, indent=4)
- # get the soledad instance
- s = get_soledad_instance(
- args.username, args.provider, passphrase, basedir)
+
+# main program
+
+@inlineCallbacks
+def _main(soledad, km, args):
+ try:
+ if args.create_doc:
+ yield soledad.create_doc({'content': args.create_doc})
+ if args.sync:
+ yield soledad.sync()
+ if args.get_all_docs:
+ yield _get_all_docs(soledad)
+ if args.export_private_key:
+ yield _export_key(args, km, args.export_private_key, private=True)
+ if args.export_public_key:
+ yield _export_key(args, km, args.expoert_public_key, private=False)
+ if args.export_incoming_messages:
+ yield _export_incoming_messages(soledad, args.export_incoming_messages)
+ except:
+ pass
+ finally:
+ reactor.stop()
+
+
+if __name__ == '__main__':
+ args = _parse_args()
+ passphrase = _get_passphrase(args)
+ basedir = _get_basedir(args)
+ uuid, server_url, cert_file, token = \
+ _get_soledad_info(args.username, args.provider, passphrase, basedir)
+ soledad = _get_soledad_instance(
+ uuid, passphrase, basedir, server_url, cert_file, token)
+ km = _get_keymanager_instance(
+ args.username,
+ args.provider,
+ soledad,
+ token,
+ uid=uuid)
+ _main(soledad, km, args)
+ reactor.run()
diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py
index 80871856..7c6d281b 100644
--- a/scripts/db_access/reset_db.py
+++ b/scripts/db_access/reset_db.py
@@ -5,20 +5,21 @@
# WARNING: running this script over a database will delete all documents but
# the one with id u1db_config (which contains db metadata) and design docs
# needed for couch backend.
+#
+# Run it like this to get some help:
+#
+# ./reset_db.py --help
-import sys
-from ConfigParser import ConfigParser
import threading
import logging
-from couchdb import Database as CouchDatabase
-
+import argparse
+import re
-if len(sys.argv) != 2:
- print 'Usage: %s <uuid>' % sys.argv[0]
- exit(1)
-uuid = sys.argv[1]
+from ConfigParser import ConfigParser
+from couchdb import Database as CouchDatabase
+from couchdb import Server as CouchServer
# create a logger
@@ -27,23 +28,6 @@ LOG_FORMAT = '%(asctime)s %(message)s'
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
-# get couch url
-cp = ConfigParser()
-cp.read('/etc/leap/soledad-server.conf')
-url = cp.get('soledad-server', 'couch_url')
-
-
-# confirm
-yes = raw_input("Are you sure you want to reset the database for user %s "
- "(type YES)? " % uuid)
-if yes != 'YES':
- print 'Bailing out...'
- exit(2)
-
-
-db = CouchDatabase('%s/user-%s' % (url, uuid))
-
-
class _DeleterThread(threading.Thread):
def __init__(self, db, doc_id, release_fun):
@@ -59,21 +43,95 @@ class _DeleterThread(threading.Thread):
self._release_fun()
-semaphore_pool = threading.BoundedSemaphore(value=20)
-
-
-threads = []
-for doc_id in db:
- if doc_id != 'u1db_config' and not doc_id.startswith('_design'):
+def get_confirmation(noconfirm, uuid, shared):
+ msg = "Are you sure you want to reset %s (type YES)? "
+ if shared:
+ msg = msg % "the shared database"
+ elif uuid:
+ msg = msg % ("the database for user %s" % uuid)
+ else:
+ msg = msg % "all databases"
+ if noconfirm is False:
+ yes = raw_input(msg)
+ if yes != 'YES':
+ print 'Bailing out...'
+ exit(2)
+
+
+def get_url(empty):
+ url = None
+ if empty is False:
+ # get couch url
+ cp = ConfigParser()
+ cp.read('/etc/leap/soledad-server.conf')
+ url = cp.get('soledad-server', 'couch_url')
+ else:
+ with open('/etc/couchdb/couchdb.netrc') as f:
+ netrc = f.read()
+ admin_password = re.match('^.* password (.*)$', netrc).groups()[0]
+ url = 'http://admin:%s@127.0.0.1:5984' % admin_password
+ return url
+
+
+def reset_all_dbs(url, empty):
+ server = CouchServer('%s' % (url))
+ for dbname in server:
+ if dbname.startswith('user-') or dbname == 'shared':
+ reset_db(url, dbname, empty)
+
+
+def reset_db(url, dbname, empty):
+ db = CouchDatabase('%s/%s' % (url, dbname))
+ semaphore_pool = threading.BoundedSemaphore(value=20)
+
+ # launch threads for deleting docs
+ threads = []
+ for doc_id in db:
+ if empty is False:
+ if doc_id == 'u1db_config' or doc_id.startswith('_design'):
+ continue
semaphore_pool.acquire()
logger.info('[main] launching thread for doc: %s' % doc_id)
t = _DeleterThread(db, doc_id, semaphore_pool.release)
t.start()
threads.append(t)
-
-logger.info('[main] waiting for threads.')
-map(lambda thread: thread.join(), threads)
-
-
-logger.info('[main] done.')
+ # wait for threads to finish
+ logger.info('[main] waiting for threads.')
+ map(lambda thread: thread.join(), threads)
+ logger.info('[main] done.')
+
+
+def _parse_args():
+ parser = argparse.ArgumentParser()
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('-u', dest='uuid', default=False,
+ help='Reset database of given user.')
+ group.add_argument('-s', dest='shared', action='store_true', default=False,
+ help='Reset the shared database.')
+ group.add_argument('-a', dest='all', action='store_true', default=False,
+ help='Reset all user databases.')
+ parser.add_argument(
+ '-e', dest='empty', action='store_true', required=False, default=False,
+ help='Empty database (do not preserve minimal set of u1db documents).')
+ parser.add_argument(
+ '-y', dest='noconfirm', action='store_true', required=False,
+ default=False,
+ help='Do not ask for confirmation.')
+ return parser.parse_args(), parser
+
+
+if __name__ == '__main__':
+ args, parser = _parse_args()
+ if not (args.uuid or args.shared or args.all):
+ parser.print_help()
+ exit(1)
+
+ url = get_url(args.empty)
+ get_confirmation(args.noconfirm, args.uuid, args.shared)
+ if args.uuid:
+ reset_db(url, "user-%s" % args.uuid, args.empty)
+ elif args.shared:
+ reset_db(url, "shared", args.empty)
+ elif args.all:
+ reset_all_dbs(url, args.empty)
diff --git a/scripts/db_access/util.py b/scripts/db_access/util.py
new file mode 120000
index 00000000..368734f7
--- /dev/null
+++ b/scripts/db_access/util.py
@@ -0,0 +1 @@
+../profiling/util.py \ No newline at end of file