diff options
Diffstat (limited to 'scripts/docker/files/bin')
| -rw-r--r-- | scripts/docker/files/bin/client_side_db.py | 322 | ||||
| -rw-r--r-- | scripts/docker/files/bin/conf/cert_default.conf | 15 | ||||
| -rw-r--r-- | scripts/docker/files/bin/conf/couchdb_default.ini | 361 | ||||
| -rw-r--r-- | scripts/docker/files/bin/conf/soledad-server_default.conf | 5 | ||||
| -rwxr-xr-x | scripts/docker/files/bin/run-client-bootstrap.sh | 20 | ||||
| -rwxr-xr-x | scripts/docker/files/bin/run-client-perf.sh | 128 | ||||
| -rwxr-xr-x | scripts/docker/files/bin/run-server.sh | 89 | ||||
| -rwxr-xr-x | scripts/docker/files/bin/run-trial-from-gitlab-ci.sh | 50 | ||||
| -rwxr-xr-x | scripts/docker/files/bin/run-trial.sh | 23 | ||||
| -rwxr-xr-x | scripts/docker/files/bin/setup-test-env.py | 641 | ||||
| -rw-r--r-- | scripts/docker/files/bin/util.py | 75 | ||||
| -rw-r--r-- | scripts/docker/files/bin/util.sh | 12 |
12 files changed, 1741 insertions, 0 deletions
diff --git a/scripts/docker/files/bin/client_side_db.py b/scripts/docker/files/bin/client_side_db.py new file mode 100644 index 00000000..4be33d13 --- /dev/null +++ b/scripts/docker/files/bin/client_side_db.py @@ -0,0 +1,322 @@ +#!/usr/bin/python + +import os +import argparse +import tempfile +import getpass +import requests +import srp._pysrp as srp +import binascii +import logging +import json +import time + +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey + +from leap.common.events import server +server.ensure_server() + +from util import ValidateUserHandle + + +""" +Script to give access to client-side Soledad database. + +This is mainly used for tests, but can also be used to recover data from a +Soledad database (public/private keys, export documents, etc). + +To speed up testing/debugging, this script can dump the auth data after +logging in. Use the --export-auth-data option to export auth data to a file. +The contents of the file is a json dictionary containing the uuid, server_url, +cert_file and token, which is enough info to instantiate a soledad client +without having to interact with the webapp again. Use the --use-auth-data +option to use the auth data stored in a file. + +Use the --help option to see available options. +""" + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def _fail(reason): + logger.error('Fail: ' + reason) + exit(2) + + +def _get_api_info(provider): + info = requests.get( + 'https://' + provider + '/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def _login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = _authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + _fail('Could not connect to server.') + if 'errors' in auth: + _fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def _authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + _fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): + # setup soledad info + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True) + + +def _get_keymanager_instance(username, provider, soledad, token, + ca_cert_path=None, api_uri=None, api_version=None, + uid=None, gpgbinary=None): + return KeyManager( + "{username}@{provider}".format(username=username, provider=provider), + "http://uri", + soledad, + token=token, + ca_cert_path=ca_cert_path, + api_uri=api_uri, + api_version=api_version, + uid=uid, + gpgbinary=gpgbinary) + + +def _parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '--basedir', '-b', default=None, + help='soledad base directory') + parser.add_argument( + '--passphrase', '-p', default=None, + help='the user passphrase') + parser.add_argument( + '--get-all-docs', '-a', action='store_true', + help='get all documents from the local database') + parser.add_argument( + '--create-docs', '-c', default=0, type=int, + help='create a number of documents') + parser.add_argument( + '--sync', '-s', action='store_true', + help='synchronize with the server replica') + parser.add_argument( + '--repeat-sync', '-r', action='store_true', + help='repeat synchronization until no new data is received') + parser.add_argument( + '--export-public-key', help="export the public key to a file") + parser.add_argument( + '--export-private-key', help="export the private key to a file") + parser.add_argument( + '--export-incoming-messages', + help="export incoming messages to a directory") + parser.add_argument( + '--export-auth-data', + help="export authentication data to a file") + parser.add_argument( + '--use-auth-data', + help="use authentication data from a file") + return parser.parse_args() + + +def _get_passphrase(args): + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + + +def _get_basedir(args): + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) + logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key( + address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) + + +# main program + +@inlineCallbacks +def _main(soledad, km, args): + try: + if args.create_docs: + for i in xrange(args.create_docs): + t = time.time() + logger.debug( + "Creating doc %d/%d..." % (i + 1, args.create_docs)) + content = { + 'datetime': time.strftime( + "%Y-%m-%d %H:%M:%S", time.gmtime(t)), + 'timestamp': t, + 'index': i, + 'total': args.create_docs, + } + yield soledad.create_doc(content) + if args.sync: + yield soledad.sync() + if args.repeat_sync: + old_gen = 0 + new_gen = yield soledad.sync() + while old_gen != new_gen: + old_gen = new_gen + new_gen = yield soledad.sync() + if args.get_all_docs: + yield _get_all_docs(soledad) + if args.export_private_key: + yield _export_key(args, km, args.export_private_key, private=True) + if args.export_public_key: + yield _export_key(args, km, args.expoert_public_key, private=False) + if args.export_incoming_messages: + yield _export_incoming_messages( + soledad, args.export_incoming_messages) + except Exception as e: + logger.error(e) + finally: + soledad.close() + reactor.callWhenRunning(reactor.stop) + + +if __name__ == '__main__': + args = _parse_args() + passphrase = _get_passphrase(args) + basedir = _get_basedir(args) + + if not args.use_auth_data: + # get auth data from server + uuid, server_url, cert_file, token = \ + _get_soledad_info( + args.username, args.provider, passphrase, basedir) + else: + # load auth data from file + with open(args.use_auth_data) as f: + auth_data = json.loads(f.read()) + uuid = auth_data['uuid'] + server_url = auth_data['server_url'] + cert_file = auth_data['cert_file'] + token = auth_data['token'] + + # export auth data to a file + if args.export_auth_data: + with open(args.export_auth_data, "w") as f: + f.write(json.dumps({ + 'uuid': uuid, + 'server_url': server_url, + 'cert_file': cert_file, + 'token': token, + })) + + soledad = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + km = _get_keymanager_instance( + args.username, + args.provider, + soledad, + token, + uid=uuid) + _main(soledad, km, args) + reactor.run() diff --git a/scripts/docker/files/bin/conf/cert_default.conf b/scripts/docker/files/bin/conf/cert_default.conf new file mode 100644 index 00000000..8043cea3 --- /dev/null +++ b/scripts/docker/files/bin/conf/cert_default.conf @@ -0,0 +1,15 @@ +[ req ] +default_bits = 1024 +default_keyfile = keyfile.pem +distinguished_name = req_distinguished_name +prompt = no +output_password = mypass + +[ req_distinguished_name ] +C = GB +ST = Test State or Province +L = Test Locality +O = Organization Name +OU = Organizational Unit Name +CN = localhost +emailAddress = test@email.address diff --git a/scripts/docker/files/bin/conf/couchdb_default.ini b/scripts/docker/files/bin/conf/couchdb_default.ini new file mode 100644 index 00000000..5ab72d7b --- /dev/null +++ b/scripts/docker/files/bin/conf/couchdb_default.ini @@ -0,0 +1,361 @@ +; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. + +; Upgrading CouchDB will overwrite this file. +[vendor] +name = The Apache Software Foundation +version = 1.6.0 + +[couchdb] +database_dir = BASEDIR +view_index_dir = BASEDIR +util_driver_dir = /usr/lib/x86_64-linux-gnu/couchdb/erlang/lib/couch-1.6.0/priv/lib +max_document_size = 4294967296 ; 4 GB +os_process_timeout = 5000 ; 5 seconds. for view and external servers. +max_dbs_open = 100 +delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned +uri_file = BASEDIR/couch.uri +; Method used to compress everything that is appended to database and view index files, except +; for attachments (see the attachments section). Available methods are: +; +; none - no compression +; snappy - use google snappy, a very fast compressor/decompressor +uuid = bc2f8b84ecb0b13a31cf7f6881a52194 + +; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest, +; lowest compression ratio) to 9 (slowest, highest compression ratio) +file_compression = snappy +; Higher values may give better read performance due to less read operations +; and/or more OS page cache hits, but they can also increase overall response +; time for writes when there are many attachment write requests in parallel. +attachment_stream_buffer_size = 4096 + +plugin_dir = /usr/lib/x86_64-linux-gnu/couchdb/plugins + +[database_compaction] +; larger buffer sizes can originate smaller files +doc_buffer_size = 524288 ; value in bytes +checkpoint_after = 5242880 ; checkpoint after every N bytes were written + +[view_compaction] +; larger buffer sizes can originate smaller files +keyvalue_buffer_size = 2097152 ; value in bytes + +[httpd] +port = 5984 +bind_address = 127.0.0.1 +authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} +default_handler = {couch_httpd_db, handle_request} +secure_rewrites = true +vhost_global_handlers = _utils, _uuids, _session, _oauth, _users +allow_jsonp = false +; Options for the MochiWeb HTTP server. +;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] +; For more socket options, consult Erlang's module 'inet' man page. +;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] +socket_options = [{recbuf, 262144}, {sndbuf, 262144}] +log_max_chunk_size = 1000000 +enable_cors = false +; CouchDB can optionally enforce a maximum uri length; +; max_uri_length = 8000 + +[ssl] +port = 6984 + +[log] +file = BASEDIR/couch.log +level = info +include_sasl = true + +[couch_httpd_auth] +authentication_db = _users +authentication_redirect = /_utils/session.html +require_valid_user = false +timeout = 600 ; number of seconds before automatic logout +auth_cache_size = 50 ; size is number of cache entries +allow_persistent_cookies = false ; set to true to allow persistent cookies +iterations = 10 ; iterations for password hashing +; min_iterations = 1 +; max_iterations = 1000000000 +; comma-separated list of public fields, 404 if empty +; public_fields = + +[cors] +credentials = false +; List of origins separated by a comma, * means accept all +; Origins must include the scheme: http://example.com +; You can’t set origins: * and credentials = true at the same time. +;origins = * +; List of accepted headers separated by a comma +; headers = +; List of accepted methods +; methods = + + +; Configuration for a vhost +;[cors:http://example.com] +; credentials = false +; List of origins separated by a comma +; Origins must include the scheme: http://example.com +; You can’t set origins: * and credentials = true at the same time. +;origins = +; List of accepted headers separated by a comma +; headers = +; List of accepted methods +; methods = + +[couch_httpd_oauth] +; If set to 'true', oauth token and consumer secrets will be looked up +; in the authentication database (_users). These secrets are stored in +; a top level property named "oauth" in user documents. Example: +; { +; "_id": "org.couchdb.user:joe", +; "type": "user", +; "name": "joe", +; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", +; "salt": "4e170ffeb6f34daecfd814dfb4001a73" +; "roles": ["foo", "bar"], +; "oauth": { +; "consumer_keys": { +; "consumerKey1": "key1Secret", +; "consumerKey2": "key2Secret" +; }, +; "tokens": { +; "token1": "token1Secret", +; "token2": "token2Secret" +; } +; } +; } +use_users_db = false + +[query_servers] +javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js +coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js + + +; Changing reduce_limit to false will disable reduce_limit. +; If you think you're hitting reduce_limit with a "good" reduce function, +; please let us know on the mailing list so we can fine tune the heuristic. +[query_server_config] +reduce_limit = true +os_process_limit = 25 + +[daemons] +index_server={couch_index_server, start_link, []} +external_manager={couch_external_manager, start_link, []} +query_servers={couch_query_servers, start_link, []} +vhosts={couch_httpd_vhost, start_link, []} +httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} +uuids={couch_uuids, start, []} +auth_cache={couch_auth_cache, start_link, []} +replicator_manager={couch_replicator_manager, start_link, []} +os_daemons={couch_os_daemons, start_link, []} +compaction_daemon={couch_compaction_daemon, start_link, []} + +[httpd_global_handlers] +/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} +favicon.ico = {couch_httpd_misc_handlers, handle_favicon_req, "/usr/share/couchdb/www"} + +_utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "/usr/share/couchdb/www"} +_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} +_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} +_config = {couch_httpd_misc_handlers, handle_config_req} +_replicate = {couch_replicator_httpd, handle_req} +_uuids = {couch_httpd_misc_handlers, handle_uuids_req} +_restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} +_log = {couch_httpd_misc_handlers, handle_log_req} +_session = {couch_httpd_auth, handle_session_req} +_oauth = {couch_httpd_oauth, handle_oauth_req} +_db_updates = {couch_dbupdates_httpd, handle_req} +_plugins = {couch_plugins_httpd, handle_req} + +[httpd_db_handlers] +_all_docs = {couch_mrview_http, handle_all_docs_req} +_changes = {couch_httpd_db, handle_changes_req} +_compact = {couch_httpd_db, handle_compact_req} +_design = {couch_httpd_db, handle_design_req} +_temp_view = {couch_mrview_http, handle_temp_view_req} +_view_cleanup = {couch_mrview_http, handle_cleanup_req} + +; The external module takes an optional argument allowing you to narrow it to a +; single script. Otherwise the script name is inferred from the first path section +; after _external's own path. +; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} +; _external = {couch_httpd_external, handle_external_req} + +[httpd_design_handlers] +_compact = {couch_mrview_http, handle_compact_req} +_info = {couch_mrview_http, handle_info_req} +_list = {couch_mrview_show, handle_view_list_req} +_rewrite = {couch_httpd_rewrite, handle_rewrite_req} +_show = {couch_mrview_show, handle_doc_show_req} +_update = {couch_mrview_show, handle_doc_update_req} +_view = {couch_mrview_http, handle_view_req} + +; enable external as an httpd handler, then link it with commands here. +; note, this api is still under consideration. +; [external] +; mykey = /path/to/mycommand + +; Here you can setup commands for CouchDB to manage +; while it is alive. It will attempt to keep each command +; alive if it exits. +; [os_daemons] +; some_daemon_name = /path/to/script -with args + + +[uuids] +; Known algorithms: +; random - 128 bits of random awesome +; All awesome, all the time. +; sequential - monotonically increasing ids with random increments +; First 26 hex characters are random. Last 6 increment in +; random amounts until an overflow occurs. On overflow, the +; random prefix is regenerated and the process starts over. +; utc_random - Time since Jan 1, 1970 UTC with microseconds +; First 14 characters are the time in hex. Last 18 are random. +; utc_id - Time since Jan 1, 1970 UTC with microseconds, plus utc_id_suffix string +; First 14 characters are the time in hex. uuids/utc_id_suffix string value is appended to these. +algorithm = sequential +; The utc_id_suffix value will be appended to uuids generated by the utc_id algorithm. +; Replicating instances should have unique utc_id_suffix values to ensure uniqueness of utc_id ids. +utc_id_suffix = +# Maximum number of UUIDs retrievable from /_uuids in a single request +max_count = 1000 + +[stats] +; rate is in milliseconds +rate = 1000 +; sample intervals are in seconds +samples = [0, 60, 300, 900] + +[attachments] +compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression +compressible_types = text/*, application/javascript, application/json, application/xml + +[replicator] +db = _replicator +; Maximum replicaton retry count can be a non-negative integer or "infinity". +max_replication_retry_count = 10 +; More worker processes can give higher network throughput but can also +; imply more disk and network IO. +worker_processes = 4 +; With lower batch sizes checkpoints are done more frequently. Lower batch sizes +; also reduce the total amount of used RAM memory. +worker_batch_size = 500 +; Maximum number of HTTP connections per replication. +http_connections = 20 +; HTTP connection timeout per replication. +; Even for very fast/reliable networks it might need to be increased if a remote +; database is too busy. +connection_timeout = 30000 +; If a request fails, the replicator will retry it up to N times. +retries_per_request = 10 +; Some socket options that might boost performance in some scenarios: +; {nodelay, boolean()} +; {sndbuf, integer()} +; {recbuf, integer()} +; {priority, integer()} +; See the `inet` Erlang module's man page for the full list of options. +socket_options = [{keepalive, true}, {nodelay, false}] +; Path to a file containing the user's certificate. +;cert_file = /full/path/to/server_cert.pem +; Path to file containing user's private PEM encoded key. +;key_file = /full/path/to/server_key.pem +; String containing the user's password. Only used if the private keyfile is password protected. +;password = somepassword +; Set to true to validate peer certificates. +verify_ssl_certificates = false +; File containing a list of peer trusted certificates (in the PEM format). +;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt +; Maximum peer certificate depth (must be set even if certificate validation is off). +ssl_certificate_max_depth = 3 + +[compaction_daemon] +; The delay, in seconds, between each check for which database and view indexes +; need to be compacted. +check_interval = 300 +; If a database or view index file is smaller then this value (in bytes), +; compaction will not happen. Very small files always have a very high +; fragmentation therefore it's not worth to compact them. +min_file_size = 131072 + +[compactions] +; List of compaction rules for the compaction daemon. +; The daemon compacts databases and their respective view groups when all the +; condition parameters are satisfied. Configuration can be per database or +; global, and it has the following format: +; +; database_name = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] +; _default = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] +; +; Possible parameters: +; +; * db_fragmentation - If the ratio (as an integer percentage), of the amount +; of old data (and its supporting metadata) over the database +; file size is equal to or greater then this value, this +; database compaction condition is satisfied. +; This value is computed as: +; +; (file_size - data_size) / file_size * 100 +; +; The data_size and file_size values can be obtained when +; querying a database's information URI (GET /dbname/). +; +; * view_fragmentation - If the ratio (as an integer percentage), of the amount +; of old data (and its supporting metadata) over the view +; index (view group) file size is equal to or greater then +; this value, then this view index compaction condition is +; satisfied. This value is computed as: +; +; (file_size - data_size) / file_size * 100 +; +; The data_size and file_size values can be obtained when +; querying a view group's information URI +; (GET /dbname/_design/groupname/_info). +; +; * from _and_ to - The period for which a database (and its view groups) compaction +; is allowed. The value for these parameters must obey the format: +; +; HH:MM - HH:MM (HH in [0..23], MM in [0..59]) +; +; * strict_window - If a compaction is still running after the end of the allowed +; period, it will be canceled if this parameter is set to 'true'. +; It defaults to 'false' and it's meaningful only if the *period* +; parameter is also specified. +; +; * parallel_view_compaction - If set to 'true', the database and its views are +; compacted in parallel. This is only useful on +; certain setups, like for example when the database +; and view index directories point to different +; disks. It defaults to 'false'. +; +; Before a compaction is triggered, an estimation of how much free disk space is +; needed is computed. This estimation corresponds to 2 times the data size of +; the database or view index. When there's not enough free disk space to compact +; a particular database or view index, a warning message is logged. +; +; Examples: +; +; 1) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}] +; The `foo` database is compacted if its fragmentation is 70% or more. +; Any view index of this database is compacted only if its fragmentation +; is 60% or more. +; +; 2) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}] +; Similar to the preceding example but a compaction (database or view index) +; is only triggered if the current time is between midnight and 4 AM. +; +; 3) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}] +; Similar to the preceding example - a compaction (database or view index) +; is only triggered if the current time is between midnight and 4 AM. If at +; 4 AM the database or one of its views is still compacting, the compaction +; process will be canceled. +; +; 4) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}, {parallel_view_compaction, true}] +; Similar to the preceding example, but a database and its views can be +; compacted in parallel. +; +;_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "23:00"}, {to, "04:00"}] diff --git a/scripts/docker/files/bin/conf/soledad-server_default.conf b/scripts/docker/files/bin/conf/soledad-server_default.conf new file mode 100644 index 00000000..5e286374 --- /dev/null +++ b/scripts/docker/files/bin/conf/soledad-server_default.conf @@ -0,0 +1,5 @@ +[soledad-server] +couch_url = http://localhost:5984 +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc +batching = 0 diff --git a/scripts/docker/files/bin/run-client-bootstrap.sh b/scripts/docker/files/bin/run-client-bootstrap.sh new file mode 100755 index 00000000..fbbb42e8 --- /dev/null +++ b/scripts/docker/files/bin/run-client-bootstrap.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Run a Soledad client connection test. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +${CMD} soledad-client test --server-url ${SOLEDAD_SERVER_URL} diff --git a/scripts/docker/files/bin/run-client-perf.sh b/scripts/docker/files/bin/run-client-perf.sh new file mode 100755 index 00000000..01b27b98 --- /dev/null +++ b/scripts/docker/files/bin/run-client-perf.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +# Start a soledad-perf test using a remote server. +# +# The script does the following: +# +# - configure a remote repository for soledad repo if SOLEDAD_REMOTE is set. +# +# - checkout a specific branch if SOLEDAD_BRANCH is set. +# +# - run the soledad-perf local twisted server that runs the client. Note +# that the actual soledad server should be running on another docker +# container. This local server is only used to measure responsiveness of +# soledad client. The script waits for the server to come up before +# continuing, or else times out after TIMEOUT seconds. +# +# - trigger the creation of documents for sync. +# +# - start the measurement of server responsiveness and sync stages. +# +# - stop the test. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +REPO="/var/local/soledad" +TIMEOUT=20 + +#----------------------------------------------------------------------------- +# configure a remote and checkout a branch +#----------------------------------------------------------------------------- + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +if [ ! -z "${SOLEDAD_PERF_REMOTE}" ]; then + git -C /var/local/soledad-perf remote set-url origin ${SOLEDAD_PERF_REMOTE} + git -C /var/local/soledad-perf fetch origin +fi + +if [ ! -z "${SOLEDAD_PERF_BRANCH}" ]; then + git -C /var/local/soledad-perf checkout ${SOLEDAD_PERF_BRANCH} +fi + +#----------------------------------------------------------------------------- +# write a configuration file for the perf test +#----------------------------------------------------------------------------- + +cd /var/local/soledad-perf + +cat > defaults.conf <<EOF +[server] +host = ${SOLEDAD_SERVER_URL} + +[client] +uuid = 1234567890abcdef +basedir = /tmp/soledad_client_test +passphrase = 12345678 + +[sync] +num_docs = ${SOLEDAD_PRELOAD_NUM} +payload = /tmp/payload +payload_size = ${SOLEDAD_PRELOAD_SIZE} +auth_token = an-auth-token + +[test] +stats_file = ./out/stats.json +EOF + +if [ "${1}" = "--drop-to-shell" ]; then + /bin/bash + exit 0 +fi + +#----------------------------------------------------------------------------- +# start the local server and wait for it to come up +#----------------------------------------------------------------------------- + +# start local test server on background +make soledad-sync-server | grep -v stats | grep -v ping & + +# wait for server until timeout +start=`date +%s` +elapsed=0 + +echo "Waiting for perf server to come up..." + +while [ ${elapsed} -lt ${TIMEOUT} ]; do + result=`curl -s http://127.0.0.1:8080/ping` + if [ ${?} -eq 0 -a "${result}" = "easy!" ]; then + echo "Perf server (running soledad client) is up!" + break + else + sleep 1 + fi + now=`date +%s` + elapsed=`expr ${now} - ${start}` +done + +# exit with an error code if timed out waiting for server +if [ ${elapsed} -ge ${TIMEOUT} ]; then + echo "Error: server unreachable at http://127.0.0.1:8080 after ${TIMEOUT} seconds." + exit 1 +fi + +#----------------------------------------------------------------------------- +# create docs and run test +#----------------------------------------------------------------------------- + +set -e + +# create documents in client +make trigger-create-docs + +# launch background series measurement +make measure-series > /dev/null & +sleep 5 # wait a bit for some data points + +# run a sync and generate a graph +make trigger-sync +make trigger-stop diff --git a/scripts/docker/files/bin/run-server.sh b/scripts/docker/files/bin/run-server.sh new file mode 100755 index 00000000..feedee7e --- /dev/null +++ b/scripts/docker/files/bin/run-server.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +# Start a soledad server inside a docker container. +# +# This script will: +# +# - eventually checkout a specific branch from a specific soledad remote. +# +# - create everything a soledad server needs to run (certificate, backend +# server database, tables, etc. +# +# - eventually preload the server database with a number of documents equal +# to SOLEDAD_PRELOAD_NUM, and with payload size equal to +# SOLEDAD_PRELOAD_SIZE. +# +# - run the soledad server. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" + +#--------------------------------------------------------------------------- +# eventually checkout a specific branch from a specific remote +#--------------------------------------------------------------------------- + +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +#--------------------------------------------------------------------------- +# setup environment for running soledad server +#--------------------------------------------------------------------------- + +${CMD} couch start +${CMD} user-db create +${CMD} token-db create +${CMD} token-db insert-token +${CMD} shared-db create +${CMD} cert create + +#--------------------------------------------------------------------------- +# write a configuration file for the perf test +#--------------------------------------------------------------------------- + +if [ "${SOLEDAD_PRELOAD_NUM}" -gt 0 ]; then + cd /var/local/soledad-perf + + cat > defaults.conf <<EOF +[server] +host = http://127.0.0.1:2424 + +[client] +uuid = 1234567890abcdef +basedir = /tmp/soledad_client_test +passphrase = 12345678 + +[sync] +num_docs = ${SOLEDAD_PRELOAD_NUM} +payload = /tmp/payload +payload_size = ${SOLEDAD_PRELOAD_SIZE} +auth_token = an-auth-token + +[test] +stats_file = ./out/stats.json +EOF + + echo "Preloading server database..." + ./scripts/preload_server_database.py +fi + +#--------------------------------------------------------------------------- +# actually run the server +#--------------------------------------------------------------------------- + +if [ "${1}" = "--drop-to-shell" ]; then + /bin/bash + exit 0 +fi + +echo "Starting soledad server..." +${CMD} soledad-server start --no-daemonize diff --git a/scripts/docker/files/bin/run-trial-from-gitlab-ci.sh b/scripts/docker/files/bin/run-trial-from-gitlab-ci.sh new file mode 100755 index 00000000..96436e26 --- /dev/null +++ b/scripts/docker/files/bin/run-trial-from-gitlab-ci.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Run Soledad trial tests in a docker container created by gitlab-ci. +# +# Gitlab-ci will copy the current test code into /builds/leap/soledad, so this +# script has to uninstall currently installed soledad packages and re-install +# from that location instead. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +BASEDIR="/builds/leap/soledad" + + +install_deps() { + # ensure all dependencies are installed + for pkg in common client server; do + testing="--testing" + if [ "${pkg}" = "server" ]; then + # soledad server doesn't currently have a requirements-testing.pip file, + # so we don't pass the option when that is the case + testing="" + fi + pip uninstall leap.soledad.${pkg} + (cd ${BASEDIR}/${pkg} \ + && ./pkg/pip_install_requirements.sh ${testing} --use-leap-wheels \ + && python setup.py develop) + done +} + + +start_couch() { + # currently soledad trial tests need a running couch on environment + ${CMD} couch start +} + + +run_tests() { + trial leap.soledad.common +} + + +main() { + install_deps + start_couch + run_tests +} + +main diff --git a/scripts/docker/files/bin/run-trial.sh b/scripts/docker/files/bin/run-trial.sh new file mode 100755 index 00000000..f38f3124 --- /dev/null +++ b/scripts/docker/files/bin/run-trial.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Run Soledad trial tests. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +# currently soledad trial tests need a running couch on environment +${CMD} couch start + +trial leap.soledad.common diff --git a/scripts/docker/files/bin/setup-test-env.py b/scripts/docker/files/bin/setup-test-env.py new file mode 100755 index 00000000..0f3ea6f4 --- /dev/null +++ b/scripts/docker/files/bin/setup-test-env.py @@ -0,0 +1,641 @@ +#!/usr/bin/env python + + +""" +This script knows how to build a minimum environment for Soledad Server, which +includes the following: + + - Couch server startup + - Token and shared database initialization + - Soledad Server startup + +Options can be passed for configuring the different environments, so this may +be used by other programs to setup different environments for arbitrary tests. +Use the --help option to get information on usage. + +For some commands you will need an environment with Soledad python packages +available, thus you might want to explicitly call python and not rely in the +shebang line. +""" + + +import time +import os +import signal +import tempfile +import psutil +from argparse import ArgumentParser +from subprocess import call +from couchdb import Server +from couchdb.http import PreconditionFailed +from couchdb.http import ResourceConflict +from couchdb.http import ResourceNotFound +from hashlib import sha512 + +from leap.soledad.common.l2db.errors import DatabaseDoesNotExist + + +# +# Utilities +# + +def get_pid(pidfile): + if not os.path.isfile(pidfile): + return 0 + try: + with open(pidfile) as f: + return int(f.read()) + except IOError: + return 0 + + +def pid_is_running(pid): + try: + psutil.Process(pid) + return True + except psutil.NoSuchProcess: + return False + + +def pidfile_is_running(pidfile): + try: + pid = get_pid(pidfile) + psutil.Process(pid) + return pid + except psutil.NoSuchProcess: + return False + + +def status_from_pidfile(args, default_basedir, name): + basedir = _get_basedir(args, default_basedir) + pidfile = os.path.join(basedir, args.pidfile) + try: + pid = get_pid(pidfile) + psutil.Process(pid) + print "[+] %s is running with pid %d" % (name, pid) + except (IOError, psutil.NoSuchProcess): + print "[-] %s stopped" % name + + +def kill_all_executables(args): + basename = os.path.basename(args.executable) + pids = [int(pid) for pid in os.listdir('/proc') if pid.isdigit()] + for pid in pids: + try: + p = psutil.Process(pid) + if p.name() == basename: + print '[!] killing - pid: %d' % pid + os.kill(pid, signal.SIGKILL) + except: + pass + + +# +# Couch Server control +# + +COUCH_EXECUTABLE = '/usr/bin/couchdb' +ERLANG_EXECUTABLE = 'beam.smp' +COUCH_TEMPLATE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + './conf/couchdb_default.ini') +COUCH_TEMPLATE +COUCH_PIDFILE = 'couchdb.pid' +COUCH_LOGFILE = 'couchdb.log' +COUCH_PORT = 5984 +COUCH_HOST = '127.0.0.1' +COUCH_BASEDIR = '/tmp/couch_test' + + +def _get_basedir(args, default): + basedir = args.basedir + if not basedir: + basedir = default + if not os.path.isdir(basedir): + os.mkdir(basedir) + return basedir + + +def couch_server_start(args): + basedir = _get_basedir(args, COUCH_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + logfile = os.path.join(basedir, args.logfile) + + # check if already running + pid = get_pid(pidfile) + if pid_is_running(pid): + print '[*] error: already running - pid: %d' % pid + exit(1) + if os.path.isfile(pidfile): + os.unlink(pidfile) + + # generate a config file from template if needed + config_file = args.config_file + if not config_file: + config_file = tempfile.mktemp(prefix='couch_config_', dir=basedir) + lines = [] + with open(args.template) as f: + lines = f.readlines() + lines = map(lambda l: l.replace('BASEDIR', basedir), lines) + with open(config_file, 'w') as f: + f.writelines(lines) + + # start couch server + try: + call([ + args.executable, + '-n', # reset configuration file chain (including system default) + '-a %s' % config_file, # add configuration FILE to chain + '-b', # spawn as a background process + '-p %s' % pidfile, # set the background PID FILE + '-o %s' % logfile, # redirect background stdout to FILE + '-e %s' % logfile]) # redirect background stderr to FILE + except Exception as e: + print '[*] error: could not start couch server - %s' % str(e) + exit(1) + + # couch may take a bit to store the pid in the pidfile, so we just wait + # until it does + pid = None + while not pid: + try: + pid = get_pid(pidfile) + break + except: + time.sleep(0.1) + + print '[+] couch is running with pid: %d' % pid + + +def couch_server_stop(args): + basedir = _get_basedir(args, COUCH_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + pid = get_pid(pidfile) + if not pid_is_running(pid): + print '[*] error: no running server found' + exit(1) + call([ + args.executable, + '-p %s' % pidfile, # set the background PID FILE + '-k']) # kill the background process, will respawn if needed + print '[-] stopped couch server with pid %d ' % pid + + +def couch_status_from_pidfile(args): + status_from_pidfile(args, COUCH_BASEDIR, 'couch') + + +# +# User DB maintenance # +# + +def user_db_create(args): + from leap.soledad.common.couch import CouchDatabase + url = 'http://localhost:%d/user-%s' % (args.port, args.uuid) + try: + CouchDatabase.open_database( + url=url, create=False, replica_uid=None, ensure_ddocs=True) + print '[*] error: database "user-%s" already exists' % args.uuid + exit(1) + except DatabaseDoesNotExist: + CouchDatabase.open_database( + url=url, create=True, replica_uid=None, ensure_ddocs=True) + print '[+] database created: user-%s' % args.uuid + + +def user_db_delete(args): + s = _couch_get_server(args) + try: + dbname = 'user-%s' % args.uuid + s.delete(dbname) + print '[-] database deleted: %s' % dbname + except ResourceNotFound: + print '[*] error: database "%s" does not exist' % dbname + exit(1) + + +# +# Soledad Server control +# + +TWISTD_EXECUTABLE = 'twistd' # use whatever is available on path + +SOLEDAD_SERVER_BASEDIR = '/tmp/soledad_server_test' +SOLEDAD_SERVER_CONFIG_FILE = './conf/soledad_default.ini' +SOLEDAD_SERVER_PIDFILE = 'soledad.pid' +SOLEDAD_SERVER_LOGFILE = 'soledad.log' +SOLEDAD_SERVER_PRIVKEY = 'soledad_privkey.pem' +SOLEDAD_SERVER_CERTKEY = 'soledad_certkey.pem' +SOLEDAD_SERVER_PORT = 2424 +SOLEDAD_SERVER_AUTH_TOKEN = 'an-auth-token' +SOLEDAD_SERVER_URL = 'https://localhost:2424' + +SOLEDAD_CLIENT_PASS = '12345678' +SOLEDAD_CLIENT_BASEDIR = '/tmp/soledad_client_test' +SOLEDAD_CLIENT_UUID = '1234567890abcdef' + + +def soledad_server_start(args): + basedir = _get_basedir(args, SOLEDAD_SERVER_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + logfile = os.path.join(basedir, args.logfile) + private_key = os.path.join(basedir, args.private_key) + cert_key = os.path.join(basedir, args.cert_key) + + pid = get_pid(pidfile) + if pid_is_running(pid): + pid = get_pid(pidfile) + print "[*] error: already running - pid: %d" % pid + exit(1) + + port = args.port + if args.tls: + port = 'ssl:%d:privateKey=%s:certKey=%s:sslmethod=SSLv23_METHOD' \ + % (args.port, private_key, cert_key) + params = [ + '--logfile=%s' % logfile, + '--pidfile=%s' % pidfile, + 'web', + '--wsgi=leap.soledad.server.application', + '--port=%s' % port + ] + if args.no_daemonize: + params.insert(0, '--nodaemon') + + call([args.executable] + params) + + pid = get_pid(pidfile) + print '[+] soledad-server is running with pid %d' % pid + + +def soledad_server_stop(args): + basedir = _get_basedir(args, SOLEDAD_SERVER_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + pid = get_pid(pidfile) + if not pid_is_running(pid): + print '[*] error: no running server found' + exit(1) + os.kill(pid, signal.SIGKILL) + print '[-] stopped - pid: %d' % pid + + +def soledad_server_status_from_pidfile(args): + status_from_pidfile(args, SOLEDAD_SERVER_BASEDIR, 'soledad-server') + + +# couch helpers + +def _couch_get_server(args): + url = 'http://%s:%d/' % (args.host, args.port) + return Server(url=url) + + +def _couch_create_db(args, dbname): + s = _couch_get_server(args) + # maybe create the database + try: + s.create(dbname) + print '[+] database created: %s' % dbname + except PreconditionFailed as e: + error_code, _ = e.message + if error_code == 'file_exists': + print '[*] error: "%s" database already exists' % dbname + exit(1) + return s + + +def _couch_delete_db(args, dbname): + s = _couch_get_server(args) + # maybe create the database + try: + s.delete(dbname) + print '[-] database deleted: %s' % dbname + except ResourceNotFound: + print '[*] error: "%s" database does not exist' % dbname + exit(1) + + +def _token_dbname(): + dbname = 'tokens_' + \ + str(int(time.time() / (30 * 24 * 3600))) + return dbname + + +def token_db_create(args): + dbname = _token_dbname() + _couch_create_db(args, dbname) + + +def token_db_insert_token(args): + s = _couch_get_server(args) + try: + dbname = _token_dbname() + db = s[dbname] + token = sha512(args.auth_token).hexdigest() + db[token] = { + 'type': 'Token', + 'user_id': args.uuid, + } + print '[+] token for uuid "%s" created in tokens database' % args.uuid + except ResourceConflict: + print '[*] error: token for uuid "%s" already exists in tokens database' \ + % args.uuid + exit(1) + + +def token_db_delete(args): + dbname = _token_dbname() + _couch_delete_db(args, dbname) + + +# +# Shared DB creation +# + +def shared_db_create(args): + _couch_create_db(args, 'shared') + + +def shared_db_delete(args): + _couch_delete_db(args, 'shared') + + +# +# Certificate creation +# + +CERT_CONFIG_FILE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + './conf/cert_default.conf') + + +def cert_create(args): + private_key = os.path.join(args.basedir, args.private_key) + cert_key = os.path.join(args.basedir, args.cert_key) + os.mkdir(args.basedir) + call([ + 'openssl', + 'req', + '-x509', + '-sha256', + '-nodes', + '-days', '365', + '-newkey', 'rsa:2048', + '-config', args.config_file, + '-keyout', private_key, + '-out', cert_key]) + + +def cert_delete(args): + private_key = os.path.join(args.basedir, args.private_key) + cert_key = os.path.join(args.basedir, args.cert_key) + os.unlink(private_key) + os.unlink(cert_key) + + +# +# Soledad Client Control +# + +def soledad_client_test(args): + + # maybe infer missing parameters + basedir = args.basedir + if not basedir: + basedir = tempfile.mkdtemp() + server_url = args.server_url + if not server_url: + server_url = 'http://127.0.0.1:%d' % args.port + + # get a soledad instance + from client_side_db import _get_soledad_instance + _get_soledad_instance( + args.uuid, + unicode(args.passphrase), + basedir, + server_url, + args.cert_key, + args.auth_token) + + +# +# Command Line Interface +# + +class Command(object): + + def __init__(self, parser=ArgumentParser()): + self.commands = [] + self.parser = parser + self.subparsers = None + + def add_command(self, *args, **kwargs): + # pop out the func parameter to use later + func = None + if 'func' in kwargs.keys(): + func = kwargs.pop('func') + # eventually create a subparser + if not self.subparsers: + self.subparsers = self.parser.add_subparsers() + # create command and associate a function with it + command = Command(self.subparsers.add_parser(*args, **kwargs)) + if func: + command.parser.set_defaults(func=func) + self.commands.append(command) + return command + + def set_func(self, func): + self.parser.set_defaults(func=func) + + def add_argument(self, *args, **kwargs): + self.parser.add_argument(*args, **kwargs) + + def add_arguments(self, arglist): + for args, kwargs in arglist: + self.add_argument(*args, **kwargs) + + def parse_args(self): + return self.parser.parse_args() + + +# +# Command Line Interface +# + +def run_cli(): + cli = Command() + + # couch command with subcommands + cmd_couch = cli.add_command('couch', help="manage couch server") + + cmd_couch_start = cmd_couch.add_command('start', func=couch_server_start) + cmd_couch_start.add_arguments([ + (['--executable', '-e'], {'default': COUCH_EXECUTABLE}), + (['--basedir', '-b'], {}), + (['--config-file', '-c'], {}), + (['--template', '-t'], {'default': COUCH_TEMPLATE}), + (['--pidfile', '-p'], {'default': COUCH_PIDFILE}), + (['--logfile', '-l'], {'default': COUCH_LOGFILE}) + ]) + + cmd_couch_stop = cmd_couch.add_command('stop', func=couch_server_stop) + cmd_couch_stop.add_arguments([ + (['--executable', '-e'], {'default': COUCH_EXECUTABLE}), + (['--basedir', '-b'], {}), + (['--pidfile', '-p'], {'default': COUCH_PIDFILE}), + ]) + + cmd_couch_status = cmd_couch.add_command( + 'status', func=couch_status_from_pidfile) + cmd_couch_status.add_arguments([ + (['--basedir', '-b'], {}), + (['--pidfile', '-p'], {'default': COUCH_PIDFILE})]) + + cmd_couch_kill = cmd_couch.add_command('kill', func=kill_all_executables) + cmd_couch_kill.add_argument( + '--executable', '-e', default=ERLANG_EXECUTABLE) + + # user database maintenance + cmd_user_db = cli.add_command('user-db') + + cmd_user_db_create = cmd_user_db.add_command('create', func=user_db_create) + cmd_user_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + ]) + + cmd_user_db_create = cmd_user_db.add_command( + 'delete', func=user_db_delete) + cmd_user_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}) + ]) + + # soledad server command with subcommands + cmd_sol_server = cli.add_command( + 'soledad-server', help="manage soledad server") + + cmd_sol_server_start = cmd_sol_server.add_command( + 'start', func=soledad_server_start) + cmd_sol_server_start.add_arguments([ + (['--executable', '-e'], {'default': TWISTD_EXECUTABLE}), + (['--config-file', '-c'], {'default': SOLEDAD_SERVER_CONFIG_FILE}), + (['--pidfile', '-p'], {'default': SOLEDAD_SERVER_PIDFILE}), + (['--logfile', '-l'], {'default': SOLEDAD_SERVER_LOGFILE}), + (['--port', '-P'], {'type': int, 'default': SOLEDAD_SERVER_PORT}), + (['--tls', '-t'], {'action': 'store_true'}), + (['--private-key', '-K'], {'default': SOLEDAD_SERVER_PRIVKEY}), + (['--cert-key', '-C'], {'default': SOLEDAD_SERVER_CERTKEY}), + (['--no-daemonize', '-n'], {'action': 'store_true'}), + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + ]) + + cmd_sol_server_stop = cmd_sol_server.add_command( + 'stop', func=soledad_server_stop) + cmd_sol_server_stop.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--pidfile', '-p'], {'default': SOLEDAD_SERVER_PIDFILE}), + ]) + + cmd_sol_server_status = cmd_sol_server.add_command( + 'status', func=soledad_server_status_from_pidfile) + cmd_sol_server_status.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--pidfile', '-p'], {'default': SOLEDAD_SERVER_PIDFILE}), + ]) + + cmd_sol_server_kill = cmd_sol_server.add_command( + 'kill', func=kill_all_executables) + cmd_sol_server_kill.add_argument( + '--executable', '-e', default=TWISTD_EXECUTABLE) + + # token db maintenance + cmd_token_db = cli.add_command('token-db') + cmd_token_db_create = cmd_token_db.add_command( + 'create', func=token_db_create) + cmd_token_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + cmd_token_db_insert_token = cmd_token_db.add_command( + 'insert-token', func=token_db_insert_token) + cmd_token_db_insert_token.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + (['--auth-token', '-a'], {'default': SOLEDAD_SERVER_AUTH_TOKEN}), + ]) + + cmd_token_db_delete = cmd_token_db.add_command( + 'delete', func=token_db_delete) + cmd_token_db_delete.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + # shared db creation + cmd_shared_db = cli.add_command('shared-db') + + cmd_shared_db_create = cmd_shared_db.add_command( + 'create', func=shared_db_create) + cmd_shared_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + cmd_shared_db_delete = cmd_shared_db.add_command( + 'delete', func=shared_db_delete) + cmd_shared_db_delete.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + # certificate generation + cmd_cert = cli.add_command('cert', help="create tls certificates") + + cmd_cert_create = cmd_cert.add_command('create', func=cert_create) + cmd_cert_create.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--config-file', '-c'], {'default': CERT_CONFIG_FILE}), + (['--private-key', '-K'], {'default': SOLEDAD_SERVER_PRIVKEY}), + (['--cert-key', '-C'], {'default': SOLEDAD_SERVER_CERTKEY}), + ]) + + cmd_cert_create = cmd_cert.add_command('delete', func=cert_delete) + cmd_cert_create.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--private-key', '-K'], {'default': SOLEDAD_SERVER_PRIVKEY}), + (['--cert-key', '-C'], {'default': SOLEDAD_SERVER_CERTKEY}), + ]) + + # soledad client command with subcommands + cmd_sol_client = cli.add_command( + 'soledad-client', help="manage soledad client") + + cmd_sol_client_test = cmd_sol_client.add_command( + 'test', func=soledad_client_test) + cmd_sol_client_test.add_arguments([ + (['--port', '-P'], {'type': int, 'default': SOLEDAD_SERVER_PORT}), + (['--tls', '-t'], {'action': 'store_true'}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--passphrase', '-k'], {'default': SOLEDAD_CLIENT_PASS}), + (['--basedir', '-b'], {'default': SOLEDAD_CLIENT_BASEDIR}), + (['--server-url', '-s'], {'default': SOLEDAD_SERVER_URL}), + (['--cert-key', '-C'], {'default': os.path.join( + SOLEDAD_SERVER_BASEDIR, + SOLEDAD_SERVER_CERTKEY)}), + (['--auth-token', '-a'], {'default': SOLEDAD_SERVER_AUTH_TOKEN}), + ]) + + # parse and run cli + args = cli.parse_args() + args.func(args) + + +if __name__ == '__main__': + run_cli() diff --git a/scripts/docker/files/bin/util.py b/scripts/docker/files/bin/util.py new file mode 100644 index 00000000..e7e2ef9a --- /dev/null +++ b/scripts/docker/files/bin/util.py @@ -0,0 +1,75 @@ +import re +import psutil +import time +import threading +import argparse +import pytz +import datetime + + +class ValidateUserHandle(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + m = re.compile('^([^@]+)@([^@]+\.[^@]+)$') + res = m.match(values) + if res == None: + parser.error('User handle should have the form user@provider.') + setattr(namespace, 'username', res.groups()[0]) + setattr(namespace, 'provider', res.groups()[1]) + + +class StatsLogger(threading.Thread): + + def __init__(self, name, fname, procs=[], interval=0.01): + threading.Thread.__init__(self) + self._stopped = True + self._name = name + self._fname = fname + self._procs = self._find_procs(procs) + self._interval = interval + + def _find_procs(self, procs): + return filter(lambda p: p.name in procs, psutil.process_iter()) + + def run(self): + self._stopped = False + with open(self._fname, 'w') as f: + self._start = time.time() + f.write(self._make_header()) + while self._stopped is False: + f.write('%s %s\n' % + (self._make_general_stats(), self._make_proc_stats())) + time.sleep(self._interval) + f.write(self._make_footer()) + + def _make_general_stats(self): + now = time.time() + stats = [] + stats.append("%f" % (now - self._start)) # elapsed time + stats.append("%f" % psutil.cpu_percent()) # total cpu + stats.append("%f" % psutil.virtual_memory().percent) # total memory + return ' '.join(stats) + + def _make_proc_stats(self): + stats = [] + for p in self._procs: + stats.append('%f' % p.get_cpu_percent()) # proc cpu + stats.append('%f' % p.get_memory_percent()) # proc memory + return ' '.join(stats) + + def _make_header(self): + header = [] + header.append('# test_name: %s' % self._name) + header.append('# start_time: %s' % datetime.datetime.now(pytz.utc)) + header.append( + '# elapsed_time total_cpu total_memory proc_cpu proc_memory ') + return '\n'.join(header) + '\n' + + def _make_footer(self): + footer = [] + footer.append('# end_time: %s' % datetime.datetime.now(pytz.utc)) + return '\n'.join(footer) + + def stop(self): + self._stopped = True + + diff --git a/scripts/docker/files/bin/util.sh b/scripts/docker/files/bin/util.sh new file mode 100644 index 00000000..77287d0d --- /dev/null +++ b/scripts/docker/files/bin/util.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +configure_soledad_repos() { + if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin + fi + + if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} + fi +} |