diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/ddocs/update_design_docs.py | 170 | ||||
| -rw-r--r-- | scripts/packaging/compile_design_docs.py | 112 |
2 files changed, 0 insertions, 282 deletions
diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py deleted file mode 100644 index 281482b8..00000000 --- a/scripts/ddocs/update_design_docs.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/python - -# This script updates Soledad's design documents in the session database and -# all user databases with contents from the installed leap.soledad.common -# package. - -import json -import logging -import argparse -import re -import threading -import binascii - -from urlparse import urlparse -from getpass import getpass -from ConfigParser import ConfigParser - -from couchdb.client import Server -from couchdb.http import Resource -from couchdb.http import Session -from couchdb.http import ResourceNotFound - -from leap.soledad.common import ddocs - - -MAX_THREADS = 20 -DESIGN_DOCS = { - '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)), - '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)), - '_design/transactions': json.loads( - binascii.a2b_base64(ddocs.transactions)), -} - - -# create a logger -logger = logging.getLogger(__name__) -LOG_FORMAT = '%(asctime)s %(message)s' -logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) - - -def _parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-u', dest='uuid', default=None, type=str, - help='the UUID of the user') - parser.add_argument('-t', dest='threads', default=MAX_THREADS, type=int, - help='the number of parallel threads') - return parser.parse_args() - - -def _get_url(): - # get couch url - cp = ConfigParser() - cp.read('/etc/soledad/soledad-server.conf') - url = urlparse(cp.get('soledad-server', 'couch_url')) - # get admin password - netloc = re.sub('^.*@', '', url.netloc) - url = url._replace(netloc=netloc) - password = getpass("Admin password for %s: " % url.geturl()) - return url._replace(netloc='admin:%s@%s' % (password, netloc)) - - -def _get_server(url): - resource = Resource( - url.geturl(), Session(retry_delays=[1, 2, 4, 8], timeout=10)) - return Server(url=resource) - - -def _confirm(url): - hidden_url = re.sub( - 'http://(.*):.*@', - 'http://\\1:xxxxx@', - url.geturl()) - - print """ - ========== - ATTENTION! - ========== - - This script will modify Soledad's shared and user databases in: - - %s - - This script does not make a backup of the couch db data, so make sure you - have a copy or you may loose data. - """ % hidden_url - confirm = raw_input("Proceed (type uppercase YES)? ") - - if confirm != "YES": - exit(1) - - -# -# Thread -# - -class DBWorkerThread(threading.Thread): - - def __init__(self, server, dbname, db_idx, db_len, release_fun): - threading.Thread.__init__(self) - self._dbname = dbname - self._cdb = server[self._dbname] - self._db_idx = db_idx - self._db_len = db_len - self._release_fun = release_fun - - def run(self): - - logger.info( - "(%d/%d) Updating db %s." - % (self._db_idx, self._db_len, self._dbname)) - - for doc_id in DESIGN_DOCS: - try: - doc = self._cdb[doc_id] - except ResourceNotFound: - doc = {'_id': doc_id} - for key in ['lists', 'views', 'updates']: - if key in DESIGN_DOCS[doc_id]: - doc[key] = DESIGN_DOCS[doc_id][key] - self._cdb.save(doc) - - # release the semaphore - self._release_fun() - - -def _launch_update_design_docs_thread( - server, dbname, db_idx, db_len, semaphore_pool): - semaphore_pool.acquire() # wait for an available working slot - thread = DBWorkerThread( - server, dbname, db_idx, db_len, semaphore_pool.release) - thread.daemon = True - thread.start() - return thread - - -def _update_design_docs(args, server): - - # find the actual databases to be updated - dbs = [] - if args.uuid: - dbs.append('user-%s' % args.uuid) - else: - for dbname in server: - if dbname.startswith('user-') or dbname == 'shared': - dbs.append(dbname) - else: - logger.info("Skipping db %s." % dbname) - - db_idx = 0 - db_len = len(dbs) - semaphore_pool = threading.BoundedSemaphore(value=args.threads) - threads = [] - - # launch the update - for db in dbs: - db_idx += 1 - threads.append( - _launch_update_design_docs_thread( - server, db, db_idx, db_len, semaphore_pool)) - - # wait for all threads to finish - map(lambda thread: thread.join(), threads) - - -if __name__ == "__main__": - args = _parse_args() - url = _get_url() - _confirm(url) - server = _get_server(url) - _update_design_docs(args, server) diff --git a/scripts/packaging/compile_design_docs.py b/scripts/packaging/compile_design_docs.py deleted file mode 100644 index b2b5729a..00000000 --- a/scripts/packaging/compile_design_docs.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/python - - -# This script builds files for the design documents represented in the -# ../common/src/soledad/common/ddocs directory structure (relative to the -# current location of the script) into a target directory. - - -import argparse -from os import listdir -from os.path import realpath, dirname, isdir, join, isfile, basename -import json - -DDOCS_REL_PATH = ('..', 'common', 'src', 'leap', 'soledad', 'common', 'ddocs') - - -def build_ddocs(): - """ - Build design documents. - - For ease of development, couch backend design documents are stored as - `.js` files in subdirectories of - `../common/src/leap/soledad/common/ddocs`. This function scans that - directory for javascript files, and builds the design documents structure. - - This funciton uses the following conventions to generate design documents: - - - Design documents are represented by directories in the form - `<prefix>/<ddoc>`, there prefix is the `src/leap/soledad/common/ddocs` - directory. - - Design document directories might contain `views`, `lists` and - `updates` subdirectories. - - Views subdirectories must contain a `map.js` file and may contain a - `reduce.js` file. - - List and updates subdirectories may contain any number of javascript - files (i.e. ending in `.js`) whose names will be mapped to the - corresponding list or update function name. - """ - ddocs = {} - - # design docs are represented by subdirectories of `DDOCS_REL_PATH` - cur_pwd = dirname(realpath(__file__)) - ddocs_path = join(cur_pwd, *DDOCS_REL_PATH) - for ddoc in [f for f in listdir(ddocs_path) - if isdir(join(ddocs_path, f))]: - - ddocs[ddoc] = {'_id': '_design/%s' % ddoc} - - for t in ['views', 'lists', 'updates']: - tdir = join(ddocs_path, ddoc, t) - if isdir(tdir): - - ddocs[ddoc][t] = {} - - if t == 'views': # handle views (with map/reduce functions) - for view in [f for f in listdir(tdir) - if isdir(join(tdir, f))]: - # look for map.js and reduce.js - mapfile = join(tdir, view, 'map.js') - reducefile = join(tdir, view, 'reduce.js') - mapfun = None - reducefun = None - try: - with open(mapfile) as f: - mapfun = f.read() - except IOError: - pass - try: - with open(reducefile) as f: - reducefun = f.read() - except IOError: - pass - ddocs[ddoc]['views'][view] = {} - - if mapfun is not None: - ddocs[ddoc]['views'][view]['map'] = mapfun - if reducefun is not None: - ddocs[ddoc]['views'][view]['reduce'] = reducefun - - else: # handle lists, updates, etc - for fun in [f for f in listdir(tdir) - if isfile(join(tdir, f))]: - funfile = join(tdir, fun) - funname = basename(funfile).replace('.js', '') - try: - with open(funfile) as f: - ddocs[ddoc][t][funname] = f.read() - except IOError: - pass - return ddocs - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - 'target', type=str, - help='the target dir where to store design documents') - args = parser.parse_args() - - # check if given target is a directory - if not isdir(args.target): - print 'Error: %s is not a directory.' % args.target - exit(1) - - # write desifgn docs files - ddocs = build_ddocs() - for ddoc in ddocs: - ddoc_filename = "%s.json" % ddoc - with open(join(args.target, ddoc_filename), 'w') as f: - f.write("%s" % json.dumps(ddocs[ddoc], indent=3)) - print "Wrote _design/%s content in %s" \ - % (ddoc, join(args.target, ddoc_filename,)) |