summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--common/setup.py125
-rw-r--r--scripts/ddocs/update_design_docs.py170
-rw-r--r--scripts/packaging/compile_design_docs.py112
3 files changed, 1 insertions, 406 deletions
diff --git a/common/setup.py b/common/setup.py
index 7191fa00..c8a543ac 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -17,13 +17,8 @@
"""
setup file for leap.soledad.common
"""
-import binascii
-import json
-from os import listdir
-from os.path import realpath, dirname, isdir, join, isfile, basename
import re
-from distutils.command.build import build as _build
from setuptools import setup
from setuptools import find_packages
from setuptools import Command
@@ -110,117 +105,6 @@ def get_versions():
with open(versioneer_cfg.versionfile_source, 'w') as f:
f.write(subst_template)
-cmdclass = versioneer.get_cmdclass()
-
-#
-# Couch backend design docs file generation.
-#
-
-old_cmd_sdist = cmdclass["sdist"]
-
-
-def build_ddocs_py(basedir=None, with_src=True):
- """
- Build `ddocs.py` file.
-
- For ease of development, couch backend design documents are stored as
- `.js` files in subdirectories of `src/leap/soledad/common/ddocs`. This
- function scans that directory for javascript files, builds the design
- documents structure, and encode those structures in the `ddocs.py` file.
-
- This function is used when installing in develop mode, building or
- generating source distributions (see the next classes and the `cmdclass`
- setuptools parameter.
-
- This funciton uses the following conventions to generate design documents:
-
- - Design documents are represented by directories in the form
- `<prefix>/<ddoc>`, there prefix is the `src/leap/soledad/common/ddocs`
- directory.
- - Design document directories might contain `views`, `lists` and
- `updates` subdirectories.
- - Views subdirectories must contain a `map.js` file and may contain a
- `reduce.js` file.
- - List and updates subdirectories may contain any number of javascript
- files (i.e. ending in `.js`) whose names will be mapped to the
- corresponding list or update function name.
- """
- cur_pwd = dirname(realpath(__file__))
- common_path = ('src', 'leap', 'soledad', 'common')
- dest_common_path = common_path
- if not with_src:
- dest_common_path = common_path[1:]
- prefix = join(cur_pwd, *common_path)
-
- dest_prefix = prefix
- if basedir is not None:
- # we're bulding a sdist
- dest_prefix = join(basedir, *dest_common_path)
-
- ddocs_prefix = join(prefix, 'ddocs')
-
- if not isdir(ddocs_prefix):
- print "No ddocs/ folder, bailing out..."
- return
-
- ddocs = {}
-
- # design docs are represented by subdirectories of `ddocs_prefix`
- for ddoc in [f for f in listdir(ddocs_prefix)
- if isdir(join(ddocs_prefix, f))]:
-
- ddocs[ddoc] = {'_id': '_design/%s' % ddoc}
-
- for t in ['views', 'lists', 'updates']:
- tdir = join(ddocs_prefix, ddoc, t)
- if isdir(tdir):
-
- ddocs[ddoc][t] = {}
-
- if t == 'views': # handle views (with map/reduce functions)
- for view in [f for f in listdir(tdir)
- if isdir(join(tdir, f))]:
- # look for map.js and reduce.js
- mapfile = join(tdir, view, 'map.js')
- reducefile = join(tdir, view, 'reduce.js')
- mapfun = None
- reducefun = None
- try:
- with open(mapfile) as f:
- mapfun = f.read()
- except IOError:
- pass
- try:
- with open(reducefile) as f:
- reducefun = f.read()
- except IOError:
- pass
- ddocs[ddoc]['views'][view] = {}
-
- if mapfun is not None:
- ddocs[ddoc]['views'][view]['map'] = mapfun
- if reducefun is not None:
- ddocs[ddoc]['views'][view]['reduce'] = reducefun
-
- else: # handle lists, updates, etc
- for fun in [f for f in listdir(tdir)
- if isfile(join(tdir, f))]:
- funfile = join(tdir, fun)
- funname = basename(funfile).replace('.js', '')
- try:
- with open(funfile) as f:
- ddocs[ddoc][t][funname] = f.read()
- except IOError:
- pass
- # write file containing design docs strings
- ddoc_filename = "ddocs.py"
- with open(join(dest_prefix, ddoc_filename), 'w') as f:
- for ddoc in ddocs:
- f.write(
- "%s = '%s'\n" %
- (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1]))
- print "Wrote design docs in %s" % (dest_prefix + '/' + ddoc_filename,)
-
class cmd_develop(_cmd_develop):
def run(self):
@@ -230,17 +114,10 @@ class cmd_develop(_cmd_develop):
# unless we update this, the command will keep using the old version
self.distribution.metadata.version = versions["version"]
_cmd_develop.run(self)
- build_ddocs_py()
-
-
-class cmd_build(_build):
- def run(self):
- _build.run(self)
- build_ddocs_py(basedir=self.build_lib, with_src=False)
+cmdclass = versioneer.get_cmdclass()
cmdclass["freeze_debianver"] = freeze_debianver
-cmdclass["build"] = cmd_build
cmdclass["develop"] = cmd_develop
diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py
deleted file mode 100644
index 281482b8..00000000
--- a/scripts/ddocs/update_design_docs.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/python
-
-# This script updates Soledad's design documents in the session database and
-# all user databases with contents from the installed leap.soledad.common
-# package.
-
-import json
-import logging
-import argparse
-import re
-import threading
-import binascii
-
-from urlparse import urlparse
-from getpass import getpass
-from ConfigParser import ConfigParser
-
-from couchdb.client import Server
-from couchdb.http import Resource
-from couchdb.http import Session
-from couchdb.http import ResourceNotFound
-
-from leap.soledad.common import ddocs
-
-
-MAX_THREADS = 20
-DESIGN_DOCS = {
- '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)),
- '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)),
- '_design/transactions': json.loads(
- binascii.a2b_base64(ddocs.transactions)),
-}
-
-
-# create a logger
-logger = logging.getLogger(__name__)
-LOG_FORMAT = '%(asctime)s %(message)s'
-logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
-
-
-def _parse_args():
- parser = argparse.ArgumentParser()
- parser.add_argument('-u', dest='uuid', default=None, type=str,
- help='the UUID of the user')
- parser.add_argument('-t', dest='threads', default=MAX_THREADS, type=int,
- help='the number of parallel threads')
- return parser.parse_args()
-
-
-def _get_url():
- # get couch url
- cp = ConfigParser()
- cp.read('/etc/soledad/soledad-server.conf')
- url = urlparse(cp.get('soledad-server', 'couch_url'))
- # get admin password
- netloc = re.sub('^.*@', '', url.netloc)
- url = url._replace(netloc=netloc)
- password = getpass("Admin password for %s: " % url.geturl())
- return url._replace(netloc='admin:%s@%s' % (password, netloc))
-
-
-def _get_server(url):
- resource = Resource(
- url.geturl(), Session(retry_delays=[1, 2, 4, 8], timeout=10))
- return Server(url=resource)
-
-
-def _confirm(url):
- hidden_url = re.sub(
- 'http://(.*):.*@',
- 'http://\\1:xxxxx@',
- url.geturl())
-
- print """
- ==========
- ATTENTION!
- ==========
-
- This script will modify Soledad's shared and user databases in:
-
- %s
-
- This script does not make a backup of the couch db data, so make sure you
- have a copy or you may loose data.
- """ % hidden_url
- confirm = raw_input("Proceed (type uppercase YES)? ")
-
- if confirm != "YES":
- exit(1)
-
-
-#
-# Thread
-#
-
-class DBWorkerThread(threading.Thread):
-
- def __init__(self, server, dbname, db_idx, db_len, release_fun):
- threading.Thread.__init__(self)
- self._dbname = dbname
- self._cdb = server[self._dbname]
- self._db_idx = db_idx
- self._db_len = db_len
- self._release_fun = release_fun
-
- def run(self):
-
- logger.info(
- "(%d/%d) Updating db %s."
- % (self._db_idx, self._db_len, self._dbname))
-
- for doc_id in DESIGN_DOCS:
- try:
- doc = self._cdb[doc_id]
- except ResourceNotFound:
- doc = {'_id': doc_id}
- for key in ['lists', 'views', 'updates']:
- if key in DESIGN_DOCS[doc_id]:
- doc[key] = DESIGN_DOCS[doc_id][key]
- self._cdb.save(doc)
-
- # release the semaphore
- self._release_fun()
-
-
-def _launch_update_design_docs_thread(
- server, dbname, db_idx, db_len, semaphore_pool):
- semaphore_pool.acquire() # wait for an available working slot
- thread = DBWorkerThread(
- server, dbname, db_idx, db_len, semaphore_pool.release)
- thread.daemon = True
- thread.start()
- return thread
-
-
-def _update_design_docs(args, server):
-
- # find the actual databases to be updated
- dbs = []
- if args.uuid:
- dbs.append('user-%s' % args.uuid)
- else:
- for dbname in server:
- if dbname.startswith('user-') or dbname == 'shared':
- dbs.append(dbname)
- else:
- logger.info("Skipping db %s." % dbname)
-
- db_idx = 0
- db_len = len(dbs)
- semaphore_pool = threading.BoundedSemaphore(value=args.threads)
- threads = []
-
- # launch the update
- for db in dbs:
- db_idx += 1
- threads.append(
- _launch_update_design_docs_thread(
- server, db, db_idx, db_len, semaphore_pool))
-
- # wait for all threads to finish
- map(lambda thread: thread.join(), threads)
-
-
-if __name__ == "__main__":
- args = _parse_args()
- url = _get_url()
- _confirm(url)
- server = _get_server(url)
- _update_design_docs(args, server)
diff --git a/scripts/packaging/compile_design_docs.py b/scripts/packaging/compile_design_docs.py
deleted file mode 100644
index b2b5729a..00000000
--- a/scripts/packaging/compile_design_docs.py
+++ /dev/null
@@ -1,112 +0,0 @@
-#!/usr/bin/python
-
-
-# This script builds files for the design documents represented in the
-# ../common/src/soledad/common/ddocs directory structure (relative to the
-# current location of the script) into a target directory.
-
-
-import argparse
-from os import listdir
-from os.path import realpath, dirname, isdir, join, isfile, basename
-import json
-
-DDOCS_REL_PATH = ('..', 'common', 'src', 'leap', 'soledad', 'common', 'ddocs')
-
-
-def build_ddocs():
- """
- Build design documents.
-
- For ease of development, couch backend design documents are stored as
- `.js` files in subdirectories of
- `../common/src/leap/soledad/common/ddocs`. This function scans that
- directory for javascript files, and builds the design documents structure.
-
- This funciton uses the following conventions to generate design documents:
-
- - Design documents are represented by directories in the form
- `<prefix>/<ddoc>`, there prefix is the `src/leap/soledad/common/ddocs`
- directory.
- - Design document directories might contain `views`, `lists` and
- `updates` subdirectories.
- - Views subdirectories must contain a `map.js` file and may contain a
- `reduce.js` file.
- - List and updates subdirectories may contain any number of javascript
- files (i.e. ending in `.js`) whose names will be mapped to the
- corresponding list or update function name.
- """
- ddocs = {}
-
- # design docs are represented by subdirectories of `DDOCS_REL_PATH`
- cur_pwd = dirname(realpath(__file__))
- ddocs_path = join(cur_pwd, *DDOCS_REL_PATH)
- for ddoc in [f for f in listdir(ddocs_path)
- if isdir(join(ddocs_path, f))]:
-
- ddocs[ddoc] = {'_id': '_design/%s' % ddoc}
-
- for t in ['views', 'lists', 'updates']:
- tdir = join(ddocs_path, ddoc, t)
- if isdir(tdir):
-
- ddocs[ddoc][t] = {}
-
- if t == 'views': # handle views (with map/reduce functions)
- for view in [f for f in listdir(tdir)
- if isdir(join(tdir, f))]:
- # look for map.js and reduce.js
- mapfile = join(tdir, view, 'map.js')
- reducefile = join(tdir, view, 'reduce.js')
- mapfun = None
- reducefun = None
- try:
- with open(mapfile) as f:
- mapfun = f.read()
- except IOError:
- pass
- try:
- with open(reducefile) as f:
- reducefun = f.read()
- except IOError:
- pass
- ddocs[ddoc]['views'][view] = {}
-
- if mapfun is not None:
- ddocs[ddoc]['views'][view]['map'] = mapfun
- if reducefun is not None:
- ddocs[ddoc]['views'][view]['reduce'] = reducefun
-
- else: # handle lists, updates, etc
- for fun in [f for f in listdir(tdir)
- if isfile(join(tdir, f))]:
- funfile = join(tdir, fun)
- funname = basename(funfile).replace('.js', '')
- try:
- with open(funfile) as f:
- ddocs[ddoc][t][funname] = f.read()
- except IOError:
- pass
- return ddocs
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
- parser.add_argument(
- 'target', type=str,
- help='the target dir where to store design documents')
- args = parser.parse_args()
-
- # check if given target is a directory
- if not isdir(args.target):
- print 'Error: %s is not a directory.' % args.target
- exit(1)
-
- # write desifgn docs files
- ddocs = build_ddocs()
- for ddoc in ddocs:
- ddoc_filename = "%s.json" % ddoc
- with open(join(args.target, ddoc_filename), 'w') as f:
- f.write("%s" % json.dumps(ddocs[ddoc], indent=3))
- print "Wrote _design/%s content in %s" \
- % (ddoc, join(args.target, ddoc_filename,))