diff options
| -rw-r--r-- | common/setup.py | 208 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/.gitignore | 1 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/couch.py | 42 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/ddocs/README.txt | 5 | ||||
| -rw-r--r-- | common/src/leap/soledad/common/ddocs/__init__.py | 138 |
5 files changed, 179 insertions, 215 deletions
diff --git a/common/setup.py b/common/setup.py index 42bf272a..e142d958 100644 --- a/common/setup.py +++ b/common/setup.py @@ -103,93 +103,163 @@ def get_versions(default={}, verbose=False): f.write(subst_template) +# +# Couch backend design docs file generation. +# + from os import listdir from os.path import realpath, dirname, isdir, join, isfile, basename import json -import logging import binascii old_cmd_sdist = cmdclass["sdist"] +def build_ddocs_py(basedir=None, with_src=True): + """ + Build `ddocs.py` file. + + For ease of development, couch backend design documents are stored as + `.js` files in subdirectories of `src/leap/soledad/common/ddocs`. This + function scans that directory for javascript files, builds the design + documents structure, and encode those structures in the `ddocs.py` file. + + This function is used when installing in develop mode, building or + generating source distributions (see the next classes and the `cmdclass` + setuptools parameter. + + This funciton uses the following conventions to generate design documents: + + - Design documents are represented by directories in the form + `<prefix>/<ddoc>`, there prefix is the `src/leap/soledad/common/ddocs` + directory. + - Design document directories might contain `views`, `lists` and + `updates` subdirectories. + - Views subdirectories must contain a `map.js` file and may contain a + `reduce.js` file. + - List and updates subdirectories may contain any number of javascript + files (i.e. ending in `.js`) whose names will be mapped to the + corresponding list or update function name. + """ + cur_pwd = dirname(realpath(__file__)) + common_path = ('src', 'leap', 'soledad', 'common') + dest_common_path = common_path + if not with_src: + dest_common_path = common_path[1:] + prefix = join(cur_pwd, *common_path) + + dest_prefix = prefix + if basedir is not None: + # we're bulding a sdist + dest_prefix = join(basedir, *dest_common_path) + + ddocs_prefix = join(prefix, 'ddocs') + ddocs = {} + + # design docs are represented by subdirectories of `ddocs_prefix` + for ddoc in [f for f in listdir(ddocs_prefix) + if isdir(join(ddocs_prefix, f))]: + + ddocs[ddoc] = {'_id': '_design/%s' % ddoc} + + for t in ['views', 'lists', 'updates']: + tdir = join(ddocs_prefix, ddoc, t) + if isdir(tdir): + + ddocs[ddoc][t] = {} + + if t == 'views': # handle views (with map/reduce functions) + for view in [f for f in listdir(tdir) + if isdir(join(tdir, f))]: + # look for map.js and reduce.js + mapfile = join(tdir, view, 'map.js') + reducefile = join(tdir, view, 'reduce.js') + mapfun = None + reducefun = None + try: + with open(mapfile) as f: + mapfun = f.read() + except IOError: + pass + try: + with open(reducefile) as f: + reducefun = f.read() + except IOError: + pass + ddocs[ddoc]['views'][view] = {} + + if mapfun is not None: + ddocs[ddoc]['views'][view]['map'] = mapfun + if reducefun is not None: + ddocs[ddoc]['views'][view]['reduce'] = reducefun + + else: # handle lists, updates, etc + for fun in [f for f in listdir(tdir) + if isfile(join(tdir, f))]: + funfile = join(tdir, fun) + funname = basename(funfile).replace('.js', '') + try: + with open(funfile) as f: + ddocs[ddoc][t][funname] = f.read() + except IOError: + pass + # write file containing design docs strings + ddoc_filename = "ddocs.py" + with open(join(dest_prefix, ddoc_filename), 'w') as f: + for ddoc in ddocs: + f.write( + "%s = '%s'\n" % + (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1])) + print "Wrote design docs in %s" % (dest_prefix + '/' + ddoc_filename,) + + +from setuptools.command.develop import develop as _cmd_develop + + +class cmd_develop(_cmd_develop): + def run(self): + # versioneer: + versions = versioneer.get_versions(verbose=True) + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old version + self.distribution.metadata.version = versions["version"] + _cmd_develop.run(self) + build_ddocs_py() + + +# versioneer powered +old_cmd_sdist = cmdclass["sdist"] + + class cmd_sdist(old_cmd_sdist): """ - Generate 'src/leap/soledad/common/ddocs.py' which contains coush design + Generate 'src/leap/soledad/common/ddocs.py' which contains couch design documents scripts. """ - def run(self): old_cmd_sdist.run(self) - self.build_ddocs_py() - - def build_ddocs_py(self): - """ - Build couch design documents based on content from subdirectories in - 'src/leap/soledad/common/ddocs'. - """ - prefix = join( - dirname(realpath(__file__)), - 'src', 'leap', 'soledad', 'common') - ddocs_prefix = join(prefix, 'ddocs') - ddocs = {} - - # design docs are represented by subdirectories of `ddocs_prefix` - for ddoc in [f for f in listdir(ddocs_prefix) - if isdir(join(ddocs_prefix, f))]: - - ddocs[ddoc] = {'_id': '_design/%s' % ddoc} - - for t in ['views', 'lists', 'updates']: - tdir = join(ddocs_prefix, ddoc, t) - if isdir(tdir): - - ddocs[ddoc][t] = {} - - if t == 'views': # handle views (with map/reduce functions) - for view in [f for f in listdir(tdir) \ - if isdir(join(tdir, f))]: - # look for map.js and reduce.js - mapfile = join(tdir, view, 'map.js') - reducefile = join(tdir, view, 'reduce.js') - mapfun = None - reducefun = None - try: - with open(mapfile) as f: - mapfun = f.read() - except IOError: - pass - try: - with open(reducefile) as f: - reducefun = f.read() - except IOError: - pass - ddocs[ddoc]['views'][view] = {} - - if mapfun is not None: - ddocs[ddoc]['views'][view]['map'] = mapfun - if reducefun is not None: - ddocs[ddoc]['views'][view]['reduce'] = reducefun - - else: # handle lists, updates, etc - for fun in [f for f in listdir(tdir) \ - if isfile(join(tdir, f))]: - funfile = join(tdir, fun) - funname = basename(funfile).replace('.js', '') - try: - with open(funfile) as f: - ddocs[ddoc][t][funname] = f.read() - except IOError: - pass - # write file containing design docs strings - with open(join(prefix, 'ddocs.py'), 'w') as f: - for ddoc in ddocs: - f.write( - "%s = '%s'\n" % - (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1])) + + def make_release_tree(self, base_dir, files): + old_cmd_sdist.make_release_tree(self, base_dir, files) + build_ddocs_py(basedir=base_dir) + + +# versioneer powered +old_cmd_build = cmdclass["build"] + + +class cmd_build(old_cmd_build): + def run(self): + old_cmd_build.run(self) + build_ddocs_py(basedir=self.build_lib, with_src=False) cmdclass["freeze_debianver"] = freeze_debianver +cmdclass["build"] = cmd_build +cmdclass["sdist"] = cmd_sdist +cmdclass["develop"] = cmd_develop + # XXX add ref to docs diff --git a/common/src/leap/soledad/common/.gitignore b/common/src/leap/soledad/common/.gitignore new file mode 100644 index 00000000..3378c78a --- /dev/null +++ b/common/src/leap/soledad/common/.gitignore @@ -0,0 +1 @@ +ddocs.py diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index b9e699f3..d2414477 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -34,9 +34,8 @@ from u1db.remote import http_app from u1db.remote.server_state import ServerState -from leap.soledad.common import USER_DB_PREFIX +from leap.soledad.common import USER_DB_PREFIX, ddocs from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.ddocs import ensure_ddocs_on_db logger = logging.getLogger(__name__) @@ -187,7 +186,7 @@ class CouchDatabase(CommonBackend): return cls(url, dbname) def __init__(self, url, dbname, replica_uid=None, full_commit=True, - session=None, ensure_ddocs=False): + session=None, ensure_ddocs=True): """ Create a new Couch data container. @@ -220,9 +219,27 @@ class CouchDatabase(CommonBackend): except ResourceNotFound: self._server.create(self._dbname) self._database = self._server[self._dbname] - self._set_replica_uid(replica_uid or uuid.uuid4().hex) + if replica_uid is not None: + self._set_replica_uid(replica_uid) if ensure_ddocs: - ensure_ddocs_on_db(self._database) + self.ensure_ddocs_on_db() + + def ensure_ddocs_on_db(self): + """ + Ensure that the design documents used by the backend exist on the + couch database. + """ + # we check for existence of one of the files, and put all of them if + # that one does not exist + try: + self._database['_design/docs'] + return + except ResourceNotFound: + for ddoc_name in ['docs', 'syncs', 'transactions']: + ddoc = json.loads( + binascii.a2b_base64( + getattr(ddocs, ddoc_name))) + self._database.save(ddoc) def get_sync_target(self): """ @@ -261,9 +278,11 @@ class CouchDatabase(CommonBackend): :type replica_uid: str """ try: + # set on existent config document doc = self._database['u1db_config'] doc['replica_uid'] = replica_uid except ResourceNotFound: + # or create the config document doc = { '_id': 'u1db_config', 'replica_uid': replica_uid, @@ -280,9 +299,16 @@ class CouchDatabase(CommonBackend): """ if self._real_replica_uid is not None: return self._real_replica_uid - doc = self._database['u1db_config'] - self._real_replica_uid = doc['replica_uid'] - return self._real_replica_uid + try: + # grab replica_uid from server + doc = self._database['u1db_config'] + self._real_replica_uid = doc['replica_uid'] + return self._real_replica_uid + except ResourceNotFound: + # create a unique replica_uid + self._real_replica_uid = uuid.uuid4().hex + self._set_replica_uid(self._real_replica_uid) + return self._real_replica_uid _replica_uid = property(_get_replica_uid, _set_replica_uid) diff --git a/common/src/leap/soledad/common/ddocs/README.txt b/common/src/leap/soledad/common/ddocs/README.txt index 37d89fbf..5569d929 100644 --- a/common/src/leap/soledad/common/ddocs/README.txt +++ b/common/src/leap/soledad/common/ddocs/README.txt @@ -1,3 +1,8 @@ +This directory holds a folder structure containing javascript files that +represent the design documents needed by the CouchDB U1DB backend. These files +are compiled into the `../ddocs.py` file by setuptools when creating the +source distribution. + The following table depicts the U1DB CouchDB backend method and the URI that is queried to obtain/update data from/to the server. diff --git a/common/src/leap/soledad/common/ddocs/__init__.py b/common/src/leap/soledad/common/ddocs/__init__.py deleted file mode 100644 index 389bdff9..00000000 --- a/common/src/leap/soledad/common/ddocs/__init__.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -CouchDB U1DB backend design documents helper. -""" - - -from os import listdir -from os.path import realpath, dirname, isdir, join, isfile, basename -import json -import logging - - -from couchdb import Document as CouchDocument - - -logger = logging.getLogger(__name__) - - -# where to search for design docs definitions -prefix = dirname(realpath(__file__)) - - -def ensure_ddocs_on_db(db, prefix=prefix): - """ - Ensure that the design documents in C{db} contain. - - :param db: The database in which to create/update the design docs. - :type db: couchdb.client.Server - :param prefix: Where to look for design documents definitions. - :type prefix: str - """ - ddocs = build_ddocs(prefix) - for ddoc_name, ddoc_content in ddocs.iteritems(): - ddoc_id = "_design/%s" % ddoc_name - ddoc = CouchDocument({'_id': ddoc_id}) - ddoc.update(ddoc_content) - # ensure revision if ddoc is already in db - doc = db.get(ddoc_id) - if doc is not None: - ddoc['_rev'] = doc.rev - db.save(ddoc) - - -def create_local_ddocs(prefix=prefix): - """ - Create local design docs based on content from subdirectories in - C{prefix}. - - :param create_local: Whether to create local .json files. - :type create_local: bool - """ - ddocs = build_ddocs(prefix) - for ddoc_name, ddoc_content in ddocs.iteritems(): - with open(join(prefix, '%s.json' % ddoc_name), 'w') as f: - f.write(json.dumps(ddoc_content, indent=4)) - - -def build_ddocs(prefix=prefix): - """ - Build design documents based on content from subdirectories in - C{prefix}. - - :param prefix: Where to look for design documents definitions. - :type prefix: str - - :return: A dictionary containing the design docs definitions. - :rtype: dict - """ - ddocs = {} - # design docs are represented by subdirectories in current directory - for ddoc in [f for f in listdir(prefix) if isdir(join(prefix, f))]: - logger.debug("Building %s.json ..." % ddoc) - - ddocs[ddoc] = {} - - for t in ['views', 'lists', 'updates']: - tdir = join(prefix, ddoc, t) - if not isdir(tdir): - logger.debug(" - no %s" % t) - else: - - ddocs[ddoc][t] = {} - - if t == 'views': # handle views (with map/reduce functions) - for view in [f for f in listdir(tdir) \ - if isdir(join(tdir, f))]: - logger.debug(" - view: %s" % view) - # look for map.js and reduce.js - mapfile = join(tdir, view, 'map.js') - reducefile = join(tdir, view, 'reduce.js') - mapfun = None - reducefun = None - try: - with open(mapfile) as f: - mapfun = f.read() - except IOError: - pass - try: - with open(reducefile) as f: - reducefun = f.read() - except IOError: - pass - ddocs[ddoc]['views'][view] = {} - - if mapfun is not None: - ddocs[ddoc]['views'][view]['map'] = mapfun - if reducefun is not None: - ddocs[ddoc]['views'][view]['reduce'] = reducefun - - else: # handle lists, updates, etc - for fun in [f for f in listdir(tdir) \ - if isfile(join(tdir, f))]: - logger.debug(" - %s: %s" % (t, fun)) - funfile = join(tdir, fun) - funname = basename(funfile).replace('.js', '') - try: - with open(funfile) as f: - ddocs[ddoc][t][funname] = f.read() - except IOError: - pass - return ddocs |