summaryrefslogtreecommitdiff
path: root/common
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2013-12-11 15:32:42 -0200
committerdrebs <drebs@leap.se>2013-12-18 14:36:59 -0200
commit96cf0af3880aa9bec8fcad422526297ef7f40964 (patch)
treedd118dff5c2d0de83aacc4e3b187f855eb422061 /common
parentaf913a77b384a97d857c37ba9c0a8730b0846373 (diff)
Add 'ddocs.py' generation on setup.py.
Diffstat (limited to 'common')
-rw-r--r--common/setup.py208
-rw-r--r--common/src/leap/soledad/common/.gitignore1
-rw-r--r--common/src/leap/soledad/common/couch.py42
-rw-r--r--common/src/leap/soledad/common/ddocs/README.txt5
-rw-r--r--common/src/leap/soledad/common/ddocs/__init__.py138
5 files changed, 179 insertions, 215 deletions
diff --git a/common/setup.py b/common/setup.py
index 42bf272a..e142d958 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -103,93 +103,163 @@ def get_versions(default={}, verbose=False):
f.write(subst_template)
+#
+# Couch backend design docs file generation.
+#
+
from os import listdir
from os.path import realpath, dirname, isdir, join, isfile, basename
import json
-import logging
import binascii
old_cmd_sdist = cmdclass["sdist"]
+def build_ddocs_py(basedir=None, with_src=True):
+ """
+ Build `ddocs.py` file.
+
+ For ease of development, couch backend design documents are stored as
+ `.js` files in subdirectories of `src/leap/soledad/common/ddocs`. This
+ function scans that directory for javascript files, builds the design
+ documents structure, and encode those structures in the `ddocs.py` file.
+
+ This function is used when installing in develop mode, building or
+ generating source distributions (see the next classes and the `cmdclass`
+ setuptools parameter.
+
+ This funciton uses the following conventions to generate design documents:
+
+ - Design documents are represented by directories in the form
+ `<prefix>/<ddoc>`, there prefix is the `src/leap/soledad/common/ddocs`
+ directory.
+ - Design document directories might contain `views`, `lists` and
+ `updates` subdirectories.
+ - Views subdirectories must contain a `map.js` file and may contain a
+ `reduce.js` file.
+ - List and updates subdirectories may contain any number of javascript
+ files (i.e. ending in `.js`) whose names will be mapped to the
+ corresponding list or update function name.
+ """
+ cur_pwd = dirname(realpath(__file__))
+ common_path = ('src', 'leap', 'soledad', 'common')
+ dest_common_path = common_path
+ if not with_src:
+ dest_common_path = common_path[1:]
+ prefix = join(cur_pwd, *common_path)
+
+ dest_prefix = prefix
+ if basedir is not None:
+ # we're bulding a sdist
+ dest_prefix = join(basedir, *dest_common_path)
+
+ ddocs_prefix = join(prefix, 'ddocs')
+ ddocs = {}
+
+ # design docs are represented by subdirectories of `ddocs_prefix`
+ for ddoc in [f for f in listdir(ddocs_prefix)
+ if isdir(join(ddocs_prefix, f))]:
+
+ ddocs[ddoc] = {'_id': '_design/%s' % ddoc}
+
+ for t in ['views', 'lists', 'updates']:
+ tdir = join(ddocs_prefix, ddoc, t)
+ if isdir(tdir):
+
+ ddocs[ddoc][t] = {}
+
+ if t == 'views': # handle views (with map/reduce functions)
+ for view in [f for f in listdir(tdir)
+ if isdir(join(tdir, f))]:
+ # look for map.js and reduce.js
+ mapfile = join(tdir, view, 'map.js')
+ reducefile = join(tdir, view, 'reduce.js')
+ mapfun = None
+ reducefun = None
+ try:
+ with open(mapfile) as f:
+ mapfun = f.read()
+ except IOError:
+ pass
+ try:
+ with open(reducefile) as f:
+ reducefun = f.read()
+ except IOError:
+ pass
+ ddocs[ddoc]['views'][view] = {}
+
+ if mapfun is not None:
+ ddocs[ddoc]['views'][view]['map'] = mapfun
+ if reducefun is not None:
+ ddocs[ddoc]['views'][view]['reduce'] = reducefun
+
+ else: # handle lists, updates, etc
+ for fun in [f for f in listdir(tdir)
+ if isfile(join(tdir, f))]:
+ funfile = join(tdir, fun)
+ funname = basename(funfile).replace('.js', '')
+ try:
+ with open(funfile) as f:
+ ddocs[ddoc][t][funname] = f.read()
+ except IOError:
+ pass
+ # write file containing design docs strings
+ ddoc_filename = "ddocs.py"
+ with open(join(dest_prefix, ddoc_filename), 'w') as f:
+ for ddoc in ddocs:
+ f.write(
+ "%s = '%s'\n" %
+ (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1]))
+ print "Wrote design docs in %s" % (dest_prefix + '/' + ddoc_filename,)
+
+
+from setuptools.command.develop import develop as _cmd_develop
+
+
+class cmd_develop(_cmd_develop):
+ def run(self):
+ # versioneer:
+ versions = versioneer.get_versions(verbose=True)
+ self._versioneer_generated_versions = versions
+ # unless we update this, the command will keep using the old version
+ self.distribution.metadata.version = versions["version"]
+ _cmd_develop.run(self)
+ build_ddocs_py()
+
+
+# versioneer powered
+old_cmd_sdist = cmdclass["sdist"]
+
+
class cmd_sdist(old_cmd_sdist):
"""
- Generate 'src/leap/soledad/common/ddocs.py' which contains coush design
+ Generate 'src/leap/soledad/common/ddocs.py' which contains couch design
documents scripts.
"""
-
def run(self):
old_cmd_sdist.run(self)
- self.build_ddocs_py()
-
- def build_ddocs_py(self):
- """
- Build couch design documents based on content from subdirectories in
- 'src/leap/soledad/common/ddocs'.
- """
- prefix = join(
- dirname(realpath(__file__)),
- 'src', 'leap', 'soledad', 'common')
- ddocs_prefix = join(prefix, 'ddocs')
- ddocs = {}
-
- # design docs are represented by subdirectories of `ddocs_prefix`
- for ddoc in [f for f in listdir(ddocs_prefix)
- if isdir(join(ddocs_prefix, f))]:
-
- ddocs[ddoc] = {'_id': '_design/%s' % ddoc}
-
- for t in ['views', 'lists', 'updates']:
- tdir = join(ddocs_prefix, ddoc, t)
- if isdir(tdir):
-
- ddocs[ddoc][t] = {}
-
- if t == 'views': # handle views (with map/reduce functions)
- for view in [f for f in listdir(tdir) \
- if isdir(join(tdir, f))]:
- # look for map.js and reduce.js
- mapfile = join(tdir, view, 'map.js')
- reducefile = join(tdir, view, 'reduce.js')
- mapfun = None
- reducefun = None
- try:
- with open(mapfile) as f:
- mapfun = f.read()
- except IOError:
- pass
- try:
- with open(reducefile) as f:
- reducefun = f.read()
- except IOError:
- pass
- ddocs[ddoc]['views'][view] = {}
-
- if mapfun is not None:
- ddocs[ddoc]['views'][view]['map'] = mapfun
- if reducefun is not None:
- ddocs[ddoc]['views'][view]['reduce'] = reducefun
-
- else: # handle lists, updates, etc
- for fun in [f for f in listdir(tdir) \
- if isfile(join(tdir, f))]:
- funfile = join(tdir, fun)
- funname = basename(funfile).replace('.js', '')
- try:
- with open(funfile) as f:
- ddocs[ddoc][t][funname] = f.read()
- except IOError:
- pass
- # write file containing design docs strings
- with open(join(prefix, 'ddocs.py'), 'w') as f:
- for ddoc in ddocs:
- f.write(
- "%s = '%s'\n" %
- (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1]))
+
+ def make_release_tree(self, base_dir, files):
+ old_cmd_sdist.make_release_tree(self, base_dir, files)
+ build_ddocs_py(basedir=base_dir)
+
+
+# versioneer powered
+old_cmd_build = cmdclass["build"]
+
+
+class cmd_build(old_cmd_build):
+ def run(self):
+ old_cmd_build.run(self)
+ build_ddocs_py(basedir=self.build_lib, with_src=False)
cmdclass["freeze_debianver"] = freeze_debianver
+cmdclass["build"] = cmd_build
+cmdclass["sdist"] = cmd_sdist
+cmdclass["develop"] = cmd_develop
+
# XXX add ref to docs
diff --git a/common/src/leap/soledad/common/.gitignore b/common/src/leap/soledad/common/.gitignore
new file mode 100644
index 00000000..3378c78a
--- /dev/null
+++ b/common/src/leap/soledad/common/.gitignore
@@ -0,0 +1 @@
+ddocs.py
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index b9e699f3..d2414477 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -34,9 +34,8 @@ from u1db.remote import http_app
from u1db.remote.server_state import ServerState
-from leap.soledad.common import USER_DB_PREFIX
+from leap.soledad.common import USER_DB_PREFIX, ddocs
from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.ddocs import ensure_ddocs_on_db
logger = logging.getLogger(__name__)
@@ -187,7 +186,7 @@ class CouchDatabase(CommonBackend):
return cls(url, dbname)
def __init__(self, url, dbname, replica_uid=None, full_commit=True,
- session=None, ensure_ddocs=False):
+ session=None, ensure_ddocs=True):
"""
Create a new Couch data container.
@@ -220,9 +219,27 @@ class CouchDatabase(CommonBackend):
except ResourceNotFound:
self._server.create(self._dbname)
self._database = self._server[self._dbname]
- self._set_replica_uid(replica_uid or uuid.uuid4().hex)
+ if replica_uid is not None:
+ self._set_replica_uid(replica_uid)
if ensure_ddocs:
- ensure_ddocs_on_db(self._database)
+ self.ensure_ddocs_on_db()
+
+ def ensure_ddocs_on_db(self):
+ """
+ Ensure that the design documents used by the backend exist on the
+ couch database.
+ """
+ # we check for existence of one of the files, and put all of them if
+ # that one does not exist
+ try:
+ self._database['_design/docs']
+ return
+ except ResourceNotFound:
+ for ddoc_name in ['docs', 'syncs', 'transactions']:
+ ddoc = json.loads(
+ binascii.a2b_base64(
+ getattr(ddocs, ddoc_name)))
+ self._database.save(ddoc)
def get_sync_target(self):
"""
@@ -261,9 +278,11 @@ class CouchDatabase(CommonBackend):
:type replica_uid: str
"""
try:
+ # set on existent config document
doc = self._database['u1db_config']
doc['replica_uid'] = replica_uid
except ResourceNotFound:
+ # or create the config document
doc = {
'_id': 'u1db_config',
'replica_uid': replica_uid,
@@ -280,9 +299,16 @@ class CouchDatabase(CommonBackend):
"""
if self._real_replica_uid is not None:
return self._real_replica_uid
- doc = self._database['u1db_config']
- self._real_replica_uid = doc['replica_uid']
- return self._real_replica_uid
+ try:
+ # grab replica_uid from server
+ doc = self._database['u1db_config']
+ self._real_replica_uid = doc['replica_uid']
+ return self._real_replica_uid
+ except ResourceNotFound:
+ # create a unique replica_uid
+ self._real_replica_uid = uuid.uuid4().hex
+ self._set_replica_uid(self._real_replica_uid)
+ return self._real_replica_uid
_replica_uid = property(_get_replica_uid, _set_replica_uid)
diff --git a/common/src/leap/soledad/common/ddocs/README.txt b/common/src/leap/soledad/common/ddocs/README.txt
index 37d89fbf..5569d929 100644
--- a/common/src/leap/soledad/common/ddocs/README.txt
+++ b/common/src/leap/soledad/common/ddocs/README.txt
@@ -1,3 +1,8 @@
+This directory holds a folder structure containing javascript files that
+represent the design documents needed by the CouchDB U1DB backend. These files
+are compiled into the `../ddocs.py` file by setuptools when creating the
+source distribution.
+
The following table depicts the U1DB CouchDB backend method and the URI that
is queried to obtain/update data from/to the server.
diff --git a/common/src/leap/soledad/common/ddocs/__init__.py b/common/src/leap/soledad/common/ddocs/__init__.py
deleted file mode 100644
index 389bdff9..00000000
--- a/common/src/leap/soledad/common/ddocs/__init__.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-CouchDB U1DB backend design documents helper.
-"""
-
-
-from os import listdir
-from os.path import realpath, dirname, isdir, join, isfile, basename
-import json
-import logging
-
-
-from couchdb import Document as CouchDocument
-
-
-logger = logging.getLogger(__name__)
-
-
-# where to search for design docs definitions
-prefix = dirname(realpath(__file__))
-
-
-def ensure_ddocs_on_db(db, prefix=prefix):
- """
- Ensure that the design documents in C{db} contain.
-
- :param db: The database in which to create/update the design docs.
- :type db: couchdb.client.Server
- :param prefix: Where to look for design documents definitions.
- :type prefix: str
- """
- ddocs = build_ddocs(prefix)
- for ddoc_name, ddoc_content in ddocs.iteritems():
- ddoc_id = "_design/%s" % ddoc_name
- ddoc = CouchDocument({'_id': ddoc_id})
- ddoc.update(ddoc_content)
- # ensure revision if ddoc is already in db
- doc = db.get(ddoc_id)
- if doc is not None:
- ddoc['_rev'] = doc.rev
- db.save(ddoc)
-
-
-def create_local_ddocs(prefix=prefix):
- """
- Create local design docs based on content from subdirectories in
- C{prefix}.
-
- :param create_local: Whether to create local .json files.
- :type create_local: bool
- """
- ddocs = build_ddocs(prefix)
- for ddoc_name, ddoc_content in ddocs.iteritems():
- with open(join(prefix, '%s.json' % ddoc_name), 'w') as f:
- f.write(json.dumps(ddoc_content, indent=4))
-
-
-def build_ddocs(prefix=prefix):
- """
- Build design documents based on content from subdirectories in
- C{prefix}.
-
- :param prefix: Where to look for design documents definitions.
- :type prefix: str
-
- :return: A dictionary containing the design docs definitions.
- :rtype: dict
- """
- ddocs = {}
- # design docs are represented by subdirectories in current directory
- for ddoc in [f for f in listdir(prefix) if isdir(join(prefix, f))]:
- logger.debug("Building %s.json ..." % ddoc)
-
- ddocs[ddoc] = {}
-
- for t in ['views', 'lists', 'updates']:
- tdir = join(prefix, ddoc, t)
- if not isdir(tdir):
- logger.debug(" - no %s" % t)
- else:
-
- ddocs[ddoc][t] = {}
-
- if t == 'views': # handle views (with map/reduce functions)
- for view in [f for f in listdir(tdir) \
- if isdir(join(tdir, f))]:
- logger.debug(" - view: %s" % view)
- # look for map.js and reduce.js
- mapfile = join(tdir, view, 'map.js')
- reducefile = join(tdir, view, 'reduce.js')
- mapfun = None
- reducefun = None
- try:
- with open(mapfile) as f:
- mapfun = f.read()
- except IOError:
- pass
- try:
- with open(reducefile) as f:
- reducefun = f.read()
- except IOError:
- pass
- ddocs[ddoc]['views'][view] = {}
-
- if mapfun is not None:
- ddocs[ddoc]['views'][view]['map'] = mapfun
- if reducefun is not None:
- ddocs[ddoc]['views'][view]['reduce'] = reducefun
-
- else: # handle lists, updates, etc
- for fun in [f for f in listdir(tdir) \
- if isfile(join(tdir, f))]:
- logger.debug(" - %s: %s" % (t, fun))
- funfile = join(tdir, fun)
- funname = basename(funfile).replace('.js', '')
- try:
- with open(funfile) as f:
- ddocs[ddoc][t][funname] = f.read()
- except IOError:
- pass
- return ddocs