Add 'ddocs.py' generation on setup.py.

5 files changed, 179 insertions, 215 deletions

diff --git a/common/setup.py b/common/setup.py
index 42bf272a..e142d958 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -103,93 +103,163 @@ def get_versions(default={}, verbose=False):
             f.write(subst_template)
 
 
+#
+# Couch backend design docs file generation.
+#
+
 from os import listdir
 from os.path import realpath, dirname, isdir, join, isfile, basename
 import json
-import logging
 import binascii
 
 
 old_cmd_sdist = cmdclass["sdist"]
 
 
+def build_ddocs_py(basedir=None, with_src=True):
+    """
+    Build `ddocs.py` file.
+
+    For ease of development, couch backend design documents are stored as
+    `.js` files in  subdirectories of `src/leap/soledad/common/ddocs`. This
+    function scans that directory for javascript files, builds the design
+    documents structure, and encode those structures in the `ddocs.py` file.
+
+    This function is used when installing in develop mode, building or
+    generating source distributions (see the next classes and the `cmdclass`
+    setuptools parameter.
+
+    This funciton uses the following conventions to generate design documents:
+
+      - Design documents are represented by directories in the form
+        `<prefix>/<ddoc>`, there prefix is the `src/leap/soledad/common/ddocs`
+        directory.
+      - Design document directories might contain `views`, `lists` and
+        `updates` subdirectories.
+      - Views subdirectories must contain a `map.js` file and may contain a
+        `reduce.js` file.
+      - List and updates subdirectories may contain any number of javascript
+        files (i.e. ending in `.js`) whose names will be mapped to the
+        corresponding list or update function name.
+    """
+    cur_pwd = dirname(realpath(__file__))
+    common_path = ('src', 'leap', 'soledad', 'common')
+    dest_common_path = common_path
+    if not with_src:
+        dest_common_path = common_path[1:]
+    prefix = join(cur_pwd, *common_path)
+
+    dest_prefix = prefix
+    if basedir is not None:
+        # we're bulding a sdist
+        dest_prefix = join(basedir, *dest_common_path)
+
+    ddocs_prefix = join(prefix, 'ddocs')
+    ddocs = {}
+
+    # design docs are represented by subdirectories of `ddocs_prefix`
+    for ddoc in [f for f in listdir(ddocs_prefix)
+                 if isdir(join(ddocs_prefix, f))]:
+
+        ddocs[ddoc] = {'_id': '_design/%s' % ddoc}
+
+        for t in ['views', 'lists', 'updates']:
+            tdir = join(ddocs_prefix, ddoc, t)
+            if isdir(tdir):
+
+                ddocs[ddoc][t] = {}
+
+                if t == 'views':  # handle views (with map/reduce functions)
+                    for view in [f for f in listdir(tdir)
+                                 if isdir(join(tdir, f))]:
+                        # look for map.js and reduce.js
+                        mapfile = join(tdir, view, 'map.js')
+                        reducefile = join(tdir, view, 'reduce.js')
+                        mapfun = None
+                        reducefun = None
+                        try:
+                            with open(mapfile) as f:
+                                mapfun = f.read()
+                        except IOError:
+                            pass
+                        try:
+                            with open(reducefile) as f:
+                                reducefun = f.read()
+                        except IOError:
+                            pass
+                        ddocs[ddoc]['views'][view] = {}
+
+                        if mapfun is not None:
+                            ddocs[ddoc]['views'][view]['map'] = mapfun
+                        if reducefun is not None:
+                            ddocs[ddoc]['views'][view]['reduce'] = reducefun
+
+                else:  # handle lists, updates, etc
+                    for fun in [f for f in listdir(tdir)
+                                if isfile(join(tdir, f))]:
+                        funfile = join(tdir, fun)
+                        funname = basename(funfile).replace('.js', '')
+                        try:
+                            with open(funfile) as f:
+                                ddocs[ddoc][t][funname] = f.read()
+                        except IOError:
+                            pass
+    # write file containing design docs strings
+    ddoc_filename = "ddocs.py"
+    with open(join(dest_prefix, ddoc_filename), 'w') as f:
+        for ddoc in ddocs:
+            f.write(
+                "%s = '%s'\n" %
+                (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1]))
+    print "Wrote design docs in %s" % (dest_prefix + '/' + ddoc_filename,)
+
+
+from setuptools.command.develop import develop as _cmd_develop
+
+
+class cmd_develop(_cmd_develop):
+    def run(self):
+        # versioneer:
+        versions = versioneer.get_versions(verbose=True)
+        self._versioneer_generated_versions = versions
+        # unless we update this, the command will keep using the old version
+        self.distribution.metadata.version = versions["version"]
+        _cmd_develop.run(self)
+        build_ddocs_py()
+
+
+# versioneer powered
+old_cmd_sdist = cmdclass["sdist"]
+
+
 class cmd_sdist(old_cmd_sdist):
     """
-    Generate 'src/leap/soledad/common/ddocs.py' which contains coush design
+    Generate 'src/leap/soledad/common/ddocs.py' which contains couch design
     documents scripts.
     """
-
     def run(self):
         old_cmd_sdist.run(self)
-        self.build_ddocs_py()
-
-    def build_ddocs_py(self):
-        """
-        Build couch design documents based on content from subdirectories in
-        'src/leap/soledad/common/ddocs'.
-        """
-        prefix = join(
-            dirname(realpath(__file__)),
-            'src', 'leap', 'soledad', 'common')
-        ddocs_prefix = join(prefix, 'ddocs')
-        ddocs = {}
-
-        # design docs are represented by subdirectories of `ddocs_prefix`
-        for ddoc in [f for f in listdir(ddocs_prefix)
-                if isdir(join(ddocs_prefix, f))]:
-
-            ddocs[ddoc] = {'_id': '_design/%s' % ddoc}
-
-            for t in ['views', 'lists', 'updates']:
-                tdir = join(ddocs_prefix, ddoc, t)
-                if isdir(tdir):
-
-                    ddocs[ddoc][t] = {}
-
-                    if t == 'views':  # handle views (with map/reduce functions)
-                        for view in [f for f in listdir(tdir) \
-                                if isdir(join(tdir, f))]:
-                            # look for map.js and reduce.js
-                            mapfile = join(tdir, view, 'map.js')
-                            reducefile = join(tdir, view, 'reduce.js')
-                            mapfun = None
-                            reducefun = None
-                            try:
-                                with open(mapfile) as f:
-                                    mapfun = f.read()
-                            except IOError:
-                                pass
-                            try:
-                                with open(reducefile) as f:
-                                    reducefun = f.read()
-                            except IOError:
-                                pass
-                            ddocs[ddoc]['views'][view] = {}
-
-                            if mapfun is not None:
-                                ddocs[ddoc]['views'][view]['map'] = mapfun
-                            if reducefun is not None:
-                                ddocs[ddoc]['views'][view]['reduce'] = reducefun
-
-                    else:  # handle lists, updates, etc
-                        for fun in [f for f in listdir(tdir) \
-                                if isfile(join(tdir, f))]:
-                            funfile = join(tdir, fun)
-                            funname = basename(funfile).replace('.js', '')
-                            try:
-                                with open(funfile) as f:
-                                    ddocs[ddoc][t][funname] = f.read()
-                            except IOError:
-                                pass
-        # write file containing design docs strings
-        with open(join(prefix, 'ddocs.py'), 'w') as f:
-            for ddoc in ddocs:
-                f.write(
-                    "%s = '%s'\n" %
-                    (ddoc, binascii.b2a_base64(json.dumps(ddocs[ddoc]))[:-1]))
+
+    def make_release_tree(self, base_dir, files):
+        old_cmd_sdist.make_release_tree(self, base_dir, files)
+        build_ddocs_py(basedir=base_dir)
+
+
+# versioneer powered
+old_cmd_build = cmdclass["build"]
+
+
+class cmd_build(old_cmd_build):
+    def run(self):
+        old_cmd_build.run(self)
+        build_ddocs_py(basedir=self.build_lib, with_src=False)
 
 
 cmdclass["freeze_debianver"] = freeze_debianver
+cmdclass["build"] = cmd_build
+cmdclass["sdist"] = cmd_sdist
+cmdclass["develop"] = cmd_develop
+
 
 # XXX add ref to docs
 
diff --git a/common/src/leap/soledad/common/.gitignore b/common/src/leap/soledad/common/.gitignore
new file mode 100644
index 00000000..3378c78a
--- /dev/null
+++ b/common/src/leap/soledad/common/.gitignore
@@ -0,0 +1 @@
+ddocs.py
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index b9e699f3..d2414477 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -34,9 +34,8 @@ from u1db.remote import http_app
 from u1db.remote.server_state import ServerState
 
 
-from leap.soledad.common import USER_DB_PREFIX
+from leap.soledad.common import USER_DB_PREFIX, ddocs
 from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.ddocs import ensure_ddocs_on_db
 
 
 logger = logging.getLogger(__name__)
@@ -187,7 +186,7 @@ class CouchDatabase(CommonBackend):
         return cls(url, dbname)
 
     def __init__(self, url, dbname, replica_uid=None, full_commit=True,
-                 session=None, ensure_ddocs=False):
+                 session=None, ensure_ddocs=True):
         """
         Create a new Couch data container.
 
@@ -220,9 +219,27 @@ class CouchDatabase(CommonBackend):
         except ResourceNotFound:
             self._server.create(self._dbname)
             self._database = self._server[self._dbname]
-        self._set_replica_uid(replica_uid or uuid.uuid4().hex)
+        if replica_uid is not None:
+            self._set_replica_uid(replica_uid)
         if ensure_ddocs:
-            ensure_ddocs_on_db(self._database)
+            self.ensure_ddocs_on_db()
+
+    def ensure_ddocs_on_db(self):
+        """
+        Ensure that the design documents used by the backend exist on the
+        couch database.
+        """
+        # we check for existence of one of the files, and put all of them if
+        # that one does not exist
+        try:
+            self._database['_design/docs']
+            return
+        except ResourceNotFound:
+            for ddoc_name in ['docs', 'syncs', 'transactions']:
+                ddoc = json.loads(
+                    binascii.a2b_base64(
+                        getattr(ddocs, ddoc_name)))
+                self._database.save(ddoc)
 
     def get_sync_target(self):
         """
@@ -261,9 +278,11 @@ class CouchDatabase(CommonBackend):
         :type replica_uid: str
         """
         try:
+            # set on existent config document
             doc = self._database['u1db_config']
             doc['replica_uid'] = replica_uid
         except ResourceNotFound:
+            # or create the config document
             doc = {
                 '_id': 'u1db_config',
                 'replica_uid': replica_uid,
@@ -280,9 +299,16 @@ class CouchDatabase(CommonBackend):
         """
         if self._real_replica_uid is not None:
             return self._real_replica_uid
-        doc = self._database['u1db_config']
-        self._real_replica_uid = doc['replica_uid']
-        return self._real_replica_uid
+        try:
+            # grab replica_uid from server
+            doc = self._database['u1db_config']
+            self._real_replica_uid = doc['replica_uid']
+            return self._real_replica_uid
+        except ResourceNotFound:
+            # create a unique replica_uid
+            self._real_replica_uid = uuid.uuid4().hex
+            self._set_replica_uid(self._real_replica_uid)
+            return self._real_replica_uid
 
     _replica_uid = property(_get_replica_uid, _set_replica_uid)
 
diff --git a/common/src/leap/soledad/common/ddocs/README.txt b/common/src/leap/soledad/common/ddocs/README.txt
index 37d89fbf..5569d929 100644
--- a/common/src/leap/soledad/common/ddocs/README.txt
+++ b/common/src/leap/soledad/common/ddocs/README.txt
@@ -1,3 +1,8 @@
+This directory holds a folder structure containing javascript files that
+represent the design documents needed by the CouchDB U1DB backend. These files
+are compiled into the `../ddocs.py` file by setuptools when creating the
+source distribution.
+
 The following table depicts the U1DB CouchDB backend method and the URI that
 is queried to obtain/update data from/to the server.
 
diff --git a/common/src/leap/soledad/common/ddocs/__init__.py b/common/src/leap/soledad/common/ddocs/__init__.py
deleted file mode 100644
index 389bdff9..00000000
--- a/common/src/leap/soledad/common/ddocs/__init__.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-CouchDB U1DB backend design documents helper.
-"""
-
-
-from os import listdir
-from os.path import realpath, dirname, isdir, join, isfile, basename
-import json
-import logging
-
-
-from couchdb import Document as CouchDocument
-
-
-logger = logging.getLogger(__name__)
-
-
-# where to search for design docs definitions
-prefix = dirname(realpath(__file__))
-
-
-def ensure_ddocs_on_db(db, prefix=prefix):
-    """
-    Ensure that the design documents in C{db} contain.
-
-    :param db: The database in which to create/update the design docs.
-    :type db: couchdb.client.Server
-    :param prefix: Where to look for design documents definitions.
-    :type prefix: str
-    """
-    ddocs = build_ddocs(prefix)
-    for ddoc_name, ddoc_content in ddocs.iteritems():
-        ddoc_id = "_design/%s" % ddoc_name
-        ddoc = CouchDocument({'_id': ddoc_id})
-        ddoc.update(ddoc_content)
-        # ensure revision if ddoc is already in db
-        doc = db.get(ddoc_id)
-        if doc is not None:
-            ddoc['_rev'] = doc.rev
-        db.save(ddoc)
-
-
-def create_local_ddocs(prefix=prefix):
-    """
-    Create local design docs based on content from subdirectories in
-    C{prefix}.
-
-    :param create_local: Whether to create local .json files.
-    :type create_local: bool
-    """
-    ddocs = build_ddocs(prefix)
-    for ddoc_name, ddoc_content in ddocs.iteritems():
-        with open(join(prefix, '%s.json' % ddoc_name), 'w') as f:
-            f.write(json.dumps(ddoc_content, indent=4))
-
-
-def build_ddocs(prefix=prefix):
-    """
-    Build design documents based on content from subdirectories in
-    C{prefix}.
-
-    :param prefix: Where to look for design documents definitions.
-    :type prefix: str
-
-    :return: A dictionary containing the design docs definitions.
-    :rtype: dict
-    """
-    ddocs = {}
-    # design docs are represented by subdirectories in current directory
-    for ddoc in [f for f in listdir(prefix) if isdir(join(prefix, f))]:
-        logger.debug("Building %s.json ..." % ddoc)
-
-        ddocs[ddoc] = {}
-
-        for t in ['views', 'lists', 'updates']:
-            tdir = join(prefix, ddoc, t)
-            if not isdir(tdir):
-                logger.debug("  - no %s" % t)
-            else:
-
-                ddocs[ddoc][t] = {}
-
-                if t == 'views':  # handle views (with map/reduce functions)
-                    for view in [f for f in listdir(tdir) \
-                            if isdir(join(tdir, f))]:
-                        logger.debug("  - view: %s" % view)
-                        # look for map.js and reduce.js
-                        mapfile = join(tdir, view, 'map.js')
-                        reducefile = join(tdir, view, 'reduce.js')
-                        mapfun = None
-                        reducefun = None
-                        try:
-                            with open(mapfile) as f:
-                                mapfun = f.read()
-                        except IOError:
-                            pass
-                        try:
-                            with open(reducefile) as f:
-                                reducefun = f.read()
-                        except IOError:
-                            pass
-                        ddocs[ddoc]['views'][view] = {}
-
-                        if mapfun is not None:
-                            ddocs[ddoc]['views'][view]['map'] = mapfun
-                        if reducefun is not None:
-                            ddocs[ddoc]['views'][view]['reduce'] = reducefun
-
-                else:  # handle lists, updates, etc
-                    for fun in [f for f in listdir(tdir) \
-                            if isfile(join(tdir, f))]:
-                        logger.debug("  - %s: %s" % (t, fun))
-                        funfile = join(tdir, fun)
-                        funname = basename(funfile).replace('.js', '')
-                        try:
-                            with open(funfile) as f:
-                                ddocs[ddoc][t][funname] = f.read()
-                        except IOError:
-                            pass
-    return ddocs