[doc] add script for copying doc to leap_se repo

author: drebs <drebs@riseup.net> 2017-10-23 10:30:15 -0200
committer: drebs <drebs@riseup.net> 2017-10-23 10:38:07 -0200
commit: d499358f0aae5e547de054b70786b726c5be65bb (patch)
tree: 70dfa828fd9f1c6b01060ebe994c4f070f5e7fb2
parent: 33e30feb2d7cedb1aecbde2f339b40f72128a585 (diff)
6 files changed, 127 insertions, 46 deletions
diff --git a/scripts/docs/copy-doc-to-leap_se.sh b/scripts/docs/copy-doc-to-leap_se.sh
new file mode 100755
index 00000000..86899943
--- /dev/null
+++ b/scripts/docs/copy-doc-to-leap_se.sh
@@ -0,0 +1,52 @@
+#!/bin/sh
+
+# This script is intended to copy a single HTML documentation page generated
+# using sphinx to the leap.se amber repository that builds the website.
+#
+# Because amber works differently than sphinx, the following modifications are
+# made in the sphinx automatically generated single html page:
+#
+#   - Remove everything from the start up to "<body>". Amber will take care of
+#     adding the HTML template to the top of the file.
+#
+#   - Remove everything from "</body>" up to "</html>". Amber will take care of
+#     adding the HTML template to the end of the file.
+#
+#   - Remove the navigation div. Amber will take care of adding it's own TOC.
+#
+#   - Remove the main <h1> tag. Amber adds it automatically.
+#
+#   - Remove all unicode paragraph characters. They are not hidden by amber and
+#     would make the page ugly.
+#
+#   - Move h2, h3, h4 to one level up (i.e. h1, h2, h3), because amber expects
+#     this kind of page organization in order to render TOC and navigation
+#     correctly.
+#
+#   - Turn h5-h7 to simple emphasized paragraphs, otherwise they would be
+#     rendered in a huge TOC.
+#
+#   - Remove the indices and tables from the end of the file, as amber does it
+#     by itself.
+
+BASEDIR=$(dirname "$0")
+
+# The following directory structure works well in my filesystem, you might have
+# to adapt to your structure/organization.
+HEADER=${BASEDIR}/amber-header.txt
+SOURCE=${BASEDIR}/../../docs/_build/singlehtml/index.html
+TARGET=${BASEDIR}/../../../leap_se/pages/docs/design/soledad.html
+
+cat ${HEADER} > ${TARGET}
+cat ${SOURCE} | sed \
+  -e '/<!DOCTYPE/,/<body>/d' \
+  -e '/role="navigation"/,+5d' \
+  -e '/<\/body>/,/<\/html>/d' \
+  -e '/<h1>Soledad.*<\/h1>/d' \
+  -e 's/¶//g' \
+  -e 's/<\(\/\)\?h2>/<\1h1>/g' \
+  -e 's/<\(\/\)\?h3>/<\1h2>/g' \
+  -e 's/<\(\/\)\?h4>/<\1h3>/g' \
+  -e 's/<h[5-7]>\(.*\)<\/h[5-7]>/<p><b>\1<\/b><\/p>/g' \
+  -e '/indices-and-tables/,$d' \
+    >> ${TARGET}
diff --git a/src/leap/soledad/common/couch/state.py b/src/leap/soledad/common/couch/state.py
index 8cbe0934..d9ff61a2 100644
--- a/src/leap/soledad/common/couch/state.py
+++ b/src/leap/soledad/common/couch/state.py
@@ -17,10 +17,12 @@
 """
 Server state using CouchDatabase as backend.
 """
-import couchdb
 import re
+import treq
 
 from six.moves.urllib.parse import urljoin
+from twisted.internet import defer
+from urlparse import urlsplit
 
 from leap.soledad.common.log import getLogger
 from leap.soledad.common.couch import CouchDatabase
@@ -51,14 +53,61 @@ def is_db_name_valid(name):
     return re.match(db_name_regex, name) is not None
 
 
+@defer.inlineCallbacks
+def _check_db(url, db, auth, agent=None):
+    # if there are documents, ensure that a config doc exists
+    db_url = urljoin(url, '%s/' % db)
+    config_doc_url = urljoin(db_url, CONFIG_DOC_ID)
+    res = yield treq.get(config_doc_url, auth=auth, agent=agent)
+    raise Exception
+
+    if res.code != 200 and res.code != 404:
+        raise Exception
+
+    if res.code == 404:
+        res = yield treq.get(urljoin(db_url, '_all_docs'), auth=auth,
+                             params={'limit': 1}, agent=agent)
+        docs = yield res.json()
+        if docs['total_rows'] != 0:
+            logger.error(
+                "Missing couch config document in database %s" % db)
+            raise MissingCouchConfigDocumentError(db)
+
+    if res.code == 200:
+        config_doc = yield res.json()
+        if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION:
+            logger.error(
+                "Unsupported database schema in database %s" % db)
+            raise WrongCouchSchemaVersionError(db)
+
+
+@defer.inlineCallbacks
+def check_schema_versions(couch_url, agent=None):
+    """
+    Check that all user databases use the correct couch schema.
+    """
+    url = urlsplit(couch_url)
+    auth = (url.username, url.password) if url.username else None
+    url = "%s://%s:%d" % (url.scheme, url.hostname, url.port)
+    res = yield treq.get(urljoin(url, '_all_dbs'), auth=auth, agent=agent)
+    dbs = yield res.json()
+    deferreds = []
+    semaphore = defer.DeferredSemaphore(20)
+    for db in dbs:
+        if not db.startswith('user-'):
+            continue
+        d = semaphore.run(_check_db, url, db, auth, agent=agent)
+        deferreds.append(d)
+    yield defer.gatherResults(deferreds)
+
+
 class CouchServerState(ServerState):
 
     """
     Inteface of the WSGI server with the CouchDB backend.
     """
 
-    def __init__(self, couch_url, create_cmd=None,
-                 check_schema_versions=False):
+    def __init__(self, couch_url, create_cmd=None):
         """
         Initialize the couch server state.
 
@@ -69,40 +118,9 @@ class CouchServerState(ServerState):
                            name and should access CouchDB with necessary
                            privileges, which server lacks for security reasons.
         :type create_cmd: str
-        :param check_schema_versions: Whether to check couch schema version of
-                                      user dbs. Set to False as this is only
-                                      intended to run once during start-up.
-        :type check_schema_versions: bool
         """
         self.couch_url = couch_url
         self.create_cmd = create_cmd
-        if check_schema_versions:
-            self._check_schema_versions()
-
-    def _check_schema_versions(self):
-        """
-        Check that all user databases use the correct couch schema.
-        """
-        server = couchdb.client.Server(self.couch_url)
-        for dbname in server:
-            if not dbname.startswith('user-'):
-                continue
-            db = server[dbname]
-
-            # if there are documents, ensure that a config doc exists
-            config_doc = db.get(CONFIG_DOC_ID)
-            if config_doc:
-                if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION:
-                    logger.error(
-                        "Unsupported database schema in database %s" % dbname)
-                    raise WrongCouchSchemaVersionError(dbname)
-            else:
-                result = db.view('_all_docs', limit=1)
-                if result.total_rows != 0:
-                    logger.error(
-                        "Missing couch config document in database %s"
-                        % dbname)
-                    raise MissingCouchConfigDocumentError(dbname)
 
     def open_database(self, dbname):
         """
diff --git a/src/leap/soledad/server/_wsgi.py b/src/leap/soledad/server/_wsgi.py
index f1b0018d..510cb7b9 100644
--- a/src/leap/soledad/server/_wsgi.py
+++ b/src/leap/soledad/server/_wsgi.py
@@ -33,8 +33,7 @@ __all__ = ['init_couch_state', 'get_sync_resource']
 
 
 def _get_couch_state(conf):
-    state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'],
-                             check_schema_versions=True)
+    state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
     SoledadBackend.BATCH_SUPPORT = conf.get('batching', False)
     return state
 
diff --git a/src/leap/soledad/server/entrypoints.py b/src/leap/soledad/server/entrypoints.py
index 7d18ca58..9bccbcf4 100644
--- a/src/leap/soledad/server/entrypoints.py
+++ b/src/leap/soledad/server/entrypoints.py
@@ -26,6 +26,7 @@ from twisted.internet import reactor
 from twisted.python import threadpool
 from twisted.logger import Logger
 
+from ..common.couch.state import check_schema_versions
 from .auth import localPortal, publicPortal
 from .session import SoledadSession
 from ._config import get_config
@@ -73,4 +74,5 @@ def check_conf():
 
 
 reactor.callWhenRunning(check_conf)
+reactor.callWhenRunning(check_schema_versions, conf['couch_url'])
 reactor.callWhenRunning(init_couch_state, conf)
diff --git a/tests/couch/test_command.py b/tests/couch/test_command.py
index 9fb2c153..52719065 100644
--- a/tests/couch/test_command.py
+++ b/tests/couch/test_command.py
@@ -10,7 +10,7 @@ class CommandBasedDBCreationTest(unittest.TestCase):
 
     def test_ensure_db_using_custom_command(self):
         state = couch_state.CouchServerState(
-            "url", create_cmd="/bin/echo", check_schema_versions=False)
+            "url", create_cmd="/bin/echo")
         mock_db = Mock()
         mock_db.replica_uid = 'replica_uid'
         state.open_database = Mock(return_value=mock_db)
@@ -20,12 +20,11 @@ class CommandBasedDBCreationTest(unittest.TestCase):
 
     def test_raises_unauthorized_on_failure(self):
         state = couch_state.CouchServerState(
-            "url", create_cmd="inexistent", check_schema_versions=False)
+            "url", create_cmd="inexistent")
         self.assertRaises(u1db_errors.Unauthorized,
                           state.ensure_database, "user-1337")
 
     def test_raises_unauthorized_by_default(self):
-        state = couch_state.CouchServerState("url",
-                                             check_schema_versions=False)
+        state = couch_state.CouchServerState("url")
         self.assertRaises(u1db_errors.Unauthorized,
                           state.ensure_database, "user-1337")
diff --git a/tests/couch/test_state.py b/tests/couch/test_state.py
index e5ac3704..07c03702 100644
--- a/tests/couch/test_state.py
+++ b/tests/couch/test_state.py
@@ -2,13 +2,15 @@ import pytest
 from leap.soledad.common.couch import CONFIG_DOC_ID
 from leap.soledad.common.couch import SCHEMA_VERSION
 from leap.soledad.common.couch import SCHEMA_VERSION_KEY
-from leap.soledad.common.couch.state import CouchServerState
+from leap.soledad.common.couch.state import check_schema_versions
 from uuid import uuid4
 
 from leap.soledad.common.errors import WrongCouchSchemaVersionError
 from leap.soledad.common.errors import MissingCouchConfigDocumentError
 from test_soledad.util import CouchDBTestCase
 
+from twisted.internet import defer
+
 
 class CouchDesignDocsTests(CouchDBTestCase):
 
@@ -16,17 +18,26 @@ class CouchDesignDocsTests(CouchDBTestCase):
         CouchDBTestCase.setUp(self)
         self.db = self.couch_server.create('user-' + uuid4().hex)
         self.addCleanup(self.delete_db, self.db.name)
+        from twisted.internet import reactor
+        from twisted.web.client import HTTPConnectionPool, Agent
+        self.pool = HTTPConnectionPool(reactor, persistent=False)
+        self.agent = Agent(reactor, pool=self.pool)
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.pool.closeCachedConnections()
 
+    @defer.inlineCallbacks
     def test_wrong_couch_version_raises(self):
         wrong_schema_version = SCHEMA_VERSION + 1
         self.db.create(
             {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version})
         with pytest.raises(WrongCouchSchemaVersionError):
-            CouchServerState(self.couch_url, create_cmd='/bin/echo',
-                             check_schema_versions=True)
+            yield check_schema_versions(self.couch_url, agent=self.agent)
 
+    @defer.inlineCallbacks
+    @pytest.mark.runme
     def test_missing_config_doc_raises(self):
         self.db.create({})
-        with pytest.raises(MissingCouchConfigDocumentError):
-            CouchServerState(self.couch_url, create_cmd='/bin/echo',
-                             check_schema_versions=True)
+        with pytest.raises(Exception):
+            yield check_schema_versions(self.couch_url, agent=self.agent)
author	drebs <drebs@riseup.net>	2017-10-23 10:30:15 -0200
committer	drebs <drebs@riseup.net>	2017-10-23 10:38:07 -0200
commit	d499358f0aae5e547de054b70786b726c5be65bb (patch)
tree	70dfa828fd9f1c6b01060ebe994c4f070f5e7fb2
parent	33e30feb2d7cedb1aecbde2f339b40f72128a585 (diff)