diff options
| -rwxr-xr-x | scripts/docs/copy-doc-to-leap_se.sh | 52 | ||||
| -rw-r--r-- | src/leap/soledad/common/couch/state.py | 86 | ||||
| -rw-r--r-- | src/leap/soledad/server/_wsgi.py | 3 | ||||
| -rw-r--r-- | src/leap/soledad/server/entrypoints.py | 2 | ||||
| -rw-r--r-- | tests/couch/test_command.py | 7 | ||||
| -rw-r--r-- | tests/couch/test_state.py | 23 |
6 files changed, 127 insertions, 46 deletions
diff --git a/scripts/docs/copy-doc-to-leap_se.sh b/scripts/docs/copy-doc-to-leap_se.sh new file mode 100755 index 00000000..86899943 --- /dev/null +++ b/scripts/docs/copy-doc-to-leap_se.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +# This script is intended to copy a single HTML documentation page generated +# using sphinx to the leap.se amber repository that builds the website. +# +# Because amber works differently than sphinx, the following modifications are +# made in the sphinx automatically generated single html page: +# +# - Remove everything from the start up to "<body>". Amber will take care of +# adding the HTML template to the top of the file. +# +# - Remove everything from "</body>" up to "</html>". Amber will take care of +# adding the HTML template to the end of the file. +# +# - Remove the navigation div. Amber will take care of adding it's own TOC. +# +# - Remove the main <h1> tag. Amber adds it automatically. +# +# - Remove all unicode paragraph characters. They are not hidden by amber and +# would make the page ugly. +# +# - Move h2, h3, h4 to one level up (i.e. h1, h2, h3), because amber expects +# this kind of page organization in order to render TOC and navigation +# correctly. +# +# - Turn h5-h7 to simple emphasized paragraphs, otherwise they would be +# rendered in a huge TOC. +# +# - Remove the indices and tables from the end of the file, as amber does it +# by itself. + +BASEDIR=$(dirname "$0") + +# The following directory structure works well in my filesystem, you might have +# to adapt to your structure/organization. +HEADER=${BASEDIR}/amber-header.txt +SOURCE=${BASEDIR}/../../docs/_build/singlehtml/index.html +TARGET=${BASEDIR}/../../../leap_se/pages/docs/design/soledad.html + +cat ${HEADER} > ${TARGET} +cat ${SOURCE} | sed \ + -e '/<!DOCTYPE/,/<body>/d' \ + -e '/role="navigation"/,+5d' \ + -e '/<\/body>/,/<\/html>/d' \ + -e '/<h1>Soledad.*<\/h1>/d' \ + -e 's/ΒΆ//g' \ + -e 's/<\(\/\)\?h2>/<\1h1>/g' \ + -e 's/<\(\/\)\?h3>/<\1h2>/g' \ + -e 's/<\(\/\)\?h4>/<\1h3>/g' \ + -e 's/<h[5-7]>\(.*\)<\/h[5-7]>/<p><b>\1<\/b><\/p>/g' \ + -e '/indices-and-tables/,$d' \ + >> ${TARGET} diff --git a/src/leap/soledad/common/couch/state.py b/src/leap/soledad/common/couch/state.py index 8cbe0934..d9ff61a2 100644 --- a/src/leap/soledad/common/couch/state.py +++ b/src/leap/soledad/common/couch/state.py @@ -17,10 +17,12 @@ """ Server state using CouchDatabase as backend. """ -import couchdb import re +import treq from six.moves.urllib.parse import urljoin +from twisted.internet import defer +from urlparse import urlsplit from leap.soledad.common.log import getLogger from leap.soledad.common.couch import CouchDatabase @@ -51,14 +53,61 @@ def is_db_name_valid(name): return re.match(db_name_regex, name) is not None +@defer.inlineCallbacks +def _check_db(url, db, auth, agent=None): + # if there are documents, ensure that a config doc exists + db_url = urljoin(url, '%s/' % db) + config_doc_url = urljoin(db_url, CONFIG_DOC_ID) + res = yield treq.get(config_doc_url, auth=auth, agent=agent) + raise Exception + + if res.code != 200 and res.code != 404: + raise Exception + + if res.code == 404: + res = yield treq.get(urljoin(db_url, '_all_docs'), auth=auth, + params={'limit': 1}, agent=agent) + docs = yield res.json() + if docs['total_rows'] != 0: + logger.error( + "Missing couch config document in database %s" % db) + raise MissingCouchConfigDocumentError(db) + + if res.code == 200: + config_doc = yield res.json() + if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION: + logger.error( + "Unsupported database schema in database %s" % db) + raise WrongCouchSchemaVersionError(db) + + +@defer.inlineCallbacks +def check_schema_versions(couch_url, agent=None): + """ + Check that all user databases use the correct couch schema. + """ + url = urlsplit(couch_url) + auth = (url.username, url.password) if url.username else None + url = "%s://%s:%d" % (url.scheme, url.hostname, url.port) + res = yield treq.get(urljoin(url, '_all_dbs'), auth=auth, agent=agent) + dbs = yield res.json() + deferreds = [] + semaphore = defer.DeferredSemaphore(20) + for db in dbs: + if not db.startswith('user-'): + continue + d = semaphore.run(_check_db, url, db, auth, agent=agent) + deferreds.append(d) + yield defer.gatherResults(deferreds) + + class CouchServerState(ServerState): """ Inteface of the WSGI server with the CouchDB backend. """ - def __init__(self, couch_url, create_cmd=None, - check_schema_versions=False): + def __init__(self, couch_url, create_cmd=None): """ Initialize the couch server state. @@ -69,40 +118,9 @@ class CouchServerState(ServerState): name and should access CouchDB with necessary privileges, which server lacks for security reasons. :type create_cmd: str - :param check_schema_versions: Whether to check couch schema version of - user dbs. Set to False as this is only - intended to run once during start-up. - :type check_schema_versions: bool """ self.couch_url = couch_url self.create_cmd = create_cmd - if check_schema_versions: - self._check_schema_versions() - - def _check_schema_versions(self): - """ - Check that all user databases use the correct couch schema. - """ - server = couchdb.client.Server(self.couch_url) - for dbname in server: - if not dbname.startswith('user-'): - continue - db = server[dbname] - - # if there are documents, ensure that a config doc exists - config_doc = db.get(CONFIG_DOC_ID) - if config_doc: - if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION: - logger.error( - "Unsupported database schema in database %s" % dbname) - raise WrongCouchSchemaVersionError(dbname) - else: - result = db.view('_all_docs', limit=1) - if result.total_rows != 0: - logger.error( - "Missing couch config document in database %s" - % dbname) - raise MissingCouchConfigDocumentError(dbname) def open_database(self, dbname): """ diff --git a/src/leap/soledad/server/_wsgi.py b/src/leap/soledad/server/_wsgi.py index f1b0018d..510cb7b9 100644 --- a/src/leap/soledad/server/_wsgi.py +++ b/src/leap/soledad/server/_wsgi.py @@ -33,8 +33,7 @@ __all__ = ['init_couch_state', 'get_sync_resource'] def _get_couch_state(conf): - state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], - check_schema_versions=True) + state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) return state diff --git a/src/leap/soledad/server/entrypoints.py b/src/leap/soledad/server/entrypoints.py index 7d18ca58..9bccbcf4 100644 --- a/src/leap/soledad/server/entrypoints.py +++ b/src/leap/soledad/server/entrypoints.py @@ -26,6 +26,7 @@ from twisted.internet import reactor from twisted.python import threadpool from twisted.logger import Logger +from ..common.couch.state import check_schema_versions from .auth import localPortal, publicPortal from .session import SoledadSession from ._config import get_config @@ -73,4 +74,5 @@ def check_conf(): reactor.callWhenRunning(check_conf) +reactor.callWhenRunning(check_schema_versions, conf['couch_url']) reactor.callWhenRunning(init_couch_state, conf) diff --git a/tests/couch/test_command.py b/tests/couch/test_command.py index 9fb2c153..52719065 100644 --- a/tests/couch/test_command.py +++ b/tests/couch/test_command.py @@ -10,7 +10,7 @@ class CommandBasedDBCreationTest(unittest.TestCase): def test_ensure_db_using_custom_command(self): state = couch_state.CouchServerState( - "url", create_cmd="/bin/echo", check_schema_versions=False) + "url", create_cmd="/bin/echo") mock_db = Mock() mock_db.replica_uid = 'replica_uid' state.open_database = Mock(return_value=mock_db) @@ -20,12 +20,11 @@ class CommandBasedDBCreationTest(unittest.TestCase): def test_raises_unauthorized_on_failure(self): state = couch_state.CouchServerState( - "url", create_cmd="inexistent", check_schema_versions=False) + "url", create_cmd="inexistent") self.assertRaises(u1db_errors.Unauthorized, state.ensure_database, "user-1337") def test_raises_unauthorized_by_default(self): - state = couch_state.CouchServerState("url", - check_schema_versions=False) + state = couch_state.CouchServerState("url") self.assertRaises(u1db_errors.Unauthorized, state.ensure_database, "user-1337") diff --git a/tests/couch/test_state.py b/tests/couch/test_state.py index e5ac3704..07c03702 100644 --- a/tests/couch/test_state.py +++ b/tests/couch/test_state.py @@ -2,13 +2,15 @@ import pytest from leap.soledad.common.couch import CONFIG_DOC_ID from leap.soledad.common.couch import SCHEMA_VERSION from leap.soledad.common.couch import SCHEMA_VERSION_KEY -from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch.state import check_schema_versions from uuid import uuid4 from leap.soledad.common.errors import WrongCouchSchemaVersionError from leap.soledad.common.errors import MissingCouchConfigDocumentError from test_soledad.util import CouchDBTestCase +from twisted.internet import defer + class CouchDesignDocsTests(CouchDBTestCase): @@ -16,17 +18,26 @@ class CouchDesignDocsTests(CouchDBTestCase): CouchDBTestCase.setUp(self) self.db = self.couch_server.create('user-' + uuid4().hex) self.addCleanup(self.delete_db, self.db.name) + from twisted.internet import reactor + from twisted.web.client import HTTPConnectionPool, Agent + self.pool = HTTPConnectionPool(reactor, persistent=False) + self.agent = Agent(reactor, pool=self.pool) + + @defer.inlineCallbacks + def tearDown(self): + yield self.pool.closeCachedConnections() + @defer.inlineCallbacks def test_wrong_couch_version_raises(self): wrong_schema_version = SCHEMA_VERSION + 1 self.db.create( {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version}) with pytest.raises(WrongCouchSchemaVersionError): - CouchServerState(self.couch_url, create_cmd='/bin/echo', - check_schema_versions=True) + yield check_schema_versions(self.couch_url, agent=self.agent) + @defer.inlineCallbacks + @pytest.mark.runme def test_missing_config_doc_raises(self): self.db.create({}) - with pytest.raises(MissingCouchConfigDocumentError): - CouchServerState(self.couch_url, create_cmd='/bin/echo', - check_schema_versions=True) + with pytest.raises(Exception): + yield check_schema_versions(self.couch_url, agent=self.agent) |