diff options
| -rw-r--r-- | src/leap/soledad/common/couch/check.py | 123 | ||||
| -rw-r--r-- | src/leap/soledad/common/couch/state.py | 98 | ||||
| -rw-r--r-- | src/leap/soledad/server/entrypoints.py | 30 | ||||
| -rw-r--r-- | src/leap/soledad/server/server.tac | 125 | ||||
| -rw-r--r-- | tests/couch/test_state.py | 4 |
5 files changed, 227 insertions, 153 deletions
diff --git a/src/leap/soledad/common/couch/check.py b/src/leap/soledad/common/couch/check.py new file mode 100644 index 00000000..f55dd6f6 --- /dev/null +++ b/src/leap/soledad/common/couch/check.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# check.py +# Copyright (C) 2015,2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Database schema version verification +""" + +import os +import treq + +from six.moves.urllib.parse import urljoin +from twisted.internet import defer +from urlparse import urlsplit + +from twisted.internet import reactor + +from leap.soledad.common.couch import CONFIG_DOC_ID +from leap.soledad.common.couch import SCHEMA_VERSION +from leap.soledad.common.couch import SCHEMA_VERSION_KEY +from leap.soledad.common.errors import WrongCouchSchemaVersionError +from leap.soledad.common.errors import MissingCouchConfigDocumentError +from leap.soledad.common.log import getLogger + + +logger = getLogger(__name__) + + +@defer.inlineCallbacks +def _check_db_schema_version(url, db, auth, agent=None): + """ + Check if the schema version is up to date for a given database. + + :param url: the server base URL. + :type url: str + :param db: the database name. + :type db: str + :param auth: a tuple with (username, password) for acessing CouchDB. + :type auth: tuple(str, str) + :param agent: an optional agent for doing requests, used in tests. + :type agent: twisted.web.client.Agent + + :raise MissingCouchConfigDocumentError: raised when a database is not empty + but has no config document in it. + + :raise WrongCouchSchemaVersionError: raised when a config document was + found but the schema version is + different from what is expected. + """ + # if there are documents, ensure that a config doc exists + db_url = urljoin(url, '%s/' % db) + config_doc_url = urljoin(db_url, CONFIG_DOC_ID) + res = yield treq.get(config_doc_url, auth=auth, agent=agent) + + if res.code != 200 and res.code != 404: + raise Exception("Unexpected HTTP response code: %d" % res.code) + + elif res.code == 404: + res = yield treq.get(urljoin(db_url, '_all_docs'), auth=auth, + params={'limit': 1}, agent=agent) + docs = yield res.json() + if docs['total_rows'] != 0: + logger.error( + "Missing couch config document in database %s" % db) + raise MissingCouchConfigDocumentError(db) + + elif res.code == 200: + config_doc = yield res.json() + if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION: + logger.error( + "Unsupported database schema in database %s" % db) + raise WrongCouchSchemaVersionError(db) + + +def _stop(failure, reactor): + logger.error("Failure while checking schema versions: %r - %s" + % (failure, failure.message)) + reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) + reactor.stop() + + +@defer.inlineCallbacks +def check_schema_versions(couch_url, agent=None, reactor=reactor): + """ + Check that all user databases use the correct couch schema. + + :param couch_url: The URL for the couch database. + :type couch_url: str + :param agent: an optional agent for doing requests, used in tests. + :type agent: twisted.web.client.Agent + :param reactor: an optional reactor for stopping in case of errors, used + in tests. + :type reactor: twisted.internet.base.ReactorBase + """ + url = urlsplit(couch_url) + auth = (url.username, url.password) if url.username else None + url = "%s://%s:%d" % (url.scheme, url.hostname, url.port) + res = yield treq.get(urljoin(url, '_all_dbs'), auth=auth, agent=agent) + dbs = yield res.json() + deferreds = [] + semaphore = defer.DeferredSemaphore(20) + logger.info('Starting schema versions check...') + for db in dbs: + if not db.startswith('user-'): + continue + d = semaphore.run(_check_db_schema_version, url, db, auth, agent=agent) + d.addErrback(_stop, reactor=reactor) + deferreds.append(d) + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: logger.info('Finished schema versions check.')) + yield d diff --git a/src/leap/soledad/common/couch/state.py b/src/leap/soledad/common/couch/state.py index 5614b32f..f3645ab6 100644 --- a/src/leap/soledad/common/couch/state.py +++ b/src/leap/soledad/common/couch/state.py @@ -18,118 +18,20 @@ Server state using CouchDatabase as backend. """ import re -import os -import treq from six.moves.urllib.parse import urljoin -from twisted.internet import defer -from urlparse import urlsplit - -from twisted.internet import reactor from leap.soledad.common.log import getLogger from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.couch import CONFIG_DOC_ID -from leap.soledad.common.couch import SCHEMA_VERSION -from leap.soledad.common.couch import SCHEMA_VERSION_KEY from leap.soledad.common.command import exec_validated_cmd from leap.soledad.common.l2db.remote.server_state import ServerState from leap.soledad.common.l2db.errors import Unauthorized -from leap.soledad.common.errors import WrongCouchSchemaVersionError -from leap.soledad.common.errors import MissingCouchConfigDocumentError logger = getLogger(__name__) # -# Database schema version verification -# - -@defer.inlineCallbacks -def _check_db_schema_version(url, db, auth, agent=None): - """ - Check if the schema version is up to date for a given database. - - :param url: the server base URL. - :type url: str - :param db: the database name. - :type db: str - :param auth: a tuple with (username, password) for acessing CouchDB. - :type auth: tuple(str, str) - :param agent: an optional agent for doing requests, used in tests. - :type agent: twisted.web.client.Agent - - :raise MissingCouchConfigDocumentError: raised when a database is not empty - but has no config document in it. - - :raise WrongCouchSchemaVersionError: raised when a config document was - found but the schema version is - different from what is expected. - """ - # if there are documents, ensure that a config doc exists - db_url = urljoin(url, '%s/' % db) - config_doc_url = urljoin(db_url, CONFIG_DOC_ID) - res = yield treq.get(config_doc_url, auth=auth, agent=agent) - - if res.code != 200 and res.code != 404: - raise Exception("Unexpected HTTP response code: %d" % res.code) - - elif res.code == 404: - res = yield treq.get(urljoin(db_url, '_all_docs'), auth=auth, - params={'limit': 1}, agent=agent) - docs = yield res.json() - if docs['total_rows'] != 0: - logger.error( - "Missing couch config document in database %s" % db) - raise MissingCouchConfigDocumentError(db) - - elif res.code == 200: - config_doc = yield res.json() - if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION: - logger.error( - "Unsupported database schema in database %s" % db) - raise WrongCouchSchemaVersionError(db) - - -def _stop(failure, reactor): - logger.error("Failure while checking schema versions: %r - %s" - % (failure, failure.message)) - reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) - reactor.stop() - - -@defer.inlineCallbacks -def check_schema_versions(couch_url, agent=None, reactor=reactor): - """ - Check that all user databases use the correct couch schema. - - :param couch_url: The URL for the couch database. - :type couch_url: str - :param agent: an optional agent for doing requests, used in tests. - :type agent: twisted.web.client.Agent - :param reactor: an optional reactor for stopping in case of errors, used - in tests. - :type reactor: twisted.internet.base.ReactorBase - """ - url = urlsplit(couch_url) - auth = (url.username, url.password) if url.username else None - url = "%s://%s:%d" % (url.scheme, url.hostname, url.port) - res = yield treq.get(urljoin(url, '_all_dbs'), auth=auth, agent=agent) - dbs = yield res.json() - deferreds = [] - semaphore = defer.DeferredSemaphore(20) - for db in dbs: - if not db.startswith('user-'): - continue - d = semaphore.run(_check_db_schema_version, url, db, auth, agent=agent) - d.addErrback(_stop, reactor=reactor) - deferreds.append(d) - d = defer.gatherResults(deferreds, consumeErrors=True) - yield d - - -# # CouchDB Server state # diff --git a/src/leap/soledad/server/entrypoints.py b/src/leap/soledad/server/entrypoints.py index fa8c3ff2..0237978e 100644 --- a/src/leap/soledad/server/entrypoints.py +++ b/src/leap/soledad/server/entrypoints.py @@ -17,20 +17,14 @@ """ Entrypoints for the Soledad server. """ -import os - from twisted.internet import reactor -from twisted.python import threadpool from twisted.logger import Logger +from twisted.python import threadpool -from ..common.couch.state import check_schema_versions from .auth import localPortal, publicPortal from .session import SoledadSession -from ._config import get_config -from ._wsgi import init_couch_state -conf = get_config() log = Logger() @@ -49,25 +43,3 @@ class ServicesEntrypoint(SoledadSession): def __init__(self): portal = localPortal() SoledadSession.__init__(self, portal) - - -def check_conf(): - path = conf['blobs_path'] - blobs_not_empty = bool(os.path.exists(path) and os.listdir(path)) - if not conf['blobs'] and blobs_not_empty: - message = """ -** WARNING: Blobs is disabled, but blobs directory isn't empty. ** -** If it was previously enabled, disabling can cause data loss due blobs ** -** documents not being accessible to users. ** -** Blobs directory: %s -** REFUSING TO START. Please double check your configuration. ** - """ - log.error(message % path) - reactor.stop() - - -reactor.callWhenRunning(check_conf) -reactor.callWhenRunning(check_schema_versions, conf['couch_url']) -# see the comments in _wsgi.py regarding why couch state has to be -# initialized when the reactor is running -reactor.callWhenRunning(init_couch_state, conf) diff --git a/src/leap/soledad/server/server.tac b/src/leap/soledad/server/server.tac index efaca790..f6f784bc 100644 --- a/src/leap/soledad/server/server.tac +++ b/src/leap/soledad/server/server.tac @@ -1,45 +1,122 @@ +# -*- coding: utf-8 -*- +# server.tac +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. import sys import os from twisted.application import service, strports +from twisted.logger import Logger from twisted.web import server -from twisted.python import log +from leap.soledad.common.couch.check import check_schema_versions from leap.soledad.server import entrypoints +from leap.soledad.server._wsgi import init_couch_state +from leap.soledad.server._config import get_config -application = service.Application('soledad-server') -# local entrypoint -local_port = os.getenv('LOCAL_SERVICES_PORT', 2525) -local_description = 'tcp:%s:interface=127.0.0.1' % local_port -local_site = server.Site(entrypoints.ServicesEntrypoint()) +logger = Logger(__name__) + + +# +# necessary checks +# + +def _check_env(local_port, public_port): + if local_port == public_port: + logger.error("LOCAL_SERVICES_PORT and HTTPS_PORT can't be the same!") + sys.exit(20) + + if public_port is None and not os.getenv('DEBUG_SERVER'): + logger.error("HTTPS_PORT env var is required to be set!") + sys.exit(20) + + +def _check_conf(conf): + path = conf['blobs_path'] + blobs_not_empty = bool(os.path.exists(path) and os.listdir(path)) + if not conf['blobs'] and blobs_not_empty: + message = """ +** WARNING: Blobs is disabled, but blobs directory isn't empty. ** +** If it was previously enabled, disabling can cause data loss due blobs ** +** documents not being accessible to users. ** +** Blobs directory: %s +** REFUSING TO START. Please double check your configuration. ** + """ + logger.error(message % path) + sys.exit(20) + + +# +# service creation functions +# + +def _create_local_service(port, application): + logger.info('Starting local Services HTTP API') + desc = 'tcp:%s:interface=127.0.0.1' % port + site = server.Site(entrypoints.ServicesEntrypoint()) + service = strports.service(desc, site) + service.setServiceParent(application) -local_server = strports.service(local_description, local_site) -local_server.setServiceParent(application) -# public entrypoint -port = os.getenv('HTTPS_PORT', None) -if port == local_port: - log.err("LOCAL_SERVICES_PORT and HTTPS_PORT can't be the same!") - sys.exit(20) -if port: +def _get_tls_service_description(port): privateKey = os.getenv('PRIVKEY_PATH', '/etc/soledad/soledad-server.key') certKey = os.getenv('CERT_PATH', '/etc/soledad/soledad-server.pem') sslmethod = os.getenv('SSL_METHOD', 'SSLv23_METHOD') - - public_description = ':'.join([ + desc = ':'.join([ 'ssl', 'port=' + str(port), 'privateKey=' + privateKey, 'certKey=' + certKey, 'sslmethod=' + sslmethod]) -elif os.getenv('DEBUG_SERVER', False): - public_description = 'tcp:port=2424:interface=0.0.0.0' -else: - log.err("HTTPS_PORT env var is required to be set!") - sys.exit(20) + return desc + + +def _create_public_service(port, application): + logger.info('Starting public Users HTTP API') + if port: + desc = _get_tls_service_description(port) + else: + logger.warn('Using plain HTTP on public Users API.') + desc = 'tcp:port=2424:interface=0.0.0.0' + + site = server.Site(entrypoints.UsersEntrypoint()) + service = strports.service(desc, site) + service.setServiceParent(application) + -public_site = server.Site(entrypoints.UsersEntrypoint()) +def _create_services(local_port, public_port, application): + _create_local_service(local_port, application) + _create_public_service(public_port, application) -public_server = strports.service(public_description, public_site) -public_server.setServiceParent(application) + +# +# the application +# + +def _run(application): + local_port = os.getenv('LOCAL_SERVICES_PORT', 2525) + public_port = os.getenv('HTTPS_PORT', None) + conf = get_config() + _check_env(local_port, public_port) + _check_conf(conf) + d = check_schema_versions(conf['couch_url']) + d.addCallback(lambda _: init_couch_state(conf)) + d.addCallback(lambda _: _create_services(local_port, public_port, + application)) + + +application = service.Application('soledad-server') +_run(application) diff --git a/tests/couch/test_state.py b/tests/couch/test_state.py index 27db5abe..d978fe56 100644 --- a/tests/couch/test_state.py +++ b/tests/couch/test_state.py @@ -4,8 +4,8 @@ import pytest from leap.soledad.common.couch import CONFIG_DOC_ID from leap.soledad.common.couch import SCHEMA_VERSION from leap.soledad.common.couch import SCHEMA_VERSION_KEY -from leap.soledad.common.couch.state import _check_db_schema_version -from leap.soledad.common.couch.state import check_schema_versions +from leap.soledad.common.couch.check import _check_db_schema_version +from leap.soledad.common.couch.check import check_schema_versions from uuid import uuid4 from leap.soledad.common.errors import WrongCouchSchemaVersionError |