From a37333302801fcc4039897839f021ba127c9b846 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 21 Nov 2016 16:49:41 -0200 Subject: [refactor] separate server application into another file Conflicts: server/src/leap/soledad/server/__init__.py testing/tests/conftest.py --- scripts/docker/files/bin/setup-test-env.py | 2 +- server/pkg/soledad-server | 2 +- server/src/leap/soledad/server/__init__.py | 44 +++---------------------- server/src/leap/soledad/server/application.py | 47 +++++++++++++++++++++++++++ testing/tests/perf/conftest.py | 2 +- 5 files changed, 54 insertions(+), 43 deletions(-) create mode 100644 server/src/leap/soledad/server/application.py diff --git a/scripts/docker/files/bin/setup-test-env.py b/scripts/docker/files/bin/setup-test-env.py index 4868fd56..bbf5267c 100755 --- a/scripts/docker/files/bin/setup-test-env.py +++ b/scripts/docker/files/bin/setup-test-env.py @@ -256,7 +256,7 @@ def soledad_server_start(args): '--logfile=%s' % logfile, '--pidfile=%s' % pidfile, 'web', - '--wsgi=leap.soledad.server.application', + '--wsgi=leap.soledad.server.application.wsgi_application', '--port=%s' % port ] if args.no_daemonize: diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server index 9dada6a0..d9dab040 100644 --- a/server/pkg/soledad-server +++ b/server/pkg/soledad-server @@ -11,7 +11,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/soledad.pid -OBJ=leap.soledad.server.application +OBJ=leap.soledad.server.application.wsgi_application HTTPS_PORT=2424 CONFDIR=/etc/soledad CERT_PATH="${CONFDIR}/soledad-server.pem" diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index d154e3fe..d8243c19 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -25,7 +25,9 @@ General information This is written as a Twisted application and intended to be run using the twistd command. To start the soledad server, run: - twistd -n web --wsgi=leap.soledad.server.application --port=X + twistd -n web \ + --wsgi=leap.soledad.server.application.wsgi_application \ + --port=X An initscript is included and will be installed system wide to make it feasible to start and stop the Soledad server service using a standard @@ -84,24 +86,17 @@ import urlparse import sys from leap.soledad.common.l2db.remote import http_app, utils +from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.server.auth import SoledadTokenAuthMiddleware -from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.server.sync import SyncResource from leap.soledad.server.sync import MAX_REQUEST_SIZE from leap.soledad.server.sync import MAX_ENTRY_SIZE -from leap.soledad.server.config import load_configuration - -from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.common.backend import SoledadBackend -from leap.soledad.common.couch.state import CouchServerState from ._version import get_versions __all__ = [ 'SoledadApp', - 'application', '__version__', ] @@ -255,36 +250,5 @@ class HTTPInvocationByMethodWithBody( http_app.HTTPInvocationByMethodWithBody = HTTPInvocationByMethodWithBody -# ---------------------------------------------------------------------------- -# Run as Twisted WSGI Resource -# ---------------------------------------------------------------------------- - - -def _load_config(): - conf = load_configuration('/etc/soledad/soledad-server.conf') - return conf['soledad-server'] - - -def _get_couch_state(): - conf = _load_config() - state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], - check_schema_versions=True) - SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) - return state - -try: - _couch_state = _get_couch_state() - # a WSGI application that may be used by `twistd -web` - application = GzipMiddleware( - SoledadTokenAuthMiddleware(SoledadApp(_couch_state))) -except: - pass - - -# another WSGI application in which we bypass token auth middleware for ease of -# mind while debugging in your local environment -# debug_local_application_do_not_use = SoledadApp(_couch_state) - - __version__ = get_versions()['version'] del get_versions diff --git a/server/src/leap/soledad/server/application.py b/server/src/leap/soledad/server/application.py new file mode 100644 index 00000000..a9ebcaf6 --- /dev/null +++ b/server/src/leap/soledad/server/application.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# application.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from leap.soledad.server.gzip_middleware import GzipMiddleware +from leap.soledad.server.config import load_configuration +from leap.soledad.common.backend import SoledadBackend +from leap.soledad.common.couch.state import CouchServerState + + +# ---------------------------------------------------------------------------- +# Run as Twisted WSGI Resource +# ---------------------------------------------------------------------------- + +def _load_config(): + conf = load_configuration('/etc/soledad/soledad-server.conf') + return conf['soledad-server'] + + +def _get_couch_state(): + conf = _load_config() + state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'], + check_schema_versions=True) + SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) + return state + + +_couch_state = _get_couch_state() + +# a WSGI application that may be used by `twistd -web` +wsgi_application = GzipMiddleware( + SoledadTokenAuthMiddleware(SoledadApp(_couch_state))) diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py index 5ac1f3c0..6fa6b2c0 100644 --- a/testing/tests/perf/conftest.py +++ b/testing/tests/perf/conftest.py @@ -162,7 +162,7 @@ class SoledadServer(object): '--logfile=%s' % self._logfile, '--pidfile=%s' % self._pidfile, 'web', - '--wsgi=leap.soledad.server.application', + '--wsgi=leap.soledad.server.application.wsgi_application', '--port=2424' ]) -- cgit v1.2.3 From bb3cc25758e7c97285195eadedb81f59a6829bec Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 22 Nov 2016 18:17:55 -0200 Subject: [feat] improve missing couch config doc error logging --- common/src/leap/soledad/common/couch/state.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py index 523ac0b0..a7f5b7b6 100644 --- a/common/src/leap/soledad/common/couch/state.py +++ b/common/src/leap/soledad/common/couch/state.py @@ -101,10 +101,15 @@ class CouchServerState(ServerState): config_doc = db.get(CONFIG_DOC_ID) if config_doc: if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION: + logger.error( + "Unsupported database schema in database %s" % dbname) raise WrongCouchSchemaVersionError(dbname) else: result = db.view('_all_docs', limit=1) if result.total_rows != 0: + logger.error( + "Missing couch config document in database %s" + % dbname) raise MissingCouchConfigDocumentError(dbname) def open_database(self, dbname): -- cgit v1.2.3 From 18380ecd9fa22e456d97a303419a6a377279a79a Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 22 Nov 2016 21:50:22 -0200 Subject: [bug] delay couch state initialization The couch state checks if all user databases have the correct schema version on initialization, and may log errors and raise exceptions if that is not the case. Because we are currently using `twistd web --wsgi`, if those errors are logged too early the reactor may have not been started and the twistd logging facilities may not have been initialized. This commit delays the state initialization until the reactor has been started, to make sure any errors raised and logged in that stage will actually reach the logfile. --- server/src/leap/soledad/server/application.py | 42 ++++++++++++++++++++++----- server/src/leap/soledad/server/auth.py | 12 +++++--- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/server/src/leap/soledad/server/application.py b/server/src/leap/soledad/server/application.py index a9ebcaf6..17296425 100644 --- a/server/src/leap/soledad/server/application.py +++ b/server/src/leap/soledad/server/application.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # application.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2016 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +A WSGI application to serve as the root resource of the webserver. + +Use it like this: + + twistd web --wsgi=leap.soledad.server.application.wsgi_application +""" +from twisted.internet import reactor from leap.soledad.server import SoledadApp from leap.soledad.server.auth import SoledadTokenAuthMiddleware @@ -21,11 +29,11 @@ from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.server.config import load_configuration from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.log import getLogger -# ---------------------------------------------------------------------------- -# Run as Twisted WSGI Resource -# ---------------------------------------------------------------------------- +__all__ = ['wsgi_application'] + def _load_config(): conf = load_configuration('/etc/soledad/soledad-server.conf') @@ -40,8 +48,26 @@ def _get_couch_state(): return state -_couch_state = _get_couch_state() +_app = SoledadTokenAuthMiddleware(SoledadApp(None)) # delay state init +wsgi_application = GzipMiddleware(_app) + + +# During its initialization, the couch state verifies if all user databases +# contain a config document with the correct couch schema version stored, and +# will log an error and raise an exception if that is not the case. +# +# If this verification made too early (i.e. before the reactor has started and +# the twistd web logging facilities have been setup), the logging will not +# work. Because of that, we delay couch state initialization until the reactor +# is running. + +def _init_couch_state(_app): + try: + _app.state = _get_couch_state() + except Exception as e: + logger = getLogger() + logger.error(str(e)) + reactor.stop() + -# a WSGI application that may be used by `twistd -web` -wsgi_application = GzipMiddleware( - SoledadTokenAuthMiddleware(SoledadApp(_couch_state))) +reactor.callWhenRunning(_init_couch_state, _app) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index b7186b3b..b0764569 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -343,9 +343,13 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." - def __init__(self, app): - self._state = app.state - super(SoledadTokenAuthMiddleware, self).__init__(app) + def _get_state(self): + return self._app.state + + def _set_state(self, state): + self._app.state = state + + state = property(_get_state, _set_state) def _verify_authentication_scheme(self, scheme): """ @@ -379,7 +383,7 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): """ token = auth_data # we expect a cleartext token at this point try: - return self._state.verify_token(uuid, token) + return self.state.verify_token(uuid, token) except Exception as e: logger.error(e) return False -- cgit v1.2.3 From 3369302f8b223c99efa0348821b9f73767ca8579 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 23 Nov 2016 10:29:19 -0200 Subject: [bug] patch twisted logger so it works with twistd --syslog --- common/src/leap/soledad/common/log.py | 40 ++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/common/src/leap/soledad/common/log.py b/common/src/leap/soledad/common/log.py index 3f026045..59a47726 100644 --- a/common/src/leap/soledad/common/log.py +++ b/common/src/leap/soledad/common/log.py @@ -25,9 +25,47 @@ to stdout, mainly for development purposes. import os import sys +import time from twisted.logger import Logger from twisted.logger import textFileLogObserver +from twisted.logger import LogLevel +from twisted.logger import InvalidLogLevelError +from twisted.python.failure import Failure + + +# What follows is a patched class to correctly log namespace and level when +# using the default formatter and --syslog option in twistd. This seems to be a +# known bug but it has not been reported to upstream yet. + +class SyslogLogger(Logger): + + def emit(self, level, format=None, **kwargs): + if level not in LogLevel.iterconstants(): + self.failure( + "Got invalid log level {invalidLevel!r} in {logger}.emit().", + Failure(InvalidLogLevelError(level)), + invalidLevel=level, + logger=self, + ) + return + + event = kwargs + event.update( + log_logger=self, log_level=level, log_namespace=self.namespace, + log_source=self.source, log_format=format, log_time=time.time(), + ) + + # ---------------------------------8<--------------------------------- + # this is a workaround for the mess between twisted's legacy log system + # and twistd's --syslog option. + event["system"] = "%s#%s" % (self.namespace, level.name) + # ---------------------------------8<--------------------------------- + + if "log_trace" in event: + event["log_trace"].append((self, self.observer)) + + self.observer(event) def getLogger(*args, **kwargs): @@ -39,7 +77,7 @@ def getLogger(*args, **kwargs): if os.environ.get('SOLEDAD_LOG_TO_STDOUT'): kwargs({'observer': textFileLogObserver(sys.stdout)}) - return Logger(*args, **kwargs) + return SyslogLogger(*args, **kwargs) __all__ = ['getLogger'] -- cgit v1.2.3 From c44783f3c4f92df2e2f59837c7cc450bf3fdc818 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 26 Nov 2016 19:31:25 -0300 Subject: [bug] fix import on create-user-db --- server/pkg/create-user-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index b955b4c3..9e2b6b50 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -22,7 +22,7 @@ import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch.state import is_db_name_valid from leap.soledad.common.couch import list_users_dbs -from leap.soledad.server import load_configuration +from leap.soledad.server.config import load_configuration BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False) -- cgit v1.2.3 From dff666e51240393ebbf2a2118e6e51130fe91f8c Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 27 Nov 2016 11:59:38 -0200 Subject: [pkg] update changelog --- CHANGELOG.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ded2cac9..12cb56ab 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,15 @@ +0.9.1 - 27 November, 2016 ++++++++++++++++++++++++++ + +Server side bug fixes +~~~~~~~~~~~~~~~~~~~~~ + +- fix import on create-user-db script +- patch twisted logger so it works with twistd --syslog +- delay couch state initialization +- improve missing couch config doc error logging +- separate server application into another file + 0.9.0 - 11 November, 2016 +++++++++++++++++++++++++ -- cgit v1.2.3