Merge tag '0.9.1'

Tag version 0.9.1

# gpg: Signature made Sun 27 Nov 2016 12:01:10 PM BRST
# gpg:                using RSA key 0x6071E70DCACC60B2
# gpg: a verificar a base de dados de confiança
# gpg: public key of ultimately trusted key 0x030F1C082D2327BE not found
# gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model
# gpg: depth: 0  valid:  11  signed:  71  trust: 0-, 0q, 0n, 0m, 0f, 11u
# gpg: depth: 1  valid:  71  signed:  64  trust: 69-, 0q, 0n, 0m, 2f, 0u
# gpg: depth: 2  valid:  55  signed:  74  trust: 55-, 0q, 0n, 0m, 0f, 0u
# gpg: proxima verificação da base de dados de confiança a 2016-12-12
# gpg: Good signature from "drebs (work key) <db@leap.se>" [ultimate]
# gpg:                 aka "drebs (work key) <drebs@leap.se>" [ultimate]
# Impressão da chave primária: 9F73 295B 6306 E06F 3151  99AE 6071 E70D CACC 60B2

10 files changed, 145 insertions, 49 deletions

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index ded2cac9..12cb56ab 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,15 @@
+0.9.1 - 27 November, 2016
++++++++++++++++++++++++++
+
+Server side bug fixes
+~~~~~~~~~~~~~~~~~~~~~
+
+- fix import on create-user-db script
+- patch twisted logger so it works with twistd --syslog
+- delay couch state initialization
+- improve missing couch config doc error logging
+- separate server application into another file
+
 0.9.0 - 11 November, 2016
 +++++++++++++++++++++++++
 
diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py
index 523ac0b0..a7f5b7b6 100644
--- a/common/src/leap/soledad/common/couch/state.py
+++ b/common/src/leap/soledad/common/couch/state.py
@@ -101,10 +101,15 @@ class CouchServerState(ServerState):
             config_doc = db.get(CONFIG_DOC_ID)
             if config_doc:
                 if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION:
+                    logger.error(
+                        "Unsupported database schema in database %s" % dbname)
                     raise WrongCouchSchemaVersionError(dbname)
             else:
                 result = db.view('_all_docs', limit=1)
                 if result.total_rows != 0:
+                    logger.error(
+                        "Missing couch config document in database %s"
+                        % dbname)
                     raise MissingCouchConfigDocumentError(dbname)
 
     def open_database(self, dbname):
diff --git a/common/src/leap/soledad/common/log.py b/common/src/leap/soledad/common/log.py
index 3f026045..59a47726 100644
--- a/common/src/leap/soledad/common/log.py
+++ b/common/src/leap/soledad/common/log.py
@@ -25,9 +25,47 @@ to stdout, mainly for development purposes.
 
 import os
 import sys
+import time
 
 from twisted.logger import Logger
 from twisted.logger import textFileLogObserver
+from twisted.logger import LogLevel
+from twisted.logger import InvalidLogLevelError
+from twisted.python.failure import Failure
+
+
+# What follows is a patched class to correctly log namespace and level when
+# using the default formatter and --syslog option in twistd. This seems to be a
+# known bug but it has not been reported to upstream yet.
+
+class SyslogLogger(Logger):
+
+    def emit(self, level, format=None, **kwargs):
+        if level not in LogLevel.iterconstants():
+            self.failure(
+                "Got invalid log level {invalidLevel!r} in {logger}.emit().",
+                Failure(InvalidLogLevelError(level)),
+                invalidLevel=level,
+                logger=self,
+            )
+            return
+
+        event = kwargs
+        event.update(
+            log_logger=self, log_level=level, log_namespace=self.namespace,
+            log_source=self.source, log_format=format, log_time=time.time(),
+        )
+
+        # ---------------------------------8<---------------------------------
+        # this is a workaround for the mess between twisted's legacy log system
+        # and twistd's --syslog option.
+        event["system"] = "%s#%s" % (self.namespace, level.name)
+        # ---------------------------------8<---------------------------------
+
+        if "log_trace" in event:
+            event["log_trace"].append((self, self.observer))
+
+        self.observer(event)
 
 
 def getLogger(*args, **kwargs):
@@ -39,7 +77,7 @@ def getLogger(*args, **kwargs):
     if os.environ.get('SOLEDAD_LOG_TO_STDOUT'):
         kwargs({'observer': textFileLogObserver(sys.stdout)})
 
-    return Logger(*args, **kwargs)
+    return SyslogLogger(*args, **kwargs)
 
 
 __all__ = ['getLogger']
diff --git a/scripts/docker/files/bin/setup-test-env.py b/scripts/docker/files/bin/setup-test-env.py
index 4868fd56..bbf5267c 100755
--- a/scripts/docker/files/bin/setup-test-env.py
+++ b/scripts/docker/files/bin/setup-test-env.py
@@ -256,7 +256,7 @@ def soledad_server_start(args):
         '--logfile=%s' % logfile,
         '--pidfile=%s' % pidfile,
         'web',
-        '--wsgi=leap.soledad.server.application',
+        '--wsgi=leap.soledad.server.application.wsgi_application',
         '--port=%s' % port
     ]
     if args.no_daemonize:
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index b955b4c3..9e2b6b50 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -22,7 +22,7 @@ import argparse
 from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.couch.state import is_db_name_valid
 from leap.soledad.common.couch import list_users_dbs
-from leap.soledad.server import load_configuration
+from leap.soledad.server.config import load_configuration
 
 
 BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False)
diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server
index 9dada6a0..d9dab040 100644
--- a/server/pkg/soledad-server
+++ b/server/pkg/soledad-server
@@ -11,7 +11,7 @@
 
 PATH=/sbin:/bin:/usr/sbin:/usr/bin
 PIDFILE=/var/run/soledad.pid
-OBJ=leap.soledad.server.application
+OBJ=leap.soledad.server.application.wsgi_application
 HTTPS_PORT=2424
 CONFDIR=/etc/soledad
 CERT_PATH="${CONFDIR}/soledad-server.pem"
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index d154e3fe..d8243c19 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -25,7 +25,9 @@ General information
 This is written as a Twisted application and intended to be run using the
 twistd command. To start the soledad server, run:
 
-    twistd -n web --wsgi=leap.soledad.server.application --port=X
+    twistd -n web \
+        --wsgi=leap.soledad.server.application.wsgi_application \
+        --port=X
 
 An initscript is included and will be installed system wide to make it
 feasible to start and stop the Soledad server service using a standard
@@ -84,24 +86,17 @@ import urlparse
 import sys
 
 from leap.soledad.common.l2db.remote import http_app, utils
+from leap.soledad.common import SHARED_DB_NAME
 
-from leap.soledad.server.auth import SoledadTokenAuthMiddleware
-from leap.soledad.server.gzip_middleware import GzipMiddleware
 from leap.soledad.server.sync import SyncResource
 from leap.soledad.server.sync import MAX_REQUEST_SIZE
 from leap.soledad.server.sync import MAX_ENTRY_SIZE
-from leap.soledad.server.config import load_configuration
-
-from leap.soledad.common import SHARED_DB_NAME
-from leap.soledad.common.backend import SoledadBackend
-from leap.soledad.common.couch.state import CouchServerState
 
 from ._version import get_versions
 
 
 __all__ = [
     'SoledadApp',
-    'application',
     '__version__',
 ]
 
@@ -255,36 +250,5 @@ class HTTPInvocationByMethodWithBody(
 http_app.HTTPInvocationByMethodWithBody = HTTPInvocationByMethodWithBody
 
 
-# ----------------------------------------------------------------------------
-# Run as Twisted WSGI Resource
-# ----------------------------------------------------------------------------
-
-
-def _load_config():
-    conf = load_configuration('/etc/soledad/soledad-server.conf')
-    return conf['soledad-server']
-
-
-def _get_couch_state():
-    conf = _load_config()
-    state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'],
-                             check_schema_versions=True)
-    SoledadBackend.BATCH_SUPPORT = conf.get('batching', False)
-    return state
-
-try:
-    _couch_state = _get_couch_state()
-    # a WSGI application that may be used by `twistd -web`
-    application = GzipMiddleware(
-        SoledadTokenAuthMiddleware(SoledadApp(_couch_state)))
-except:
-    pass
-
-
-# another WSGI application in which we bypass token auth middleware for ease of
-# mind while debugging in your local environment
-# debug_local_application_do_not_use = SoledadApp(_couch_state)
-
-
 __version__ = get_versions()['version']
 del get_versions
diff --git a/server/src/leap/soledad/server/application.py b/server/src/leap/soledad/server/application.py
new file mode 100644
index 00000000..17296425
--- /dev/null
+++ b/server/src/leap/soledad/server/application.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# application.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+A WSGI application to serve as the root resource of the webserver.
+
+Use it like this:
+
+  twistd web --wsgi=leap.soledad.server.application.wsgi_application
+"""
+from twisted.internet import reactor
+
+from leap.soledad.server import SoledadApp
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+from leap.soledad.server.gzip_middleware import GzipMiddleware
+from leap.soledad.server.config import load_configuration
+from leap.soledad.common.backend import SoledadBackend
+from leap.soledad.common.couch.state import CouchServerState
+from leap.soledad.common.log import getLogger
+
+
+__all__ = ['wsgi_application']
+
+
+def _load_config():
+    conf = load_configuration('/etc/soledad/soledad-server.conf')
+    return conf['soledad-server']
+
+
+def _get_couch_state():
+    conf = _load_config()
+    state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'],
+                             check_schema_versions=True)
+    SoledadBackend.BATCH_SUPPORT = conf.get('batching', False)
+    return state
+
+
+_app = SoledadTokenAuthMiddleware(SoledadApp(None))  # delay state init
+wsgi_application = GzipMiddleware(_app)
+
+
+# During its initialization, the couch state verifies if all user databases
+# contain a config document with the correct couch schema version stored, and
+# will log an error and raise an exception if that is not the case.
+#
+# If this verification made too early (i.e.  before the reactor has started and
+# the twistd web logging facilities have been setup), the logging will not
+# work.  Because of that, we delay couch state initialization until the reactor
+# is running.
+
+def _init_couch_state(_app):
+    try:
+        _app.state = _get_couch_state()
+    except Exception as e:
+        logger = getLogger()
+        logger.error(str(e))
+        reactor.stop()
+
+
+reactor.callWhenRunning(_init_couch_state, _app)
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index b7186b3b..b0764569 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -343,9 +343,13 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
 
     TOKEN_AUTH_ERROR_STRING = "Incorrect address or token."
 
-    def __init__(self, app):
-        self._state = app.state
-        super(SoledadTokenAuthMiddleware, self).__init__(app)
+    def _get_state(self):
+        return self._app.state
+
+    def _set_state(self, state):
+        self._app.state = state
+
+    state = property(_get_state, _set_state)
 
     def _verify_authentication_scheme(self, scheme):
         """
@@ -379,7 +383,7 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
         """
         token = auth_data  # we expect a cleartext token at this point
         try:
-            return self._state.verify_token(uuid, token)
+            return self.state.verify_token(uuid, token)
         except Exception as e:
             logger.error(e)
             return False
diff --git a/testing/tests/perf/conftest.py b/testing/tests/perf/conftest.py
index 5ac1f3c0..6fa6b2c0 100644
--- a/testing/tests/perf/conftest.py
+++ b/testing/tests/perf/conftest.py
@@ -162,7 +162,7 @@ class SoledadServer(object):
             '--logfile=%s' % self._logfile,
             '--pidfile=%s' % self._pidfile,
             'web',
-            '--wsgi=leap.soledad.server.application',
+            '--wsgi=leap.soledad.server.application.wsgi_application',
             '--port=2424'
         ])