From 1b07d2b54208d6b19135e771162f7e39e02c1f97 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 14 Oct 2015 11:50:48 -0300 Subject: [bug] reduce overall sync session caching to 120s It was 3600s, but closing connections seems to yet depend on garbage collection and now causes server to leak file handlers. 120s should be enough to a sync session finish. Also, lowering this value will only make very long syncs use more of couch every 2 minutes, while raising this value will keep memory busy for useless time. --- server/src/leap/soledad/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 619be565..b3fb2b84 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -185,7 +185,7 @@ class SyncResource(http_app.SyncResource): :type ensure: bool """ # create or open the database - cache = get_cache_for('db-' + sync_id + self.dbname) + cache = get_cache_for('db-' + sync_id + self.dbname, expire=120) if ensure: db, self.replica_uid = self.state.ensure_database(self.dbname) elif cache and 'instance' in cache: -- cgit v1.2.3 From 0f8364b6a38793f9b0f8596c1b98a590131fbb41 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 19 Oct 2015 16:27:43 -0300 Subject: [bug] remove instance caching couchdb library relies on garbage collector to close remaining connections. Somehow, caching the instance is avoiding gc to call __del__ on underlying couchdb sessions. --- server/src/leap/soledad/server/sync.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index b3fb2b84..92b29102 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -188,12 +188,9 @@ class SyncResource(http_app.SyncResource): cache = get_cache_for('db-' + sync_id + self.dbname, expire=120) if ensure: db, self.replica_uid = self.state.ensure_database(self.dbname) - elif cache and 'instance' in cache: - db = cache['instance'] else: db = self.state.open_database(self.dbname) db.init_caching(cache) - cache['instance'] = db # validate the information the client has about server replica db.validate_gen_and_trans_id( last_known_generation, last_known_trans_id) -- cgit v1.2.3 From f8d38125098829fe50199725545365d6d2a889a6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 26 Oct 2015 18:50:20 -0300 Subject: [feat] read security doc from configuration LEAP Platform needs to granularly allow access on user database for other services, like mx. This is now possible by editing soledad-server.conf file. A new section 'database-security' was added and it is parsed during 'create-user-db' to be set on security design document, present on every per-user database. --- server/src/leap/soledad/server/__init__.py | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index f64d07bf..4d03c82a 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -272,6 +272,20 @@ http_app.HTTPInvocationByMethodWithBody = HTTPInvocationByMethodWithBody # ---------------------------------------------------------------------------- # Auxiliary functions # ---------------------------------------------------------------------------- +CONFIG_DEFAULTS = { + 'soledad-server': { + 'couch_url': 'http://localhost:5984', + 'create_cmd': None, + 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', + }, + 'database-security': { + 'members': ['soledad'], + 'members_roles': [], + 'admins': [], + 'admins_roles': [] + } +} + def load_configuration(file_path): """ @@ -283,17 +297,18 @@ def load_configuration(file_path): @return: A dictionary with the configuration. @rtype: dict """ - defaults = { - 'couch_url': 'http://localhost:5984', - 'create_cmd': None, - 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', - } + defaults = dict(CONFIG_DEFAULTS) config = configparser.ConfigParser() config.read(file_path) - if 'soledad-server' in config: - for key in defaults: - if key in config['soledad-server']: - defaults[key] = config['soledad-server'][key] + for section in defaults.keys(): + if section in config: + for key in defaults[section]: + if key in config[section]: + defaults[section][key] = config[section][key] + for key, value in defaults['database-security'].iteritems(): + if type(value) is not unicode: continue + defaults['database-security'][key] = \ + [item.strip() for item in value.split(',')] # TODO: implement basic parsing/sanitization of options comming from # config file. return defaults @@ -305,6 +320,7 @@ def load_configuration(file_path): def application(environ, start_response): conf = load_configuration('/etc/soledad/soledad-server.conf') + conf = conf['soledad-server'] state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( -- cgit v1.2.3 From 7c50a100a46c69f759dc22165cec9b8098cac3a4 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Tue, 27 Oct 2015 10:20:56 +0100 Subject: [style] fix pep8 warnigs --- server/src/leap/soledad/server/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 4d03c82a..fe67e45f 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -306,7 +306,8 @@ def load_configuration(file_path): if key in config[section]: defaults[section][key] = config[section][key] for key, value in defaults['database-security'].iteritems(): - if type(value) is not unicode: continue + if type(value) is not unicode: + continue defaults['database-security'][key] = \ [item.strip() for item in value.split(',')] # TODO: implement basic parsing/sanitization of options comming from -- cgit v1.2.3 From 3b869fb7ffedc88c738e0a17347b9506d242cabe Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 27 Oct 2015 13:58:56 -0300 Subject: [feat] remove SSL tsafe monkeypatch This was added for a Twisted 12 bug that should be gone by now. --- server/src/leap/soledad/server/__init__.py | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index fe67e45f..618ccb2b 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -94,12 +94,6 @@ from u1db.remote import http_app, utils from ._version import get_versions -# Keep OpenSSL's tsafe before importing Twisted submodules so we can put -# it back if Twisted==12.0.0 messes with it. -from OpenSSL import tsafe - -from twisted import version - from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.server.lock_resource import LockResource @@ -112,13 +106,6 @@ from leap.soledad.server.sync import ( from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import CouchServerState -old_tsafe = tsafe - -if version.base() == "12.0.0": - # Put OpenSSL's tsafe back into place. This can probably be removed if we - # come to use Twisted>=12.3.0. - sys.modules['OpenSSL.tsafe'] = old_tsafe - # ---------------------------------------------------------------------------- # Soledad WSGI application # ---------------------------------------------------------------------------- -- cgit v1.2.3 From b0557f9c1d5e6f153f926ba3cb5876453ef23a10 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 1 Oct 2015 15:07:25 -0300 Subject: [refactor] separate SoledadBackend from CouchDatabase CouchDatabase was renamed to SoledadBackend and a new class CouchDatabase was created to hold all couchdb code. This should make SoledadBackend less tied to database implementation. A few more separations are needed to split into modules. --- server/src/leap/soledad/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 92b29102..db25c406 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -32,7 +32,7 @@ class SyncExchange(sync.SyncExchange): def __init__(self, db, source_replica_uid, last_known_generation, sync_id): """ :param db: The target syncing database. - :type db: CouchDatabase + :type db: SoledadBackend :param source_replica_uid: The uid of the source syncing replica. :type source_replica_uid: str :param last_known_generation: The last target replica generation the -- cgit v1.2.3 From f0b96af943dcb6c8cde4f6d4280186d78c78096c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 13 Oct 2015 21:34:40 -0300 Subject: [refactor] split out backend from couch database First step of splitting classes across files on common. backend.py holds SoledadBackend (generic backend logic) couch/ is now a directory with old code inside __init__.py and CouchServerState on state.py Also removed mock IndexedSoledadBackend, since Soledad does not support indexing due to encryption on server side. Also fixed DesignDocUnknownError to show up what is the message of the original exception. It was being lost. --- server/src/leap/soledad/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 618ccb2b..00e1e9fb 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -104,7 +104,7 @@ from leap.soledad.server.sync import ( ) from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.common.couch import CouchServerState +from leap.soledad.common.couch.state import CouchServerState # ---------------------------------------------------------------------------- # Soledad WSGI application -- cgit v1.2.3 From 421691ef71019d0bcd4447a773efa5e9b15b0c71 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 27 Oct 2015 16:48:39 -0300 Subject: [refactor] token verification moved to couch module + tests Added tests for this token verification as it wasn't covered. Then moved it to the new couch module that implements a couch storage. The ServerState was chosen to hold the verify_token method. CouchServerState holds the current implementation, which is called on authentication middleware as the new test shows. --- server/src/leap/soledad/server/auth.py | 52 ++-------------------------------- 1 file changed, 2 insertions(+), 50 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 02b54cca..01baf1ce 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -21,20 +21,16 @@ Authentication facilities for Soledad Server. """ -import time import httplib import json from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper -from couchdb.client import Server from twisted.python import log -from hashlib import sha512 from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import USER_DB_PREFIX -from leap.soledad.common.errors import InvalidAuthTokenError class URLToAuthorization(object): @@ -193,6 +189,7 @@ class SoledadAuthMiddleware(object): @type prefix: str """ self._app = app + self._state = app.state def _error(self, start_response, status, description, message=None): """ @@ -351,12 +348,6 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): Token based authentication. """ - TOKENS_DB_PREFIX = "tokens_" - TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds - TOKENS_TYPE_KEY = "type" - TOKENS_TYPE_DEF = "Token" - TOKENS_USER_ID_KEY = "user_id" - TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." def _verify_authentication_scheme(self, scheme): @@ -391,50 +382,11 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): """ token = auth_data # we expect a cleartext token at this point try: - return self._verify_token_in_couch(uuid, token) - except InvalidAuthTokenError: - raise + return self._state.verify_token(uuid, token) except Exception as e: log.err(e) return False - def _verify_token_in_couch(self, uuid, token): - """ - Query couchdb to decide if C{token} is valid for C{uuid}. - - @param uuid: The user uuid. - @type uuid: str - @param token: The token. - @type token: str - - @raise InvalidAuthTokenError: Raised when token received from user is - either missing in the tokens db or is - invalid. - """ - server = Server(url=self._app.state.couch_url) - # the tokens db rotates every 30 days, and the current db name is - # "tokens_NNN", where NNN is the number of seconds since epoch divided - # by the rotate period in seconds. When rotating, old and new tokens - # db coexist during a certain window of time and valid tokens are - # replicated from the old db to the new one. See: - # https://leap.se/code/issues/6785 - dbname = self.TOKENS_DB_PREFIX + \ - str(int(time.time() / self.TOKENS_DB_EXPIRE)) - db = server[dbname] - # lookup key is a hash of the token to prevent timing attacks. - token = db.get(sha512(token).hexdigest()) - if token is None: - raise InvalidAuthTokenError() - # we compare uuid hashes to avoid possible timing attacks that - # might exploit python's builtin comparison operator behaviour, - # which fails immediatelly when non-matching bytes are found. - couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest() - req_uuid_hash = sha512(uuid).digest() - if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ - or couch_uuid_hash != req_uuid_hash: - raise InvalidAuthTokenError() - return True - def _get_auth_error_string(self): """ Get the error string for token auth. -- cgit v1.2.3 From eaf19626a57d5f5325653fee3aea9db27c9531fe Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 28 Oct 2015 17:58:27 -0300 Subject: [refactor] resource logic encapsulation Creating a resource from a path to use get_json causes a lot of dirty code and unexplained things like response[2]. This commit extracts that logic into a helper to let it more clear about what is happening. --- server/src/leap/soledad/server/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 01baf1ce..ccbd6fbd 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -189,7 +189,6 @@ class SoledadAuthMiddleware(object): @type prefix: str """ self._app = app - self._state = app.state def _error(self, start_response, status, description, message=None): """ @@ -350,6 +349,10 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." + def __init__(self, app): + self._state = app.state + super(SoledadTokenAuthMiddleware, self).__init__(app) + def _verify_authentication_scheme(self, scheme): """ Verify if authentication scheme is valid. -- cgit v1.2.3