From a10ae06bb33c6caaaf3a8474cc6d7b01c33abc9f Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 30 Aug 2015 05:17:20 -0300 Subject: [feat] first draft of sync_state in memory This commit changes sync_state to be in memory, with all tests passing. The memory variable for now is a dict with each key composed by source_replica_uid and sync_id, replicating CouchDB implementation. Next steps includes migrating this to Beaker and refactor/clean up code. Changed the module's INFO dict to use Beaker's caching and adapted methods to get and save from it. Still needs refactoring, all tests passes. Beaker is now using memory as default; It is configurable, but we aren't opening the possibility of config now for security. We need to check what can be misconfigured first. We are not sure if beaker will be the definitive solution for server side caching. This change isolates it with more granularity. In order to replace it, just change get_cache_for to return the proper caching object using another implementation. This caching object is supposed to behave as a dict. --- server/src/leap/soledad/server/caching.py | 32 ++++++ server/src/leap/soledad/server/state.py | 143 ++++++++++++++++++++++++ server/src/leap/soledad/server/sync.py | 173 +----------------------------- 3 files changed, 177 insertions(+), 171 deletions(-) create mode 100644 server/src/leap/soledad/server/caching.py create mode 100644 server/src/leap/soledad/server/state.py (limited to 'server/src') diff --git a/server/src/leap/soledad/server/caching.py b/server/src/leap/soledad/server/caching.py new file mode 100644 index 00000000..cd5d8dd4 --- /dev/null +++ b/server/src/leap/soledad/server/caching.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# caching.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Server side caching. Using beaker for now. +""" +from beaker.cache import CacheManager + + +def setup_caching(): + _cache_manager = CacheManager(type='memory') + return _cache_manager + + +_cache_manager = setup_caching() + + +def get_cache_for(key): + return _cache_manager.get_cache(key) diff --git a/server/src/leap/soledad/server/state.py b/server/src/leap/soledad/server/state.py new file mode 100644 index 00000000..d1225170 --- /dev/null +++ b/server/src/leap/soledad/server/state.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# state.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Server side synchronization infrastructure. +""" +from leap.soledad.server import caching + + +class ServerSyncState(object): + """ + The state of one sync session, as stored on backend server. + + On server side, the ongoing syncs metadata is maintained in a caching layer. + """ + + def __init__(self, source_replica_uid, sync_id): + """ + Initialize the sync state object. + + :param sync_id: The id of current sync + :type sync_id: str + :param source_replica_uid: The source replica uid + :type source_replica_uid: str + """ + self._source_replica_uid = source_replica_uid + self._sync_id = sync_id + self._storage = caching.get_cache_for(source_replica_uid + sync_id) + + def _put_dict_info(self, key, value): + """ + Put some information about the sync state. + + :param key: The key for the info to be put. + :type key: str + :param value: The value for the info to be put. + :type value: str + """ + if key not in self._storage: + self._storage[key] = [] + info_list = self._storage.get(key) + info_list.append(value) + self._storage[key] = info_list + + def put_seen_id(self, seen_id, gen): + """ + Put one seen id on the sync state. + + :param seen_id: The doc_id of a document seen during sync. + :type seen_id: str + :param gen: The corresponding db generation for that document. + :type gen: int + """ + self._put_dict_info( + 'seen_id', + (seen_id, gen)) + + def seen_ids(self): + """ + Return all document ids seen during the sync. + + :return: A dict with doc ids seen during the sync. + :rtype: dict + """ + if 'seen_id' in self._storage: + seen_ids = self._storage.get('seen_id') + else: + seen_ids = [] + return dict(seen_ids) + + def put_changes_to_return(self, gen, trans_id, changes_to_return): + """ + Put the calculated changes to return in the backend sync state. + + :param gen: The target database generation that will be synced. + :type gen: int + :param trans_id: The target database transaction id that will be + synced. + :type trans_id: str + :param changes_to_return: A list of tuples with the changes to be + returned during the sync process. + :type changes_to_return: list + """ + self._put_dict_info( + 'changes_to_return', + { + 'gen': gen, + 'trans_id': trans_id, + 'changes_to_return': changes_to_return, + } + ) + + def sync_info(self): + """ + Return information about the current sync state. + + :return: The generation and transaction id of the target database + which will be synced, and the number of documents to return, + or a tuple of Nones if those have not already been sent to + server. + :rtype: tuple + """ + info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None + gen = None + trans_id = None + number_of_changes = None + if info: + info = info[0] + gen = info['gen'] + trans_id = info['trans_id'] + number_of_changes = len(info['changes_to_return']) + return gen, trans_id, number_of_changes + + def next_change_to_return(self, received): + """ + Return the next change to be returned to the source syncing replica. + + :param received: How many documents the source replica has already + received during the current sync process. + :type received: int + """ + info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None + gen = trans_id = next_change_to_return = None + if info: + info = info[0] + gen = info['gen'] + trans_id = info['trans_id'] + if received < len(info['changes_to_return']): + next_change_to_return = tuple(info['changes_to_return'][received]) + return gen, trans_id, next_change_to_return diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 18c4ee40..64f7e4e7 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -17,183 +17,15 @@ """ Server side synchronization infrastructure. """ -import json - -from leap.soledad.common.couch import CouchDatabase from u1db import sync, Document from u1db.remote import http_app +from leap.soledad.server.state import ServerSyncState MAX_REQUEST_SIZE = 200 # in Mb MAX_ENTRY_SIZE = 200 # in Mb -class ServerSyncState(object): - """ - The state of one sync session, as stored on backend server. - - This object performes queries to distinct design documents: - - _design/syncs/_update/state - _design/syncs/_view/state - _design/syncs/_view/seen_ids - _design/syncs/_view/changes_to_return - - On server side, the ongoing syncs metadata is maintained in a document - called 'u1db_sync_state'. - """ - - def __init__(self, db, source_replica_uid, sync_id): - """ - Initialize the sync state object. - - :param db: The target syncing database. - :type db: CouchDatabase. - :param source_replica_uid: CouchDatabase - :type source_replica_uid: str - """ - self._db = db - self._source_replica_uid = source_replica_uid - self._sync_id = sync_id - - def _key(self, key): - """ - Format a key to be used on couch views. - - :param key: The lookup key. - :type key: json serializable object - - :return: The properly formatted key. - :rtype: str - """ - return json.dumps(key, separators=(',', ':')) - - def _put_info(self, key, value): - """ - Put some information on the sync state document. - - This method works in conjunction with the - _design/syncs/_update/state update handler couch backend. - - :param key: The key for the info to be put. - :type key: str - :param value: The value for the info to be put. - :type value: str - """ - ddoc_path = [ - '_design', 'syncs', '_update', 'state', - 'u1db_sync_state'] - res = self._db._database.resource(*ddoc_path) - with CouchDatabase.sync_info_lock[self._db.replica_uid]: - res.put_json( - body={ - 'sync_id': self._sync_id, - 'source_replica_uid': self._source_replica_uid, - key: value, - }, - headers={'content-type': 'application/json'}) - - def put_seen_id(self, seen_id, gen): - """ - Put one seen id on the sync state document. - - :param seen_id: The doc_id of a document seen during sync. - :type seen_id: str - :param gen: The corresponding db generation for that document. - :type gen: int - """ - self._put_info( - 'seen_id', - [seen_id, gen]) - - def seen_ids(self): - """ - Return all document ids seen during the sync. - - :return: A list with doc ids seen during the sync. - :rtype: list - """ - ddoc_path = ['_design', 'syncs', '_view', 'seen_ids'] - resource = self._db._database.resource(*ddoc_path) - response = resource.get_json( - key=self._key([self._source_replica_uid, self._sync_id])) - data = response[2] - if data['rows']: - entry = data['rows'].pop() - return entry['value']['seen_ids'] - return [] - - def put_changes_to_return(self, gen, trans_id, changes_to_return): - """ - Put the calculated changes to return in the backend sync state - document. - - :param gen: The target database generation that will be synced. - :type gen: int - :param trans_id: The target database transaction id that will be - synced. - :type trans_id: str - :param changes_to_return: A list of tuples with the changes to be - returned during the sync process. - :type changes_to_return: list - """ - self._put_info( - 'changes_to_return', - { - 'gen': gen, - 'trans_id': trans_id, - 'changes_to_return': changes_to_return, - } - ) - - def sync_info(self): - """ - Return information about the current sync state. - - :return: The generation and transaction id of the target database - which will be synced, and the number of documents to return, - or a tuple of Nones if those have not already been sent to - server. - :rtype: tuple - """ - ddoc_path = ['_design', 'syncs', '_view', 'state'] - resource = self._db._database.resource(*ddoc_path) - response = resource.get_json( - key=self._key([self._source_replica_uid, self._sync_id])) - data = response[2] - gen = None - trans_id = None - number_of_changes = None - if data['rows'] and data['rows'][0]['value'] is not None: - value = data['rows'][0]['value'] - gen = value['gen'] - trans_id = value['trans_id'] - number_of_changes = value['number_of_changes'] - return gen, trans_id, number_of_changes - - def next_change_to_return(self, received): - """ - Return the next change to be returned to the source syncing replica. - - :param received: How many documents the source replica has already - received during the current sync process. - :type received: int - """ - ddoc_path = ['_design', 'syncs', '_view', 'changes_to_return'] - resource = self._db._database.resource(*ddoc_path) - response = resource.get_json( - key=self._key( - [self._source_replica_uid, self._sync_id, received])) - data = response[2] - if not data['rows']: - return None, None, None - value = data['rows'][0]['value'] - gen = value['gen'] - trans_id = value['trans_id'] - next_change_to_return = value['next_change_to_return'] - return gen, trans_id, tuple(next_change_to_return) - - class SyncExchange(sync.SyncExchange): def __init__(self, db, source_replica_uid, last_known_generation, sync_id): @@ -216,8 +48,7 @@ class SyncExchange(sync.SyncExchange): self.new_trans_id = None self._trace_hook = None # recover sync state - self._sync_state = ServerSyncState( - self._db, self.source_replica_uid, sync_id) + self._sync_state = ServerSyncState(self.source_replica_uid, sync_id) def find_changes_to_return(self, received): """ -- cgit v1.2.3 From f9faa007bc0d5d681f85aa246ea05ec5fe35ccc5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 6 Sep 2015 05:39:05 -0300 Subject: [feat] adds caching for other gen and trans id There are two functions in couch.py used to save and retrieve the last know gen and trans id for the syncing replica. The get function is called very often, but is only set on one point. Added a simple caching to avoid queying couch for a value that we already have. If cache is empty, it just query as usual and fills it. --- server/src/leap/soledad/server/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 1b795016..58f2b0b1 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -111,6 +111,7 @@ from leap.soledad.server.sync import ( from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import CouchServerState +from leap.soledad.server.caching import get_cache_for old_tsafe = tsafe @@ -303,7 +304,8 @@ def load_configuration(file_path): def application(environ, start_response): conf = load_configuration('/etc/leap/soledad-server.conf') - state = CouchServerState(conf['couch_url']) + cache = get_cache_for('replica_cache') + state = CouchServerState(conf['couch_url'], cache=cache) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) -- cgit v1.2.3 From 14300959a12e4908df7b00281d2590e627a47ba9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 7 Sep 2015 03:56:02 -0300 Subject: [feat] full caching for each sync_id session The CouchDB backend implementation was accessing CouchDB too many times for the same values. Those values are known inside the same sync_id, which is the id of current sync session. This commit adds caching for all redundant calls to Couch inside the same sync_id for each replica. Refactoring is still needed, but for now couch.py works normally as if caching is not present, while sync.py injects the cache as a attribute to enable it. This needs a simpler implementation. --- server/src/leap/soledad/server/__init__.py | 4 +--- server/src/leap/soledad/server/sync.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 58f2b0b1..1b795016 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -111,7 +111,6 @@ from leap.soledad.server.sync import ( from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import CouchServerState -from leap.soledad.server.caching import get_cache_for old_tsafe = tsafe @@ -304,8 +303,7 @@ def load_configuration(file_path): def application(environ, start_response): conf = load_configuration('/etc/leap/soledad-server.conf') - cache = get_cache_for('replica_cache') - state = CouchServerState(conf['couch_url'], cache=cache) + state = CouchServerState(conf['couch_url']) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 64f7e4e7..262b6769 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -20,6 +20,7 @@ Server side synchronization infrastructure. from u1db import sync, Document from u1db.remote import http_app from leap.soledad.server.state import ServerSyncState +from leap.soledad.server.caching import get_cache_for MAX_REQUEST_SIZE = 200 # in Mb @@ -188,6 +189,8 @@ class SyncResource(http_app.SyncResource): db, self.replica_uid = self.state.ensure_database(self.dbname) else: db = self.state.open_database(self.dbname) + cache = get_cache_for('db-' + sync_id + db.replica_uid) + db._cache = cache # validate the information the client has about server replica db.validate_gen_and_trans_id( last_known_generation, last_known_trans_id) -- cgit v1.2.3 From 0e954f3328b7b8c31c88e0bee796230e87bca829 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 20 Sep 2015 17:47:07 -0300 Subject: [feat] adds cache expiration Now each backend object will be retrieved from cache for sync.py and values will live for 3600 by default. That is changed via parameter if needed. --- server/src/leap/soledad/server/caching.py | 4 ++-- server/src/leap/soledad/server/sync.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/caching.py b/server/src/leap/soledad/server/caching.py index cd5d8dd4..9a049a39 100644 --- a/server/src/leap/soledad/server/caching.py +++ b/server/src/leap/soledad/server/caching.py @@ -28,5 +28,5 @@ def setup_caching(): _cache_manager = setup_caching() -def get_cache_for(key): - return _cache_manager.get_cache(key) +def get_cache_for(key, expire=3600): + return _cache_manager.get_cache(key, expire=expire) diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 262b6769..e4fd1260 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -185,12 +185,15 @@ class SyncResource(http_app.SyncResource): :type ensure: bool """ # create or open the database + cache = get_cache_for('db-' + sync_id + self.dbname) if ensure: db, self.replica_uid = self.state.ensure_database(self.dbname) + elif cache and 'instance' in cache: + db = cache['instance'] else: db = self.state.open_database(self.dbname) - cache = get_cache_for('db-' + sync_id + db.replica_uid) db._cache = cache + cache['instance'] = db # validate the information the client has about server replica db.validate_gen_and_trans_id( last_known_generation, last_known_trans_id) -- cgit v1.2.3 From df8c794395ee695ea1dfb30603a073b32b24ee58 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 21 Sep 2015 17:19:18 -0300 Subject: [refactor] init_caching instead of setting attr As meskio found commented, setting this attribute directly is ugly, CouchDatabase now has a init_caching method for setting up cache instance. --- server/src/leap/soledad/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index e4fd1260..619be565 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -192,7 +192,7 @@ class SyncResource(http_app.SyncResource): db = cache['instance'] else: db = self.state.open_database(self.dbname) - db._cache = cache + db.init_caching(cache) cache['instance'] = db # validate the information the client has about server replica db.validate_gen_and_trans_id( -- cgit v1.2.3 From 5b20b469f22ab99533135beefd49129db49064e5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 23 Sep 2015 15:43:06 -0300 Subject: [style] pep8 --- server/src/leap/soledad/server/state.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/state.py b/server/src/leap/soledad/server/state.py index d1225170..f269b77e 100644 --- a/server/src/leap/soledad/server/state.py +++ b/server/src/leap/soledad/server/state.py @@ -24,7 +24,8 @@ class ServerSyncState(object): """ The state of one sync session, as stored on backend server. - On server side, the ongoing syncs metadata is maintained in a caching layer. + On server side, the ongoing syncs metadata is maintained in + a caching layer. """ def __init__(self, source_replica_uid, sync_id): @@ -38,7 +39,8 @@ class ServerSyncState(object): """ self._source_replica_uid = source_replica_uid self._sync_id = sync_id - self._storage = caching.get_cache_for(source_replica_uid + sync_id) + caching_key = source_replica_uid + sync_id + self._storage = caching.get_cache_for(caching_key) def _put_dict_info(self, key, value): """ @@ -61,7 +63,7 @@ class ServerSyncState(object): :param seen_id: The doc_id of a document seen during sync. :type seen_id: str - :param gen: The corresponding db generation for that document. + :param gen: The corresponding db generation. :type gen: int """ self._put_dict_info( @@ -113,12 +115,9 @@ class ServerSyncState(object): server. :rtype: tuple """ - info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None - gen = None - trans_id = None - number_of_changes = None - if info: - info = info[0] + gen = trans_id = number_of_changes = None + if 'changes_to_return' in self._storage: + info = self._storage.get('changes_to_return')[0] gen = info['gen'] trans_id = info['trans_id'] number_of_changes = len(info['changes_to_return']) @@ -132,12 +131,11 @@ class ServerSyncState(object): received during the current sync process. :type received: int """ - info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None gen = trans_id = next_change_to_return = None - if info: - info = info[0] + if 'changes_to_return' in self._storage: + info = self._storage.get('changes_to_return')[0] gen = info['gen'] trans_id = info['trans_id'] if received < len(info['changes_to_return']): - next_change_to_return = tuple(info['changes_to_return'][received]) + next_change_to_return = (info['changes_to_return'][received]) return gen, trans_id, next_change_to_return -- cgit v1.2.3 From e76ce03ad29b5582c4763d61a5acaed1aeba87e5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 15 Sep 2015 19:59:14 -0300 Subject: [feat] conf for enabling db creation via custom sh We can now use a custom script to create databases by setting a parameter 'create_cmd' on soledad configuration. This will set CouchServerState to use it on ensure_database. --- server/src/leap/soledad/server/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 1b795016..bb1c6db0 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -285,6 +285,7 @@ def load_configuration(file_path): """ conf = { 'couch_url': 'http://localhost:5984', + 'create_cmd': None } config = configparser.ConfigParser() config.read(file_path) @@ -303,7 +304,7 @@ def load_configuration(file_path): def application(environ, start_response): conf = load_configuration('/etc/leap/soledad-server.conf') - state = CouchServerState(conf['couch_url']) + state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) -- cgit v1.2.3 From 7a0dba8b7008aca8652f7b334352d9ca63cb2054 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 30 Sep 2015 15:27:21 -0300 Subject: [feat] read netrc path from configuration file netrc file was hardcoded inside create-user-db. Now it reads the path from /etc/leap/soledad-server.conf as done on server process. The new configuration property is called 'admin_netrc'. --- server/src/leap/soledad/server/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index bb1c6db0..1273d15c 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -283,19 +283,20 @@ def load_configuration(file_path): @return: A dictionary with the configuration. @rtype: dict """ - conf = { + defaults = { 'couch_url': 'http://localhost:5984', - 'create_cmd': None + 'create_cmd': None, + 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', } config = configparser.ConfigParser() config.read(file_path) if 'soledad-server' in config: - for key in conf: + for key in defaults: if key in config['soledad-server']: - conf[key] = config['soledad-server'][key] + defaults[key] = config['soledad-server'][key] # TODO: implement basic parsing/sanitization of options comming from # config file. - return conf + return defaults # ---------------------------------------------------------------------------- -- cgit v1.2.3 From 147986547f400f79e8bc0d50af5ae7b5d2a140b4 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 5 Oct 2015 11:54:56 +0200 Subject: [feat] Move config dir to /etc/soledad - Resolves: #7509 --- server/src/leap/soledad/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 1273d15c..f64d07bf 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -304,7 +304,7 @@ def load_configuration(file_path): # ---------------------------------------------------------------------------- def application(environ, start_response): - conf = load_configuration('/etc/leap/soledad-server.conf') + conf = load_configuration('/etc/soledad/soledad-server.conf') state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( -- cgit v1.2.3 From 1b07d2b54208d6b19135e771162f7e39e02c1f97 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 14 Oct 2015 11:50:48 -0300 Subject: [bug] reduce overall sync session caching to 120s It was 3600s, but closing connections seems to yet depend on garbage collection and now causes server to leak file handlers. 120s should be enough to a sync session finish. Also, lowering this value will only make very long syncs use more of couch every 2 minutes, while raising this value will keep memory busy for useless time. --- server/src/leap/soledad/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 619be565..b3fb2b84 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -185,7 +185,7 @@ class SyncResource(http_app.SyncResource): :type ensure: bool """ # create or open the database - cache = get_cache_for('db-' + sync_id + self.dbname) + cache = get_cache_for('db-' + sync_id + self.dbname, expire=120) if ensure: db, self.replica_uid = self.state.ensure_database(self.dbname) elif cache and 'instance' in cache: -- cgit v1.2.3 From 0f8364b6a38793f9b0f8596c1b98a590131fbb41 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 19 Oct 2015 16:27:43 -0300 Subject: [bug] remove instance caching couchdb library relies on garbage collector to close remaining connections. Somehow, caching the instance is avoiding gc to call __del__ on underlying couchdb sessions. --- server/src/leap/soledad/server/sync.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index b3fb2b84..92b29102 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -188,12 +188,9 @@ class SyncResource(http_app.SyncResource): cache = get_cache_for('db-' + sync_id + self.dbname, expire=120) if ensure: db, self.replica_uid = self.state.ensure_database(self.dbname) - elif cache and 'instance' in cache: - db = cache['instance'] else: db = self.state.open_database(self.dbname) db.init_caching(cache) - cache['instance'] = db # validate the information the client has about server replica db.validate_gen_and_trans_id( last_known_generation, last_known_trans_id) -- cgit v1.2.3 From f8d38125098829fe50199725545365d6d2a889a6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 26 Oct 2015 18:50:20 -0300 Subject: [feat] read security doc from configuration LEAP Platform needs to granularly allow access on user database for other services, like mx. This is now possible by editing soledad-server.conf file. A new section 'database-security' was added and it is parsed during 'create-user-db' to be set on security design document, present on every per-user database. --- server/src/leap/soledad/server/__init__.py | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index f64d07bf..4d03c82a 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -272,6 +272,20 @@ http_app.HTTPInvocationByMethodWithBody = HTTPInvocationByMethodWithBody # ---------------------------------------------------------------------------- # Auxiliary functions # ---------------------------------------------------------------------------- +CONFIG_DEFAULTS = { + 'soledad-server': { + 'couch_url': 'http://localhost:5984', + 'create_cmd': None, + 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', + }, + 'database-security': { + 'members': ['soledad'], + 'members_roles': [], + 'admins': [], + 'admins_roles': [] + } +} + def load_configuration(file_path): """ @@ -283,17 +297,18 @@ def load_configuration(file_path): @return: A dictionary with the configuration. @rtype: dict """ - defaults = { - 'couch_url': 'http://localhost:5984', - 'create_cmd': None, - 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', - } + defaults = dict(CONFIG_DEFAULTS) config = configparser.ConfigParser() config.read(file_path) - if 'soledad-server' in config: - for key in defaults: - if key in config['soledad-server']: - defaults[key] = config['soledad-server'][key] + for section in defaults.keys(): + if section in config: + for key in defaults[section]: + if key in config[section]: + defaults[section][key] = config[section][key] + for key, value in defaults['database-security'].iteritems(): + if type(value) is not unicode: continue + defaults['database-security'][key] = \ + [item.strip() for item in value.split(',')] # TODO: implement basic parsing/sanitization of options comming from # config file. return defaults @@ -305,6 +320,7 @@ def load_configuration(file_path): def application(environ, start_response): conf = load_configuration('/etc/soledad/soledad-server.conf') + conf = conf['soledad-server'] state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( -- cgit v1.2.3 From 7c50a100a46c69f759dc22165cec9b8098cac3a4 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Tue, 27 Oct 2015 10:20:56 +0100 Subject: [style] fix pep8 warnigs --- server/src/leap/soledad/server/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 4d03c82a..fe67e45f 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -306,7 +306,8 @@ def load_configuration(file_path): if key in config[section]: defaults[section][key] = config[section][key] for key, value in defaults['database-security'].iteritems(): - if type(value) is not unicode: continue + if type(value) is not unicode: + continue defaults['database-security'][key] = \ [item.strip() for item in value.split(',')] # TODO: implement basic parsing/sanitization of options comming from -- cgit v1.2.3 From 3b869fb7ffedc88c738e0a17347b9506d242cabe Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 27 Oct 2015 13:58:56 -0300 Subject: [feat] remove SSL tsafe monkeypatch This was added for a Twisted 12 bug that should be gone by now. --- server/src/leap/soledad/server/__init__.py | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index fe67e45f..618ccb2b 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -94,12 +94,6 @@ from u1db.remote import http_app, utils from ._version import get_versions -# Keep OpenSSL's tsafe before importing Twisted submodules so we can put -# it back if Twisted==12.0.0 messes with it. -from OpenSSL import tsafe - -from twisted import version - from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.server.lock_resource import LockResource @@ -112,13 +106,6 @@ from leap.soledad.server.sync import ( from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import CouchServerState -old_tsafe = tsafe - -if version.base() == "12.0.0": - # Put OpenSSL's tsafe back into place. This can probably be removed if we - # come to use Twisted>=12.3.0. - sys.modules['OpenSSL.tsafe'] = old_tsafe - # ---------------------------------------------------------------------------- # Soledad WSGI application # ---------------------------------------------------------------------------- -- cgit v1.2.3 From b0557f9c1d5e6f153f926ba3cb5876453ef23a10 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 1 Oct 2015 15:07:25 -0300 Subject: [refactor] separate SoledadBackend from CouchDatabase CouchDatabase was renamed to SoledadBackend and a new class CouchDatabase was created to hold all couchdb code. This should make SoledadBackend less tied to database implementation. A few more separations are needed to split into modules. --- server/src/leap/soledad/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 92b29102..db25c406 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -32,7 +32,7 @@ class SyncExchange(sync.SyncExchange): def __init__(self, db, source_replica_uid, last_known_generation, sync_id): """ :param db: The target syncing database. - :type db: CouchDatabase + :type db: SoledadBackend :param source_replica_uid: The uid of the source syncing replica. :type source_replica_uid: str :param last_known_generation: The last target replica generation the -- cgit v1.2.3 From f0b96af943dcb6c8cde4f6d4280186d78c78096c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 13 Oct 2015 21:34:40 -0300 Subject: [refactor] split out backend from couch database First step of splitting classes across files on common. backend.py holds SoledadBackend (generic backend logic) couch/ is now a directory with old code inside __init__.py and CouchServerState on state.py Also removed mock IndexedSoledadBackend, since Soledad does not support indexing due to encryption on server side. Also fixed DesignDocUnknownError to show up what is the message of the original exception. It was being lost. --- server/src/leap/soledad/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 618ccb2b..00e1e9fb 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -104,7 +104,7 @@ from leap.soledad.server.sync import ( ) from leap.soledad.common import SHARED_DB_NAME -from leap.soledad.common.couch import CouchServerState +from leap.soledad.common.couch.state import CouchServerState # ---------------------------------------------------------------------------- # Soledad WSGI application -- cgit v1.2.3 From 421691ef71019d0bcd4447a773efa5e9b15b0c71 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 27 Oct 2015 16:48:39 -0300 Subject: [refactor] token verification moved to couch module + tests Added tests for this token verification as it wasn't covered. Then moved it to the new couch module that implements a couch storage. The ServerState was chosen to hold the verify_token method. CouchServerState holds the current implementation, which is called on authentication middleware as the new test shows. --- server/src/leap/soledad/server/auth.py | 52 ++-------------------------------- 1 file changed, 2 insertions(+), 50 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 02b54cca..01baf1ce 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -21,20 +21,16 @@ Authentication facilities for Soledad Server. """ -import time import httplib import json from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper -from couchdb.client import Server from twisted.python import log -from hashlib import sha512 from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import USER_DB_PREFIX -from leap.soledad.common.errors import InvalidAuthTokenError class URLToAuthorization(object): @@ -193,6 +189,7 @@ class SoledadAuthMiddleware(object): @type prefix: str """ self._app = app + self._state = app.state def _error(self, start_response, status, description, message=None): """ @@ -351,12 +348,6 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): Token based authentication. """ - TOKENS_DB_PREFIX = "tokens_" - TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds - TOKENS_TYPE_KEY = "type" - TOKENS_TYPE_DEF = "Token" - TOKENS_USER_ID_KEY = "user_id" - TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." def _verify_authentication_scheme(self, scheme): @@ -391,50 +382,11 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): """ token = auth_data # we expect a cleartext token at this point try: - return self._verify_token_in_couch(uuid, token) - except InvalidAuthTokenError: - raise + return self._state.verify_token(uuid, token) except Exception as e: log.err(e) return False - def _verify_token_in_couch(self, uuid, token): - """ - Query couchdb to decide if C{token} is valid for C{uuid}. - - @param uuid: The user uuid. - @type uuid: str - @param token: The token. - @type token: str - - @raise InvalidAuthTokenError: Raised when token received from user is - either missing in the tokens db or is - invalid. - """ - server = Server(url=self._app.state.couch_url) - # the tokens db rotates every 30 days, and the current db name is - # "tokens_NNN", where NNN is the number of seconds since epoch divided - # by the rotate period in seconds. When rotating, old and new tokens - # db coexist during a certain window of time and valid tokens are - # replicated from the old db to the new one. See: - # https://leap.se/code/issues/6785 - dbname = self.TOKENS_DB_PREFIX + \ - str(int(time.time() / self.TOKENS_DB_EXPIRE)) - db = server[dbname] - # lookup key is a hash of the token to prevent timing attacks. - token = db.get(sha512(token).hexdigest()) - if token is None: - raise InvalidAuthTokenError() - # we compare uuid hashes to avoid possible timing attacks that - # might exploit python's builtin comparison operator behaviour, - # which fails immediatelly when non-matching bytes are found. - couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest() - req_uuid_hash = sha512(uuid).digest() - if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \ - or couch_uuid_hash != req_uuid_hash: - raise InvalidAuthTokenError() - return True - def _get_auth_error_string(self): """ Get the error string for token auth. -- cgit v1.2.3 From eaf19626a57d5f5325653fee3aea9db27c9531fe Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 28 Oct 2015 17:58:27 -0300 Subject: [refactor] resource logic encapsulation Creating a resource from a path to use get_json causes a lot of dirty code and unexplained things like response[2]. This commit extracts that logic into a helper to let it more clear about what is happening. --- server/src/leap/soledad/server/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 01baf1ce..ccbd6fbd 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -189,7 +189,6 @@ class SoledadAuthMiddleware(object): @type prefix: str """ self._app = app - self._state = app.state def _error(self, start_response, status, description, message=None): """ @@ -350,6 +349,10 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): TOKEN_AUTH_ERROR_STRING = "Incorrect address or token." + def __init__(self, app): + self._state = app.state + super(SoledadTokenAuthMiddleware, self).__init__(app) + def _verify_authentication_scheme(self, scheme): """ Verify if authentication scheme is valid. -- cgit v1.2.3 From 577abee147c98592753bcdc68e1693d1f4ab5a08 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 13 Nov 2015 23:02:28 -0300 Subject: [feat] prepare server to handle batches Created two methods on the backend to start and finish a batch. A dict of callbacks is available to defer actions for the last document, allowing temporary (changing often) metadata to be recorded only once. Using those methods we will also be able to put all docs in one go on the CouchDatabase implementation, but that is another step. --- server/src/leap/soledad/server/sync.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index db25c406..96f65912 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -112,6 +112,14 @@ class SyncExchange(sync.SyncExchange): doc = self._db.get_doc(changed_doc_id, include_deleted=True) return_doc_cb(doc, gen, trans_id) + def batched_insert_from_source(self, entries, sync_id): + self._db.batch_start() + for entry in entries: + doc, gen, trans_id, number_of_docs, doc_idx = entry + self.insert_doc_from_source(doc, gen, trans_id, number_of_docs, + doc_idx, sync_id) + self._db.batch_end() + def insert_doc_from_source( self, doc, source_gen, trans_id, number_of_docs=None, doc_idx=None, sync_id=None): @@ -198,6 +206,7 @@ class SyncResource(http_app.SyncResource): self.sync_exch = self.sync_exchange_class( db, self.source_replica_uid, last_known_generation, sync_id) self._sync_id = sync_id + self._staging = [] @http_app.http_method(content_as_args=True) def post_put( @@ -225,9 +234,7 @@ class SyncResource(http_app.SyncResource): :type doc_idx: int """ doc = Document(id, rev, content) - self.sync_exch.insert_doc_from_source( - doc, gen, trans_id, number_of_docs=number_of_docs, - doc_idx=doc_idx, sync_id=self._sync_id) + self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) @http_app.http_method(received=int, content_as_args=True) def post_get(self, received): @@ -266,6 +273,7 @@ class SyncResource(http_app.SyncResource): Return the current generation and transaction_id after inserting one incoming document. """ + self.sync_exch.batched_insert_from_source(self._staging, self._sync_id) self.responder.content_type = 'application/x-soledad-sync-response' self.responder.start_response(200) self.responder.start_stream(), -- cgit v1.2.3 From f1497b92aff3b953eca572c08d85d8ddffb36391 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 17 Nov 2015 21:27:38 -0300 Subject: [feat] add configuration to disable batching Batch support is optional. This commit adds a 'batching' configuration option to disable it. --- server/src/leap/soledad/server/__init__.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 00e1e9fb..7320c133 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -104,6 +104,7 @@ from leap.soledad.server.sync import ( ) from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState # ---------------------------------------------------------------------------- @@ -264,6 +265,7 @@ CONFIG_DEFAULTS = { 'couch_url': 'http://localhost:5984', 'create_cmd': None, 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', + 'batching': True }, 'database-security': { 'members': ['soledad'], @@ -285,18 +287,20 @@ def load_configuration(file_path): @rtype: dict """ defaults = dict(CONFIG_DEFAULTS) - config = configparser.ConfigParser() + config = configparser.SafeConfigParser() config.read(file_path) - for section in defaults.keys(): - if section in config: - for key in defaults[section]: - if key in config[section]: - defaults[section][key] = config[section][key] - for key, value in defaults['database-security'].iteritems(): - if type(value) is not unicode: + for section in defaults: + if not config.has_section(section): continue - defaults['database-security'][key] = \ - [item.strip() for item in value.split(',')] + for key, value in defaults[section].items(): + if type(value) == bool: + defaults[section][key] = config.getboolean(section, key) + elif type(value) == list: + values = config.get(section, key).split(',') + values = [v.strip() for v in values] + defaults[section][key] = values + else: + defaults[section][key] = config.get(section, key) # TODO: implement basic parsing/sanitization of options comming from # config file. return defaults @@ -310,6 +314,7 @@ def application(environ, start_response): conf = load_configuration('/etc/soledad/soledad-server.conf') conf = conf['soledad-server'] state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) + SoledadBackend.BATCH_SUPPORT = conf['batching'] # WSGI application that may be used by `twistd -web` application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) -- cgit v1.2.3 From 7208d8bc5e5f23d0773533b15763f64d236489b4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 3 Dec 2015 19:34:56 -0300 Subject: [feat] set default to False on batching for now All batching code has no effect by default with this commit. Since we know that this is a dangerous new feature we will enable them only on our test servers and check them manually before setting it as default or adding more configuration features. Use SyncTarget and server conf file to enable it for testing. --- server/src/leap/soledad/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 7320c133..39edcc1b 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -265,7 +265,7 @@ CONFIG_DEFAULTS = { 'couch_url': 'http://localhost:5984', 'create_cmd': None, 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', - 'batching': True + 'batching': False }, 'database-security': { 'members': ['soledad'], -- cgit v1.2.3 From 27bda0ac201e236e3a2c9671462a337f2970e993 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 Dec 2015 15:47:53 -0300 Subject: [bug] skip missing keys on existing sections While parsing the configuration file, if a key doesnt exist we need to skip it. --- server/src/leap/soledad/server/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 39edcc1b..22894dac 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -293,7 +293,9 @@ def load_configuration(file_path): if not config.has_section(section): continue for key, value in defaults[section].items(): - if type(value) == bool: + if not config.has_option(section, key): + continue + elif type(value) == bool: defaults[section][key] = config.getboolean(section, key) elif type(value) == list: values = config.get(section, key).split(',') -- cgit v1.2.3 From b5fd4060831b82f1e4ea26d6ef3792793be76d77 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 1 Apr 2016 17:54:58 -0400 Subject: [pkg] updated to versioneer 0.16 (patched) --- server/src/leap/soledad/server/_version.py | 548 +++++++++++++++++++++-------- 1 file changed, 408 insertions(+), 140 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/_version.py b/server/src/leap/soledad/server/_version.py index 61bb57d9..8c27440f 100644 --- a/server/src/leap/soledad/server/_version.py +++ b/server/src/leap/soledad/server/_version.py @@ -1,74 +1,157 @@ + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) +# versioneer-0.16 (https://github.com/warner/python-versioneer) -# these strings will be replaced by git during git-archive +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -import re -import os.path -IN_LONG_VERSION_PY = True -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/soledad/server/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %s" % args[0]) - print(e) + print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -def get_expanded_variables(versionfile_source): +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source, "r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -84,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) + print("discarding '%s', no digits" % ",".join(refs-tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -94,123 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): if verbose: print("picking %s" % r) return {"version": r, - "full": variables["full"].strip()} - # no suitable tags, so we use the full revision id + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return {"version": variables["full"].strip(), - "full": variables["full"].strip()} - - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname( - os.path.join('..', here)) - ###################################################### - # XXX patch for our specific configuration with - # the three projects leap.soledad.{common, client, server} - # inside the same repo. - ###################################################### - root = os.path.dirname(os.path.join('..', root)) +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % - (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start " - "with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap.soledad.server-" -versionfile_source = "src/leap/soledad/server/_version.py" - - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = {"refnames": git_refnames, "full": git_full} - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} -- cgit v1.2.3