From 577abee147c98592753bcdc68e1693d1f4ab5a08 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 13 Nov 2015 23:02:28 -0300 Subject: [feat] prepare server to handle batches Created two methods on the backend to start and finish a batch. A dict of callbacks is available to defer actions for the last document, allowing temporary (changing often) metadata to be recorded only once. Using those methods we will also be able to put all docs in one go on the CouchDatabase implementation, but that is another step. --- server/changes/feat_handle_send_batch_better | 1 + server/src/leap/soledad/server/sync.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 server/changes/feat_handle_send_batch_better (limited to 'server') diff --git a/server/changes/feat_handle_send_batch_better b/server/changes/feat_handle_send_batch_better new file mode 100644 index 00000000..6ee8688a --- /dev/null +++ b/server/changes/feat_handle_send_batch_better @@ -0,0 +1 @@ +o Added two methods to start and finish a batch on backend. They can be used to change database behaviour, allowing batch operations to be optimized. diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index db25c406..96f65912 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -112,6 +112,14 @@ class SyncExchange(sync.SyncExchange): doc = self._db.get_doc(changed_doc_id, include_deleted=True) return_doc_cb(doc, gen, trans_id) + def batched_insert_from_source(self, entries, sync_id): + self._db.batch_start() + for entry in entries: + doc, gen, trans_id, number_of_docs, doc_idx = entry + self.insert_doc_from_source(doc, gen, trans_id, number_of_docs, + doc_idx, sync_id) + self._db.batch_end() + def insert_doc_from_source( self, doc, source_gen, trans_id, number_of_docs=None, doc_idx=None, sync_id=None): @@ -198,6 +206,7 @@ class SyncResource(http_app.SyncResource): self.sync_exch = self.sync_exchange_class( db, self.source_replica_uid, last_known_generation, sync_id) self._sync_id = sync_id + self._staging = [] @http_app.http_method(content_as_args=True) def post_put( @@ -225,9 +234,7 @@ class SyncResource(http_app.SyncResource): :type doc_idx: int """ doc = Document(id, rev, content) - self.sync_exch.insert_doc_from_source( - doc, gen, trans_id, number_of_docs=number_of_docs, - doc_idx=doc_idx, sync_id=self._sync_id) + self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) @http_app.http_method(received=int, content_as_args=True) def post_get(self, received): @@ -266,6 +273,7 @@ class SyncResource(http_app.SyncResource): Return the current generation and transaction_id after inserting one incoming document. """ + self.sync_exch.batched_insert_from_source(self._staging, self._sync_id) self.responder.content_type = 'application/x-soledad-sync-response' self.responder.start_response(200) self.responder.start_stream(), -- cgit v1.2.3 From f1497b92aff3b953eca572c08d85d8ddffb36391 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 17 Nov 2015 21:27:38 -0300 Subject: [feat] add configuration to disable batching Batch support is optional. This commit adds a 'batching' configuration option to disable it. --- server/src/leap/soledad/server/__init__.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'server') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 00e1e9fb..7320c133 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -104,6 +104,7 @@ from leap.soledad.server.sync import ( ) from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState # ---------------------------------------------------------------------------- @@ -264,6 +265,7 @@ CONFIG_DEFAULTS = { 'couch_url': 'http://localhost:5984', 'create_cmd': None, 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', + 'batching': True }, 'database-security': { 'members': ['soledad'], @@ -285,18 +287,20 @@ def load_configuration(file_path): @rtype: dict """ defaults = dict(CONFIG_DEFAULTS) - config = configparser.ConfigParser() + config = configparser.SafeConfigParser() config.read(file_path) - for section in defaults.keys(): - if section in config: - for key in defaults[section]: - if key in config[section]: - defaults[section][key] = config[section][key] - for key, value in defaults['database-security'].iteritems(): - if type(value) is not unicode: + for section in defaults: + if not config.has_section(section): continue - defaults['database-security'][key] = \ - [item.strip() for item in value.split(',')] + for key, value in defaults[section].items(): + if type(value) == bool: + defaults[section][key] = config.getboolean(section, key) + elif type(value) == list: + values = config.get(section, key).split(',') + values = [v.strip() for v in values] + defaults[section][key] = values + else: + defaults[section][key] = config.get(section, key) # TODO: implement basic parsing/sanitization of options comming from # config file. return defaults @@ -310,6 +314,7 @@ def application(environ, start_response): conf = load_configuration('/etc/soledad/soledad-server.conf') conf = conf['soledad-server'] state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) + SoledadBackend.BATCH_SUPPORT = conf['batching'] # WSGI application that may be used by `twistd -web` application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) -- cgit v1.2.3 From 7208d8bc5e5f23d0773533b15763f64d236489b4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 3 Dec 2015 19:34:56 -0300 Subject: [feat] set default to False on batching for now All batching code has no effect by default with this commit. Since we know that this is a dangerous new feature we will enable them only on our test servers and check them manually before setting it as default or adding more configuration features. Use SyncTarget and server conf file to enable it for testing. --- server/src/leap/soledad/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 7320c133..39edcc1b 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -265,7 +265,7 @@ CONFIG_DEFAULTS = { 'couch_url': 'http://localhost:5984', 'create_cmd': None, 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', - 'batching': True + 'batching': False }, 'database-security': { 'members': ['soledad'], -- cgit v1.2.3 From 27bda0ac201e236e3a2c9671462a337f2970e993 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 Dec 2015 15:47:53 -0300 Subject: [bug] skip missing keys on existing sections While parsing the configuration file, if a key doesnt exist we need to skip it. --- server/src/leap/soledad/server/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'server') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 39edcc1b..22894dac 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -293,7 +293,9 @@ def load_configuration(file_path): if not config.has_section(section): continue for key, value in defaults[section].items(): - if type(value) == bool: + if not config.has_option(section, key): + continue + elif type(value) == bool: defaults[section][key] = config.getboolean(section, key) elif type(value) == list: values = config.get(section, key).split(',') -- cgit v1.2.3