From 32d73ec50d6147d2511d6679bb12c17dc01210e4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 22 Sep 2016 05:03:59 -0300 Subject: [feature] batch based on payload size batch is slower than usual insert for a single doc, so, if a document exceeds the buffer, commit the batch (if any) and put the huge load by traditional insert. refactor coming. --- server/src/leap/soledad/server/sync.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'server/src') diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 6fcfe240..e12ebf8a 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -118,6 +118,8 @@ class SyncExchange(sync.SyncExchange): return_doc_cb(doc, gen, trans_id) def batched_insert_from_source(self, entries, sync_id): + if not entries: + return self._db.batch_start() for entry in entries: doc, gen, trans_id, number_of_docs, doc_idx = entry @@ -241,8 +243,19 @@ class SyncResource(http_app.SyncResource): """ doc = ServerDocument(id, rev) doc._json = content - self._staging_size += len(content or '') - self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) + if (len(content or '') > (8192 * 1024) / 4) or number_of_docs < 4: + self.sync_exch.batched_insert_from_source(self._staging, + self._sync_id) + self._staging = [] + self._staging_size = 0 + self.sync_exch.insert_doc_from_source( + doc, gen, trans_id, + number_of_docs=number_of_docs, + doc_idx=doc_idx, + sync_id=self._sync_id) + else: + self._staging_size += len(content or '') + self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) if self._staging_size > 8192 * 1024 or doc_idx == number_of_docs: self.sync_exch.batched_insert_from_source(self._staging, self._sync_id) -- cgit v1.2.3