summaryrefslogtreecommitdiff
path: root/server/src
diff options
context:
space:
mode:
authorVictor Shyba <victor.shyba@gmail.com>2016-09-22 05:03:59 -0300
committerdrebs <drebs@leap.se>2016-12-12 09:11:59 -0200
commit32d73ec50d6147d2511d6679bb12c17dc01210e4 (patch)
tree22e8a7ad38e51ff8725946947007ad216d04fec6 /server/src
parent9ea98145abd130227b33d691b82dbcca76ef70de (diff)
[feature] batch based on payload size
batch is slower than usual insert for a single doc, so, if a document exceeds the buffer, commit the batch (if any) and put the huge load by traditional insert. refactor coming.
Diffstat (limited to 'server/src')
-rw-r--r--server/src/leap/soledad/server/sync.py17
1 files changed, 15 insertions, 2 deletions
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 6fcfe240..e12ebf8a 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -118,6 +118,8 @@ class SyncExchange(sync.SyncExchange):
return_doc_cb(doc, gen, trans_id)
def batched_insert_from_source(self, entries, sync_id):
+ if not entries:
+ return
self._db.batch_start()
for entry in entries:
doc, gen, trans_id, number_of_docs, doc_idx = entry
@@ -241,8 +243,19 @@ class SyncResource(http_app.SyncResource):
"""
doc = ServerDocument(id, rev)
doc._json = content
- self._staging_size += len(content or '')
- self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx))
+ if (len(content or '') > (8192 * 1024) / 4) or number_of_docs < 4:
+ self.sync_exch.batched_insert_from_source(self._staging,
+ self._sync_id)
+ self._staging = []
+ self._staging_size = 0
+ self.sync_exch.insert_doc_from_source(
+ doc, gen, trans_id,
+ number_of_docs=number_of_docs,
+ doc_idx=doc_idx,
+ sync_id=self._sync_id)
+ else:
+ self._staging_size += len(content or '')
+ self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx))
if self._staging_size > 8192 * 1024 or doc_idx == number_of_docs:
self.sync_exch.batched_insert_from_source(self._staging,
self._sync_id)