diff options
| author | Victor Shyba <victor.shyba@gmail.com> | 2016-09-22 05:03:59 -0300 |
|---|---|---|
| committer | drebs <drebs@leap.se> | 2016-12-12 09:11:59 -0200 |
| commit | 32d73ec50d6147d2511d6679bb12c17dc01210e4 (patch) | |
| tree | 22e8a7ad38e51ff8725946947007ad216d04fec6 /server/src | |
| parent | 9ea98145abd130227b33d691b82dbcca76ef70de (diff) | |
[feature] batch based on payload size
batch is slower than usual insert for a single doc, so, if a document
exceeds the buffer, commit the batch (if any) and put the huge load by
traditional insert.
refactor coming.
Diffstat (limited to 'server/src')
| -rw-r--r-- | server/src/leap/soledad/server/sync.py | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 6fcfe240..e12ebf8a 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -118,6 +118,8 @@ class SyncExchange(sync.SyncExchange): return_doc_cb(doc, gen, trans_id) def batched_insert_from_source(self, entries, sync_id): + if not entries: + return self._db.batch_start() for entry in entries: doc, gen, trans_id, number_of_docs, doc_idx = entry @@ -241,8 +243,19 @@ class SyncResource(http_app.SyncResource): """ doc = ServerDocument(id, rev) doc._json = content - self._staging_size += len(content or '') - self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) + if (len(content or '') > (8192 * 1024) / 4) or number_of_docs < 4: + self.sync_exch.batched_insert_from_source(self._staging, + self._sync_id) + self._staging = [] + self._staging_size = 0 + self.sync_exch.insert_doc_from_source( + doc, gen, trans_id, + number_of_docs=number_of_docs, + doc_idx=doc_idx, + sync_id=self._sync_id) + else: + self._staging_size += len(content or '') + self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) if self._staging_size > 8192 * 1024 or doc_idx == number_of_docs: self.sync_exch.batched_insert_from_source(self._staging, self._sync_id) |