diff options
| author | Victor Shyba <victor.shyba@gmail.com> | 2016-09-22 01:00:23 -0300 |
|---|---|---|
| committer | drebs <drebs@leap.se> | 2016-12-12 09:11:59 -0200 |
| commit | 7680ec18f26ce6bab48c8a57a05e08cba7c6ba5e (patch) | |
| tree | 12b8478e6ae9c23a1dc8c68269f66be9a4df8f6b /server/src | |
| parent | a302322e53878a6212532d33ac0a0f9e0c34b176 (diff) | |
[feature] stream content in a separate line
This allow different paths for raw data and metadata, avoiding
unnecessary json parsing.
Diffstat (limited to 'server/src')
| -rw-r--r-- | server/src/leap/soledad/server/__init__.py | 7 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/sync.py | 11 |
2 files changed, 12 insertions, 6 deletions
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 889bf48f..7ba95543 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -221,14 +221,15 @@ class HTTPInvocationByMethodWithBody( meth_put = self._lookup('%s_put' % method) meth_end = self._lookup('%s_end' % method) while True: - line = body_getline() - entry = line.strip() + entry = body_getline().strip() if entry == ']': # end of incoming document stream break if not entry or not comma: # empty or no prec comma raise http_app.BadRequest entry, comma = utils.check_and_strip_comma(entry) - meth_put({}, entry) + content = body_getline().strip() + content, comma = utils.check_and_strip_comma(content) + meth_put({'content': content or None}, entry) if comma or body_getline(): # extra comma or data raise http_app.BadRequest return meth_end() diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index bc977210..6fcfe240 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -212,6 +212,7 @@ class SyncResource(http_app.SyncResource): db, self.source_replica_uid, last_known_generation, sync_id) self._sync_id = sync_id self._staging = [] + self._staging_size = 0 @http_app.http_method(content_as_args=True) def post_put( @@ -240,9 +241,13 @@ class SyncResource(http_app.SyncResource): """ doc = ServerDocument(id, rev) doc._json = content - self.sync_exch.insert_doc_from_source( - doc, gen, trans_id, number_of_docs=None, - doc_idx=None, sync_id=None) + self._staging_size += len(content or '') + self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx)) + if self._staging_size > 8192 * 1024 or doc_idx == number_of_docs: + self.sync_exch.batched_insert_from_source(self._staging, + self._sync_id) + self._staging = [] + self._staging_size = 0 @http_app.http_method(received=int, content_as_args=True) def post_get(self, received): |