From 23ea0193a521a1f5cb539a342be594b7b7acedcf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 21 Aug 2015 12:52:12 -0300 Subject: [bug] reduce memory usage by consuming single doc Preparing many docs is useful for batching only. As we are sendind one by one I will leave prepare_one_doc method to do the encrypt as the docs goes to upload. Also fixes method name as kaliy suggested. --- client/src/leap/soledad/client/http_target/send.py | 25 ++++++++++------------ .../src/leap/soledad/client/http_target/support.py | 4 ++-- 2 files changed, 13 insertions(+), 16 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index fe3a753f..72c33c6c 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -37,20 +37,19 @@ class HTTPDocSender(object): defer.returnValue([None, None]) # add remote replica metadata to the request - metadata = RequestBody( + body = RequestBody( last_known_generation=last_known_generation, last_known_trans_id=last_known_trans_id, sync_id=sync_id, ensure=self._ensure_callback is not None) total = len(docs_by_generation) - entries = yield self._entries_from_docs(metadata, docs_by_generation) - while len(entries): + for idx, entry in enumerate(docs_by_generation, 1): + yield self._prepare_one_doc(entry, body, idx, total) result = yield self._http_request( self._url, method='POST', - body=entries.remove(1), + body=body.pop(1), content_type='application/x-soledad-sync-put') - idx = total - len(entries) if self._defer_encryption: self._delete_sent(idx, docs_by_generation) _emit_send_status(idx, total) @@ -65,15 +64,13 @@ class HTTPDocSender(object): doc.doc_id, doc.rev) @defer.inlineCallbacks - def _entries_from_docs(self, initial_body, docs_by_generation): - number_of_docs = len(docs_by_generation) - for idx, (doc, gen, trans_id) in enumerate(docs_by_generation, 1): - content = yield self._encrypt_doc(doc) - initial_body.insert_info( - id=doc.doc_id, rev=doc.rev, content=content, gen=gen, - trans_id=trans_id, number_of_docs=number_of_docs, - doc_idx=idx) - defer.returnValue(initial_body) + def _prepare_one_doc(self, entry, body, idx, total): + doc, gen, trans_id = entry + content = yield self._encrypt_doc(doc) + body.insert_info( + id=doc.doc_id, rev=doc.rev, content=content, gen=gen, + trans_id=trans_id, number_of_docs=total, + doc_idx=idx) def _encrypt_doc(self, doc): d = None diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py index 5daabb61..44cd7089 100644 --- a/client/src/leap/soledad/client/http_target/support.py +++ b/client/src/leap/soledad/client/http_target/support.py @@ -167,12 +167,12 @@ class RequestBody(object): self.entries.append(entry) return len(entry) - def remove(self, number=1): + def pop(self, number=1): """ Removes an amount of entries and returns it formatted and ready to be sent. - :param number: number of entries to remove and format + :param number: number of entries to pop and format :type number: int :return: formatted body ready to be sent -- cgit v1.2.3