summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVictor Shyba <victor.shyba@gmail.com>2015-08-21 12:52:12 -0300
committerVictor Shyba <victor.shyba@gmail.com>2015-08-26 17:20:22 -0300
commit23ea0193a521a1f5cb539a342be594b7b7acedcf (patch)
tree9b13ff068df7601de6947a8f28808c9b7f67ee85
parentde73fc6969433a69ec6ba12ec508c3c93b83fcc6 (diff)
[bug] reduce memory usage by consuming single doc
Preparing many docs is useful for batching only. As we are sendind one by one I will leave prepare_one_doc method to do the encrypt as the docs goes to upload. Also fixes method name as kaliy suggested.
-rw-r--r--client/src/leap/soledad/client/http_target/send.py25
-rw-r--r--client/src/leap/soledad/client/http_target/support.py4
2 files changed, 13 insertions, 16 deletions
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index fe3a753f..72c33c6c 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -37,20 +37,19 @@ class HTTPDocSender(object):
defer.returnValue([None, None])
# add remote replica metadata to the request
- metadata = RequestBody(
+ body = RequestBody(
last_known_generation=last_known_generation,
last_known_trans_id=last_known_trans_id,
sync_id=sync_id,
ensure=self._ensure_callback is not None)
total = len(docs_by_generation)
- entries = yield self._entries_from_docs(metadata, docs_by_generation)
- while len(entries):
+ for idx, entry in enumerate(docs_by_generation, 1):
+ yield self._prepare_one_doc(entry, body, idx, total)
result = yield self._http_request(
self._url,
method='POST',
- body=entries.remove(1),
+ body=body.pop(1),
content_type='application/x-soledad-sync-put')
- idx = total - len(entries)
if self._defer_encryption:
self._delete_sent(idx, docs_by_generation)
_emit_send_status(idx, total)
@@ -65,15 +64,13 @@ class HTTPDocSender(object):
doc.doc_id, doc.rev)
@defer.inlineCallbacks
- def _entries_from_docs(self, initial_body, docs_by_generation):
- number_of_docs = len(docs_by_generation)
- for idx, (doc, gen, trans_id) in enumerate(docs_by_generation, 1):
- content = yield self._encrypt_doc(doc)
- initial_body.insert_info(
- id=doc.doc_id, rev=doc.rev, content=content, gen=gen,
- trans_id=trans_id, number_of_docs=number_of_docs,
- doc_idx=idx)
- defer.returnValue(initial_body)
+ def _prepare_one_doc(self, entry, body, idx, total):
+ doc, gen, trans_id = entry
+ content = yield self._encrypt_doc(doc)
+ body.insert_info(
+ id=doc.doc_id, rev=doc.rev, content=content, gen=gen,
+ trans_id=trans_id, number_of_docs=total,
+ doc_idx=idx)
def _encrypt_doc(self, doc):
d = None
diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py
index 5daabb61..44cd7089 100644
--- a/client/src/leap/soledad/client/http_target/support.py
+++ b/client/src/leap/soledad/client/http_target/support.py
@@ -167,12 +167,12 @@ class RequestBody(object):
self.entries.append(entry)
return len(entry)
- def remove(self, number=1):
+ def pop(self, number=1):
"""
Removes an amount of entries and returns it formatted and ready
to be sent.
- :param number: number of entries to remove and format
+ :param number: number of entries to pop and format
:type number: int
:return: formatted body ready to be sent