summaryrefslogtreecommitdiff
path: root/src/leap/soledad/client
diff options
context:
space:
mode:
authorVictor Shyba <victor1984@riseup.net>2017-09-25 14:15:20 -0300
committerVictor Shyba <victor1984@riseup.net>2017-10-05 05:41:40 -0300
commit0c9bbaf5519e76841d141865f6ca76cba8d01d67 (patch)
treeb2ef44dfe8af7045626204de9c454fdeb7709498 /src/leap/soledad/client
parent0e2dca49d70082f51c43cd6873f36fee6a9b62ad (diff)
[feature] send/fetch missing using local statuses
Instead of querying the server, fetch_missing and send_missing now uses the PENDING_DOWNLOAD and PENDING_UPLOAD statuses to guide itself on what to do. This allows the sync mechanism to control when/how to query data from server and reuse the query data during the sync. -- Related: #8822
Diffstat (limited to 'src/leap/soledad/client')
-rw-r--r--src/leap/soledad/client/_db/blobs.py30
1 files changed, 17 insertions, 13 deletions
diff --git a/src/leap/soledad/client/_db/blobs.py b/src/leap/soledad/client/_db/blobs.py
index 75f196c7..76746c71 100644
--- a/src/leap/soledad/client/_db/blobs.py
+++ b/src/leap/soledad/client/_db/blobs.py
@@ -275,13 +275,15 @@ class BlobManager(object):
return self.local.list(namespace, sync_status)
@defer.inlineCallbacks
- def refresh_sync_status_from_server(self):
- d1 = self.remote_list()
- d2 = self.local_list()
+ def refresh_sync_status_from_server(self, namespace=''):
+ d1 = self.remote_list(namespace=namespace)
+ d2 = self.local_list(namespace)
remote_list, local_list = yield defer.gatherResults([d1, d2])
pending_download_ids = tuple(set(remote_list) - set(local_list))
yield self.local.update_batch_sync_status(
- pending_download_ids, sync_status=SyncStatus.PENDING_DOWNLOAD)
+ pending_download_ids,
+ sync_status=SyncStatus.PENDING_DOWNLOAD,
+ namespace=namespace)
@defer.inlineCallbacks
def send_missing(self, namespace=''):
@@ -292,11 +294,9 @@ class BlobManager(object):
Optional parameter to restrict operation to a given namespace.
:type namespace: str
"""
- our_blobs = yield self.local_list(namespace)
- server_blobs = yield self.remote_list(namespace=namespace)
- missing = [b_id for b_id in our_blobs if b_id not in server_blobs]
+ d = self.local_list(namespace, SyncStatus.PENDING_UPLOAD)
+ missing = yield d
logger.info("Amount of documents missing on server: %s" % len(missing))
- # TODO: Send concurrently when we are able to stream directly from db
while missing:
deferreds = []
for _ in range(min(self.concurrency_limit, len(missing))):
@@ -330,18 +330,22 @@ class BlobManager(object):
:type namespace: str
"""
# TODO: Use something to prioritize user requests over general new docs
- our_blobs = yield self.local_list(namespace)
- server_blobs = yield self.remote_list(namespace=namespace)
- docs_we_want = [b_id for b_id in server_blobs if b_id not in our_blobs]
+ d = self.local_list(namespace, SyncStatus.PENDING_DOWNLOAD)
+ docs_we_want = yield d
logger.info("Fetching new docs from server: %s" % len(docs_we_want))
- # TODO: Fetch concurrently when we are able to stream directly into db
while docs_we_want:
deferreds = []
for _ in range(min(self.concurrency_limit, len(docs_we_want))):
blob_id = docs_we_want.pop()
logger.info("Fetching new doc: %s" % blob_id)
deferreds.append(self.get(blob_id, namespace))
- yield defer.DeferredList(deferreds)
+ yield defer.gatherResults(deferreds)
+
+ @defer.inlineCallbacks
+ def sync(self, namespace=''):
+ yield self.refresh_sync_status_from_server(namespace)
+ yield self.fetch_missing(namespace)
+ yield self.send_missing(namespace)
@defer.inlineCallbacks
def put(self, doc, size, namespace=''):