summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/leap/soledad/client/_db/blobs/__init__.py21
-rw-r--r--src/leap/soledad/client/_db/blobs/sync.py16
-rw-r--r--tests/blobs/test_blob_manager.py45
3 files changed, 14 insertions, 68 deletions
diff --git a/src/leap/soledad/client/_db/blobs/__init__.py b/src/leap/soledad/client/_db/blobs/__init__.py
index 3c8facba..2df315dc 100644
--- a/src/leap/soledad/client/_db/blobs/__init__.py
+++ b/src/leap/soledad/client/_db/blobs/__init__.py
@@ -107,6 +107,9 @@ class BlobManager(BlobsSynchronizer):
The BlobManager can list, put, get, set flags and synchronize blobs stored
in local and remote storages.
"""
+ max_decrypt_retries = 3
+ concurrent_transfers_limit = 3
+ concurrent_writes_limit = 100
def __init__(
self, local_path, remote, key, secret, user, token=None,
@@ -128,9 +131,6 @@ class BlobManager(BlobsSynchronizer):
:type cert_file: str
"""
super(BlobsSynchronizer, self).__init__()
- self.max_retries = 3
- self.concurrent_transfers_limit = 3
- self.concurrent_writes_limit = 100
if local_path:
mkdir_p(os.path.dirname(local_path))
self.local = SQLiteBlobBackend(local_path, key=key, user=user)
@@ -307,22 +307,23 @@ class BlobManager(BlobsSynchronizer):
_, retries = yield self.local.get_sync_status(blob_id)
if isinstance(e, InvalidBlob):
+ max_retries = self.max_decrypt_retries
message = "Corrupted blob received from server! ID: %s\n"
message += "Error: %r\n"
message += "Retries: %s - Attempts left: %s\n"
message += "This is either a bug or the contents of the "
message += "blob have been tampered with. Please, report to "
message += "your provider's sysadmin and submit a bug report."
- message %= (blob_id, e, retries, (self.max_retries - retries))
+ message %= (blob_id, e, retries, (max_retries - retries))
logger.error(message)
- yield self.local.increment_retries(blob_id)
+ yield self.local.increment_retries(blob_id)
- if (retries + 1) >= self.max_retries:
- failed_download = SyncStatus.FAILED_DOWNLOAD
- yield self.local.update_sync_status(
- blob_id, failed_download, namespace=namespace)
- raise MaximumRetriesError(e)
+ if (retries + 1) >= max_retries:
+ failed_download = SyncStatus.FAILED_DOWNLOAD
+ yield self.local.update_sync_status(
+ blob_id, failed_download, namespace=namespace)
+ raise MaximumRetriesError(e)
raise RetriableTransferError(e)
diff --git a/src/leap/soledad/client/_db/blobs/sync.py b/src/leap/soledad/client/_db/blobs/sync.py
index a1b5cc4c..ee10443d 100644
--- a/src/leap/soledad/client/_db/blobs/sync.py
+++ b/src/leap/soledad/client/_db/blobs/sync.py
@@ -22,7 +22,7 @@ from twisted.internet import reactor
from twisted.logger import Logger
from twisted.internet import error
from .sql import SyncStatus
-from .errors import MaximumRetriesError, RetriableTransferError
+from .errors import RetriableTransferError
logger = Logger()
@@ -90,18 +90,8 @@ class BlobsSynchronizer(object):
logger.info("Sending blob to server (%d/%d): %s"
% (i, total, blob_id))
fd = yield self.local.get(blob_id, namespace=namespace)
- try:
- yield self._encrypt_and_upload(blob_id, fd)
- yield self.local.update_sync_status(blob_id, SyncStatus.SYNCED)
- except Exception as e:
- yield self.local.increment_retries(blob_id)
- res = yield self.local.get_sync_status(blob_id)
- _, retries = res
- if (retries + 1) > self.max_retries:
- failed_upload = SyncStatus.FAILED_UPLOAD
- yield self.local.update_sync_status(blob_id, failed_upload)
- raise MaximumRetriesError(e)
- raise e
+ yield self._encrypt_and_upload(blob_id, fd)
+ yield self.local.update_sync_status(blob_id, SyncStatus.SYNCED)
@defer.inlineCallbacks
def fetch_missing(self, namespace=''):
diff --git a/tests/blobs/test_blob_manager.py b/tests/blobs/test_blob_manager.py
index 4b2b1135..f1872ab1 100644
--- a/tests/blobs/test_blob_manager.py
+++ b/tests/blobs/test_blob_manager.py
@@ -23,7 +23,6 @@ from leap.soledad.client._document import BlobDoc
from leap.soledad.client._db.blobs import BlobManager, FIXED_REV
from leap.soledad.client._db.blobs import BlobAlreadyExistsError
from leap.soledad.client._db.blobs import SyncStatus
-from leap.soledad.client._db.blobs import RetriableTransferError
from io import BytesIO
from mock import Mock
from uuid import uuid4
@@ -169,50 +168,6 @@ class BlobManagerTestCase(unittest.TestCase):
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
- def test_upload_retry_limit(self):
- # prepare the manager to fail accordingly
- self.manager.remote_list = Mock(return_value=[])
- self.manager._encrypt_and_upload = Mock(
- side_effect=RetriableTransferError)
- # put a blob in local storage
- content, blob_id = "Blob content", uuid4().hex
- yield self.manager.local.put(blob_id, BytesIO(content), len(content))
- pending = SyncStatus.PENDING_UPLOAD
- yield self.manager.local.update_sync_status(blob_id, pending)
- # try to send missing
- with pytest.raises(defer.FirstError):
- yield self.manager.send_missing()
- # assert failed state and number of retries
- failed_upload = SyncStatus.FAILED_UPLOAD
- local_list = yield self.manager.local_list_status(failed_upload)
- self.assertIn(blob_id, local_list)
- sync_status, retries = \
- yield self.manager.local.get_sync_status(blob_id)
- self.assertEqual(failed_upload, sync_status)
- self.assertEqual(self.manager.max_retries, retries)
-
- @defer.inlineCallbacks
- @pytest.mark.usefixtures("method_tmpdir")
- def test_download_retry_limit(self):
- # prepare the manager to fail accordingly
- blob_id = uuid4().hex
- self.manager.local_list_status = Mock(return_value=[blob_id])
- self.manager._download_and_decrypt = Mock(
- side_effect=RetriableTransferError)
- # try to fetch missing
- with pytest.raises(defer.FirstError):
- yield self.manager.fetch_missing()
- # assert failed state and number of retries
- failed_download = SyncStatus.FAILED_DOWNLOAD
- local_list = yield self.manager.local.list_status(failed_download)
- self.assertIn(blob_id, local_list)
- sync_status, retries = \
- yield self.manager.local.get_sync_status(blob_id)
- self.assertEqual(failed_download, sync_status)
- self.assertEqual(self.manager.max_retries, retries)
-
- @defer.inlineCallbacks
- @pytest.mark.usefixtures("method_tmpdir")
def test_local_list_doesnt_include_unavailable_blobs(self):
local = self.manager.local
unavailable_ids, deferreds = [], []