summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--common/changes/feature_4836_allow-sync-of-large-files1
-rw-r--r--common/src/leap/soledad/common/couch.py7
-rw-r--r--common/src/leap/soledad/common/tests/couchdb.ini.template2
-rw-r--r--common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py1
-rw-r--r--common/src/leap/soledad/common/tests/test_server.py37
-rw-r--r--server/changes/feature_4836_allow-sync-of-large-files1
-rw-r--r--server/src/leap/soledad/server/__init__.py9
7 files changed, 56 insertions, 2 deletions
diff --git a/common/changes/feature_4836_allow-sync-of-large-files b/common/changes/feature_4836_allow-sync-of-large-files
new file mode 100644
index 00000000..f124e899
--- /dev/null
+++ b/common/changes/feature_4836_allow-sync-of-large-files
@@ -0,0 +1 @@
+ o Allow sync of large files (~100MB) (#4836).
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 45ca4282..3ca3f408 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -27,7 +27,7 @@ import socket
from couchdb.client import Server
-from couchdb.http import ResourceNotFound, Unauthorized, ServerError
+from couchdb.http import ResourceNotFound, Unauthorized, ServerError, Session
from u1db import query_parser, vectorclock
from u1db.errors import (
DatabaseDoesNotExist,
@@ -50,6 +50,9 @@ from leap.soledad.common.document import SoledadDocument
logger = logging.getLogger(__name__)
+COUCH_TIMEOUT = 120 # timeout for transfers between Soledad server and Couch
+
+
class InvalidURLError(Exception):
"""
Exception raised when Soledad encounters a malformed URL.
@@ -275,6 +278,8 @@ class CouchDatabase(CommonBackend):
# save params
self._url = url
self._full_commit = full_commit
+ if session is None:
+ session = Session(timeout=COUCH_TIMEOUT)
self._session = session
self._factory = CouchDocument
self._real_replica_uid = None
diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template
index 217ae201..1fc2205b 100644
--- a/common/src/leap/soledad/common/tests/couchdb.ini.template
+++ b/common/src/leap/soledad/common/tests/couchdb.ini.template
@@ -6,7 +6,7 @@
database_dir = %(tempdir)s/lib
view_index_dir = %(tempdir)s/lib
max_document_size = 4294967296 ; 4 GB
-os_process_timeout = 5000 ; 5 seconds. for view and external servers.
+os_process_timeout = 120000 ; 120 seconds. for view and external servers.
max_dbs_open = 100
delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned
uri_file = %(tempdir)s/lib/couch.uri
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 8b001859..5384d465 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -100,6 +100,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
def tearDown(self):
+ self.db.delete_database()
CouchDBTestCase.tearDown(self)
TestCaseWithServer.tearDown(self)
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index 83df192b..ff699486 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -25,6 +25,7 @@ import tempfile
import simplejson as json
import mock
import time
+import binascii
from leap.common.testing.basetest import BaseLeapTest
@@ -436,6 +437,42 @@ class EncryptedSyncTestCase(
self.assertEqual(doc1, doc2)
+ def test_sync_very_large_files(self):
+ """
+ Test if Soledad can sync very large files.
+ """
+ # define the size of the "very large file"
+ length = 100*(10**6) # 100 MB
+ self.startServer()
+ # instantiate soledad and create a document
+ sol1 = self._soledad_instance(
+ # token is verified in test_target.make_token_soledad_app
+ auth_token='auth-token'
+ )
+ _, doclist = sol1.get_all_docs()
+ self.assertEqual([], doclist)
+ content = binascii.hexlify(os.urandom(length/2)) # len() == length
+ doc1 = sol1.create_doc({'data': content})
+ # sync with server
+ sol1._server_url = self.getURL()
+ sol1.sync()
+ # instantiate soledad with empty db, but with same secrets path
+ sol2 = self._soledad_instance(prefix='x', auth_token='auth-token')
+ _, doclist = sol2.get_all_docs()
+ self.assertEqual([], doclist)
+ sol2._secrets_path = sol1.secrets_path
+ sol2._load_secrets()
+ sol2._set_secret_id(sol1._secret_id)
+ # sync the new instance
+ sol2._server_url = self.getURL()
+ sol2.sync()
+ _, doclist = sol2.get_all_docs()
+ self.assertEqual(1, len(doclist))
+ doc2 = doclist[0]
+ # assert incoming doc is equal to the first sent doc
+ self.assertEqual(doc1, doc2)
+
+
class LockResourceTestCase(
CouchDBTestCase, TestCaseWithServer):
"""
diff --git a/server/changes/feature_4836_allow-sync-of-large-files b/server/changes/feature_4836_allow-sync-of-large-files
new file mode 100644
index 00000000..f124e899
--- /dev/null
+++ b/server/changes/feature_4836_allow-sync-of-large-files
@@ -0,0 +1 @@
+ o Allow sync of large files (~100MB) (#4836).
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index b2d55fcd..84f6a849 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -114,6 +114,10 @@ from leap.soledad.common.couch import CouchServerState
# Soledad WSGI application
#-----------------------------------------------------------------------------
+MAX_REQUEST_SIZE = 200 # in Mb
+MAX_ENTRY_SIZE = 200 # in Mb
+
+
class SoledadApp(http_app.HTTPApp):
"""
Soledad WSGI application
@@ -124,6 +128,9 @@ class SoledadApp(http_app.HTTPApp):
The name of the shared database that holds user's encrypted secrets.
"""
+ max_request_size = MAX_REQUEST_SIZE * 1024 * 1024
+ max_entry_size = MAX_ENTRY_SIZE * 1024 * 1024
+
def __call__(self, environ, start_response):
"""
Handle a WSGI call to the Soledad application.
@@ -143,6 +150,8 @@ class SoledadApp(http_app.HTTPApp):
http_app.url_to_resource.register(LockResource)
+http_app.SyncResource.max_request_size = MAX_REQUEST_SIZE * 1024 * 1024
+http_app.SyncResource.max_entry_size = MAX_ENTRY_SIZE * 1024 * 1024
#-----------------------------------------------------------------------------