From e0e427728848ce8bb33b4a4d6f8937ec5788d2c6 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 11 Oct 2017 11:20:22 -0300 Subject: [feature] make concurrent blob writes configurable --- docs/server.rst | 34 ++++++++++++++++++---------------- src/leap/soledad/server/_blobs.py | 5 +++-- src/leap/soledad/server/_config.py | 1 + src/leap/soledad/server/auth.py | 7 +++++-- tests/server/test_auth.py | 2 +- tests/server/test_config.py | 3 ++- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/docs/server.rst b/docs/server.rst index 90321922..f06668d7 100644 --- a/docs/server.rst +++ b/docs/server.rst @@ -15,22 +15,24 @@ Soledad Server looks for a configuration file in ``/etc/soledad/soledad-server.conf`` and will read the following configuration options from the ``[soledad-server]`` section: -==================== =============================================== ================================ -Option Description Default value -==================== =============================================== ================================ -couch_url The URL of the CouchDB backend storage. ``http://localhost:5984`` -create_cmd The shell command to create user databases. None -admin_netrc The netrc file to be used for authenticating ``/etc/couchdb/couchdb.netrc`` - with the CouchDB backend storage. -batching Whether to use batching capabilities for ``true`` - synchronization. -blobs Whether to provide the Blobs functionality or ``false`` - not. -blobs_path The path for blobs storage in the server's file ``/var/lib/soledad/blobs`` - system. -services_tokens_file The file containing authentication tokens for ``/etc/soledad/services.tokens`` - services provided through the Services API. -==================== =============================================== ================================ +====================== =============================================== ================================ +Option Description Default value +====================== =============================================== ================================ +couch_url The URL of the CouchDB backend storage. ``http://localhost:5984`` +create_cmd The shell command to create user databases. None +admin_netrc The netrc file to be used for authenticating ``/etc/couchdb/couchdb.netrc`` + with the CouchDB backend storage. +batching Whether to use batching capabilities for ``true`` + synchronization. +blobs Whether to provide the Blobs functionality or ``false`` + not. +blobs_path The path for blobs storage in the server's file ``/var/lib/soledad/blobs`` + system. +concurrent_blob_writes Limit of concurrent blob writes to the 50 + filesystem. +services_tokens_file The file containing authentication tokens for ``/etc/soledad/services.tokens`` + services provided through the Services API. +====================== =============================================== ================================ Running ------- diff --git a/src/leap/soledad/server/_blobs.py b/src/leap/soledad/server/_blobs.py index cd47098d..22277fc4 100644 --- a/src/leap/soledad/server/_blobs.py +++ b/src/leap/soledad/server/_blobs.py @@ -62,9 +62,10 @@ VALID_STRINGS = re.compile('^[a-zA-Z0-9_-]+$') @implementer(interfaces.IIncomingBoxBackend) class FilesystemBlobsBackend(object): - def __init__(self, blobs_path='/tmp/blobs/', quota=200 * 1024): + def __init__(self, blobs_path='/tmp/blobs/', quota=200 * 1024, + concurrent_writes=50): self.quota = quota - self.semaphore = defer.DeferredSemaphore(50) # TODO: make configurable + self.semaphore = defer.DeferredSemaphore(concurrent_writes) if not os.path.isdir(blobs_path): os.makedirs(blobs_path) self.path = blobs_path diff --git a/src/leap/soledad/server/_config.py b/src/leap/soledad/server/_config.py index c23f1eae..0d37879e 100644 --- a/src/leap/soledad/server/_config.py +++ b/src/leap/soledad/server/_config.py @@ -33,6 +33,7 @@ CONFIG_DEFAULTS = { 'blobs': False, 'blobs_path': '/var/lib/soledad/blobs', 'services_tokens_file': '/etc/soledad/services.tokens', + 'concurrent_blob_writes': 50, }, 'database-security': { 'members': ['soledad'], diff --git a/src/leap/soledad/server/auth.py b/src/leap/soledad/server/auth.py index da13dc8c..39eb09f6 100644 --- a/src/leap/soledad/server/auth.py +++ b/src/leap/soledad/server/auth.py @@ -56,8 +56,11 @@ class SoledadRealm(object): if conf is None: conf = get_config() blobs = conf['blobs'] - blobs_resource = BlobsResource("filesystem", - conf['blobs_path']) if blobs else None + concurrent_writes = conf['concurrent_blob_writes'] + blobs_resource = BlobsResource( + "filesystem", + conf['blobs_path'], + concurrent_writes=concurrent_writes) if blobs else None self.anon_resource = AnonymousResource( enable_blobs=blobs) self.auth_resource = PublicResource( diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index 78cf20ab..a313c934 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -43,7 +43,7 @@ class SoledadRealmTestCase(unittest.TestCase): def test_returned_resource(self): # we have to pass a pool to the realm , otherwise tests will hang - conf = {'blobs': False} + conf = {'blobs': False, 'concurrent_blob_writes': 50} pool = reactor.getThreadPool() realm = SoledadRealm(conf=conf, sync_pool=pool) iface, avatar, logout = realm.requestAvatar('any', None, IResource) diff --git a/tests/server/test_config.py b/tests/server/test_config.py index dfb09f4c..92b0f67f 100644 --- a/tests/server/test_config.py +++ b/tests/server/test_config.py @@ -66,5 +66,6 @@ class ConfigurationParsingTest(unittest.TestCase): 'batching': False, 'blobs': False, 'services_tokens_file': '/etc/soledad/services.tokens', - 'blobs_path': '/var/lib/soledad/blobs'} + 'blobs_path': '/var/lib/soledad/blobs', + 'concurrent_blob_writes': 50} self.assertDictEqual(expected, config['soledad-server']) -- cgit v1.2.3