From 78621bb742cd0a816dc507010743a7d765d84538 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 10 Aug 2017 00:15:01 -0300 Subject: [feature] add path partitioning to namespaces All blobs were being stored in a single folder when using namespaces, this commits adds path partitioning as discussed on #8882, which should help with a large number of files (each folder will hold a smaller subset, allowing the use of better filesystem walk strategies). Also, the default empty namespace is now called 'default' to prevent it from listing other namespaces contents. So everything will always use namespaces, with the option to use it explicitly or just fall to the default one. -- Related: #8882 --- src/leap/soledad/server/_blobs.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'src/leap/soledad/server') diff --git a/src/leap/soledad/server/_blobs.py b/src/leap/soledad/server/_blobs.py index 7d51e1b5..dc2c204c 100644 --- a/src/leap/soledad/server/_blobs.py +++ b/src/leap/soledad/server/_blobs.py @@ -135,7 +135,7 @@ class FilesystemBlobsBackend(object): raise NotImplementedError def count(self, user, request, namespace=''): - base_path = self._get_path(user, custom_preffix=namespace) + base_path = self._get_path(user, namespace=namespace) count = 0 for _, _, filenames in os.walk(base_path): count += len(filter(lambda i: not i.endswith('.flags'), filenames)) @@ -144,7 +144,7 @@ class FilesystemBlobsBackend(object): def list_blobs(self, user, request, namespace='', order_by=None, filter_flag=False): blob_ids = [] - base_path = self._get_path(user, custom_preffix=namespace) + base_path = self._get_path(user, namespace=namespace) for root, dirs, filenames in os.walk(base_path): blob_ids += [os.path.join(root, name) for name in filenames if not name.endswith('.flags')] @@ -200,18 +200,22 @@ class FilesystemBlobsBackend(object): raise Exception(err) return desired_path - def _get_path(self, user, blob_id='', custom_preffix=''): + def _get_path(self, user, blob_id='', namespace=''): parts = [user] - parts += self._get_preffix(blob_id, custom_preffix) if blob_id: - parts += [blob_id] + namespace = namespace or 'default' + parts += self._get_path_parts(blob_id, namespace) + elif namespace and not blob_id: + parts += [namespace] # namespace path + else: + pass # root path path = os.path.join(self.path, *parts) return self._validate_path(path, user, blob_id) - def _get_preffix(self, blob_id, custom=''): - if custom or not blob_id: + def _get_path_parts(self, blob_id, custom): + if custom and not blob_id: return [custom] - return [blob_id[0], blob_id[0:3], blob_id[0:6]] + return [custom] + [blob_id[0], blob_id[0:3], blob_id[0:6]] + [blob_id] class ImproperlyConfiguredException(Exception): -- cgit v1.2.3