diff options
| -rw-r--r-- | client/src/leap/soledad/client/_database/blobs.py | 9 | ||||
| -rw-r--r-- | server/src/leap/soledad/server/_blobs.py | 19 | ||||
| -rw-r--r-- | testing/tests/blobs/test_blob_manager.py (renamed from testing/tests/blobs/test_local_backend.py) | 17 | ||||
| -rw-r--r-- | testing/tests/blobs/test_decrypter_buffer.py (renamed from testing/tests/blobs/test_blobs.py) | 7 | ||||
| -rw-r--r-- | testing/tests/blobs/test_fs_backend.py | 15 | ||||
| -rw-r--r-- | testing/tests/blobs/test_sqlcipher_client_backend.py | 63 |
6 files changed, 122 insertions, 8 deletions
diff --git a/client/src/leap/soledad/client/_database/blobs.py b/client/src/leap/soledad/client/_database/blobs.py index de0b2d08..79404bf3 100644 --- a/client/src/leap/soledad/client/_database/blobs.py +++ b/client/src/leap/soledad/client/_database/blobs.py @@ -211,6 +211,9 @@ class BlobManager(object): @defer.inlineCallbacks def put(self, doc, size): + if (yield self.local.exists(doc.blob_id)): + error_message = "Blob already exists: %s" % doc.blob_id + raise BlobAlreadyExistsError(error_message) fd = doc.blob_fd # TODO this is a tee really, but ok... could do db and upload # concurrently. not sure if we'd gain something. @@ -350,6 +353,12 @@ class SQLiteBlobBackend(object): else: defer.returnValue([]) + @defer.inlineCallbacks + def exists(self, blob_id): + query = 'SELECT blob_id from blobs WHERE blob_id = ?' + result = yield self.dbpool.runQuery(query, (blob_id,)) + defer.returnValue(bool(len(result))) + def _init_blob_table(conn): maybe_create = ( diff --git a/server/src/leap/soledad/server/_blobs.py b/server/src/leap/soledad/server/_blobs.py index 84056c0a..72125d5a 100644 --- a/server/src/leap/soledad/server/_blobs.py +++ b/server/src/leap/soledad/server/_blobs.py @@ -43,6 +43,8 @@ __all__ = ['BlobsResource'] logger = Logger() +# Used for sanitizers, we accept only letters, numbers, '-' and '_' +VALID_STRINGS = re.compile('^[a-zA-Z0-9_-]+$') # TODO some error handling needed @@ -112,7 +114,9 @@ class FilesystemBlobsBackend(object): def list_blobs(self, user, request): blob_ids = [] + assert VALID_STRINGS.match(user) base_path = os.path.join(self.path, user) + assert self._valid_subdir(base_path) for _, _, filenames in os.walk(base_path): blob_ids += filenames return json.dumps(blob_ids) @@ -153,6 +157,7 @@ class FilesystemBlobsBackend(object): yield fbp.startProducing(open(path, 'wb')) def get_total_storage(self, user): + assert VALID_STRINGS.match(user) return self._get_disk_usage(os.path.join(self.path, user)) def delete_blob(user, blob_id): @@ -165,16 +170,26 @@ class FilesystemBlobsBackend(object): def _get_disk_usage(self, start_path): if not os.path.isdir(start_path): defer.returnValue(0) + assert self._valid_subdir(start_path) cmd = ['/usr/bin/du', '-s', '-c', start_path] output = yield utils.getProcessOutput(cmd[0], cmd[1:]) size = output.split()[0] defer.returnValue(int(size)) + def _valid_subdir(self, desired_path): + desired_path = os.path.realpath(desired_path) # expand path references + root = os.path.realpath(self.path) + return desired_path.startswith(root + os.sep) + def _get_path(self, user, blob_id): + assert VALID_STRINGS.match(user) + assert VALID_STRINGS.match(blob_id) parts = [user] parts += [blob_id[0], blob_id[0:3], blob_id[0:6]] parts += [blob_id] - return os.path.join(self.path, *parts) + path = os.path.join(self.path, *parts) + assert self._valid_subdir(path) + return path class BlobsResource(resource.Resource): @@ -217,7 +232,7 @@ class BlobsResource(resource.Resource): def _validate(self, request): for arg in request.postpath: - if arg and not re.match('^[a-zA-Z0-9_-]+$', arg): + if arg and not VALID_STRINGS.match(arg): raise Exception('Invalid blob resource argument: %s' % arg) return request.postpath diff --git a/testing/tests/blobs/test_local_backend.py b/testing/tests/blobs/test_blob_manager.py index 202d0611..69a272c8 100644 --- a/testing/tests/blobs/test_local_backend.py +++ b/testing/tests/blobs/test_blob_manager.py @@ -15,11 +15,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Tests for sqlcipher backend on blobs client. +Tests for BlobManager. """ from twisted.trial import unittest from twisted.internet import defer from leap.soledad.client._database.blobs import BlobManager, BlobDoc, FIXED_REV +from leap.soledad.client._database.blobs import BlobAlreadyExistsError from io import BytesIO from mock import Mock import pytest @@ -115,3 +116,17 @@ class BlobManagerTestCase(unittest.TestCase): call_blob_id, call_fd = call_list[0][0] self.assertEquals('missing_id', call_blob_id) self.assertEquals('test', call_fd.getvalue()) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_duplicated_blob_error_on_put(self): + self.manager._encrypt_and_upload = Mock(return_value=None) + content = "Blob content" + doc1 = BlobDoc(BytesIO(content), 'existing_id') + yield self.manager.put(doc1, len(content)) + doc2 = BlobDoc(BytesIO(content), 'existing_id') + # reset mock, so we can check that upload wasnt called + self.manager._encrypt_and_upload = Mock(return_value=None) + with pytest.raises(BlobAlreadyExistsError): + yield self.manager.put(doc2, len(content)) + self.assertFalse(self.manager._encrypt_and_upload.called) diff --git a/testing/tests/blobs/test_blobs.py b/testing/tests/blobs/test_decrypter_buffer.py index 9a0feb61..edaa66e2 100644 --- a/testing/tests/blobs/test_blobs.py +++ b/testing/tests/blobs/test_decrypter_buffer.py @@ -15,7 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Tests for blobs handling. +Tests for blobs decrypter buffer. A component which is used as a decryption +sink during blob stream download. """ from io import BytesIO from mock import Mock @@ -29,7 +30,7 @@ from leap.soledad.client._database.blobs import FIXED_REV from leap.soledad.client import _crypto -class BlobTestCase(unittest.TestCase): +class DecrypterBufferCase(unittest.TestCase): class doc_info: doc_id = 'D-BLOB-ID' @@ -53,7 +54,7 @@ class BlobTestCase(unittest.TestCase): self.assertEquals(fd.getvalue(), 'rosa de foc') @defer.inlineCallbacks - def test_blob_manager_encrypted_upload(self): + def test_decrypt_uploading_encrypted_blob(self): @defer.inlineCallbacks def _check_result(uri, data, *args, **kwargs): diff --git a/testing/tests/blobs/test_fs_backend.py b/testing/tests/blobs/test_fs_backend.py index 6da22621..0d7e9789 100644 --- a/testing/tests/blobs/test_fs_backend.py +++ b/testing/tests/blobs/test_fs_backend.py @@ -18,6 +18,7 @@ Tests for blobs backend on server side. """ from twisted.trial import unittest +from twisted.internet import defer from twisted.web.test.test_web import DummyRequest from leap.soledad.server import _blobs from io import BytesIO @@ -57,17 +58,19 @@ class FilesystemBackendTestCase(unittest.TestCase): render_mock.render_GET.assert_called_once_with(request) @mock.patch.object(os.path, 'isfile') + @defer.inlineCallbacks def test_cannot_overwrite(self, isfile): isfile.return_value = True backend = _blobs.FilesystemBlobsBackend() backend._get_path = Mock(return_value='path') request = DummyRequest(['']) - result = yield backend.write_blob('user', 'blob_id', request) - self.assertEquals(result, "Blob already exists: blob_id") + yield backend.write_blob('user', 'blob_id', request) + self.assertEquals(request.written[0], "Blob already exists: blob_id") self.assertEquals(request.responseCode, 409) @pytest.mark.usefixtures("method_tmpdir") @mock.patch.object(os.path, 'isfile') + @defer.inlineCallbacks def test_write_cannot_exceed_quota(self, isfile): isfile.return_value = False backend = _blobs.FilesystemBlobsBackend() @@ -94,3 +97,11 @@ class FilesystemBackendTestCase(unittest.TestCase): walk_mock.return_value = [(_, _, ['blob_0']), (_, _, ['blob_1'])] result = json.loads(backend.list_blobs('user', DummyRequest(['']))) self.assertEquals(result, ['blob_0', 'blob_1']) + + @pytest.mark.usefixtures("method_tmpdir") + def test_path_validation_for_subdirectories(self): + blobs_path = self.tempdir + backend = _blobs.FilesystemBlobsBackend(blobs_path) + self.assertFalse(backend._valid_subdir('/')) + self.assertFalse(backend._valid_subdir(blobs_path + '../../../../../')) + self.assertTrue(backend._valid_subdir(os.path.join(blobs_path, 'x'))) diff --git a/testing/tests/blobs/test_sqlcipher_client_backend.py b/testing/tests/blobs/test_sqlcipher_client_backend.py new file mode 100644 index 00000000..865a64e1 --- /dev/null +++ b/testing/tests/blobs/test_sqlcipher_client_backend.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher_client_backend.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for sqlcipher backend on blobs client. +""" +from twisted.trial import unittest +from twisted.internet import defer +from leap.soledad.client._database.blobs import SQLiteBlobBackend +from io import BytesIO +import pytest + + +class SQLBackendTestCase(unittest.TestCase): + + def setUp(self): + self.key = "A" * 96 + self.local = SQLiteBlobBackend(self.tempdir, self.key) + self.addCleanup(self.local.close) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_inexistent(self): + bad_blob_id = 'inexsistent_id' + self.assertFalse((yield self.local.exists(bad_blob_id))) + result = yield self.local.get(bad_blob_id) + self.assertIsNone(result) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_existent(self): + blob_id = 'blob_id' + content = "x" + yield self.local.put(blob_id, BytesIO(content), len(content)) + result = yield self.local.get(blob_id) + self.assertTrue((yield self.local.exists(blob_id))) + self.assertEquals(result.getvalue(), content) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list(self): + blob_ids = [('blob_id%s' % i) for i in range(10)] + content = "x" + deferreds = [] + for blob_id in blob_ids: + deferreds.append(self.local.put(blob_id, BytesIO(content), + len(content))) + yield defer.gatherResults(deferreds) + result = yield self.local.list() + self.assertEquals(blob_ids, result) |