diff options
Diffstat (limited to 'service')
10 files changed, 214 insertions, 99 deletions
diff --git a/service/pixelated/adapter/mailstore/leap_attachment_store.py b/service/pixelated/adapter/mailstore/leap_attachment_store.py new file mode 100644 index 00000000..86121db9 --- /dev/null +++ b/service/pixelated/adapter/mailstore/leap_attachment_store.py @@ -0,0 +1,61 @@ + +import quopri +import base64 +from email import encoders +from leap.mail.adaptors.soledad import SoledadMailAdaptor, ContentDocWrapper +from twisted.internet import defer +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.multipart import MIMEMultipart +from leap.mail.mail import Message + + +class LeapAttachmentStore(object): + + def __init__(self, soledad): + self.soledad = soledad + + @defer.inlineCallbacks + def get_mail_attachment(self, attachment_id): + results = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', attachment_id) if attachment_id else [] + if len(results): + content = ContentDocWrapper(**results[0].content) + defer.returnValue({'content-type': content.content_type, 'content': self._try_decode( + content.raw, content.content_transfer_encoding)}) + else: + raise ValueError('No attachment with id %s found!' % attachment_id) + + @defer.inlineCallbacks + def add_attachment(self, content, content_type): + cdoc = self._attachment_to_cdoc(content, content_type) + yield self.soledad.create_doc(cdoc.serialize(), doc_id=cdoc.phash) + defer.returnValue(cdoc.phash) + + def _try_decode(self, raw, encoding): + encoding = encoding.lower() + if encoding == 'base64': + data = base64.decodestring(raw) + elif encoding == 'quoted-printable': + data = quopri.decodestring(raw) + else: + data = str(raw) + + return bytearray(data) + + def _attachment_to_cdoc(self, content, content_type, encoder=encoders.encode_base64): + major, sub = content_type.split('/') + attachment = MIMENonMultipart(major, sub) + attachment.set_payload(content) + encoder(attachment) + attachment.add_header('Content-Disposition', 'attachment', filename='does_not_matter.txt') + + pseudo_mail = MIMEMultipart() + pseudo_mail.attach(attachment) + + tmp_mail = SoledadMailAdaptor().get_msg_from_string(MessageClass=Message, raw_msg=pseudo_mail.as_string()) + + cdoc = tmp_mail.get_wrapper().cdocs[1] + return cdoc + + def _calc_attachment_id_(self, content, content_type, encoder=encoders.encode_base64): + cdoc = self._attachment_to_cdoc(content, content_type, encoder) + return cdoc.phash diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py index 14b0e417..2660a208 100644 --- a/service/pixelated/adapter/mailstore/leap_mailstore.py +++ b/service/pixelated/adapter/mailstore/leap_mailstore.py @@ -16,7 +16,6 @@ import base64 from email.header import decode_header from email.utils import parseaddr -import quopri from uuid import uuid4 import re @@ -222,27 +221,6 @@ class LeapMailStore(MailStore): return defer.gatherResults(deferreds, consumeErrors=True) @defer.inlineCallbacks - def get_mail_attachment(self, attachment_id): - results = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', attachment_id) if attachment_id else [] - if len(results): - content = ContentDocWrapper(**results[0].content) - defer.returnValue({'content-type': content.content_type, 'content': self._try_decode( - content.raw, content.content_transfer_encoding)}) - else: - raise ValueError('No attachment with id %s found!' % attachment_id) - - def _try_decode(self, raw, encoding): - encoding = encoding.lower() - if encoding == 'base64': - data = base64.decodestring(raw) - elif encoding == 'quoted-printable': - data = quopri.decodestring(raw) - else: - data = str(raw) - - return bytearray(data) - - @defer.inlineCallbacks def update_mail(self, mail): message = yield self._fetch_msg_from_soledad(mail.mail_id) message.get_wrapper().set_tags(tuple(mail.tags)) diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py index 4c1c1c0d..75c0808e 100644 --- a/service/pixelated/adapter/services/mail_service.py +++ b/service/pixelated/adapter/services/mail_service.py @@ -28,39 +28,20 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor class MailService(object): - def __init__(self, mail_sender, mail_store, search_engine, account_email): + def __init__(self, mail_sender, mail_store, search_engine, account_email, attachment_store): self.mail_store = mail_store self.search_engine = search_engine self.mail_sender = mail_sender self.account_email = account_email + self.attchment_store = attachment_store @defer.inlineCallbacks def all_mails(self): mails = yield self.mail_store.all_mails() defer.returnValue(mails) - def _attachment_to_cdoc(self, content, content_type, encoder=encoders.encode_base64): - major, sub = content_type.split('/') - attachment = MIMENonMultipart(major, sub) - attachment.set_payload(content) - encoder(attachment) - attachment.add_header('Content-Disposition', 'attachment', filename='does_not_matter.txt') - - pseudo_mail = MIMEMultipart() - pseudo_mail.attach(attachment) - - tmp_mail = SoledadMailAdaptor().get_msg_from_string(MessageClass=Message, raw_msg=pseudo_mail.as_string()) - - cdoc = tmp_mail.get_wrapper().cdocs[1] - return cdoc - - def _calc_attachment_id_(self, content, content_type, encoder=encoders.encode_base64): - cdoc = self._attachment_to_cdoc(content, content_type, encoder) - - return cdoc.phash - - def attachment_id(self, content, content_type): - return self._calc_attachment_id_(content, content_type) + def save_attachment(self, content, content_type): + return self.attchment_store.add_attachment(content, content_type) @defer.inlineCallbacks def mails(self, query, window_size, page): @@ -103,7 +84,7 @@ class MailService(object): return self.mail_store.get_mail(mail_id, include_body=True) def attachment(self, attachment_id): - return self.mail_store.get_mail_attachment(attachment_id) + return self.attchment_store.get_mail_attachment(attachment_id) @defer.inlineCallbacks def mail_exists(self, mail_id): diff --git a/service/pixelated/config/services.py b/service/pixelated/config/services.py index b70bb3f9..b8b355b0 100644 --- a/service/pixelated/config/services.py +++ b/service/pixelated/config/services.py @@ -1,3 +1,4 @@ +from pixelated.adapter.mailstore.leap_attachment_store import LeapAttachmentStore from pixelated.adapter.mailstore.searchable_mailstore import SearchableMailStore from pixelated.adapter.services.mail_service import MailService from pixelated.adapter.model.mail import InputMail @@ -61,7 +62,8 @@ class Services(object): pixelated_mail_sender, leap_session.mail_store, search_engine, - leap_session.account_email()) + leap_session.account_email(), + LeapAttachmentStore(leap_session.soledad)) def setup_draft_service(self, mail_store): return DraftService(mail_store) diff --git a/service/pixelated/resources/attachments_resource.py b/service/pixelated/resources/attachments_resource.py index 911fc43d..05bc923f 100644 --- a/service/pixelated/resources/attachments_resource.py +++ b/service/pixelated/resources/attachments_resource.py @@ -90,7 +90,7 @@ class AttachmentsResource(Resource): fields = cgi.FieldStorage(fp=request.content, headers=(request.getAllHeaders()), environ={'REQUEST_METHOD': 'POST'}) _file = fields['attachment'] - deferred = defer.maybeDeferred(self.mail_service.attachment_id, _file.value, _file.type) + deferred = self.mail_service.save_attachment(_file.value, _file.type) def send_location(attachment_id): request.headers['Location'] = '/%s/%s' % (self.BASE_URL, attachment_id) diff --git a/service/test/support/integration/app_test_client.py b/service/test/support/integration/app_test_client.py index 5f7eb90a..89f42532 100644 --- a/service/test/support/integration/app_test_client.py +++ b/service/test/support/integration/app_test_client.py @@ -31,6 +31,7 @@ from twisted.internet import reactor, defer from twisted.internet.defer import succeed from twisted.web.resource import getChildForRequest # from twisted.web.server import Site as PixelatedSite +from pixelated.adapter.mailstore.leap_attachment_store import LeapAttachmentStore from pixelated.adapter.services.feedback_service import FeedbackService from pixelated.config.site import PixelatedSite @@ -70,6 +71,7 @@ class AppTestClient(object): self.mail_sender = self._create_mail_sender() self.mail_store = SearchableMailStore(LeapMailStore(self.soledad), self.search_engine) + self.attachment_store = LeapAttachmentStore(self.soledad) account_ready_cb = defer.Deferred() self.account = IMAPAccount(self.ACCOUNT, self.soledad, account_ready_cb) @@ -78,7 +80,7 @@ class AppTestClient(object): self.leap_session = mock() self.feedback_service = FeedbackService(self.leap_session) - self.mail_service = self._create_mail_service(self.mail_sender, self.mail_store, self.search_engine) + self.mail_service = self._create_mail_service(self.mail_sender, self.mail_store, self.search_engine, self.attachment_store) mails = yield self.mail_service.all_mails() self.search_engine.index_mails(mails) @@ -166,8 +168,8 @@ class AppTestClient(object): mail_sender.sendmail.side_effect = lambda mail: succeed(mail) return mail_sender - def _create_mail_service(self, mail_sender, mail_store, search_engine): - mail_service = MailService(mail_sender, mail_store, search_engine, self.MAIL_ADDRESS) + def _create_mail_service(self, mail_sender, mail_store, search_engine, attachment_store): + mail_service = MailService(mail_sender, mail_store, search_engine, self.MAIL_ADDRESS, attachment_store) return mail_service def _generate_soledad_test_folder_name(self, soledad_test_folder='/tmp/soledad-test/test'): diff --git a/service/test/unit/adapter/mailstore/test_leap_attachment_store.py b/service/test/unit/adapter/mailstore/test_leap_attachment_store.py new file mode 100644 index 00000000..172cceb6 --- /dev/null +++ b/service/test/unit/adapter/mailstore/test_leap_attachment_store.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import json +from uuid import uuid4 + +from leap.mail.adaptors.soledad_indexes import MAIL_INDEXES +from leap.soledad.common.document import SoledadDocument +from mockito import mock, when, verify +import test.support.mockito +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from leap.mail.adaptors.soledad import SoledadMailAdaptor, MailboxWrapper, ContentDocWrapper + +from pixelated.adapter.mailstore.leap_attachment_store import LeapAttachmentStore + + +class TestLeapAttachmentStore(TestCase): + def setUp(self): + self.soledad = mock() + self.mbox_uuid = str(uuid4()) + self.doc_by_id = {} + self.mbox_uuid_by_name = {} + self.mbox_soledad_docs = [] + + when(self.soledad).get_from_index('by-type', 'mbox').thenAnswer(lambda: defer.succeed(self.mbox_soledad_docs)) + self._mock_get_mailbox('INBOX') + + @defer.inlineCallbacks + def test_get_mail_attachment(self): + attachment_id = 'AAAA9AAD9E153D24265395203C53884506ABA276394B9FEC02B214BF9E77E48E' + doc = SoledadDocument(json=json.dumps({'content_type': 'foo/bar', 'raw': 'asdf'})) + when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt', attachment_id).thenReturn(defer.succeed([doc])) + store = LeapAttachmentStore(self.soledad) + + attachment = yield store.get_mail_attachment(attachment_id) + + self.assertEqual({'content-type': 'foo/bar', 'content': bytearray('asdf')}, attachment) + + @defer.inlineCallbacks + def test_store_attachment(self): + content = 'this is some attachment content' + content_type = 'text/plain' + cdoc_serialized = {'content_transfer_encoding': 'base64', 'lkf': [], 'content_disposition': 'attachment', + 'ctype': '', 'raw': 'dGhpcyBpcyBzb21lIGF0dGFjaG1lbnQgY29udGVudA==', + 'phash': '9863729729D2E2EE8E52F0A7115CE33AD18DDA4B58E49AE08DD092D1C8E699B0', + 'content_type': 'text/plain', 'type': 'cnt'} + + store = LeapAttachmentStore(self.soledad) + + attachment_id = yield store.add_attachment(content, content_type) + + self.assertEqual('9863729729D2E2EE8E52F0A7115CE33AD18DDA4B58E49AE08DD092D1C8E699B0', attachment_id) + + verify(self.soledad).create_doc(cdoc_serialized, doc_id=attachment_id) + + @defer.inlineCallbacks + def test_get_mail_attachment_different_content_encodings(self): + attachment_id = '1B0A9AAD9E153D24265395203C53884506ABA276394B9FEC02B214BF9E77E48E' + encoding_examples = [('', 'asdf', 'asdf'), + ('base64', 'asdf', 'YXNkZg=='), + ('quoted-printable', 'äsdf', '=C3=A4sdf')] + + for transfer_encoding, data, encoded_data in encoding_examples: + doc = SoledadDocument(json=json.dumps({'content_type': 'foo/bar', 'raw': encoded_data, + 'content_transfer_encoding': transfer_encoding})) + when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt', attachment_id).thenReturn(defer.succeed([doc])) + store = LeapAttachmentStore(self.soledad) + + attachment = yield store.get_mail_attachment(attachment_id) + + self.assertEqual(bytearray(data), attachment['content']) + + @defer.inlineCallbacks + def test_get_mail_attachment_throws_exception_if_attachment_does_not_exist(self): + attachment_id = '1B0A9AAD9E153D24265395203C53884506ABA276394B9FEC02B214BF9E77E48E' + when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt', attachment_id).thenReturn(defer.succeed([])) + store = LeapAttachmentStore(self.soledad) + try: + yield store.get_mail_attachment(attachment_id) + self.fail('ValueError exception expected') + except ValueError: + pass + + def _mock_get_mailbox(self, mailbox_name, create_new_uuid=False): + mbox_uuid = self.mbox_uuid if not create_new_uuid else str(uuid4()) + when(self.soledad).list_indexes().thenReturn(defer.succeed(MAIL_INDEXES)).thenReturn( + defer.succeed(MAIL_INDEXES)) + doc_id = str(uuid4()) + mbox = MailboxWrapper(doc_id=doc_id, mbox=mailbox_name, uuid=mbox_uuid) + soledad_doc = SoledadDocument(doc_id, json=json.dumps(mbox.serialize())) + when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', mailbox_name).thenReturn(defer.succeed([soledad_doc])) + self._mock_get_soledad_doc(doc_id, mbox) + + self.mbox_uuid_by_name[mailbox_name] = mbox_uuid + self.mbox_soledad_docs.append(soledad_doc) + + return mbox, soledad_doc + + def _mock_get_soledad_doc(self, doc_id, doc): + soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize())) + + # when(self.soledad).get_doc(doc_id).thenReturn(defer.succeed(soledad_doc)) + when(self.soledad).get_doc(doc_id).thenAnswer(lambda: defer.succeed(soledad_doc)) + + self.doc_by_id[doc_id] = soledad_doc diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py index 4eabc144..b5b6a742 100644 --- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py +++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py @@ -135,45 +135,6 @@ class TestLeapMailStore(TestCase): self.assertEqual(expeted_body, mail.body) @defer.inlineCallbacks - def test_get_mail_attachment(self): - attachment_id = 'AAAA9AAD9E153D24265395203C53884506ABA276394B9FEC02B214BF9E77E48E' - doc = SoledadDocument(json=json.dumps({'content_type': 'foo/bar', 'raw': 'asdf'})) - when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt', attachment_id).thenReturn(defer.succeed([doc])) - store = LeapMailStore(self.soledad) - - attachment = yield store.get_mail_attachment(attachment_id) - - self.assertEqual({'content-type': 'foo/bar', 'content': bytearray('asdf')}, attachment) - - @defer.inlineCallbacks - def test_get_mail_attachment_different_content_encodings(self): - attachment_id = '1B0A9AAD9E153D24265395203C53884506ABA276394B9FEC02B214BF9E77E48E' - encoding_examples = [('', 'asdf', 'asdf'), - ('base64', 'asdf', 'YXNkZg=='), - ('quoted-printable', 'äsdf', '=C3=A4sdf')] - - for transfer_encoding, data, encoded_data in encoding_examples: - doc = SoledadDocument(json=json.dumps({'content_type': 'foo/bar', 'raw': encoded_data, - 'content_transfer_encoding': transfer_encoding})) - when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt', attachment_id).thenReturn(defer.succeed([doc])) - store = LeapMailStore(self.soledad) - - attachment = yield store.get_mail_attachment(attachment_id) - - self.assertEqual(bytearray(data), attachment['content']) - - @defer.inlineCallbacks - def test_get_mail_attachment_throws_exception_if_attachment_does_not_exist(self): - attachment_id = '1B0A9AAD9E153D24265395203C53884506ABA276394B9FEC02B214BF9E77E48E' - when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt', attachment_id).thenReturn(defer.succeed([])) - store = LeapMailStore(self.soledad) - try: - yield store.get_mail_attachment(attachment_id) - self.fail('ValueError exception expected') - except ValueError: - pass - - @defer.inlineCallbacks def test_update_mail(self): mdoc_id, fdoc_id = self._add_mail_fixture_to_soledad_from_file('mbox00000000') soledad_fdoc = self.doc_by_id[fdoc_id] diff --git a/service/test/unit/adapter/test_mail_service.py b/service/test/unit/adapter/test_mail_service.py index d14a0eb0..27c80fde 100644 --- a/service/test/unit/adapter/test_mail_service.py +++ b/service/test/unit/adapter/test_mail_service.py @@ -28,6 +28,7 @@ class TestMailService(unittest.TestCase): def setUp(self): self.drafts = mock() self.mail_store = mock() + self.attachment_store = mock() self.mailboxes = mock() self.mailboxes.drafts = defer.succeed(self.drafts) @@ -37,7 +38,7 @@ class TestMailService(unittest.TestCase): self.mail_sender = mock() self.search_engine = mock() - self.mail_service = MailService(self.mail_sender, self.mail_store, self.search_engine, 'acount@email') + self.mail_service = MailService(self.mail_sender, self.mail_store, self.search_engine, 'acount@email', self.attachment_store) def tearDown(self): unstub() @@ -158,7 +159,7 @@ class TestMailService(unittest.TestCase): @defer.inlineCallbacks def test_get_attachment(self): attachment_dict = {'content': bytearray('data'), 'content-type': 'text/plain'} - when(self.mail_store).get_mail_attachment('some attachment id').thenReturn(defer.succeed(attachment_dict)) + when(self.attachment_store).get_mail_attachment('some attachment id').thenReturn(defer.succeed(attachment_dict)) attachment = yield self.mail_service.attachment('some attachment id') diff --git a/service/test/unit/resources/test_attachments_resource.py b/service/test/unit/resources/test_attachments_resource.py index 837f5324..2afa208c 100644 --- a/service/test/unit/resources/test_attachments_resource.py +++ b/service/test/unit/resources/test_attachments_resource.py @@ -2,6 +2,7 @@ import json import unittest from mock import patch, MagicMock +from mockito import mock, when, verify from twisted.internet import defer from twisted.web.test.requesthelper import DummyRequest @@ -12,19 +13,22 @@ from test.unit.resources import DummySite class AttachmentsResourceTest(unittest.TestCase): def setUp(self): - self.mail_service = MagicMock() + self.mail_service = mock() self.mails_resource = AttachmentsResource(self.mail_service) self.mails_resource.isLeaf = True self.web = DummySite(self.mails_resource) - @patch('twisted.internet.defer.maybeDeferred') @patch('cgi.FieldStorage') - def test_post_new_attachment(self, mock_fields, mock_maybe_deferred): + def test_post_new_attachment(self, mock_fields): request = DummyRequest(['/attachment']) request.method = 'POST' request.content = 'mocked' attachment_id = 'B5B4ED80AC3B894523D72E375DACAA2FC6606C18EDF680FE95903086C8B5E14A' - mock_maybe_deferred.return_value = defer.succeed(attachment_id) + _file = MagicMock() + _file.value = 'some mocked value' + _file.type = 'some mocked type' + mock_fields.return_value = {'attachment': _file} + when(self.mail_service).save_attachment('some mocked value', 'some mocked type').thenReturn(defer.succeed(attachment_id)) d = self.web.get(request) @@ -32,24 +36,30 @@ class AttachmentsResourceTest(unittest.TestCase): self.assertEqual(201, request.code) self.assertEqual('/attachment/%s' % attachment_id, request.headers['Location']) self.assertEqual({'attachment_id': attachment_id}, json.loads(request.written[0])) + verify(self.mail_service).save_attachment('some mocked value', 'some mocked type') d.addCallback(assert_response) return d - @patch('twisted.internet.defer.maybeDeferred') @patch('cgi.FieldStorage') - def test_post_attachment_fails(self, mock_fields, mock_maybe_deferred): - mock_maybe_deferred.return_value = defer.fail(Exception) + def test_post_attachment_fails(self, mock_fields): request = DummyRequest(['/attachment']) request.method = 'POST' request.content = 'mocked' + _file = MagicMock() + _file.value = 'some mocked value' + _file.type = 'some mocked type' + mock_fields.return_value = {'attachment': _file} + when(self.mail_service).save_attachment('some mocked value', 'some mocked type').thenReturn(defer.fail(Exception)) + d = self.web.get(request) def assert_response(_): self.assertEqual(500, request.code) self.assertFalse('Location' in request.headers) self.assertEqual({"message": "Something went wrong. Attachement not saved."}, json.loads(request.written[0])) + verify(self.mail_service).save_attachment('some mocked value', 'some mocked type') d.addCallback(assert_response) return d |