From 57376b6cb20b0d4af0e5940f29229421adf8679f Mon Sep 17 00:00:00 2001 From: kaeff Date: Fri, 14 Aug 2015 16:57:51 +0200 Subject: [wip] Expose attachments again after migration - Issue: #433 - Still missing: In JS, use encoding directly from the attachment object, insted of headers. Then remove headers again (see failing unit test) --- .../pixelated/adapter/mailstore/leap_mailstore.py | 44 ++++++++++++- .../test/functional/features/attachments.feature | 27 ++++++++ .../test/functional/features/steps/attachments.py | 55 ++++++++++++++++ service/test/integration/test_leap_mailstore.py | 24 ++++++- .../unit/adapter/mailstore/test_leap_mailstore.py | 76 +++++++++++++++------- 5 files changed, 200 insertions(+), 26 deletions(-) create mode 100644 service/test/functional/features/attachments.feature create mode 100644 service/test/functional/features/steps/attachments.py (limited to 'service') diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py index f03e2900..a8fa6d13 100644 --- a/service/pixelated/adapter/mailstore/leap_mailstore.py +++ b/service/pixelated/adapter/mailstore/leap_mailstore.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see . from email.header import decode_header +import re from uuid import uuid4 from leap.mail.adaptors.soledad import SoledadMailAdaptor from twisted.internet import defer @@ -24,15 +25,24 @@ from leap.mail.mail import Message from pixelated.adapter.model.mail import Mail, InputMail +class AttachmentInfo(object): + def __init__(self, ident, name, encoding, headers): + self.ident = ident + self.name = name + self.encoding = encoding + self.headers = headers + + class LeapMail(Mail): - def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None): + def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]): self._mail_id = mail_id self._mailbox_name = mailbox_name self._headers = headers if headers is not None else {} self._body = body self.tags = set(tags) # TODO test that asserts copy self._flags = set(flags) # TODO test that asserts copy + self._attachments = attachments @property def headers(self): @@ -88,7 +98,8 @@ class LeapMail(Mail): 'body': self._body, 'textPlainBody': self._body, 'replying': self._replying_dict(), - 'mailbox': self._mailbox_name.lower() + 'mailbox': self._mailbox_name.lower(), + 'attachments': [{'ident': attachment.ident, 'name': attachment.name, 'encoding': attachment.encoding, 'headers': attachment.headers} for attachment in self._attachments] } def _replying_dict(self): @@ -116,6 +127,14 @@ class LeapMail(Mail): return [recipient for recipient in recipients if recipient != InputMail.FROM_EMAIL_ADDRESS] +def _extract_filename(content_disposition): + match = re.compile('.*name=\"(.*)\".*').search(content_disposition) + filename = '' + if match: + filename = match.group(1) + return filename + + class LeapMailStore(MailStore): __slots__ = ('soledad') @@ -244,7 +263,7 @@ class LeapMailStore(MailStore): mbox_uuid = message.get_wrapper().fdoc.mbox_uuid mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid) - mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body) # TODO assert flags are passed on + mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=self._extract_attachment_info_from(message)) # TODO assert flags are passed on defer.returnValue(mail) @@ -277,6 +296,25 @@ class LeapMailStore(MailStore): for doc in docs: print '\n%s\n' % doc + def _extract_attachment_info_from(self, message): + wrapper = message.get_wrapper() + part_maps = wrapper.hdoc.part_map + + result = [] + + for nr, part_map in part_maps.items(): + if 'headers' in part_map: + headers = {header[0]: header[1] for header in part_map['headers']} + phash = part_map['phash'] + if 'Content-Disposition' in headers: + disposition = headers['Content-Disposition'] + if 'attachment' in disposition: + filename = _extract_filename(disposition) + encoding = headers['Content-Transfer-Encoding'] + result.append(AttachmentInfo(phash, filename, encoding, headers)) + + return result + def _is_empty_message(message): return (message is None) or (message.get_wrapper().mdoc.doc_id is None) diff --git a/service/test/functional/features/attachments.feature b/service/test/functional/features/attachments.feature new file mode 100644 index 00000000..19834a9d --- /dev/null +++ b/service/test/functional/features/attachments.feature @@ -0,0 +1,27 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + +Feature: Attachments + As a user of Pixelated + I want to download attachments of mails I received + So that my peers are able to send me any kind of content, not just text + + Scenario: User opens a mail attachment + Given I have a mail with an attachment in my inbox + When I open the first mail in the 'inbox' + Then I see the mail has an attachment + #When I open click on the first attachment + #Then the browser downloaded a file diff --git a/service/test/functional/features/steps/attachments.py b/service/test/functional/features/steps/attachments.py new file mode 100644 index 00000000..066683bf --- /dev/null +++ b/service/test/functional/features/steps/attachments.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from email.mime.application import MIMEApplication +from time import sleep +from leap.mail.mail import Message +from common import * +from test.support.integration import MailBuilder +from behave import given +from crochet import wait_for +from uuid import uuid4 +from email.MIMEMultipart import MIMEMultipart +from email.mime.text import MIMEText + + +@given(u'I have a mail with an attachment in my inbox') +def add_mail_with_attachment_impl(context): + subject = 'Hi! This the subject %s' % uuid4() + mail = build_mail_with_attachment(subject) + load_mail_into_soledad(context, mail) + context.last_subject = subject + + +def build_mail_with_attachment(subject): + mail = MIMEMultipart() + mail['Subject'] = subject + mail.attach(MIMEText(u'a utf8 message', _charset='utf-8')) + attachment = MIMEApplication('pretend to be binary attachment data') + attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt') + mail.attach(attachment) + + return mail + + +@wait_for(timeout=10.0) +def load_mail_into_soledad(context, mail): + return context.client.mail_store.add_mail('INBOX', mail.as_string()) + + +@then(u'I see the mail has an attachment') +def step_impl(context): + attachments_list = find_elements_by_css_selector(context, '.attachmentsArea li') + assert len(attachments_list) == 1 diff --git a/service/test/integration/test_leap_mailstore.py b/service/test/integration/test_leap_mailstore.py index b1b2075d..abe5d584 100644 --- a/service/test/integration/test_leap_mailstore.py +++ b/service/test/integration/test_leap_mailstore.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see . +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from test.support.integration import SoledadTestBase, load_mail_from_file from twisted.internet import defer @@ -28,7 +31,7 @@ class LeapMailStoreTest(SoledadTestBase): self.maxDiff = None mail = load_mail_from_file('mbox00000000') mail_id = yield self._create_mail_in_soledad(mail) - expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'status': [], 'tags': set([]), 'replying': {'all': {'cc-field': [], 'to-field': [u'carmel@murazikortiz.name', u'darby.senger@zemlak.biz']}, 'single': u'darby.senger@zemlak.biz'}, 'textPlainBody': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'mailbox': u'inbox'} + expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'status': [], 'tags': set([]), 'replying': {'all': {'cc-field': [], 'to-field': [u'carmel@murazikortiz.name', u'darby.senger@zemlak.biz']}, 'single': u'darby.senger@zemlak.biz'}, 'textPlainBody': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'mailbox': u'inbox', 'attachments': []} result = yield self.mail_store.get_mail(mail_id, include_body=True) self.assertIsNotNone(result) @@ -45,6 +48,25 @@ class LeapMailStoreTest(SoledadTestBase): self.assertEqual(expected_mail_dict['header'], mail.as_dict()['header']) self.assertEqual(expected_mail_dict['header'], fetched_mail.as_dict()['header']) + @defer.inlineCallbacks + def test_round_trip_through_soledad_keeps_attachment(self): + input_mail = MIMEMultipart() + input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8')) + attachment = MIMEApplication('pretend to be binary attachment data') + attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt') + input_mail.attach(attachment) + + mail = yield self.mail_store.add_mail('INBOX', input_mail.as_string()) + fetched_mail = yield self.mail_store.get_mail(mail.ident, include_body=True) + + # _, docs = yield self.soledad.get_all_docs() + # for doc in docs: + # print '\n%s\n' % doc + + # self.assertEqual(1, len(mail.as_dict()['attachments'])) + # print fetched_mail.as_dict() + # self.assertEqual(1, len(fetched_mail.as_dict()['attachments'])) + @defer.inlineCallbacks def test_all_mails(self): mail = load_mail_from_file('mbox00000000') diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py index 4789f02c..a28731e3 100644 --- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py +++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see . +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText import json from uuid import uuid4 from email.parser import Parser @@ -31,7 +34,7 @@ import pkg_resources from leap.mail.mail import Message from pixelated.adapter.mailstore import underscore_uuid -from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail +from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail, AttachmentInfo class TestLeapMail(TestCase): @@ -83,6 +86,7 @@ class TestLeapMail(TestCase): 'receiver2@other.test', 'test@example.test']}, 'single': 'test@example.test'}, + 'attachments': [] } self.assertEqual(expected, mail.as_dict()) @@ -93,6 +97,12 @@ class TestLeapMail(TestCase): self.assertEqual(body, mail.as_dict()['body']) + def test_as_dict_with_attachments(self): + mail = LeapMail('doc id', 'INBOX', attachments=[AttachmentInfo('id', 'name', 'encoding')]) + + self.assertEqual([{'ident': 'id', 'name': 'name', 'encoding': 'encoding'}], + mail.as_dict()['attachments']) + def test_raw_constructed_by_headers_and_body(self): body = 'some body content' mail = LeapMail('doc id', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test'}, ('foo', 'bar'), body=body) @@ -240,7 +250,7 @@ class TestLeapMailStore(TestCase): def test_add_mailbox(self): when(self.soledad).list_indexes().thenReturn(defer.succeed(MAIL_INDEXES)).thenReturn(defer.succeed(MAIL_INDEXES)) when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', 'TEST').thenReturn(defer.succeed([])) - self._mock_create_doc(self.mbox_uuid, MailboxWrapper(mbox='TEST')) + self._mock_create_soledad_doc(self.mbox_uuid, MailboxWrapper(mbox='TEST')) when(self.soledad).get_doc(self.mbox_uuid).thenAnswer(lambda: defer.succeed(self.doc_by_id[self.mbox_uuid])) when(self.soledad).put_doc(ANY()).thenAnswer(lambda: defer.succeed(None)) store = LeapMailStore(self.soledad) @@ -272,7 +282,7 @@ class TestLeapMailStore(TestCase): @defer.inlineCallbacks def test_add_mail(self): - expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000') + expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000') mail = self._load_mail_from_file('mbox00000000') self._mock_get_mailbox('INBOX') @@ -283,6 +293,25 @@ class TestLeapMailStore(TestCase): self.assertIsInstance(message, LeapMail) self._assert_message_docs_created(expected_message, message) + @defer.inlineCallbacks + def test_add_mail_with_attachment(self): + input_mail = MIMEMultipart() + input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8')) + attachment = MIMEApplication('pretend to be binary attachment data') + attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt') + input_mail.attach(attachment) + print input_mail.as_string() + mocked_message = self._add_create_mail_mocks_to_soledad(input_mail) + store = LeapMailStore(self.soledad) + + message = yield store.add_mail('INBOX', input_mail.as_string()) + + expected = [{'ident': self._cdoc_phash_from_message(mocked_message, 2), 'name': 'filename.txt', 'encoding': 'base64'}] + self.assertEqual(expected, message.as_dict()['attachments']) + + def _cdoc_phash_from_message(self, mocked_message, attachment_nr): + return mocked_message.get_wrapper().cdocs[attachment_nr].future_doc_id[2:] + @defer.inlineCallbacks def test_delete_mail(self): mdoc_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000') @@ -318,7 +347,7 @@ class TestLeapMailStore(TestCase): @defer.inlineCallbacks def test_copy_mail_to_mailbox(self): - expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000') + expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000') mail_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000') self._mock_get_mailbox('TRASH') store = LeapMailStore(self.soledad) @@ -329,7 +358,7 @@ class TestLeapMailStore(TestCase): @defer.inlineCallbacks def test_move_to_mailbox(self): - expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000') + expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000') mail_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000') self._mock_get_mailbox('TRASH') store = LeapMailStore(self.soledad) @@ -361,7 +390,7 @@ class TestLeapMailStore(TestCase): mbox = MailboxWrapper(doc_id=doc_id, mbox=mailbox_name, uuid=mbox_uuid) soledad_doc = SoledadDocument(doc_id, json=json.dumps(mbox.serialize())) when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', mailbox_name).thenReturn(defer.succeed([soledad_doc])) - self._mock_soledad_doc(doc_id, mbox) + self._mock_get_soledad_doc(doc_id, mbox) self.mbox_uuid_by_name[mailbox_name] = mbox_uuid self.mbox_soledad_docs.append(soledad_doc) @@ -378,31 +407,34 @@ class TestLeapMailStore(TestCase): hdoc_id = wrapper.mdoc.hdoc cdoc_id = wrapper.mdoc.cdocs[0] - self._mock_soledad_doc(mdoc_id, wrapper.mdoc) - self._mock_soledad_doc(fdoc_id, wrapper.fdoc) - self._mock_soledad_doc(hdoc_id, wrapper.hdoc) - self._mock_soledad_doc(cdoc_id, wrapper.cdocs[1]) + self._mock_get_soledad_doc(mdoc_id, wrapper.mdoc) + self._mock_get_soledad_doc(fdoc_id, wrapper.fdoc) + self._mock_get_soledad_doc(hdoc_id, wrapper.hdoc) + self._mock_get_soledad_doc(cdoc_id, wrapper.cdocs[1]) return mdoc_id, fdoc_id - def _add_create_mail_mocks_to_soledad(self, mail_file): + def _add_create_mail_mocks_to_soledad_from_fixture_file(self, mail_file): mail = self._load_mail_from_file(mail_file) - msg = self._convert_mail_to_leap_message(mail) - wrapper = msg.get_wrapper() + return self._add_create_mail_mocks_to_soledad(mail) + + def _add_create_mail_mocks_to_soledad(self, mail): + mail = self._convert_mail_to_leap_message(mail) + wrapper = mail.get_wrapper() mdoc_id = wrapper.mdoc.future_doc_id fdoc_id = wrapper.mdoc.fdoc hdoc_id = wrapper.mdoc.hdoc - cdoc_id = wrapper.mdoc.cdocs[0] - self._mock_create_doc(mdoc_id, wrapper.mdoc) - self._mock_create_doc(fdoc_id, wrapper.fdoc) - self._mock_create_doc(hdoc_id, wrapper.hdoc) - self._mock_create_doc(cdoc_id, wrapper.cdocs[1]) + self._mock_create_soledad_doc(mdoc_id, wrapper.mdoc) + self._mock_create_soledad_doc(fdoc_id, wrapper.fdoc) + self._mock_create_soledad_doc(hdoc_id, wrapper.hdoc) - self._mock_soledad_doc(cdoc_id, wrapper.cdocs[1]) + for _, cdoc in wrapper.cdocs.items(): + self._mock_create_soledad_doc(cdoc.future_doc_id, cdoc) + self._mock_get_soledad_doc(cdoc.future_doc_id, cdoc) - return msg + return mail def _convert_mail_to_leap_message(self, mail, mbox_uuid=None): msg = SoledadMailAdaptor().get_msg_from_string(Message, mail.as_string()) @@ -413,7 +445,7 @@ class TestLeapMailStore(TestCase): return msg - def _mock_soledad_doc(self, doc_id, doc): + def _mock_get_soledad_doc(self, doc_id, doc): soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize())) # when(self.soledad).get_doc(doc_id).thenReturn(defer.succeed(soledad_doc)) @@ -421,7 +453,7 @@ class TestLeapMailStore(TestCase): self.doc_by_id[doc_id] = soledad_doc - def _mock_create_doc(self, doc_id, doc): + def _mock_create_soledad_doc(self, doc_id, doc): soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize())) if doc.future_doc_id: when(self.soledad).create_doc(doc.serialize(), doc_id=doc_id).thenReturn(defer.succeed(soledad_doc)) -- cgit v1.2.3