[wip] Expose attachments again after migration

- Issue: #433
- Still missing: In JS, use encoding directly from the attachment
  object, insted of headers. Then remove headers again (see failing unit
test)

5 files changed, 200 insertions, 26 deletions

diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py
index f03e2900..a8fa6d13 100644
--- a/service/pixelated/adapter/mailstore/leap_mailstore.py
+++ b/service/pixelated/adapter/mailstore/leap_mailstore.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 from email.header import decode_header
+import re
 from uuid import uuid4
 from leap.mail.adaptors.soledad import SoledadMailAdaptor
 from twisted.internet import defer
@@ -24,15 +25,24 @@ from leap.mail.mail import Message
 from pixelated.adapter.model.mail import Mail, InputMail
 
 
+class AttachmentInfo(object):
+    def __init__(self, ident, name, encoding, headers):
+        self.ident = ident
+        self.name = name
+        self.encoding = encoding
+        self.headers = headers
+
+
 class LeapMail(Mail):
 
-    def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None):
+    def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]):
         self._mail_id = mail_id
         self._mailbox_name = mailbox_name
         self._headers = headers if headers is not None else {}
         self._body = body
         self.tags = set(tags)   # TODO test that asserts copy
         self._flags = set(flags)  # TODO test that asserts copy
+        self._attachments = attachments
 
     @property
     def headers(self):
@@ -88,7 +98,8 @@ class LeapMail(Mail):
             'body': self._body,
             'textPlainBody': self._body,
             'replying': self._replying_dict(),
-            'mailbox': self._mailbox_name.lower()
+            'mailbox': self._mailbox_name.lower(),
+            'attachments': [{'ident': attachment.ident, 'name': attachment.name, 'encoding': attachment.encoding, 'headers': attachment.headers} for attachment in self._attachments]
         }
 
     def _replying_dict(self):
@@ -116,6 +127,14 @@ class LeapMail(Mail):
         return [recipient for recipient in recipients if recipient != InputMail.FROM_EMAIL_ADDRESS]
 
 
+def _extract_filename(content_disposition):
+    match = re.compile('.*name=\"(.*)\".*').search(content_disposition)
+    filename = ''
+    if match:
+        filename = match.group(1)
+    return filename
+
+
 class LeapMailStore(MailStore):
     __slots__ = ('soledad')
 
@@ -244,7 +263,7 @@ class LeapMailStore(MailStore):
         mbox_uuid = message.get_wrapper().fdoc.mbox_uuid
         mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid)
 
-        mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body)   # TODO assert flags are passed on
+        mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=self._extract_attachment_info_from(message))   # TODO assert flags are passed on
 
         defer.returnValue(mail)
 
@@ -277,6 +296,25 @@ class LeapMailStore(MailStore):
         for doc in docs:
             print '\n%s\n' % doc
 
+    def _extract_attachment_info_from(self, message):
+        wrapper = message.get_wrapper()
+        part_maps = wrapper.hdoc.part_map
+
+        result = []
+
+        for nr, part_map in part_maps.items():
+            if 'headers' in part_map:
+                headers = {header[0]: header[1] for header in part_map['headers']}
+                phash = part_map['phash']
+                if 'Content-Disposition' in headers:
+                    disposition = headers['Content-Disposition']
+                    if 'attachment' in disposition:
+                        filename = _extract_filename(disposition)
+                        encoding = headers['Content-Transfer-Encoding']
+                        result.append(AttachmentInfo(phash, filename, encoding, headers))
+
+        return result
+
 
 def _is_empty_message(message):
     return (message is None) or (message.get_wrapper().mdoc.doc_id is None)
diff --git a/service/test/functional/features/attachments.feature b/service/test/functional/features/attachments.feature
new file mode 100644
index 00000000..19834a9d
--- /dev/null
+++ b/service/test/functional/features/attachments.feature
@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+Feature: Attachments
+  As a user of Pixelated
+  I want to download attachments of mails I received
+  So that my peers are able to send me any kind of content, not just text
+
+  Scenario: User opens a mail attachment
+    Given I have a mail with an attachment in my inbox
+    When I open the first mail in the 'inbox'
+    Then I see the mail has an attachment
+    #When I open click on the first attachment
+    #Then the browser downloaded a file
diff --git a/service/test/functional/features/steps/attachments.py b/service/test/functional/features/steps/attachments.py
new file mode 100644
index 00000000..066683bf
--- /dev/null
+++ b/service/test/functional/features/steps/attachments.py
@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.mime.application import MIMEApplication
+from time import sleep
+from leap.mail.mail import Message
+from common import *
+from test.support.integration import MailBuilder
+from behave import given
+from crochet import wait_for
+from uuid import uuid4
+from email.MIMEMultipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+
+@given(u'I have a mail with an attachment in my inbox')
+def add_mail_with_attachment_impl(context):
+    subject = 'Hi! This the subject %s' % uuid4()
+    mail = build_mail_with_attachment(subject)
+    load_mail_into_soledad(context, mail)
+    context.last_subject = subject
+
+
+def build_mail_with_attachment(subject):
+    mail = MIMEMultipart()
+    mail['Subject'] = subject
+    mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+    attachment = MIMEApplication('pretend to be binary attachment data')
+    attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt')
+    mail.attach(attachment)
+
+    return mail
+
+
+@wait_for(timeout=10.0)
+def load_mail_into_soledad(context, mail):
+    return context.client.mail_store.add_mail('INBOX', mail.as_string())
+
+
+@then(u'I see the mail has an attachment')
+def step_impl(context):
+    attachments_list = find_elements_by_css_selector(context, '.attachmentsArea li')
+    assert len(attachments_list) == 1
diff --git a/service/test/integration/test_leap_mailstore.py b/service/test/integration/test_leap_mailstore.py
index b1b2075d..abe5d584 100644
--- a/service/test/integration/test_leap_mailstore.py
+++ b/service/test/integration/test_leap_mailstore.py
@@ -13,6 +13,9 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 from test.support.integration import SoledadTestBase, load_mail_from_file
 from twisted.internet import defer
 
@@ -28,7 +31,7 @@ class LeapMailStoreTest(SoledadTestBase):
         self.maxDiff = None
         mail = load_mail_from_file('mbox00000000')
         mail_id = yield self._create_mail_in_soledad(mail)
-        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'status': [], 'tags': set([]), 'replying': {'all': {'cc-field': [], 'to-field': [u'carmel@murazikortiz.name', u'darby.senger@zemlak.biz']}, 'single': u'darby.senger@zemlak.biz'}, 'textPlainBody': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'mailbox': u'inbox'}
+        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'status': [], 'tags': set([]), 'replying': {'all': {'cc-field': [], 'to-field': [u'carmel@murazikortiz.name', u'darby.senger@zemlak.biz']}, 'single': u'darby.senger@zemlak.biz'}, 'textPlainBody': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'mailbox': u'inbox', 'attachments': []}
 
         result = yield self.mail_store.get_mail(mail_id, include_body=True)
         self.assertIsNotNone(result)
@@ -46,6 +49,25 @@ class LeapMailStoreTest(SoledadTestBase):
         self.assertEqual(expected_mail_dict['header'], fetched_mail.as_dict()['header'])
 
     @defer.inlineCallbacks
+    def test_round_trip_through_soledad_keeps_attachment(self):
+        input_mail = MIMEMultipart()
+        input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+        attachment = MIMEApplication('pretend to be binary attachment data')
+        attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt')
+        input_mail.attach(attachment)
+
+        mail = yield self.mail_store.add_mail('INBOX', input_mail.as_string())
+        fetched_mail = yield self.mail_store.get_mail(mail.ident, include_body=True)
+
+        # _, docs = yield self.soledad.get_all_docs()
+        # for doc in docs:
+        #     print '\n%s\n' % doc
+
+        # self.assertEqual(1, len(mail.as_dict()['attachments']))
+        # print fetched_mail.as_dict()
+        # self.assertEqual(1, len(fetched_mail.as_dict()['attachments']))
+
+    @defer.inlineCallbacks
     def test_all_mails(self):
         mail = load_mail_from_file('mbox00000000')
         yield self._create_mail_in_soledad(mail)
diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
index 4789f02c..a28731e3 100644
--- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py
+++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
@@ -13,6 +13,9 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 import json
 from uuid import uuid4
 from email.parser import Parser
@@ -31,7 +34,7 @@ import pkg_resources
 from leap.mail.mail import Message
 from pixelated.adapter.mailstore import underscore_uuid
 
-from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail
+from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail, AttachmentInfo
 
 
 class TestLeapMail(TestCase):
@@ -83,6 +86,7 @@ class TestLeapMail(TestCase):
                                               'receiver2@other.test',
                                               'test@example.test']},
                          'single': 'test@example.test'},
+            'attachments': []
         }
 
         self.assertEqual(expected, mail.as_dict())
@@ -93,6 +97,12 @@ class TestLeapMail(TestCase):
 
         self.assertEqual(body, mail.as_dict()['body'])
 
+    def test_as_dict_with_attachments(self):
+        mail = LeapMail('doc id', 'INBOX', attachments=[AttachmentInfo('id', 'name', 'encoding')])
+
+        self.assertEqual([{'ident': 'id', 'name': 'name', 'encoding': 'encoding'}],
+                         mail.as_dict()['attachments'])
+
     def test_raw_constructed_by_headers_and_body(self):
         body = 'some body content'
         mail = LeapMail('doc id', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test'}, ('foo', 'bar'), body=body)
@@ -240,7 +250,7 @@ class TestLeapMailStore(TestCase):
     def test_add_mailbox(self):
         when(self.soledad).list_indexes().thenReturn(defer.succeed(MAIL_INDEXES)).thenReturn(defer.succeed(MAIL_INDEXES))
         when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', 'TEST').thenReturn(defer.succeed([]))
-        self._mock_create_doc(self.mbox_uuid, MailboxWrapper(mbox='TEST'))
+        self._mock_create_soledad_doc(self.mbox_uuid, MailboxWrapper(mbox='TEST'))
         when(self.soledad).get_doc(self.mbox_uuid).thenAnswer(lambda: defer.succeed(self.doc_by_id[self.mbox_uuid]))
         when(self.soledad).put_doc(ANY()).thenAnswer(lambda: defer.succeed(None))
         store = LeapMailStore(self.soledad)
@@ -272,7 +282,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_add_mail(self):
-        expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000')
+        expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000')
         mail = self._load_mail_from_file('mbox00000000')
         self._mock_get_mailbox('INBOX')
 
@@ -284,6 +294,25 @@ class TestLeapMailStore(TestCase):
         self._assert_message_docs_created(expected_message, message)
 
     @defer.inlineCallbacks
+    def test_add_mail_with_attachment(self):
+        input_mail = MIMEMultipart()
+        input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+        attachment = MIMEApplication('pretend to be binary attachment data')
+        attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt')
+        input_mail.attach(attachment)
+        print input_mail.as_string()
+        mocked_message = self._add_create_mail_mocks_to_soledad(input_mail)
+        store = LeapMailStore(self.soledad)
+
+        message = yield store.add_mail('INBOX', input_mail.as_string())
+
+        expected = [{'ident': self._cdoc_phash_from_message(mocked_message, 2), 'name': 'filename.txt', 'encoding': 'base64'}]
+        self.assertEqual(expected, message.as_dict()['attachments'])
+
+    def _cdoc_phash_from_message(self, mocked_message, attachment_nr):
+        return mocked_message.get_wrapper().cdocs[attachment_nr].future_doc_id[2:]
+
+    @defer.inlineCallbacks
     def test_delete_mail(self):
         mdoc_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000')
 
@@ -318,7 +347,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_copy_mail_to_mailbox(self):
-        expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000')
+        expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000')
         mail_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000')
         self._mock_get_mailbox('TRASH')
         store = LeapMailStore(self.soledad)
@@ -329,7 +358,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_move_to_mailbox(self):
-        expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000')
+        expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000')
         mail_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000')
         self._mock_get_mailbox('TRASH')
         store = LeapMailStore(self.soledad)
@@ -361,7 +390,7 @@ class TestLeapMailStore(TestCase):
         mbox = MailboxWrapper(doc_id=doc_id, mbox=mailbox_name, uuid=mbox_uuid)
         soledad_doc = SoledadDocument(doc_id, json=json.dumps(mbox.serialize()))
         when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', mailbox_name).thenReturn(defer.succeed([soledad_doc]))
-        self._mock_soledad_doc(doc_id, mbox)
+        self._mock_get_soledad_doc(doc_id, mbox)
 
         self.mbox_uuid_by_name[mailbox_name] = mbox_uuid
         self.mbox_soledad_docs.append(soledad_doc)
@@ -378,31 +407,34 @@ class TestLeapMailStore(TestCase):
         hdoc_id = wrapper.mdoc.hdoc
         cdoc_id = wrapper.mdoc.cdocs[0]
 
-        self._mock_soledad_doc(mdoc_id, wrapper.mdoc)
-        self._mock_soledad_doc(fdoc_id, wrapper.fdoc)
-        self._mock_soledad_doc(hdoc_id, wrapper.hdoc)
-        self._mock_soledad_doc(cdoc_id, wrapper.cdocs[1])
+        self._mock_get_soledad_doc(mdoc_id, wrapper.mdoc)
+        self._mock_get_soledad_doc(fdoc_id, wrapper.fdoc)
+        self._mock_get_soledad_doc(hdoc_id, wrapper.hdoc)
+        self._mock_get_soledad_doc(cdoc_id, wrapper.cdocs[1])
 
         return mdoc_id, fdoc_id
 
-    def _add_create_mail_mocks_to_soledad(self, mail_file):
+    def _add_create_mail_mocks_to_soledad_from_fixture_file(self, mail_file):
         mail = self._load_mail_from_file(mail_file)
-        msg = self._convert_mail_to_leap_message(mail)
-        wrapper = msg.get_wrapper()
+        return self._add_create_mail_mocks_to_soledad(mail)
+
+    def _add_create_mail_mocks_to_soledad(self, mail):
+        mail = self._convert_mail_to_leap_message(mail)
+        wrapper = mail.get_wrapper()
 
         mdoc_id = wrapper.mdoc.future_doc_id
         fdoc_id = wrapper.mdoc.fdoc
         hdoc_id = wrapper.mdoc.hdoc
-        cdoc_id = wrapper.mdoc.cdocs[0]
 
-        self._mock_create_doc(mdoc_id, wrapper.mdoc)
-        self._mock_create_doc(fdoc_id, wrapper.fdoc)
-        self._mock_create_doc(hdoc_id, wrapper.hdoc)
-        self._mock_create_doc(cdoc_id, wrapper.cdocs[1])
+        self._mock_create_soledad_doc(mdoc_id, wrapper.mdoc)
+        self._mock_create_soledad_doc(fdoc_id, wrapper.fdoc)
+        self._mock_create_soledad_doc(hdoc_id, wrapper.hdoc)
 
-        self._mock_soledad_doc(cdoc_id, wrapper.cdocs[1])
+        for _, cdoc in wrapper.cdocs.items():
+            self._mock_create_soledad_doc(cdoc.future_doc_id, cdoc)
+            self._mock_get_soledad_doc(cdoc.future_doc_id, cdoc)
 
-        return msg
+        return mail
 
     def _convert_mail_to_leap_message(self, mail, mbox_uuid=None):
         msg = SoledadMailAdaptor().get_msg_from_string(Message, mail.as_string())
@@ -413,7 +445,7 @@ class TestLeapMailStore(TestCase):
 
         return msg
 
-    def _mock_soledad_doc(self, doc_id, doc):
+    def _mock_get_soledad_doc(self, doc_id, doc):
         soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
 
         # when(self.soledad).get_doc(doc_id).thenReturn(defer.succeed(soledad_doc))
@@ -421,7 +453,7 @@ class TestLeapMailStore(TestCase):
 
         self.doc_by_id[doc_id] = soledad_doc
 
-    def _mock_create_doc(self, doc_id, doc):
+    def _mock_create_soledad_doc(self, doc_id, doc):
         soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
         if doc.future_doc_id:
             when(self.soledad).create_doc(doc.serialize(), doc_id=doc_id).thenReturn(defer.succeed(soledad_doc))