[wip] Expose attachments again after migration

- Issue: #433
- Still missing: In JS, use encoding directly from the attachment
  object, insted of headers. Then remove headers again (see failing unit
test)

4 files changed, 159 insertions, 23 deletions

diff --git a/service/test/functional/features/attachments.feature b/service/test/functional/features/attachments.feature
new file mode 100644
index 00000000..19834a9d
--- /dev/null
+++ b/service/test/functional/features/attachments.feature
@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+Feature: Attachments
+  As a user of Pixelated
+  I want to download attachments of mails I received
+  So that my peers are able to send me any kind of content, not just text
+
+  Scenario: User opens a mail attachment
+    Given I have a mail with an attachment in my inbox
+    When I open the first mail in the 'inbox'
+    Then I see the mail has an attachment
+    #When I open click on the first attachment
+    #Then the browser downloaded a file
diff --git a/service/test/functional/features/steps/attachments.py b/service/test/functional/features/steps/attachments.py
new file mode 100644
index 00000000..066683bf
--- /dev/null
+++ b/service/test/functional/features/steps/attachments.py
@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.mime.application import MIMEApplication
+from time import sleep
+from leap.mail.mail import Message
+from common import *
+from test.support.integration import MailBuilder
+from behave import given
+from crochet import wait_for
+from uuid import uuid4
+from email.MIMEMultipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+
+@given(u'I have a mail with an attachment in my inbox')
+def add_mail_with_attachment_impl(context):
+    subject = 'Hi! This the subject %s' % uuid4()
+    mail = build_mail_with_attachment(subject)
+    load_mail_into_soledad(context, mail)
+    context.last_subject = subject
+
+
+def build_mail_with_attachment(subject):
+    mail = MIMEMultipart()
+    mail['Subject'] = subject
+    mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+    attachment = MIMEApplication('pretend to be binary attachment data')
+    attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt')
+    mail.attach(attachment)
+
+    return mail
+
+
+@wait_for(timeout=10.0)
+def load_mail_into_soledad(context, mail):
+    return context.client.mail_store.add_mail('INBOX', mail.as_string())
+
+
+@then(u'I see the mail has an attachment')
+def step_impl(context):
+    attachments_list = find_elements_by_css_selector(context, '.attachmentsArea li')
+    assert len(attachments_list) == 1
diff --git a/service/test/integration/test_leap_mailstore.py b/service/test/integration/test_leap_mailstore.py
index b1b2075d..abe5d584 100644
--- a/service/test/integration/test_leap_mailstore.py
+++ b/service/test/integration/test_leap_mailstore.py
@@ -13,6 +13,9 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 from test.support.integration import SoledadTestBase, load_mail_from_file
 from twisted.internet import defer
 
@@ -28,7 +31,7 @@ class LeapMailStoreTest(SoledadTestBase):
         self.maxDiff = None
         mail = load_mail_from_file('mbox00000000')
         mail_id = yield self._create_mail_in_soledad(mail)
-        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'status': [], 'tags': set([]), 'replying': {'all': {'cc-field': [], 'to-field': [u'carmel@murazikortiz.name', u'darby.senger@zemlak.biz']}, 'single': u'darby.senger@zemlak.biz'}, 'textPlainBody': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'mailbox': u'inbox'}
+        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'status': [], 'tags': set([]), 'replying': {'all': {'cc-field': [], 'to-field': [u'carmel@murazikortiz.name', u'darby.senger@zemlak.biz']}, 'single': u'darby.senger@zemlak.biz'}, 'textPlainBody': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'mailbox': u'inbox', 'attachments': []}
 
         result = yield self.mail_store.get_mail(mail_id, include_body=True)
         self.assertIsNotNone(result)
@@ -46,6 +49,25 @@ class LeapMailStoreTest(SoledadTestBase):
         self.assertEqual(expected_mail_dict['header'], fetched_mail.as_dict()['header'])
 
     @defer.inlineCallbacks
+    def test_round_trip_through_soledad_keeps_attachment(self):
+        input_mail = MIMEMultipart()
+        input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+        attachment = MIMEApplication('pretend to be binary attachment data')
+        attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt')
+        input_mail.attach(attachment)
+
+        mail = yield self.mail_store.add_mail('INBOX', input_mail.as_string())
+        fetched_mail = yield self.mail_store.get_mail(mail.ident, include_body=True)
+
+        # _, docs = yield self.soledad.get_all_docs()
+        # for doc in docs:
+        #     print '\n%s\n' % doc
+
+        # self.assertEqual(1, len(mail.as_dict()['attachments']))
+        # print fetched_mail.as_dict()
+        # self.assertEqual(1, len(fetched_mail.as_dict()['attachments']))
+
+    @defer.inlineCallbacks
     def test_all_mails(self):
         mail = load_mail_from_file('mbox00000000')
         yield self._create_mail_in_soledad(mail)
diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
index 4789f02c..a28731e3 100644
--- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py
+++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
@@ -13,6 +13,9 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 import json
 from uuid import uuid4
 from email.parser import Parser
@@ -31,7 +34,7 @@ import pkg_resources
 from leap.mail.mail import Message
 from pixelated.adapter.mailstore import underscore_uuid
 
-from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail
+from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail, AttachmentInfo
 
 
 class TestLeapMail(TestCase):
@@ -83,6 +86,7 @@ class TestLeapMail(TestCase):
                                               'receiver2@other.test',
                                               'test@example.test']},
                          'single': 'test@example.test'},
+            'attachments': []
         }
 
         self.assertEqual(expected, mail.as_dict())
@@ -93,6 +97,12 @@ class TestLeapMail(TestCase):
 
         self.assertEqual(body, mail.as_dict()['body'])
 
+    def test_as_dict_with_attachments(self):
+        mail = LeapMail('doc id', 'INBOX', attachments=[AttachmentInfo('id', 'name', 'encoding')])
+
+        self.assertEqual([{'ident': 'id', 'name': 'name', 'encoding': 'encoding'}],
+                         mail.as_dict()['attachments'])
+
     def test_raw_constructed_by_headers_and_body(self):
         body = 'some body content'
         mail = LeapMail('doc id', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test'}, ('foo', 'bar'), body=body)
@@ -240,7 +250,7 @@ class TestLeapMailStore(TestCase):
     def test_add_mailbox(self):
         when(self.soledad).list_indexes().thenReturn(defer.succeed(MAIL_INDEXES)).thenReturn(defer.succeed(MAIL_INDEXES))
         when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', 'TEST').thenReturn(defer.succeed([]))
-        self._mock_create_doc(self.mbox_uuid, MailboxWrapper(mbox='TEST'))
+        self._mock_create_soledad_doc(self.mbox_uuid, MailboxWrapper(mbox='TEST'))
         when(self.soledad).get_doc(self.mbox_uuid).thenAnswer(lambda: defer.succeed(self.doc_by_id[self.mbox_uuid]))
         when(self.soledad).put_doc(ANY()).thenAnswer(lambda: defer.succeed(None))
         store = LeapMailStore(self.soledad)
@@ -272,7 +282,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_add_mail(self):
-        expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000')
+        expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000')
         mail = self._load_mail_from_file('mbox00000000')
         self._mock_get_mailbox('INBOX')
 
@@ -284,6 +294,25 @@ class TestLeapMailStore(TestCase):
         self._assert_message_docs_created(expected_message, message)
 
     @defer.inlineCallbacks
+    def test_add_mail_with_attachment(self):
+        input_mail = MIMEMultipart()
+        input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+        attachment = MIMEApplication('pretend to be binary attachment data')
+        attachment.add_header('Content-Disposition', 'attachment', filename='filename.txt')
+        input_mail.attach(attachment)
+        print input_mail.as_string()
+        mocked_message = self._add_create_mail_mocks_to_soledad(input_mail)
+        store = LeapMailStore(self.soledad)
+
+        message = yield store.add_mail('INBOX', input_mail.as_string())
+
+        expected = [{'ident': self._cdoc_phash_from_message(mocked_message, 2), 'name': 'filename.txt', 'encoding': 'base64'}]
+        self.assertEqual(expected, message.as_dict()['attachments'])
+
+    def _cdoc_phash_from_message(self, mocked_message, attachment_nr):
+        return mocked_message.get_wrapper().cdocs[attachment_nr].future_doc_id[2:]
+
+    @defer.inlineCallbacks
     def test_delete_mail(self):
         mdoc_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000')
 
@@ -318,7 +347,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_copy_mail_to_mailbox(self):
-        expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000')
+        expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000')
         mail_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000')
         self._mock_get_mailbox('TRASH')
         store = LeapMailStore(self.soledad)
@@ -329,7 +358,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_move_to_mailbox(self):
-        expected_message = self._add_create_mail_mocks_to_soledad('mbox00000000')
+        expected_message = self._add_create_mail_mocks_to_soledad_from_fixture_file('mbox00000000')
         mail_id, fdoc_id = self._add_mail_fixture_to_soledad('mbox00000000')
         self._mock_get_mailbox('TRASH')
         store = LeapMailStore(self.soledad)
@@ -361,7 +390,7 @@ class TestLeapMailStore(TestCase):
         mbox = MailboxWrapper(doc_id=doc_id, mbox=mailbox_name, uuid=mbox_uuid)
         soledad_doc = SoledadDocument(doc_id, json=json.dumps(mbox.serialize()))
         when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', mailbox_name).thenReturn(defer.succeed([soledad_doc]))
-        self._mock_soledad_doc(doc_id, mbox)
+        self._mock_get_soledad_doc(doc_id, mbox)
 
         self.mbox_uuid_by_name[mailbox_name] = mbox_uuid
         self.mbox_soledad_docs.append(soledad_doc)
@@ -378,31 +407,34 @@ class TestLeapMailStore(TestCase):
         hdoc_id = wrapper.mdoc.hdoc
         cdoc_id = wrapper.mdoc.cdocs[0]
 
-        self._mock_soledad_doc(mdoc_id, wrapper.mdoc)
-        self._mock_soledad_doc(fdoc_id, wrapper.fdoc)
-        self._mock_soledad_doc(hdoc_id, wrapper.hdoc)
-        self._mock_soledad_doc(cdoc_id, wrapper.cdocs[1])
+        self._mock_get_soledad_doc(mdoc_id, wrapper.mdoc)
+        self._mock_get_soledad_doc(fdoc_id, wrapper.fdoc)
+        self._mock_get_soledad_doc(hdoc_id, wrapper.hdoc)
+        self._mock_get_soledad_doc(cdoc_id, wrapper.cdocs[1])
 
         return mdoc_id, fdoc_id
 
-    def _add_create_mail_mocks_to_soledad(self, mail_file):
+    def _add_create_mail_mocks_to_soledad_from_fixture_file(self, mail_file):
         mail = self._load_mail_from_file(mail_file)
-        msg = self._convert_mail_to_leap_message(mail)
-        wrapper = msg.get_wrapper()
+        return self._add_create_mail_mocks_to_soledad(mail)
+
+    def _add_create_mail_mocks_to_soledad(self, mail):
+        mail = self._convert_mail_to_leap_message(mail)
+        wrapper = mail.get_wrapper()
 
         mdoc_id = wrapper.mdoc.future_doc_id
         fdoc_id = wrapper.mdoc.fdoc
         hdoc_id = wrapper.mdoc.hdoc
-        cdoc_id = wrapper.mdoc.cdocs[0]
 
-        self._mock_create_doc(mdoc_id, wrapper.mdoc)
-        self._mock_create_doc(fdoc_id, wrapper.fdoc)
-        self._mock_create_doc(hdoc_id, wrapper.hdoc)
-        self._mock_create_doc(cdoc_id, wrapper.cdocs[1])
+        self._mock_create_soledad_doc(mdoc_id, wrapper.mdoc)
+        self._mock_create_soledad_doc(fdoc_id, wrapper.fdoc)
+        self._mock_create_soledad_doc(hdoc_id, wrapper.hdoc)
 
-        self._mock_soledad_doc(cdoc_id, wrapper.cdocs[1])
+        for _, cdoc in wrapper.cdocs.items():
+            self._mock_create_soledad_doc(cdoc.future_doc_id, cdoc)
+            self._mock_get_soledad_doc(cdoc.future_doc_id, cdoc)
 
-        return msg
+        return mail
 
     def _convert_mail_to_leap_message(self, mail, mbox_uuid=None):
         msg = SoledadMailAdaptor().get_msg_from_string(Message, mail.as_string())
@@ -413,7 +445,7 @@ class TestLeapMailStore(TestCase):
 
         return msg
 
-    def _mock_soledad_doc(self, doc_id, doc):
+    def _mock_get_soledad_doc(self, doc_id, doc):
         soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
 
         # when(self.soledad).get_doc(doc_id).thenReturn(defer.succeed(soledad_doc))
@@ -421,7 +453,7 @@ class TestLeapMailStore(TestCase):
 
         self.doc_by_id[doc_id] = soledad_doc
 
-    def _mock_create_doc(self, doc_id, doc):
+    def _mock_create_soledad_doc(self, doc_id, doc):
         soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
         if doc.future_doc_id:
             when(self.soledad).create_doc(doc.serialize(), doc_id=doc_id).thenReturn(defer.succeed(soledad_doc))