Make indexing recepients work again by fixing serialization bug

10 files changed, 86 insertions, 15 deletions

diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py
index c34d2b16..049af9ee 100644
--- a/service/pixelated/adapter/mailstore/leap_mailstore.py
+++ b/service/pixelated/adapter/mailstore/leap_mailstore.py
@@ -27,12 +27,21 @@ class LeapMail(Mail):
     def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None):
         self._mail_id = mail_id
         self._mailbox_name = mailbox_name
-        self.headers = headers if headers is not None else {}
+        self._headers = headers if headers is not None else {}
         self._body = body
         self.tags = tags
         self._flags = flags
 
     @property
+    def headers(self):
+        cpy = dict(self._headers)
+
+        for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']):
+            cpy[name] = self._headers[name].split(',') if self._headers[name] else None
+
+        return cpy
+
+    @property
     def ident(self):
         return self._mail_id
 
@@ -55,7 +64,7 @@ class LeapMail(Mail):
     @property
     def raw(self):
         result = ''
-        for k, v in self.headers.items():
+        for k, v in self._headers.items():
             result = result + '%s: %s\n' % (k, v)
         result = result + '\n'
         if self._body:
@@ -197,7 +206,7 @@ class LeapMailStore(MailStore):
         mbox_uuid = message.get_wrapper().fdoc.mbox_uuid
         mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid)
 
-        mail = LeapMail(mail_id, mbox_name, message.get_headers(), set(message.get_tags()), body=body)
+        mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), body=body)
 
         defer.returnValue(mail)
 
diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py
index 61e56f33..388263ca 100644
--- a/service/pixelated/adapter/model/mail.py
+++ b/service/pixelated/adapter/model/mail.py
@@ -95,7 +95,10 @@ class Mail(object):
             return self._mime
         mime = MIMEMultipart()
         for key, value in self.headers.items():
-            mime[str(key)] = str(value)
+            if isinstance(value, list):
+                mime[str(key)] = ', '.join(value)
+            else:
+                mime[str(key)] = str(value)
 
         try:
             body_to_use = self.body
diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py
index 5e4d7a6d..6b011e11 100644
--- a/service/pixelated/adapter/services/mail_service.py
+++ b/service/pixelated/adapter/services/mail_service.py
@@ -112,8 +112,7 @@ class MailService(object):
         if mail.mailbox_name.upper() == u'TRASH':
             yield self.mail_store.delete_mail(mail_id)
         else:
-            trashed_mail = yield self.mail_store.move_mail_to_mailbox(mail_id, 'TRASH')
-            self.search_engine.index_mail(trashed_mail)
+            yield self.mail_store.move_mail_to_mailbox(mail_id, 'TRASH')
 
     def recover_mail(self, mail_id):
         recovered_mail = self.mailboxes.move_to_inbox(mail_id)
diff --git a/service/pixelated/resources/contacts_resource.py b/service/pixelated/resources/contacts_resource.py
index 5ec39761..c9b81f54 100644
--- a/service/pixelated/resources/contacts_resource.py
+++ b/service/pixelated/resources/contacts_resource.py
@@ -29,8 +29,16 @@ class ContactsResource(Resource):
         self._search_engine = search_engine
 
     def render_GET(self, request):
-        query = request.args.get('q', [''])[0]
+        query = request.args.get('q', [''])
         d = deferToThread(lambda: self._search_engine.contacts(query))
         d.addCallback(lambda tags: respond_json_deferred(tags, request))
 
+        def handle_error(error):
+            print 'Something went wrong'
+            import traceback
+            traceback.print_exc()
+            print error
+
+        d.addErrback(handle_error)
+
         return server.NOT_DONE_YET
diff --git a/service/test/integration/test_contacts.py b/service/test/integration/test_contacts.py
index d2e2ac2a..7e849a87 100644
--- a/service/test/integration/test_contacts.py
+++ b/service/test/integration/test_contacts.py
@@ -34,7 +34,7 @@ class ContactsTest(SoledadTestBase):
 
     @defer.inlineCallbacks
     def test_FROM_address_is_being_searched(self):
-        input_mail = MailBuilder().with_tags(['important']).build_input_mail()
+        input_mail = MailBuilder().with_tags(['important']).with_from('Formatted Sender <sender@from.com>').build_input_mail()
         yield self.add_mail_to_inbox(input_mail)
 
         contacts = yield self.get_contacts(query='Sender')
diff --git a/service/test/integration/test_leap_mailstore.py b/service/test/integration/test_leap_mailstore.py
index f111d2b1..94a5aa6b 100644
--- a/service/test/integration/test_leap_mailstore.py
+++ b/service/test/integration/test_leap_mailstore.py
@@ -27,13 +27,24 @@ class LeapMailStoreTest(SoledadTestBase):
     def test_get_mail_with_body(self):
         mail = load_mail_from_file('mbox00000000')
         mail_id = yield self._create_mail_in_soledad(mail)
-        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': u'carmel@murazikortiz.name', u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'tags': set([])}
+        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'tags': set([])}
 
         result = yield self.mail_store.get_mail(mail_id, include_body=True)
         self.assertIsNotNone(result)
         self.assertEqual(expected_mail_dict, result.as_dict())
 
     @defer.inlineCallbacks
+    def test_round_trip_through_soledad_does_not_modify_content(self):
+        mail = load_mail_from_file('mbox00000000')
+        mail_id = yield self._create_mail_in_soledad(mail)
+        expected_mail_dict = {'body': u'Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.\n\nEarum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.\n\nUt recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.\n\n', 'header': {u'date': u'Tue, 21 Apr 2015 08:43:27 +0000 (UTC)', u'to': [u'carmel@murazikortiz.name'], u'x-tw-pixelated-tags': u'nite, macro, trash', u'from': u'darby.senger@zemlak.biz', u'subject': u'Itaque consequatur repellendus provident sunt quia.'}, 'ident': mail_id, 'tags': set([])}
+
+        mail = yield self.mail_store.add_mail('INBOX', mail.as_string())
+        fetched_mail = yield self.mail_store.get_mail(mail_id, include_body=True)
+        self.assertEqual(expected_mail_dict['header'], mail.as_dict()['header'])
+        self.assertEqual(expected_mail_dict['header'], fetched_mail.as_dict()['header'])
+
+    @defer.inlineCallbacks
     def test_all_mails(self):
         mail = load_mail_from_file('mbox00000000')
         yield self._create_mail_in_soledad(mail)
diff --git a/service/test/support/integration/model.py b/service/test/support/integration/model.py
index e90a3ec5..93b5969b 100644
--- a/service/test/support/integration/model.py
+++ b/service/test/support/integration/model.py
@@ -45,6 +45,10 @@ class MailBuilder:
         self.mail['header']['subject'] = subject
         return self
 
+    def with_from(self, sender):
+        self.mail['header']['from'] = sender
+        return self
+
     def with_to(self, to):
         self.mail['header']['to'] = to
         return self
diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
index 429f1683..eec40ced 100644
--- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py
+++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
@@ -17,6 +17,7 @@ import json
 from uuid import uuid4
 from email.parser import Parser
 import os
+from leap.mail.utils import CaseInsensitiveDict
 from leap.soledad.common.document import SoledadDocument
 from leap.mail.adaptors.soledad_indexes import MAIL_INDEXES
 from twisted.internet.defer import FirstError
@@ -31,24 +32,44 @@ from leap.mail.mail import Message
 from pixelated.adapter.mailstore import underscore_uuid
 
 from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore, LeapMail
-import test.support.mockito
+
 
 class TestLeapMail(TestCase):
     def test_leap_mail(self):
         mail = LeapMail('', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test'})
 
         self.assertEqual('test@example.test', mail.from_sender)
-        self.assertEqual('receiver@example.test', mail.to)
+        self.assertEqual(['receiver@example.test'], mail.to)
         self.assertEqual('A test Mail', mail.subject)
 
+    def test_email_addresses_in_to_are_split_into_a_list(self):
+        mail = LeapMail('', 'INBOX', {'To': 'first@example.test,second@example.test'})
+
+        self.assertEqual(['first@example.test', 'second@example.test'],mail.headers['To'])
+
+    def test_email_addresses_in_cc_are_split_into_a_list(self):
+        mail = LeapMail('', 'INBOX', {'Cc': 'first@example.test,second@example.test'})
+
+        self.assertEqual(['first@example.test', 'second@example.test'],mail.headers['Cc'])
+
+    def test_email_addresses_in_bcc_are_split_into_a_list(self):
+        mail = LeapMail('', 'INBOX', {'Bcc': 'first@example.test,second@example.test'})
+
+        self.assertEqual(['first@example.test', 'second@example.test'],mail.headers['Bcc'])
+
+    def test_email_addresses_might_be_none(self):
+        mail = LeapMail('', 'INBOX', {'Cc': None})
+
+        self.assertEqual(None, mail.headers['Cc'])
+
     def test_as_dict(self):
-        mail = LeapMail('doc id', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test'}, ('foo', 'bar'))
+        mail = LeapMail('doc id', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test,receiver2@other.test'}, ('foo', 'bar'))
 
         expected = {
             'header': {
                 'from': 'test@example.test',
                 'subject': 'A test Mail',
-                'to': 'receiver@example.test',
+                'to': ['receiver@example.test', 'receiver2@other.test'],
 
             },
             'ident': 'doc id',
@@ -87,6 +108,7 @@ class TestLeapMailStore(TestCase):
 
     @defer.inlineCallbacks
     def test_get_mail_not_exist(self):
+        when(self.soledad).get_doc(ANY()).thenAnswer(lambda: defer.succeed(None))
         store = LeapMailStore(self.soledad)
 
         mail = yield store.get_mail(_format_mdoc_id(uuid4(), 1))
@@ -103,7 +125,7 @@ class TestLeapMailStore(TestCase):
 
         self.assertIsInstance(mail, LeapMail)
         self.assertEqual('darby.senger@zemlak.biz', mail.from_sender)
-        self.assertEqual('carmel@murazikortiz.name', mail.to)
+        self.assertEqual(['carmel@murazikortiz.name'], mail.to)
         self.assertEqual('Itaque consequatur repellendus provident sunt quia.', mail.subject)
         self.assertIsNone(mail.body)
         self.assertEqual('INBOX', mail.mailbox_name)
diff --git a/service/test/unit/adapter/test_mail.py b/service/test/unit/adapter/test_mail.py
index 4751bfdd..d30e2485 100644
--- a/service/test/unit/adapter/test_mail.py
+++ b/service/test/unit/adapter/test_mail.py
@@ -438,6 +438,21 @@ class InputMailTest(unittest.TestCase):
         self.assertNotRegexpMatches(mime_multipart.as_string(), "\nCc: \n")
         self.assertNotRegexpMatches(mime_multipart.as_string(), "\nSubject: \n")
 
+    def test_single_recipient(self):
+        mail_single_recipient = {
+            'body': '',
+            'header': {
+                'to': ['to@pixelated.org'],
+                'cc': [''],
+                'bcc': [''],
+                'subject': 'Oi'
+            }
+        }
+
+        result = InputMail.from_dict(mail_single_recipient).raw
+
+        self.assertRegexpMatches(result, 'To: to@pixelated.org')
+
     def test_to_mime_multipart(self):
         pixelated.support.date.iso_now = lambda: 'date now'
 
diff --git a/service/test/unit/adapter/test_mail_service.py b/service/test/unit/adapter/test_mail_service.py
index 9d1521b8..361f28f9 100644
--- a/service/test/unit/adapter/test_mail_service.py
+++ b/service/test/unit/adapter/test_mail_service.py
@@ -89,7 +89,7 @@ class TestMailService(unittest.TestCase):
 
     def test_delete_mail(self):
         mail_to_delete = LeapMail(1, 'INBOX')
-        when(self.mail_store).get_mail(1).thenReturn(mail_to_delete)
+        when(self.mail_store).get_mail(1).thenReturn(defer.succeed(mail_to_delete))
 
         self.mail_service.delete_mail(1)