diff options
4 files changed, 46 insertions, 8 deletions
diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py index c18748b6..73ce543f 100644 --- a/service/pixelated/adapter/mailstore/leap_mailstore.py +++ b/service/pixelated/adapter/mailstore/leap_mailstore.py @@ -88,9 +88,19 @@ class LeapMail(Mail): return result + def _decoded_header_utf_8(self, header_value): + if isinstance(header_value, list): + return [self._decoded_header_utf_8(v) for v in header_value] + else: + content, encoding = decode_header(header_value)[0] + if encoding: + return unicode(content, encoding=encoding) + else: + return unicode(content, encoding='ascii') + def as_dict(self): return { - 'header': {k.lower(): v for k, v in self.headers.items()}, + 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()}, 'ident': self._mail_id, 'tags': self.tags, 'status': list(self.status), diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py index 46691926..84e69a9a 100644 --- a/service/pixelated/adapter/search/__init__.py +++ b/service/pixelated/adapter/search/__init__.py @@ -126,8 +126,8 @@ class SearchEngine(object): bounced = mail.bounced if mail.bounced else [''] index_data = { - 'sender': self._unicode_header_field(header.get('from', '')), - 'subject': self._unicode_header_field(header.get('subject', '')), + 'sender': self._empty_string_to_none(header.get('from', '')), + 'subject': self._empty_string_to_none(header.get('subject', '')), 'date': milliseconds(header.get('date', '')), 'to': self._format_recipient(header, 'to'), 'cc': self._format_recipient(header, 'cc'), @@ -146,11 +146,11 @@ class SearchEngine(object): list = headers.get(name, ['']) return u','.join(list) if list else u'' - def _unicode_header_field(self, field_value): + def _empty_string_to_none(self, field_value): if not field_value: return None - - return unicode(field_value.decode('utf-8')) + else: + return field_value def index_mails(self, mails, callback=None): try: diff --git a/service/test/unit/adapter/mailstore/test_leap_mail.py b/service/test/unit/adapter/mailstore/test_leap_mail.py index dd749aa3..9dab4e28 100644 --- a/service/test/unit/adapter/mailstore/test_leap_mail.py +++ b/service/test/unit/adapter/mailstore/test_leap_mail.py @@ -84,6 +84,20 @@ class TestLeapMail(TestCase): self.assertEqual([{'ident': 'id', 'name': 'name', 'encoding': 'encoding'}], mail.as_dict()['attachments']) + def test_as_dict_headers_with_special_chars(self): + expected_address = u'"\xc4lbert \xdcbr\xf6" <\xe4\xfc\xf6@example.mail>' + expected_subject = u'H\xe4ll\xf6 W\xf6rld' + mail = LeapMail('', 'INBOX', + {'From': '=?iso-8859-1?q?=22=C4lbert_=DCbr=F6=22_=3C=E4=FC=F6=40example=2Email=3E?=', + 'To': '=?iso-8859-1?q?=22=C4lbert_=DCbr=F6=22_=3C=E4=FC=F6=40example=2Email=3E?=', + 'Cc': '=?iso-8859-1?q?=22=C4lbert_=DCbr=F6=22_=3C=E4=FC=F6=40example=2Email=3E?=', + 'Subject': '=?iso-8859-1?q?H=E4ll=F6_W=F6rld?='}) + + self.assertEqual(expected_address, mail.as_dict()['header']['from']) + self.assertEqual([expected_address], mail.as_dict()['header']['to']) + self.assertEqual([expected_address], mail.as_dict()['header']['cc']) + self.assertEqual(expected_subject, mail.as_dict()['header']['subject']) + def test_raw_constructed_by_headers_and_body(self): body = 'some body content' mail = LeapMail('doc id', 'INBOX', {'From': 'test@example.test', 'Subject': 'A test Mail', 'To': 'receiver@example.test'}, ('foo', 'bar'), body=body) diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py index c173d87c..09c92980 100644 --- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py +++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # # Copyright (c) 2015 ThoughtWorks, Inc. # @@ -13,6 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from email.header import Header from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -241,6 +243,18 @@ class TestLeapMailStore(TestCase): expected = [{'ident': self._cdoc_phash_from_message(mocked_message, 2), 'name': 'filename.txt', 'encoding': 'base64'}] self.assertEqual(expected, message.as_dict()['attachments']) + @defer.inlineCallbacks + def test_add_mail_with_special_chars(self): + input_mail = MIMEText(u'a utf8 message', _charset='utf-8') + input_mail['From'] = Header(u'"Älbert Übrö" <äüö@example.mail>', 'iso-8859-1') + input_mail['Subject'] = Header(u'Hällö Wörld', 'iso-8859-1') + self._add_create_mail_mocks_to_soledad(input_mail) + store = LeapMailStore(self.soledad) + + message = yield store.add_mail('INBOX', input_mail.as_string()) + + self.assertEqual(u'"\xc4lbert \xdcbr\xf6" <\xe4\xfc\xf6@example.mail>', message.as_dict()['header']['from']) + def _cdoc_phash_from_message(self, mocked_message, attachment_nr): return mocked_message.get_wrapper().cdocs[attachment_nr].future_doc_id[2:] @@ -350,8 +364,8 @@ class TestLeapMailStore(TestCase): mail = self._load_mail_from_file(mail_file) return self._add_create_mail_mocks_to_soledad(mail) - def _add_create_mail_mocks_to_soledad(self, mail): - mail = self._convert_mail_to_leap_message(mail) + def _add_create_mail_mocks_to_soledad(self, example_mail): + mail = self._convert_mail_to_leap_message(example_mail) wrapper = mail.get_wrapper() mdoc_id = wrapper.mdoc.future_doc_id |