diff options
| -rw-r--r-- | service/pixelated/adapter/mailstore/body_parser.py | 41 | ||||
| -rw-r--r-- | service/pixelated/adapter/mailstore/leap_mailstore.py | 18 | ||||
| -rw-r--r-- | service/pixelated/adapter/search/__init__.py | 2 | ||||
| -rw-r--r-- | service/test/integration/test_search.py | 64 | ||||
| -rw-r--r-- | service/test/support/integration/app_test_client.py | 6 | ||||
| -rw-r--r-- | service/test/unit/adapter/mailstore/test_body_parser.py | 30 |
6 files changed, 136 insertions, 25 deletions
diff --git a/service/pixelated/adapter/mailstore/body_parser.py b/service/pixelated/adapter/mailstore/body_parser.py new file mode 100644 index 00000000..893daac9 --- /dev/null +++ b/service/pixelated/adapter/mailstore/body_parser.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from email.parser import Parser + + +class BodyParser(object): + + def __init__(self, content, content_type='text/plain; charset="us-ascii"', content_transfer_encoding=None): + self._content = content + self._content_type = content_type + self._content_transfer_encoding = content_transfer_encoding + + def parsed_content(self): + parser = Parser() + + text='' + text += 'Content-Type: %s\n' % self._content_type + if self._content_transfer_encoding is not None: + text += 'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding + text += '\n' + text += self._content + + parsed_body = parser.parsestr(text) + + result = unicode(parsed_body.get_payload(decode=True), encoding='utf-8') + + return unicode(result)
\ No newline at end of file diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py index 9261f877..8831ae84 100644 --- a/service/pixelated/adapter/mailstore/leap_mailstore.py +++ b/service/pixelated/adapter/mailstore/leap_mailstore.py @@ -16,6 +16,7 @@ from uuid import uuid4 from leap.mail.adaptors.soledad import SoledadMailAdaptor from twisted.internet import defer +from pixelated.adapter.mailstore.body_parser import BodyParser from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid from leap.mail.mail import Message @@ -65,8 +66,9 @@ class LeapMail(Mail): def raw(self): result = '' for k, v in self._headers.items(): - result = result + '%s: %s\n' % (k, v) - result = result + '\n' + result += '%s: %s\n' % (k, v) + result += '\n' + if self._body: result = result + self._body @@ -151,10 +153,11 @@ class LeapMailStore(MailStore): mailbox = yield self._get_or_create_mailbox(mailbox_name) message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg) message.get_wrapper().set_mbox_uuid(mailbox.uuid) + yield message.get_wrapper().create(self.soledad) # add behavious from insert_mdoc_id from mail.py - mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=False) + mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body defer.returnValue(mail) @defer.inlineCallbacks @@ -200,7 +203,8 @@ class LeapMailStore(MailStore): @defer.inlineCallbacks def _leap_message_to_leap_mail(self, mail_id, message, include_body): if include_body: - body = (yield message.get_wrapper().get_body(self.soledad)).raw + # TODO use body from message if available + body = yield self._raw_message_body(message) else: body = None @@ -213,6 +217,12 @@ class LeapMailStore(MailStore): defer.returnValue(mail) @defer.inlineCallbacks + def _raw_message_body(self, message): + content_doc = (yield message.get_wrapper().get_body(self.soledad)) + parser = BodyParser(content_doc.raw, content_type=content_doc.content_type, content_transfer_encoding=content_doc.content_transfer_encoding) + defer.returnValue(parser.parsed_content()) + + @defer.inlineCallbacks def _mailbox_name_from_uuid(self, uuid): map = (yield self._mailbox_uuid_to_name_map()) defer.returnValue(map[uuid]) diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py index 759ad5ef..a95055d2 100644 --- a/service/pixelated/adapter/search/__init__.py +++ b/service/pixelated/adapter/search/__init__.py @@ -137,7 +137,7 @@ class SearchEngine(object): 'body': unicode(mdict['textPlainBody'] if 'textPlainBody' in mdict else mdict['body']), 'ident': unicode(mdict['ident']), 'flags': unicode(','.join(unique(mail.flags))), - 'raw': unicode(mail.raw.decode('utf-8')) + 'raw': unicode(mail.raw) } writer.update_document(**index_data) diff --git a/service/test/integration/test_search.py b/service/test/integration/test_search.py index 38abb7f8..23f2bba8 100644 --- a/service/test/integration/test_search.py +++ b/service/test/integration/test_search.py @@ -13,17 +13,39 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from email.parser import Parser +from leap.mail import walk +from leap.mail.utils import first +from leap.mail.walk import get_hash, get_payloads from test.support.integration import SoledadTestBase, MailBuilder from twisted.internet import defer +def get_raw_docs(msg, parts): + return ( + { + "type": "cnt", # type content they'll be + "raw": payload, + "phash": get_hash(payload), + "content-disposition": first(headers.get( + 'content-disposition', '').split(';')), + "content-type": headers.get( + 'content-type', ''), + "content-transfer-encoding": headers.get( + 'content-transfer-encoding', '') + } for payload, headers in get_payloads(msg) + if not isinstance(payload, list)) + +walk.get_raw_docs = get_raw_docs + class SearchTest(SoledadTestBase): @defer.inlineCallbacks def test_that_tags_returns_all_tags(self): - input_mail = MailBuilder().with_tags(['important']).build_input_mail() - yield self.add_mail_to_inbox(input_mail) + input_mail = MailBuilder().build_input_mail() + mail = yield self.add_mail_to_inbox(input_mail) + yield self.mail_service.update_tags(mail.ident, ['important']) all_tags = yield self.get_tags() @@ -36,8 +58,9 @@ class SearchTest(SoledadTestBase): @defer.inlineCallbacks def test_that_tags_are_filtered_by_query(self): - input_mail = MailBuilder().with_tags(['ateu', 'catoa', 'luat', 'zuado']).build_input_mail() - yield self.add_mail_to_inbox(input_mail) + input_mail = MailBuilder().build_input_mail() + mail = yield self.add_mail_to_inbox(input_mail) + yield self.mail_service.update_tags(mail.ident, ['ateu', 'catoa', 'luat', 'zuado']) all_tags = yield self.get_tags(q=["at"], skipDefaultTags=["true"]) @@ -49,8 +72,9 @@ class SearchTest(SoledadTestBase): @defer.inlineCallbacks def test_tags_with_multiple_words_are_searchable(self): - input_mail = MailBuilder().with_tags(['one tag four words']).build_input_mail() - yield self.add_mail_to_inbox(input_mail) + input_mail = MailBuilder().build_input_mail() + mail = yield self.add_mail_to_inbox(input_mail) + yield self.mail_service.update_tags(mail.ident, ['one tag four words']) first_page = yield self.get_mails_by_tag('"one tag four words"', page=1, window=1) @@ -58,8 +82,9 @@ class SearchTest(SoledadTestBase): @defer.inlineCallbacks def test_that_default_tags_are_ignorable(self): - input_mail = MailBuilder().with_tags(['sometag']).build_input_mail() - yield self.add_mail_to_inbox(input_mail) + input_mail = MailBuilder().build_input_mail() + mail = yield self.add_mail_to_inbox(input_mail) + yield self.mail_service.update_tags(mail.ident, ['sometag']) all_tags = yield self.get_tags(skipDefaultTags=["true"]) @@ -96,13 +121,13 @@ class SearchTest(SoledadTestBase): def test_search_mails_with_multiple_pages(self): input_mail = MailBuilder().build_input_mail() input_mail2 = MailBuilder().build_input_mail() - yield self.add_mail_to_inbox(input_mail) - yield self.add_mail_to_inbox(input_mail2) + mail1 = yield self.add_mail_to_inbox(input_mail) + mail2 = yield self.add_mail_to_inbox(input_mail2) first_page = yield self.get_mails_by_tag('inbox', page=1, window=1) second_page = yield self.get_mails_by_tag('inbox', page=2, window=1) - idents = [input_mail.ident, input_mail2.ident] + idents = [mail1.ident, mail2.ident] self.assertIn(first_page[0].ident, idents) self.assertIn(second_page[0].ident, idents) @@ -110,9 +135,9 @@ class SearchTest(SoledadTestBase): @defer.inlineCallbacks def test_page_zero_fetches_first_page(self): input_mail = MailBuilder().build_input_mail() - yield self.add_mail_to_inbox(input_mail) + mail = yield self.add_mail_to_inbox(input_mail) page = yield self.get_mails_by_tag('inbox', page=0, window=1) - self.assertEqual(page[0].ident, input_mail.ident) + self.assertEqual(page[0].ident, mail.ident) def get_count(self, tags_count, mailbox): for tag in tags_count: @@ -124,19 +149,20 @@ class SearchTest(SoledadTestBase): input_mail = MailBuilder().with_date('2014-10-15T15:15').build_input_mail() input_mail2 = MailBuilder().with_date('2014-10-15T15:16').build_input_mail() - yield self.add_mail_to_inbox(input_mail) - yield self.add_mail_to_inbox(input_mail2) + mail1 = yield self.add_mail_to_inbox(input_mail) + mail2 = yield self.add_mail_to_inbox(input_mail2) results = yield self.get_mails_by_tag('inbox') - self.assertEqual(results[0].ident, input_mail2.ident) - self.assertEqual(results[1].ident, input_mail.ident) + self.assertEqual(results[0].ident, mail2.ident) + self.assertEqual(results[1].ident, mail1.ident) @defer.inlineCallbacks def test_search_base64_body(self): body = u'bl\xe1' input_mail = MailBuilder().with_body(body.encode('utf-8')).build_input_mail() - yield self.add_mail_to_inbox(input_mail) + + mail = yield self.add_mail_to_inbox(input_mail) results = yield self.search(body) self.assertGreater(len(results), 0, 'No results returned from search') - self.assertEquals(results[0].ident, input_mail.ident) + self.assertEquals(results[0].ident, mail.ident) diff --git a/service/test/support/integration/app_test_client.py b/service/test/support/integration/app_test_client.py index 31ffcd77..2c1343ab 100644 --- a/service/test/support/integration/app_test_client.py +++ b/service/test/support/integration/app_test_client.py @@ -159,7 +159,11 @@ class AppTestClient(object): input_mail = builder.build_input_mail() mail = yield self.mail_store.add_mail(mailbox, input_mail.raw) if tags: - mail.tags.add(tags) + mail.tags |= set(tags) + if flags: + for flag in flags: + mail.flags.add(flag) + if tags or flags: yield self.mail_store.update_mail(mail) mails.append(mail) diff --git a/service/test/unit/adapter/mailstore/test_body_parser.py b/service/test/unit/adapter/mailstore/test_body_parser.py new file mode 100644 index 00000000..1af4ce50 --- /dev/null +++ b/service/test/unit/adapter/mailstore/test_body_parser.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import unittest +from pixelated.adapter.mailstore.body_parser import BodyParser + + +class BodyParserTest(unittest.TestCase): + + def test_simple_text(self): + parser = BodyParser('simple text') + + self.assertEqual('simple text', parser.parsed_content()) + + def test_base64_text(self): + parser = BodyParser('dGVzdCB0ZXh0\n', content_type='text/plain; charset="utf-8"', content_transfer_encoding='base64') + + self.assertEqual('test text', parser.parsed_content())
\ No newline at end of file |