summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFolker Bernitt <fbernitt@thoughtworks.com>2015-08-10 13:37:37 +0200
committerFolker Bernitt <fbernitt@thoughtworks.com>2015-08-11 17:00:34 +0200
commitb8410e1e199a581155566c6c3b9d40dbda32b082 (patch)
treea9dbd66030b1a070192b264630a0512d6cac175b
parent4ae8a4bb0dedae1ab5a6b66eea3e1b9598a72e9f (diff)
Fixed test_search integration test.
-rw-r--r--service/pixelated/adapter/mailstore/body_parser.py41
-rw-r--r--service/pixelated/adapter/mailstore/leap_mailstore.py18
-rw-r--r--service/pixelated/adapter/search/__init__.py2
-rw-r--r--service/test/integration/test_search.py64
-rw-r--r--service/test/support/integration/app_test_client.py6
-rw-r--r--service/test/unit/adapter/mailstore/test_body_parser.py30
6 files changed, 136 insertions, 25 deletions
diff --git a/service/pixelated/adapter/mailstore/body_parser.py b/service/pixelated/adapter/mailstore/body_parser.py
new file mode 100644
index 00000000..893daac9
--- /dev/null
+++ b/service/pixelated/adapter/mailstore/body_parser.py
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from email.parser import Parser
+
+
+class BodyParser(object):
+
+ def __init__(self, content, content_type='text/plain; charset="us-ascii"', content_transfer_encoding=None):
+ self._content = content
+ self._content_type = content_type
+ self._content_transfer_encoding = content_transfer_encoding
+
+ def parsed_content(self):
+ parser = Parser()
+
+ text=''
+ text += 'Content-Type: %s\n' % self._content_type
+ if self._content_transfer_encoding is not None:
+ text += 'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding
+ text += '\n'
+ text += self._content
+
+ parsed_body = parser.parsestr(text)
+
+ result = unicode(parsed_body.get_payload(decode=True), encoding='utf-8')
+
+ return unicode(result) \ No newline at end of file
diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py
index 9261f877..8831ae84 100644
--- a/service/pixelated/adapter/mailstore/leap_mailstore.py
+++ b/service/pixelated/adapter/mailstore/leap_mailstore.py
@@ -16,6 +16,7 @@
from uuid import uuid4
from leap.mail.adaptors.soledad import SoledadMailAdaptor
from twisted.internet import defer
+from pixelated.adapter.mailstore.body_parser import BodyParser
from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid
from leap.mail.mail import Message
@@ -65,8 +66,9 @@ class LeapMail(Mail):
def raw(self):
result = ''
for k, v in self._headers.items():
- result = result + '%s: %s\n' % (k, v)
- result = result + '\n'
+ result += '%s: %s\n' % (k, v)
+ result += '\n'
+
if self._body:
result = result + self._body
@@ -151,10 +153,11 @@ class LeapMailStore(MailStore):
mailbox = yield self._get_or_create_mailbox(mailbox_name)
message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg)
message.get_wrapper().set_mbox_uuid(mailbox.uuid)
+
yield message.get_wrapper().create(self.soledad)
# add behavious from insert_mdoc_id from mail.py
- mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=False)
+ mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body
defer.returnValue(mail)
@defer.inlineCallbacks
@@ -200,7 +203,8 @@ class LeapMailStore(MailStore):
@defer.inlineCallbacks
def _leap_message_to_leap_mail(self, mail_id, message, include_body):
if include_body:
- body = (yield message.get_wrapper().get_body(self.soledad)).raw
+ # TODO use body from message if available
+ body = yield self._raw_message_body(message)
else:
body = None
@@ -213,6 +217,12 @@ class LeapMailStore(MailStore):
defer.returnValue(mail)
@defer.inlineCallbacks
+ def _raw_message_body(self, message):
+ content_doc = (yield message.get_wrapper().get_body(self.soledad))
+ parser = BodyParser(content_doc.raw, content_type=content_doc.content_type, content_transfer_encoding=content_doc.content_transfer_encoding)
+ defer.returnValue(parser.parsed_content())
+
+ @defer.inlineCallbacks
def _mailbox_name_from_uuid(self, uuid):
map = (yield self._mailbox_uuid_to_name_map())
defer.returnValue(map[uuid])
diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py
index 759ad5ef..a95055d2 100644
--- a/service/pixelated/adapter/search/__init__.py
+++ b/service/pixelated/adapter/search/__init__.py
@@ -137,7 +137,7 @@ class SearchEngine(object):
'body': unicode(mdict['textPlainBody'] if 'textPlainBody' in mdict else mdict['body']),
'ident': unicode(mdict['ident']),
'flags': unicode(','.join(unique(mail.flags))),
- 'raw': unicode(mail.raw.decode('utf-8'))
+ 'raw': unicode(mail.raw)
}
writer.update_document(**index_data)
diff --git a/service/test/integration/test_search.py b/service/test/integration/test_search.py
index 38abb7f8..23f2bba8 100644
--- a/service/test/integration/test_search.py
+++ b/service/test/integration/test_search.py
@@ -13,17 +13,39 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.parser import Parser
+from leap.mail import walk
+from leap.mail.utils import first
+from leap.mail.walk import get_hash, get_payloads
from test.support.integration import SoledadTestBase, MailBuilder
from twisted.internet import defer
+def get_raw_docs(msg, parts):
+ return (
+ {
+ "type": "cnt", # type content they'll be
+ "raw": payload,
+ "phash": get_hash(payload),
+ "content-disposition": first(headers.get(
+ 'content-disposition', '').split(';')),
+ "content-type": headers.get(
+ 'content-type', ''),
+ "content-transfer-encoding": headers.get(
+ 'content-transfer-encoding', '')
+ } for payload, headers in get_payloads(msg)
+ if not isinstance(payload, list))
+
+walk.get_raw_docs = get_raw_docs
+
class SearchTest(SoledadTestBase):
@defer.inlineCallbacks
def test_that_tags_returns_all_tags(self):
- input_mail = MailBuilder().with_tags(['important']).build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
+ input_mail = MailBuilder().build_input_mail()
+ mail = yield self.add_mail_to_inbox(input_mail)
+ yield self.mail_service.update_tags(mail.ident, ['important'])
all_tags = yield self.get_tags()
@@ -36,8 +58,9 @@ class SearchTest(SoledadTestBase):
@defer.inlineCallbacks
def test_that_tags_are_filtered_by_query(self):
- input_mail = MailBuilder().with_tags(['ateu', 'catoa', 'luat', 'zuado']).build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
+ input_mail = MailBuilder().build_input_mail()
+ mail = yield self.add_mail_to_inbox(input_mail)
+ yield self.mail_service.update_tags(mail.ident, ['ateu', 'catoa', 'luat', 'zuado'])
all_tags = yield self.get_tags(q=["at"], skipDefaultTags=["true"])
@@ -49,8 +72,9 @@ class SearchTest(SoledadTestBase):
@defer.inlineCallbacks
def test_tags_with_multiple_words_are_searchable(self):
- input_mail = MailBuilder().with_tags(['one tag four words']).build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
+ input_mail = MailBuilder().build_input_mail()
+ mail = yield self.add_mail_to_inbox(input_mail)
+ yield self.mail_service.update_tags(mail.ident, ['one tag four words'])
first_page = yield self.get_mails_by_tag('"one tag four words"', page=1, window=1)
@@ -58,8 +82,9 @@ class SearchTest(SoledadTestBase):
@defer.inlineCallbacks
def test_that_default_tags_are_ignorable(self):
- input_mail = MailBuilder().with_tags(['sometag']).build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
+ input_mail = MailBuilder().build_input_mail()
+ mail = yield self.add_mail_to_inbox(input_mail)
+ yield self.mail_service.update_tags(mail.ident, ['sometag'])
all_tags = yield self.get_tags(skipDefaultTags=["true"])
@@ -96,13 +121,13 @@ class SearchTest(SoledadTestBase):
def test_search_mails_with_multiple_pages(self):
input_mail = MailBuilder().build_input_mail()
input_mail2 = MailBuilder().build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
- yield self.add_mail_to_inbox(input_mail2)
+ mail1 = yield self.add_mail_to_inbox(input_mail)
+ mail2 = yield self.add_mail_to_inbox(input_mail2)
first_page = yield self.get_mails_by_tag('inbox', page=1, window=1)
second_page = yield self.get_mails_by_tag('inbox', page=2, window=1)
- idents = [input_mail.ident, input_mail2.ident]
+ idents = [mail1.ident, mail2.ident]
self.assertIn(first_page[0].ident, idents)
self.assertIn(second_page[0].ident, idents)
@@ -110,9 +135,9 @@ class SearchTest(SoledadTestBase):
@defer.inlineCallbacks
def test_page_zero_fetches_first_page(self):
input_mail = MailBuilder().build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
+ mail = yield self.add_mail_to_inbox(input_mail)
page = yield self.get_mails_by_tag('inbox', page=0, window=1)
- self.assertEqual(page[0].ident, input_mail.ident)
+ self.assertEqual(page[0].ident, mail.ident)
def get_count(self, tags_count, mailbox):
for tag in tags_count:
@@ -124,19 +149,20 @@ class SearchTest(SoledadTestBase):
input_mail = MailBuilder().with_date('2014-10-15T15:15').build_input_mail()
input_mail2 = MailBuilder().with_date('2014-10-15T15:16').build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
- yield self.add_mail_to_inbox(input_mail2)
+ mail1 = yield self.add_mail_to_inbox(input_mail)
+ mail2 = yield self.add_mail_to_inbox(input_mail2)
results = yield self.get_mails_by_tag('inbox')
- self.assertEqual(results[0].ident, input_mail2.ident)
- self.assertEqual(results[1].ident, input_mail.ident)
+ self.assertEqual(results[0].ident, mail2.ident)
+ self.assertEqual(results[1].ident, mail1.ident)
@defer.inlineCallbacks
def test_search_base64_body(self):
body = u'bl\xe1'
input_mail = MailBuilder().with_body(body.encode('utf-8')).build_input_mail()
- yield self.add_mail_to_inbox(input_mail)
+
+ mail = yield self.add_mail_to_inbox(input_mail)
results = yield self.search(body)
self.assertGreater(len(results), 0, 'No results returned from search')
- self.assertEquals(results[0].ident, input_mail.ident)
+ self.assertEquals(results[0].ident, mail.ident)
diff --git a/service/test/support/integration/app_test_client.py b/service/test/support/integration/app_test_client.py
index 31ffcd77..2c1343ab 100644
--- a/service/test/support/integration/app_test_client.py
+++ b/service/test/support/integration/app_test_client.py
@@ -159,7 +159,11 @@ class AppTestClient(object):
input_mail = builder.build_input_mail()
mail = yield self.mail_store.add_mail(mailbox, input_mail.raw)
if tags:
- mail.tags.add(tags)
+ mail.tags |= set(tags)
+ if flags:
+ for flag in flags:
+ mail.flags.add(flag)
+ if tags or flags:
yield self.mail_store.update_mail(mail)
mails.append(mail)
diff --git a/service/test/unit/adapter/mailstore/test_body_parser.py b/service/test/unit/adapter/mailstore/test_body_parser.py
new file mode 100644
index 00000000..1af4ce50
--- /dev/null
+++ b/service/test/unit/adapter/mailstore/test_body_parser.py
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import unittest
+from pixelated.adapter.mailstore.body_parser import BodyParser
+
+
+class BodyParserTest(unittest.TestCase):
+
+ def test_simple_text(self):
+ parser = BodyParser('simple text')
+
+ self.assertEqual('simple text', parser.parsed_content())
+
+ def test_base64_text(self):
+ parser = BodyParser('dGVzdCB0ZXh0\n', content_type='text/plain; charset="utf-8"', content_transfer_encoding='base64')
+
+ self.assertEqual('test text', parser.parsed_content()) \ No newline at end of file