From fa1299aca064d2ac347a3e14d4c2169335a8f40c Mon Sep 17 00:00:00 2001 From: mnandri Date: Mon, 7 Dec 2015 16:53:39 +0100 Subject: #454 added utf-8 encoding on message body, and corresponding query search term --- service/test/unit/adapter/search/test_search.py | 52 +++++++++++++++++++++- service/test/unit/resources/test_mails_resource.py | 51 +++++++++++++++++++++ service/test/unit/test_utils.py | 36 +++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 service/test/unit/resources/test_mails_resource.py create mode 100755 service/test/unit/test_utils.py (limited to 'service/test') diff --git a/service/test/unit/adapter/search/test_search.py b/service/test/unit/adapter/search/test_search.py index 76e704b6..2e7badff 100644 --- a/service/test/unit/adapter/search/test_search.py +++ b/service/test/unit/adapter/search/test_search.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # # Copyright (c) 2014 ThoughtWorks, Inc. # @@ -21,6 +22,8 @@ from pixelated.adapter.search import SearchEngine from tempdir import TempDir from test.support import test_helper +from pixelated.utils import to_unicode + INDEX_KEY = '\xde3?\x87\xff\xd9\xd3\x14\xf0\xa7>\x1f%C{\x16.\\\xae\x8c\x13\xa7\xfb\x04\xd4]+\x8d_\xed\xd1\x8d\x0bI' \ '\x8a\x0e\xa4tm\xab\xbf\xb4\xa5\x99\x00d\xd5w\x9f\x18\xbc\x1d\xd4_W\xd2\xb6\xe8H\x83\x1b\xd8\x9d\xad' @@ -45,7 +48,7 @@ class SearchEngineTest(unittest.TestCase): def tearDown(self): self.tempdir.dissolve() - def test_encoding(self): + def test_headers_encoding(self): # given se = SearchEngine(INDEX_KEY, self.agent_home) @@ -62,3 +65,50 @@ class SearchEngineTest(unittest.TestCase): result = se.search('folker') self.assertEqual((['mailid'], 1), result) + + def test_contents_encoding_accents(self): + # given + se = SearchEngine(INDEX_KEY, self.agent_home) + + headers = { + 'From': 'foo@bar.tld', + 'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=', + 'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=', + 'Subject': 'Some test mail', + } + + body = "When doing the search, it's not possible to find words with graphical accents, e.g.: 'coração', 'é', 'Fièvre', La Pluie d'été, 'não'." + + # when + se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid')) + + result = se.search(u"'coração', 'é',") + self.assertEqual((['mailid'], 1), result) + + result = se.search(u"Fièvre") + self.assertEqual((['mailid'], 1), result) + + result = se.search(u"été") + self.assertEqual((['mailid'], 1), result) + + def test_contents_encoding_special_characters(self): + # given + se = SearchEngine(INDEX_KEY, self.agent_home) + + headers = { + 'From': 'foo@bar.tld', + 'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=', + 'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=', + 'Subject': 'Some test mail', + } + + body = "When doing the search, 您好 أهلا" + + # when + se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid')) + + result = se.search(u"您好") + self.assertEqual((['mailid'], 1), result) + + result = se.search(u"أهلا") + self.assertEqual((['mailid'], 1), result) diff --git a/service/test/unit/resources/test_mails_resource.py b/service/test/unit/resources/test_mails_resource.py new file mode 100644 index 00000000..02b17bf1 --- /dev/null +++ b/service/test/unit/resources/test_mails_resource.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +import unittest +import json +from mockito import mock, when, verify +from test.unit.resources import DummySite +from twisted.web.test.requesthelper import DummyRequest +from pixelated.resources.mails_resource import MailsResource +from twisted.internet import defer +from mock import patch + + +class TestArchiveResource(unittest.TestCase): + def setUp(self): + self.mail_service = mock() + + @patch('leap.common.events.register') + def test_render_GET_should_unicode_mails_search_query(self, mock_register): + request = DummyRequest(['/mails']) + non_unicode_search_term = 'coração' + request.addArg('q', non_unicode_search_term) + request.addArg('w', 25) + request.addArg('p', 1) + + unicodified_search_term = u'coração' + when(self.mail_service).mails(unicodified_search_term, 25, 1).thenReturn(defer.Deferred()) + + mails_resource = MailsResource(self.mail_service, mock()) + mails_resource.isLeaf = True + web = DummySite(mails_resource) + d = web.get(request) + + def assert_response(_): + verify(self.mail_service).mails(unicodified_search_term, 25, 1) + + d.addCallback(assert_response) + return d diff --git a/service/test/unit/test_utils.py b/service/test/unit/test_utils.py new file mode 100755 index 00000000..ffaf3c8d --- /dev/null +++ b/service/test/unit/test_utils.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# along with Pixelated. If not, see . +# You should have received a copy of the GNU Affero General Public License + + +import unittest + +from pixelated.utils import to_unicode + + +class UtilsTest(unittest.TestCase): + + def test_to_unicode_guesses_encoding_and_unicode_text(self): + text = 'coração' + self.assertEqual(u'coração', to_unicode(text)) + + def test_to_unicode_self(self): + text = u'already unicode' + self.assertEqual(text, to_unicode(text)) + + def test_to_unicode_empty_string(self): + text = '' + self.assertEqual(text, to_unicode(text)) -- cgit v1.2.3