#454 added utf-8 encoding on message body, and corresponding query search term

3 files changed, 138 insertions, 1 deletions

diff --git a/service/test/unit/adapter/search/test_search.py b/service/test/unit/adapter/search/test_search.py
index 76e704b6..2e7badff 100644
--- a/service/test/unit/adapter/search/test_search.py
+++ b/service/test/unit/adapter/search/test_search.py
@@ -1,3 +1,4 @@
+#  -*- coding: utf-8 -*-
 #
 # Copyright (c) 2014 ThoughtWorks, Inc.
 #
@@ -21,6 +22,8 @@ from pixelated.adapter.search import SearchEngine
 from tempdir import TempDir
 from test.support import test_helper
 
+from pixelated.utils import to_unicode
+
 INDEX_KEY = '\xde3?\x87\xff\xd9\xd3\x14\xf0\xa7>\x1f%C{\x16.\\\xae\x8c\x13\xa7\xfb\x04\xd4]+\x8d_\xed\xd1\x8d\x0bI' \
     '\x8a\x0e\xa4tm\xab\xbf\xb4\xa5\x99\x00d\xd5w\x9f\x18\xbc\x1d\xd4_W\xd2\xb6\xe8H\x83\x1b\xd8\x9d\xad'
 
@@ -45,7 +48,7 @@ class SearchEngineTest(unittest.TestCase):
     def tearDown(self):
         self.tempdir.dissolve()
 
-    def test_encoding(self):
+    def test_headers_encoding(self):
         # given
         se = SearchEngine(INDEX_KEY, self.agent_home)
 
@@ -62,3 +65,50 @@ class SearchEngineTest(unittest.TestCase):
         result = se.search('folker')
 
         self.assertEqual((['mailid'], 1), result)
+
+    def test_contents_encoding_accents(self):
+        # given
+        se = SearchEngine(INDEX_KEY, self.agent_home)
+
+        headers = {
+            'From': 'foo@bar.tld',
+            'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Subject': 'Some test mail',
+        }
+
+        body = "When doing the search, it's not possible to find words with graphical accents, e.g.: 'coração', 'é',  'Fièvre', La Pluie d'été, 'não'."
+
+        # when
+        se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body))   # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
+
+        result = se.search(u"'coração', 'é',")
+        self.assertEqual((['mailid'], 1), result)
+
+        result = se.search(u"Fièvre")
+        self.assertEqual((['mailid'], 1), result)
+
+        result = se.search(u"été")
+        self.assertEqual((['mailid'], 1), result)
+
+    def test_contents_encoding_special_characters(self):
+        # given
+        se = SearchEngine(INDEX_KEY, self.agent_home)
+
+        headers = {
+            'From': 'foo@bar.tld',
+            'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Subject': 'Some test mail',
+        }
+
+        body = "When doing the search, 您好  أهلا"
+
+        # when
+        se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body))   # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
+
+        result = se.search(u"您好")
+        self.assertEqual((['mailid'], 1), result)
+
+        result = se.search(u"أهلا")
+        self.assertEqual((['mailid'], 1), result)
diff --git a/service/test/unit/resources/test_mails_resource.py b/service/test/unit/resources/test_mails_resource.py
new file mode 100644
index 00000000..02b17bf1
--- /dev/null
+++ b/service/test/unit/resources/test_mails_resource.py
@@ -0,0 +1,51 @@
+#  -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import unittest
+import json
+from mockito import mock, when, verify
+from test.unit.resources import DummySite
+from twisted.web.test.requesthelper import DummyRequest
+from pixelated.resources.mails_resource import MailsResource
+from twisted.internet import defer
+from mock import patch
+
+
+class TestArchiveResource(unittest.TestCase):
+    def setUp(self):
+        self.mail_service = mock()
+
+    @patch('leap.common.events.register')
+    def test_render_GET_should_unicode_mails_search_query(self, mock_register):
+        request = DummyRequest(['/mails'])
+        non_unicode_search_term = 'coração'
+        request.addArg('q', non_unicode_search_term)
+        request.addArg('w', 25)
+        request.addArg('p', 1)
+
+        unicodified_search_term = u'coração'
+        when(self.mail_service).mails(unicodified_search_term, 25, 1).thenReturn(defer.Deferred())
+
+        mails_resource = MailsResource(self.mail_service, mock())
+        mails_resource.isLeaf = True
+        web = DummySite(mails_resource)
+        d = web.get(request)
+
+        def assert_response(_):
+            verify(self.mail_service).mails(unicodified_search_term, 25, 1)
+
+        d.addCallback(assert_response)
+        return d
diff --git a/service/test/unit/test_utils.py b/service/test/unit/test_utils.py
new file mode 100755
index 00000000..ffaf3c8d
--- /dev/null
+++ b/service/test/unit/test_utils.py
@@ -0,0 +1,36 @@
+#  -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+
+
+import unittest
+
+from pixelated.utils import to_unicode
+
+
+class UtilsTest(unittest.TestCase):
+
+    def test_to_unicode_guesses_encoding_and_unicode_text(self):
+        text = 'coração'
+        self.assertEqual(u'coração', to_unicode(text))
+
+    def test_to_unicode_self(self):
+        text = u'already unicode'
+        self.assertEqual(text, to_unicode(text))
+
+    def test_to_unicode_empty_string(self):
+        text = ''
+        self.assertEqual(text, to_unicode(text))