#454 added utf-8 encoding on message body, and corresponding query search term

1 files changed, 51 insertions, 1 deletions

diff --git a/service/test/unit/adapter/search/test_search.py b/service/test/unit/adapter/search/test_search.py
index 76e704b6..2e7badff 100644
--- a/service/test/unit/adapter/search/test_search.py
+++ b/service/test/unit/adapter/search/test_search.py
@@ -1,3 +1,4 @@
+#  -*- coding: utf-8 -*-
 #
 # Copyright (c) 2014 ThoughtWorks, Inc.
 #
@@ -21,6 +22,8 @@ from pixelated.adapter.search import SearchEngine
 from tempdir import TempDir
 from test.support import test_helper
 
+from pixelated.utils import to_unicode
+
 INDEX_KEY = '\xde3?\x87\xff\xd9\xd3\x14\xf0\xa7>\x1f%C{\x16.\\\xae\x8c\x13\xa7\xfb\x04\xd4]+\x8d_\xed\xd1\x8d\x0bI' \
     '\x8a\x0e\xa4tm\xab\xbf\xb4\xa5\x99\x00d\xd5w\x9f\x18\xbc\x1d\xd4_W\xd2\xb6\xe8H\x83\x1b\xd8\x9d\xad'
 
@@ -45,7 +48,7 @@ class SearchEngineTest(unittest.TestCase):
     def tearDown(self):
         self.tempdir.dissolve()
 
-    def test_encoding(self):
+    def test_headers_encoding(self):
         # given
         se = SearchEngine(INDEX_KEY, self.agent_home)
 
@@ -62,3 +65,50 @@ class SearchEngineTest(unittest.TestCase):
         result = se.search('folker')
 
         self.assertEqual((['mailid'], 1), result)
+
+    def test_contents_encoding_accents(self):
+        # given
+        se = SearchEngine(INDEX_KEY, self.agent_home)
+
+        headers = {
+            'From': 'foo@bar.tld',
+            'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Subject': 'Some test mail',
+        }
+
+        body = "When doing the search, it's not possible to find words with graphical accents, e.g.: 'coração', 'é',  'Fièvre', La Pluie d'été, 'não'."
+
+        # when
+        se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body))   # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
+
+        result = se.search(u"'coração', 'é',")
+        self.assertEqual((['mailid'], 1), result)
+
+        result = se.search(u"Fièvre")
+        self.assertEqual((['mailid'], 1), result)
+
+        result = se.search(u"été")
+        self.assertEqual((['mailid'], 1), result)
+
+    def test_contents_encoding_special_characters(self):
+        # given
+        se = SearchEngine(INDEX_KEY, self.agent_home)
+
+        headers = {
+            'From': 'foo@bar.tld',
+            'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+            'Subject': 'Some test mail',
+        }
+
+        body = "When doing the search, 您好  أهلا"
+
+        # when
+        se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body))   # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
+
+        result = se.search(u"您好")
+        self.assertEqual((['mailid'], 1), result)
+
+        result = se.search(u"أهلا")
+        self.assertEqual((['mailid'], 1), result)