summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--service/pixelated/adapter/mailstore/leap_mailstore.py4
-rw-r--r--service/pixelated/adapter/search/__init__.py4
-rw-r--r--service/pixelated/resources/mails_resource.py12
-rwxr-xr-xservice/pixelated/utils.py22
-rw-r--r--service/test/unit/adapter/search/test_search.py52
-rw-r--r--service/test/unit/resources/test_mails_resource.py51
-rwxr-xr-xservice/test/unit/test_utils.py36
7 files changed, 172 insertions, 9 deletions
diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py
index 90035d20..14b0e417 100644
--- a/service/pixelated/adapter/mailstore/leap_mailstore.py
+++ b/service/pixelated/adapter/mailstore/leap_mailstore.py
@@ -27,6 +27,8 @@ from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid
from leap.mail.mail import Message
from pixelated.adapter.model.mail import Mail, InputMail
+from pixelated.utils import to_unicode
+
class AttachmentInfo(object):
def __init__(self, ident, name, encoding):
@@ -41,7 +43,7 @@ class LeapMail(Mail):
self._mail_id = mail_id
self._mailbox_name = mailbox_name
self._headers = headers if headers is not None else {}
- self._body = body
+ self._body = to_unicode(body)
self.tags = set(tags) # TODO test that asserts copy
self._flags = set(flags) # TODO test that asserts copy
self._attachments = attachments
diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py
index 065dd5e5..8d0b1f4e 100644
--- a/service/pixelated/adapter/search/__init__.py
+++ b/service/pixelated/adapter/search/__init__.py
@@ -31,6 +31,8 @@ from whoosh import sorting
from pixelated.support.functional import unique
import traceback
+from pixelated.utils import to_unicode
+
class SearchEngine(object):
DEFAULT_INDEX_HOME = os.path.join(os.environ['HOME'], '.leap')
@@ -132,7 +134,7 @@ class SearchEngine(object):
'cc': self._format_recipient(header, 'cc'),
'bcc': self._format_recipient(header, 'bcc'),
'tag': u','.join(unique(tags)),
- 'body': unicode(mdict['textPlainBody'] if 'textPlainBody' in mdict else mdict['body']),
+ 'body': to_unicode(mdict.get('textPlainBody', mdict.get('body', ''))),
'ident': unicode(mdict['ident']),
'flags': unicode(','.join(unique(mail.flags))),
'raw': unicode(mail.raw)
diff --git a/service/pixelated/resources/mails_resource.py b/service/pixelated/resources/mails_resource.py
index 34dd042a..613c5e6f 100644
--- a/service/pixelated/resources/mails_resource.py
+++ b/service/pixelated/resources/mails_resource.py
@@ -7,10 +7,9 @@ from twisted.web.resource import Resource
from twisted.web import server
from twisted.internet import defer
from twisted.python.log import err
-from leap.common.events import (
- register,
- catalog as events
-)
+from leap.common import events
+
+from pixelated.utils import to_unicode
class MailsUnreadResource(Resource):
@@ -120,7 +119,7 @@ class MailsResource(Resource):
delivery_error_mail = InputMail.delivery_error_template(delivery_address=event.content)
self._mail_service.mailboxes.inbox.add(delivery_error_mail)
- register(events.SMTP_SEND_MESSAGE_ERROR, callback=on_error)
+ events.register(events.catalog.SMTP_SEND_MESSAGE_ERROR, callback=on_error)
def __init__(self, mail_service, draft_service):
Resource.__init__(self)
@@ -136,7 +135,8 @@ class MailsResource(Resource):
def render_GET(self, request):
query, window_size, page = request.args.get('q')[0], request.args.get('w')[0], request.args.get('p')[0]
- d = self._mail_service.mails(query, window_size, page)
+ unicode_query = to_unicode(query)
+ d = self._mail_service.mails(unicode_query, window_size, page)
d.addCallback(lambda (mails, total): {
"stats": {
diff --git a/service/pixelated/utils.py b/service/pixelated/utils.py
new file mode 100755
index 00000000..2e0f1f88
--- /dev/null
+++ b/service/pixelated/utils.py
@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+
+def to_unicode(text):
+ if text and not isinstance(text, unicode):
+ encoding = 'utf-8'
+ return unicode(text, encoding=encoding)
+ return text
diff --git a/service/test/unit/adapter/search/test_search.py b/service/test/unit/adapter/search/test_search.py
index 76e704b6..2e7badff 100644
--- a/service/test/unit/adapter/search/test_search.py
+++ b/service/test/unit/adapter/search/test_search.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
#
# Copyright (c) 2014 ThoughtWorks, Inc.
#
@@ -21,6 +22,8 @@ from pixelated.adapter.search import SearchEngine
from tempdir import TempDir
from test.support import test_helper
+from pixelated.utils import to_unicode
+
INDEX_KEY = '\xde3?\x87\xff\xd9\xd3\x14\xf0\xa7>\x1f%C{\x16.\\\xae\x8c\x13\xa7\xfb\x04\xd4]+\x8d_\xed\xd1\x8d\x0bI' \
'\x8a\x0e\xa4tm\xab\xbf\xb4\xa5\x99\x00d\xd5w\x9f\x18\xbc\x1d\xd4_W\xd2\xb6\xe8H\x83\x1b\xd8\x9d\xad'
@@ -45,7 +48,7 @@ class SearchEngineTest(unittest.TestCase):
def tearDown(self):
self.tempdir.dissolve()
- def test_encoding(self):
+ def test_headers_encoding(self):
# given
se = SearchEngine(INDEX_KEY, self.agent_home)
@@ -62,3 +65,50 @@ class SearchEngineTest(unittest.TestCase):
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
+
+ def test_contents_encoding_accents(self):
+ # given
+ se = SearchEngine(INDEX_KEY, self.agent_home)
+
+ headers = {
+ 'From': 'foo@bar.tld',
+ 'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+ 'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+ 'Subject': 'Some test mail',
+ }
+
+ body = "When doing the search, it's not possible to find words with graphical accents, e.g.: 'coração', 'é', 'Fièvre', La Pluie d'été, 'não'."
+
+ # when
+ se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
+
+ result = se.search(u"'coração', 'é',")
+ self.assertEqual((['mailid'], 1), result)
+
+ result = se.search(u"Fièvre")
+ self.assertEqual((['mailid'], 1), result)
+
+ result = se.search(u"été")
+ self.assertEqual((['mailid'], 1), result)
+
+ def test_contents_encoding_special_characters(self):
+ # given
+ se = SearchEngine(INDEX_KEY, self.agent_home)
+
+ headers = {
+ 'From': 'foo@bar.tld',
+ 'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+ 'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
+ 'Subject': 'Some test mail',
+ }
+
+ body = "When doing the search, 您好 أهلا"
+
+ # when
+ se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
+
+ result = se.search(u"您好")
+ self.assertEqual((['mailid'], 1), result)
+
+ result = se.search(u"أهلا")
+ self.assertEqual((['mailid'], 1), result)
diff --git a/service/test/unit/resources/test_mails_resource.py b/service/test/unit/resources/test_mails_resource.py
new file mode 100644
index 00000000..02b17bf1
--- /dev/null
+++ b/service/test/unit/resources/test_mails_resource.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import unittest
+import json
+from mockito import mock, when, verify
+from test.unit.resources import DummySite
+from twisted.web.test.requesthelper import DummyRequest
+from pixelated.resources.mails_resource import MailsResource
+from twisted.internet import defer
+from mock import patch
+
+
+class TestArchiveResource(unittest.TestCase):
+ def setUp(self):
+ self.mail_service = mock()
+
+ @patch('leap.common.events.register')
+ def test_render_GET_should_unicode_mails_search_query(self, mock_register):
+ request = DummyRequest(['/mails'])
+ non_unicode_search_term = 'coração'
+ request.addArg('q', non_unicode_search_term)
+ request.addArg('w', 25)
+ request.addArg('p', 1)
+
+ unicodified_search_term = u'coração'
+ when(self.mail_service).mails(unicodified_search_term, 25, 1).thenReturn(defer.Deferred())
+
+ mails_resource = MailsResource(self.mail_service, mock())
+ mails_resource.isLeaf = True
+ web = DummySite(mails_resource)
+ d = web.get(request)
+
+ def assert_response(_):
+ verify(self.mail_service).mails(unicodified_search_term, 25, 1)
+
+ d.addCallback(assert_response)
+ return d
diff --git a/service/test/unit/test_utils.py b/service/test/unit/test_utils.py
new file mode 100755
index 00000000..ffaf3c8d
--- /dev/null
+++ b/service/test/unit/test_utils.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+
+
+import unittest
+
+from pixelated.utils import to_unicode
+
+
+class UtilsTest(unittest.TestCase):
+
+ def test_to_unicode_guesses_encoding_and_unicode_text(self):
+ text = 'coração'
+ self.assertEqual(u'coração', to_unicode(text))
+
+ def test_to_unicode_self(self):
+ text = u'already unicode'
+ self.assertEqual(text, to_unicode(text))
+
+ def test_to_unicode_empty_string(self):
+ text = ''
+ self.assertEqual(text, to_unicode(text))
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-28 15:36:10 +0000