Improved body decoding when parsing mails.

- Issue #450
- does not fix 8bit encoded bodies - special chars are replaced before
  we are in pixelated code

2 files changed, 46 insertions, 8 deletions

diff --git a/service/pixelated/adapter/mailstore/body_parser.py b/service/pixelated/adapter/mailstore/body_parser.py
index c79e11a4..d8bf3264 100644
--- a/service/pixelated/adapter/mailstore/body_parser.py
+++ b/service/pixelated/adapter/mailstore/body_parser.py
@@ -15,6 +15,14 @@
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 
 from email.parser import Parser
+import re
+
+
+def _parse_charset_header(content_type_and_charset_header, default_charset='us-ascii'):
+    try:
+        return re.compile('.*charset="?([a-zA-Z0-9-]+)"?', re.MULTILINE | re.DOTALL).match(content_type_and_charset_header).group(1)
+    except:
+        return default_charset
 
 
 class BodyParser(object):
@@ -25,17 +33,29 @@ class BodyParser(object):
         self._content_transfer_encoding = content_transfer_encoding
 
     def parsed_content(self):
-        parser = Parser()
+        charset = _parse_charset_header(self._content_type)
+        text = self._serialize_for_parser(charset)
+
+        decoded_body = self._parse_and_decode(text)
+
+        return unicode(decoded_body, encoding=charset)
+
+    def _parse_and_decode(self, text):
+        parsed_body = Parser().parsestr(text)
+        decoded_body = self._unwrap_content_transfer_encoding(parsed_body)
+        return decoded_body
 
+    def _unwrap_content_transfer_encoding(self, parsed_body):
+        return parsed_body.get_payload(decode=True)
+
+    def _serialize_for_parser(self, charset):
         text = ''
         text += 'Content-Type: %s\n' % self._content_type
         if self._content_transfer_encoding is not None:
             text += 'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding
         text += '\n'
-        text += self._content
-
-        parsed_body = parser.parsestr(text)
-
-        result = unicode(parsed_body.get_payload(decode=True), encoding='utf-8')
-
-        return unicode(result)
+        if isinstance(self._content, unicode):
+            text += self._content.encode(charset)
+        else:
+            text += self._content
+        return text
diff --git a/service/test/unit/adapter/mailstore/test_body_parser.py b/service/test/unit/adapter/mailstore/test_body_parser.py
index eeb1a2ed..3c2d17fb 100644
--- a/service/test/unit/adapter/mailstore/test_body_parser.py
+++ b/service/test/unit/adapter/mailstore/test_body_parser.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #
 # Copyright (c) 2015 ThoughtWorks, Inc.
 #
@@ -28,3 +29,20 @@ class BodyParserTest(unittest.TestCase):
         parser = BodyParser('dGVzdCB0ZXh0\n', content_type='text/plain; charset="utf-8"', content_transfer_encoding='base64')
 
         self.assertEqual('test text', parser.parsed_content())
+
+    def test_8bit_transfer_encoding_with_iso_8859_1_str_input(self):
+        data = 'Hmm, here are \xdcml\xe4\xfcts again!'
+        parser = BodyParser(data, content_type='text/plain; charset=iso-8859-1', content_transfer_encoding='8bit')
+
+        self.assertEqual(u'Hmm, here are Ümläüts again!', parser.parsed_content())
+
+    def test_8bit_transfer_encoding_with_iso_8859_1_unicode_input(self):
+        data = u'Hmm, here are \xdcml\xe4\xfcts again!'
+        parser = BodyParser(data, content_type='text/plain; charset=iso-8859-1', content_transfer_encoding='8bit')
+
+        self.assertEqual(u'Hmm, here are Ümläüts again!', parser.parsed_content())
+
+    def test_base64_with_default_us_ascii_encoding(self):
+        parser = BodyParser('dGVzdCB0ZXh0\n', content_type='text/plain', content_transfer_encoding='base64')
+
+        self.assertEqual('test text', parser.parsed_content())