Improve charset handling for email

2 files changed, 28 insertions, 2 deletions

diff --git a/mail/changes/better_charset_handling b/mail/changes/better_charset_handling
new file mode 100644
index 00000000..832f7b2c
--- /dev/null
+++ b/mail/changes/better_charset_handling
@@ -0,0 +1,2 @@
+  o Improve charset handling when exposing mails to the mail client. Related to
+    #3660.
diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py
index ae76833b..10d338a1 100644
--- a/mail/src/leap/mail/imap/server.py
+++ b/mail/src/leap/mail/imap/server.py
@@ -18,7 +18,9 @@
 Soledad-backed IMAP Server.
 """
 import copy
+import email
 import logging
+import re
 import StringIO
 import cStringIO
 import time
@@ -693,6 +695,26 @@ class LeapMessage(WithMsgFields):
     the more complex MIME-based interface.
     """
 
+    def _get_charset(self, content):
+        """
+        Mini parser to retrieve the charset of an email
+
+        :param content: mail contents
+        :type content: unicode
+
+        :returns: the charset as parsed from the contents
+        :rtype: str
+        """
+        charset = "UTF-8"
+        try:
+            em = email.message_from_string(content.encode("utf-8"))
+            # Miniparser for: Content-Type: <something>; charset=<charset>
+            charset_re = r'''charset=(?P<charset>[\w|\d|-]*)'''
+            charset = re.findall(charset_re, em["Content-Type"])[0]
+        except Exception:
+            pass
+        return charset
+
     def open(self):
         """
         Return an file-like object opened for reading.
@@ -704,7 +726,8 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = cStringIO.StringIO()
-        fd.write(str(self._doc.content.get(self.RAW_KEY, '')))
+        charset = self._get_charset(self._doc.content.get(self.RAW_KEY, ''))
+        fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset))
         fd.seek(0)
         return fd
 
@@ -723,7 +746,8 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        fd.write(str(self._doc.content.get(self.RAW_KEY, '')))
+        charset = self._get_charset(self._doc.content.get(self.RAW_KEY, ''))
+        fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset))
         # SHOULD use a separate BODY FIELD ...
         fd.seek(0)
         return fd