diff options
| -rw-r--r-- | mail/changes/VERSION_COMPAT | 3 | ||||
| -rw-r--r-- | mail/changes/handle-unicode-characters | 1 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/fetch.py | 19 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/messages.py | 4 | ||||
| -rw-r--r-- | mail/src/leap/mail/utils.py | 101 |
5 files changed, 110 insertions, 18 deletions
diff --git a/mail/changes/VERSION_COMPAT b/mail/changes/VERSION_COMPAT index 1d5643f9..03caa3eb 100644 --- a/mail/changes/VERSION_COMPAT +++ b/mail/changes/VERSION_COMPAT @@ -9,4 +9,5 @@ # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z leap.soledad.client 0.5.0 # get_count_by_index - +leap.common 0.3.7 # get_email_charset +leap.keymanager 0.3.8 # openpgp.decrypt diff --git a/mail/changes/handle-unicode-characters b/mail/changes/handle-unicode-characters new file mode 100644 index 00000000..052c5438 --- /dev/null +++ b/mail/changes/handle-unicode-characters @@ -0,0 +1 @@ + o Handle correctly unicode characters in emails. Closes #4838. diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py index 604a2ea1..817ad6a2 100644 --- a/mail/src/leap/mail/imap/fetch.py +++ b/mail/src/leap/mail/imap/fetch.py @@ -18,9 +18,7 @@ Incoming mail fetcher. """ import copy -import json import logging -#import ssl import threading import time import sys @@ -34,7 +32,6 @@ from StringIO import StringIO from twisted.python import log from twisted.internet import defer from twisted.internet.task import LoopingCall -#from twisted.internet.threads import deferToThread from zope.proxy import sameProxiedObjects from leap.common import events as leap_events @@ -49,6 +46,7 @@ from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey from leap.mail.decorators import deferred +from leap.mail.utils import json_loads from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -321,7 +319,8 @@ class LeapIncomingMail(object): """ log.msg('processing decrypted doc') doc, data = msgtuple - msg = json.loads(data) + msg = json_loads(data) + if not isinstance(msg, dict): defer.returnValue(False) if not msg.get(self.INCOMING_KEY, False): @@ -338,16 +337,15 @@ class LeapIncomingMail(object): Tries to decrypt a gpg message if data looks like one. :param data: the text to be decrypted. - :type data: unicode + :type data: str :return: data, possibly descrypted. :rtype: str """ + leap_assert_type(data, str) log.msg('maybe decrypting doc') - leap_assert_type(data, unicode) # parse the original message encoding = get_email_charset(data) - data = data.encode(encoding) msg = self._parser.parsestr(data) # try to obtain sender public key @@ -420,13 +418,6 @@ class LeapIncomingMail(object): # Bailing out! return (msg, False) - # decrypted successully, now fix encoding and parse - try: - decrdata = decrdata.encode(encoding) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - decrdata = decrdata.encode(encoding, 'replace') - decrmsg = self._parser.parsestr(decrdata) # remove original message's multipart/encrypted content-type del(msg['content-type']) diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py index 7a210099..d2c09504 100644 --- a/mail/src/leap/mail/imap/messages.py +++ b/mail/src/leap/mail/imap/messages.py @@ -532,8 +532,8 @@ class LeapMessage(fields, MailParser, MBoxParser): if not charset: charset = self._get_charset(body) try: - body = body.decode(charset).encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: + body = body.encode(charset) + except UnicodeError as e: logger.error("Unicode error {0}".format(e)) body = body.encode(charset, 'replace') diff --git a/mail/src/leap/mail/utils.py b/mail/src/leap/mail/utils.py index 2480efc8..93388d31 100644 --- a/mail/src/leap/mail/utils.py +++ b/mail/src/leap/mail/utils.py @@ -15,8 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Small utilities. +Mail utilities. """ +import json +import traceback def first(things): @@ -27,3 +29,100 @@ def first(things): return things[0] except (IndexError, TypeError): return None + + +class CustomJsonScanner(object): + """ + This class is a context manager definition used to monkey patch the default + json string parsing behavior. + The emails can have more than one encoding, so the `str` objects have more + than one encoding and json does not support direct work with `str` + (only `unicode`). + """ + + def _parse_string_str(self, s, idx, *args, **kwargs): + """ + Parses the string "s" starting at the point idx and returns an `str` + object. Which basically means it works exactly the same as the regular + JSON string parsing, except that it doesn't try to decode utf8. + We need this because mail raw strings might have bytes in multiple + encodings. + + :param s: the string we want to parse + :type s: str + :param idx: the starting point for parsing + :type idx: int + + :returns: the parsed string and the index where the + string ends. + :rtype: tuple (str, int) + """ + # NOTE: we just want to use this monkey patched version if we are + # calling the loads from our custom method. Otherwise, we use the + # json's default parser. + monkey_patched = False + for i in traceback.extract_stack(): + # look for json_loads method in the call stack + if i[2] == json_loads.__name__: + monkey_patched = True + break + + if not monkey_patched: + return self._orig_scanstring(s, idx, *args, **kwargs) + + found = False + end = s.find("\"", idx) + while not found: + try: + if s[end-1] != "\\": + found = True + else: + end = s.find("\"", end+1) + except Exception: + found = True + return s[idx:end].decode("string-escape"), end+1 + + def __enter__(self): + """ + Replace the json methods with the needed ones. + Also make a backup to restore them later. + """ + # backup original values + self._orig_make_scanner = json.scanner.make_scanner + self._orig_scanstring = json.decoder.scanstring + + # We need the make_scanner function to be the python one so we can + # monkey_patch the json string parsing + json.scanner.make_scanner = json.scanner.py_make_scanner + + # And now we monkey patch the money method + json.decoder.scanstring = self._parse_string_str + + def __exit__(self, exc_type, exc_value, traceback): + """ + Restores the backuped methods. + """ + # restore original values + json.scanner.make_scanner = self._orig_make_scanner + json.decoder.scanstring = self._orig_scanstring + + +def json_loads(data): + """ + It works as json.loads but supporting multiple encodings in the same + string and accepting an `str` parameter that won't be converted to unicode. + + :param data: the string to load the objects from + :type data: str + + :returns: the corresponding python object result of parsing 'data', this + behaves similarly as json.loads, with the exception of that + returns always `str` instead of `unicode`. + """ + obj = None + with CustomJsonScanner(): + # We need to use the cls parameter in order to trigger the code + # that will let us control the string parsing method. + obj = json.loads(data, cls=json.JSONDecoder) + + return obj |