From 8fdaf603e313b9cb938f2f41713862e75f40d583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Mon, 21 Oct 2013 10:18:54 -0300 Subject: Default to UTF-8 when there is not charset parsed from the mail contents --- src/leap/mail/imap/server.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 10d338a..df510ce 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -711,6 +711,8 @@ class LeapMessage(WithMsgFields): # Miniparser for: Content-Type: ; charset= charset_re = r'''charset=(?P[\w|\d|-]*)''' charset = re.findall(charset_re, em["Content-Type"])[0] + if charset is None or len(charset) == 0: + charset = "UTF-8" except Exception: pass return charset -- cgit v1.2.3 From c0834048d564dfdc3e2cedc1a0f81788d14f3cab Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 23 Oct 2013 10:17:21 -0300 Subject: Move charset parser to a utils module. --- src/leap/mail/imap/server.py | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index df510ce..b9a041d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -18,9 +18,7 @@ Soledad-backed IMAP Server. """ import copy -import email import logging -import re import StringIO import cStringIO import time @@ -41,6 +39,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.soledad.client import Soledad +from leap.mail.utils import get_email_charset logger = logging.getLogger(__name__) @@ -695,28 +694,6 @@ class LeapMessage(WithMsgFields): the more complex MIME-based interface. """ - def _get_charset(self, content): - """ - Mini parser to retrieve the charset of an email - - :param content: mail contents - :type content: unicode - - :returns: the charset as parsed from the contents - :rtype: str - """ - charset = "UTF-8" - try: - em = email.message_from_string(content.encode("utf-8")) - # Miniparser for: Content-Type: ; charset= - charset_re = r'''charset=(?P[\w|\d|-]*)''' - charset = re.findall(charset_re, em["Content-Type"])[0] - if charset is None or len(charset) == 0: - charset = "UTF-8" - except Exception: - pass - return charset - def open(self): """ Return an file-like object opened for reading. @@ -728,7 +705,7 @@ class LeapMessage(WithMsgFields): :rtype: StringIO """ fd = cStringIO.StringIO() - charset = self._get_charset(self._doc.content.get(self.RAW_KEY, '')) + charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset)) fd.seek(0) return fd @@ -748,7 +725,7 @@ class LeapMessage(WithMsgFields): :rtype: StringIO """ fd = StringIO.StringIO() - charset = self._get_charset(self._doc.content.get(self.RAW_KEY, '')) + charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset)) # SHOULD use a separate BODY FIELD ... fd.seek(0) -- cgit v1.2.3 From f864f21b51b9f767afa8ccb3b3f39967b2f08f60 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 23 Oct 2013 10:18:34 -0300 Subject: Add encoding exception catch to avoid crashes. --- src/leap/mail/imap/server.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b9a041d..7ae3c45 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -706,7 +706,13 @@ class LeapMessage(WithMsgFields): """ fd = cStringIO.StringIO() charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) - fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset)) + content = self._doc.content.get(self.RAW_KEY, '') + try: + content = content.encode(charset) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + content = content.encode(charset, 'replace') + fd.write(content) fd.seek(0) return fd @@ -726,7 +732,13 @@ class LeapMessage(WithMsgFields): """ fd = StringIO.StringIO() charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) - fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset)) + content = self._doc.content.get(self.RAW_KEY, '') + try: + content = content.encode(charset) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + content = content.encode(charset, 'replace') + fd.write(content) # SHOULD use a separate BODY FIELD ... fd.seek(0) return fd -- cgit v1.2.3 From 7dd2e6ed4d1980626f486a6d5065a4fd5ffdfeb3 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 23 Oct 2013 10:19:44 -0300 Subject: Use correct encoding and data type in mails. --- src/leap/mail/imap/fetch.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 0a71f53..a776ac7 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -40,6 +40,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.mail.utils import get_email_charset logger = logging.getLogger(__name__) @@ -296,12 +297,17 @@ class LeapIncomingMail(object): Tries to decrypt a gpg message if data looks like one. :param data: the text to be decrypted. - :type data: str + :type data: unicode :return: data, possibly descrypted. :rtype: str """ + leap_assert_type(data, unicode) + parser = Parser() + encoding = get_email_charset(data) + data = data.encode(encoding) origmsg = parser.parsestr(data) + # handle multipart/encrypted messages if origmsg.get_content_type() == 'multipart/encrypted': # sanity check @@ -320,13 +326,21 @@ class LeapIncomingMail(object): "Multipart/encrypted messages' second body part should " "have content type equal to 'octet-stream' (instead of " "%s)." % payload[1].get_content_type()) + # parse message and get encrypted content pgpencmsg = origmsg.get_payload()[1] encdata = pgpencmsg.get_payload() + # decrypt and parse decrypted message decrdata = self._keymanager.decrypt( encdata, self._pkey, passphrase=self._soledad.passphrase) + try: + decrdata = decrdata.encode(encoding) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + decrdata = decrdata.encode(encoding, 'replace') + decrmsg = parser.parsestr(decrdata) # replace headers back in original message for hkey, hval in decrmsg.items(): @@ -335,6 +349,7 @@ class LeapIncomingMail(object): origmsg.replace_header(hkey, hval) except KeyError: origmsg[hkey] = hval + # replace payload by unencrypted payload origmsg.set_payload(decrmsg.get_payload()) return origmsg.as_string(unixfrom=False) @@ -352,6 +367,10 @@ class LeapIncomingMail(object): # replace encrypted by decrypted content data = data.replace(pgp_message, decrdata) # if message is not encrypted, return raw data + + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return data def _add_message_locally(self, msgtuple): -- cgit v1.2.3 From 2e5dbaec5189603615c7b4e9e93bd6129af9c3b1 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 23 Oct 2013 10:22:47 -0300 Subject: Remove commented imports. --- src/leap/mail/imap/server.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 7ae3c45..6fc4db3 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -31,10 +31,6 @@ from twisted.mail import imap4 from twisted.internet import defer from twisted.python import log -#from twisted import cred - -#import u1db - from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type -- cgit v1.2.3 From 755d4a18ce80db745118edb9b2b27d359ed839d2 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 23 Oct 2013 10:24:39 -0300 Subject: pep8 fix: line too long. --- src/leap/mail/imap/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 6fc4db3..5a98315 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -179,7 +179,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): # messages TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(recent)', 'bool(seen)'], + TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, + 'bool(recent)', 'bool(seen)'], } INBOX_NAME = "INBOX" -- cgit v1.2.3 From 971058ec42c117d4173f1800ad4457047fc08a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Wed, 30 Oct 2013 15:02:16 -0300 Subject: Return port and factory from the imap launch method --- src/leap/mail/imap/service/imap.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index b840e86..b641d2e 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -141,7 +141,9 @@ def run_service(*args, **kwargs): Main entry point to run the service from the client. :returns: the LoopingCall instance that will have to be stoppped - before shutting down the client. + before shutting down the client, the port as returned by + the reactor when starts listening, and the factory for + the protocol. """ leap_assert(len(args) == 2) soledad, keymanager = args @@ -157,8 +159,8 @@ def run_service(*args, **kwargs): from twisted.internet import reactor try: - reactor.listenTCP(port, factory, - interface="localhost") + tport = reactor.listenTCP(port, factory, + interface="localhost") fetcher = LeapIncomingMail( keymanager, soledad, @@ -174,7 +176,7 @@ def run_service(*args, **kwargs): fetcher.start_loop() logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) - return fetcher + return fetcher, tport, factory # not ok, signal error. leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) -- cgit v1.2.3 From 4efecad56eae8d777d2fa0d39c4de13b022c7bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 31 Oct 2013 10:18:47 -0300 Subject: Refactor out get_email_charset to leap.common --- src/leap/mail/imap/fetch.py | 2 +- src/leap/mail/imap/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index a776ac7..2775d71 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -40,7 +40,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.mail.utils import get_email_charset +from leap.common.mail import get_email_charset logger = logging.getLogger(__name__) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 5a98315..9e3e23e 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -34,8 +34,8 @@ from twisted.python import log from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type +from leap.common.mail import get_email_charset from leap.soledad.client import Soledad -from leap.mail.utils import get_email_charset logger = logging.getLogger(__name__) -- cgit v1.2.3 From 64cbb0d02e312a91c324b49ddb99bfb691b4e2cd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 31 Oct 2013 22:34:50 -0200 Subject: notify MUA of new mail as it gets added to mailbox --- src/leap/mail/imap/fetch.py | 44 ++++++++++++++++++++++++++++++---- src/leap/mail/imap/server.py | 48 ++++++++++++++++++++++++++++++-------- src/leap/mail/imap/service/imap.py | 4 ++-- 3 files changed, 79 insertions(+), 17 deletions(-) (limited to 'src/leap/mail/imap') diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 2775d71..dd65def 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -31,9 +31,6 @@ from twisted.internet.threads import deferToThread from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type -from leap.soledad.client import Soledad -from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY - from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED @@ -41,6 +38,9 @@ from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.mail import get_email_charset +from leap.keymanager import errors as keymanager_errors +from leap.soledad.client import Soledad +from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY logger = logging.getLogger(__name__) @@ -198,6 +198,29 @@ class LeapIncomingMail(object): logger.warning('Unknown error while ' 'syncing soledad: %r' % (err,)) + def _log_err(self, failure): + """ + Generic errback + """ + err = failure.value + logger.error("error!: %r" % (err,)) + + def _decryption_error(self, failure): + """ + Errback for decryption errors. + """ + # XXX should signal unrecoverable maybe. + err = failure.value + logger.error("error decrypting msg: %s" % (err,)) + + def _saving_error(self, failure): + """ + Errback for local save errors. + """ + # XXX should signal unrecoverable maybe. + err = failure.value + logger.error("error saving msg locally: %s" % (err,)) + def _process_doclist(self, doclist): """ Iterates through the doclist, checks if each doc @@ -228,7 +251,13 @@ class LeapIncomingMail(object): # Deferred chain for individual messages d = deferToThread(self._decrypt_msg, doc, encdata) d.addCallback(self._process_decrypted) + d.addErrback(self._log_err) d.addCallback(self._add_message_locally) + d.addErrback(self._log_err) + # XXX check this, add_locally should not get called if we + # get an error in process + #d.addCallbacks(self._process_decrypted, self._decryption_error) + #d.addCallbacks(self._add_message_locally, self._saving_error) docs_cb.append(d) else: # Ooops, this does not. @@ -289,8 +318,12 @@ class LeapIncomingMail(object): rawmsg = msg.get(self.CONTENT_KEY, None) if not rawmsg: return False - data = self._maybe_decrypt_gpg_msg(rawmsg) - return doc, data + try: + data = self._maybe_decrypt_gpg_msg(rawmsg) + return doc, data + except keymanager_errors.EncryptionDecryptionFailed as exc: + logger.error(exc) + raise def _maybe_decrypt_gpg_msg(self, data): """ @@ -384,6 +417,7 @@ class LeapIncomingMail(object): incoming message :type msgtuple: (SoledadDocument, str) """ + print "adding message locally....." doc, data = msgtuple self._inbox.addMessage(data, (self.RECENT_FLAG,)) leap_events.signal(IMAP_MSG_SAVED_LOCALLY) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 9e3e23e..7a9f810 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -573,7 +573,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): return "" % self._account_name ####################################### -# Soledad Message, MessageCollection +# LeapMessage, MessageCollection # and Mailbox ####################################### @@ -1099,6 +1099,7 @@ class SoledadMailbox(WithMsgFields): which we instantiate and make accessible in the `messages` attribute. """ implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) + # XXX should finish the implementation of IMailboxListener messages = None _closed = False @@ -1115,6 +1116,8 @@ class SoledadMailbox(WithMsgFields): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" + listeners = [] + def __init__(self, mbox, soledad=None, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a @@ -1147,15 +1150,35 @@ class SoledadMailbox(WithMsgFields): if not self.getFlags(): self.setFlags(self.INIT_FLAGS) - # XXX what is/was this used for? -------- - # ---> mail/imap4.py +1155, - # _cbSelectWork makes use of this - # probably should implement hooks here - # using leap.common.events - self.listeners = [] - self.addListener = self.listeners.append - self.removeListener = self.listeners.remove - #------------------------------------------ + # the server itself is a listener to the mailbox. + # so we can notify it (and should!) after chanes in flags + # and number of messages. + print "emptying the listeners" + map(lambda i: self.listeners.remove(i), self.listeners) + + def addListener(self, listener): + """ + Rdds a listener to the listeners queue. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + logger.debug('adding mailbox listener: %s' % listener) + self.listeners.append(listener) + + def removeListener(self, listener): + """ + Removes a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + logger.debug('removing mailbox listener: %s' % listener) + try: + self.listeners.remove(listener) + except ValueError: + logger.error( + "removeListener: cannot remove listener %s" % listener) def _get_mbox(self): """ @@ -1180,6 +1203,7 @@ class SoledadMailbox(WithMsgFields): #return map(str, self.INIT_FLAGS) # XXX CHECK against thunderbird XXX + # XXX I think this is slightly broken.. :/ mbox = self._get_mbox() if not mbox: @@ -1352,6 +1376,10 @@ class SoledadMailbox(WithMsgFields): self.messages.add_msg(message, flags=flags, date=date, uid=uid_next) + exists = len(self.messages) + recent = len(self.messages.get_recent()) + for listener in self.listeners: + listener.newMessages(exists, recent) return defer.succeed(None) # commands, do not rename methods diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index b641d2e..5f7322a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -77,7 +77,7 @@ class LeapIMAPServer(imap4.IMAP4Server): #self.theAccount = theAccount def lineReceived(self, line): - if "login" in line: + if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. msg = line[:7] + " [...]" @@ -160,7 +160,7 @@ def run_service(*args, **kwargs): try: tport = reactor.listenTCP(port, factory, - interface="localhost") + interface="localhost") fetcher = LeapIncomingMail( keymanager, soledad, -- cgit v1.2.3