Merge branch 'release-0.3.6'0.3.6

3 files changed, 123 insertions, 52 deletions

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 0a71f53..dd65def 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -31,15 +31,16 @@ from twisted.internet.threads import deferToThread
 
 from leap.common import events as leap_events
 from leap.common.check import leap_assert, leap_assert_type
-from leap.soledad.client import Soledad
-from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
-
 from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING
 from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING
 from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED
 from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY
 from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
+from leap.common.mail import get_email_charset
+from leap.keymanager import errors as keymanager_errors
+from leap.soledad.client import Soledad
+from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
 
 logger = logging.getLogger(__name__)
@@ -197,6 +198,29 @@ class LeapIncomingMail(object):
             logger.warning('Unknown error while '
                            'syncing soledad: %r' % (err,))
 
+    def _log_err(self, failure):
+        """
+        Generic errback
+        """
+        err = failure.value
+        logger.error("error!: %r" % (err,))
+
+    def _decryption_error(self, failure):
+        """
+        Errback for decryption errors.
+        """
+        # XXX should signal unrecoverable maybe.
+        err = failure.value
+        logger.error("error decrypting msg: %s" % (err,))
+
+    def _saving_error(self, failure):
+        """
+        Errback for local save errors.
+        """
+        # XXX should signal unrecoverable maybe.
+        err = failure.value
+        logger.error("error saving msg locally: %s" % (err,))
+
     def _process_doclist(self, doclist):
         """
         Iterates through the doclist, checks if each doc
@@ -227,7 +251,13 @@ class LeapIncomingMail(object):
                 # Deferred chain for individual messages
                 d = deferToThread(self._decrypt_msg, doc, encdata)
                 d.addCallback(self._process_decrypted)
+                d.addErrback(self._log_err)
                 d.addCallback(self._add_message_locally)
+                d.addErrback(self._log_err)
+                # XXX check this, add_locally should not get called if we
+                # get an error in process
+                #d.addCallbacks(self._process_decrypted, self._decryption_error)
+                #d.addCallbacks(self._add_message_locally, self._saving_error)
                 docs_cb.append(d)
             else:
                 # Ooops, this does not.
@@ -288,20 +318,29 @@ class LeapIncomingMail(object):
         rawmsg = msg.get(self.CONTENT_KEY, None)
         if not rawmsg:
             return False
-        data = self._maybe_decrypt_gpg_msg(rawmsg)
-        return doc, data
+        try:
+            data = self._maybe_decrypt_gpg_msg(rawmsg)
+            return doc, data
+        except keymanager_errors.EncryptionDecryptionFailed as exc:
+            logger.error(exc)
+            raise
 
     def _maybe_decrypt_gpg_msg(self, data):
         """
         Tries to decrypt a gpg message if data looks like one.
 
         :param data: the text to be decrypted.
-        :type data: str
+        :type data: unicode
         :return: data, possibly descrypted.
         :rtype: str
         """
+        leap_assert_type(data, unicode)
+
         parser = Parser()
+        encoding = get_email_charset(data)
+        data = data.encode(encoding)
         origmsg = parser.parsestr(data)
+
         # handle multipart/encrypted messages
         if origmsg.get_content_type() == 'multipart/encrypted':
             # sanity check
@@ -320,13 +359,21 @@ class LeapIncomingMail(object):
                     "Multipart/encrypted messages' second body part should "
                     "have content type equal to 'octet-stream' (instead of "
                     "%s)." % payload[1].get_content_type())
+
             # parse message and get encrypted content
             pgpencmsg = origmsg.get_payload()[1]
             encdata = pgpencmsg.get_payload()
+
             # decrypt and parse decrypted message
             decrdata = self._keymanager.decrypt(
                 encdata, self._pkey,
                 passphrase=self._soledad.passphrase)
+            try:
+                decrdata = decrdata.encode(encoding)
+            except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                logger.error("Unicode error {0}".format(e))
+                decrdata = decrdata.encode(encoding, 'replace')
+
             decrmsg = parser.parsestr(decrdata)
             # replace headers back in original message
             for hkey, hval in decrmsg.items():
@@ -335,6 +382,7 @@ class LeapIncomingMail(object):
                     origmsg.replace_header(hkey, hval)
                 except KeyError:
                     origmsg[hkey] = hval
+
             # replace payload by unencrypted payload
             origmsg.set_payload(decrmsg.get_payload())
             return origmsg.as_string(unixfrom=False)
@@ -352,6 +400,10 @@ class LeapIncomingMail(object):
                 # replace encrypted by decrypted content
                 data = data.replace(pgp_message, decrdata)
         # if message is not encrypted, return raw data
+
+        if isinstance(data, unicode):
+            data = data.encode(encoding, 'replace')
+
         return data
 
     def _add_message_locally(self, msgtuple):
@@ -365,6 +417,7 @@ class LeapIncomingMail(object):
                          incoming message
         :type msgtuple: (SoledadDocument, str)
         """
+        print "adding message locally....."
         doc, data = msgtuple
         self._inbox.addMessage(data, (self.RECENT_FLAG,))
         leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 10d338a..7a9f810 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -18,9 +18,7 @@
 Soledad-backed IMAP Server.
 """
 import copy
-import email
 import logging
-import re
 import StringIO
 import cStringIO
 import time
@@ -33,13 +31,10 @@ from twisted.mail import imap4
 from twisted.internet import defer
 from twisted.python import log
 
-#from twisted import cred
-
-#import u1db
-
 from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
+from leap.common.mail import get_email_charset
 from leap.soledad.client import Soledad
 
 logger = logging.getLogger(__name__)
@@ -184,7 +179,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         # messages
         TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
         TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
-        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(recent)', 'bool(seen)'],
+        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
+                                  'bool(recent)', 'bool(seen)'],
     }
 
     INBOX_NAME = "INBOX"
@@ -577,7 +573,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         return "<SoledadBackedAccount (%s)>" % self._account_name
 
 #######################################
-# Soledad Message, MessageCollection
+# LeapMessage, MessageCollection
 # and Mailbox
 #######################################
 
@@ -695,26 +691,6 @@ class LeapMessage(WithMsgFields):
     the more complex MIME-based interface.
     """
 
-    def _get_charset(self, content):
-        """
-        Mini parser to retrieve the charset of an email
-
-        :param content: mail contents
-        :type content: unicode
-
-        :returns: the charset as parsed from the contents
-        :rtype: str
-        """
-        charset = "UTF-8"
-        try:
-            em = email.message_from_string(content.encode("utf-8"))
-            # Miniparser for: Content-Type: <something>; charset=<charset>
-            charset_re = r'''charset=(?P<charset>[\w|\d|-]*)'''
-            charset = re.findall(charset_re, em["Content-Type"])[0]
-        except Exception:
-            pass
-        return charset
-
     def open(self):
         """
         Return an file-like object opened for reading.
@@ -726,8 +702,14 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = cStringIO.StringIO()
-        charset = self._get_charset(self._doc.content.get(self.RAW_KEY, ''))
-        fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset))
+        charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
+        content = self._doc.content.get(self.RAW_KEY, '')
+        try:
+            content = content.encode(charset)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            content = content.encode(charset, 'replace')
+        fd.write(content)
         fd.seek(0)
         return fd
 
@@ -746,8 +728,14 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        charset = self._get_charset(self._doc.content.get(self.RAW_KEY, ''))
-        fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset))
+        charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
+        content = self._doc.content.get(self.RAW_KEY, '')
+        try:
+            content = content.encode(charset)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            content = content.encode(charset, 'replace')
+        fd.write(content)
         # SHOULD use a separate BODY FIELD ...
         fd.seek(0)
         return fd
@@ -1111,6 +1099,7 @@ class SoledadMailbox(WithMsgFields):
     which we instantiate and make accessible in the `messages` attribute.
     """
     implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
+    # XXX should finish the implementation of IMailboxListener
 
     messages = None
     _closed = False
@@ -1127,6 +1116,8 @@ class SoledadMailbox(WithMsgFields):
     CMD_UIDVALIDITY = "UIDVALIDITY"
     CMD_UNSEEN = "UNSEEN"
 
+    listeners = []
+
     def __init__(self, mbox, soledad=None, rw=1):
         """
         SoledadMailbox constructor. Needs to get passed a name, plus a
@@ -1159,15 +1150,35 @@ class SoledadMailbox(WithMsgFields):
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
 
-        # XXX what is/was this used for? --------
-        # ---> mail/imap4.py +1155,
-        #      _cbSelectWork makes use of this
-        # probably should implement hooks here
-        # using leap.common.events
-        self.listeners = []
-        self.addListener = self.listeners.append
-        self.removeListener = self.listeners.remove
-        #------------------------------------------
+        # the server itself is a listener to the mailbox.
+        # so we can notify it (and should!) after chanes in flags
+        # and number of messages.
+        print "emptying the listeners"
+        map(lambda i: self.listeners.remove(i), self.listeners)
+
+    def addListener(self, listener):
+        """
+        Rdds a listener to the listeners queue.
+
+        :param listener: listener to add
+        :type listener: an object that implements IMailboxListener
+        """
+        logger.debug('adding mailbox listener: %s' % listener)
+        self.listeners.append(listener)
+
+    def removeListener(self, listener):
+        """
+        Removes a listener from the listeners queue.
+
+        :param listener: listener to remove
+        :type listener: an object that implements IMailboxListener
+        """
+        logger.debug('removing mailbox listener: %s' % listener)
+        try:
+            self.listeners.remove(listener)
+        except ValueError:
+            logger.error(
+                "removeListener: cannot remove listener %s" % listener)
 
     def _get_mbox(self):
         """
@@ -1192,6 +1203,7 @@ class SoledadMailbox(WithMsgFields):
         #return map(str, self.INIT_FLAGS)
 
         # XXX CHECK against thunderbird XXX
+        # XXX I think this is slightly broken.. :/
 
         mbox = self._get_mbox()
         if not mbox:
@@ -1364,6 +1376,10 @@ class SoledadMailbox(WithMsgFields):
 
         self.messages.add_msg(message, flags=flags, date=date,
                               uid=uid_next)
+        exists = len(self.messages)
+        recent = len(self.messages.get_recent())
+        for listener in self.listeners:
+            listener.newMessages(exists, recent)
         return defer.succeed(None)
 
     # commands, do not rename methods
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index b840e86..5f7322a 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -77,7 +77,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         #self.theAccount = theAccount
 
     def lineReceived(self, line):
-        if "login" in line:
+        if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
             msg = line[:7] + " [...]"
@@ -141,7 +141,9 @@ def run_service(*args, **kwargs):
     Main entry point to run the service from the client.
 
     :returns: the LoopingCall instance that will have to be stoppped
-              before shutting down the client.
+              before shutting down the client, the port as returned by
+              the reactor when starts listening, and the factory for
+              the protocol.
     """
     leap_assert(len(args) == 2)
     soledad, keymanager = args
@@ -157,8 +159,8 @@ def run_service(*args, **kwargs):
     from twisted.internet import reactor
 
     try:
-        reactor.listenTCP(port, factory,
-                          interface="localhost")
+        tport = reactor.listenTCP(port, factory,
+                                  interface="localhost")
         fetcher = LeapIncomingMail(
             keymanager,
             soledad,
@@ -174,7 +176,7 @@ def run_service(*args, **kwargs):
         fetcher.start_loop()
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
         leap_events.signal(IMAP_SERVICE_STARTED, str(port))
-        return fetcher
+        return fetcher, tport, factory
 
     # not ok, signal error.
     leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port))