Add a header that reflects the validity of incoming signatures. Closes #4354.

3 files changed, 198 insertions, 90 deletions

diff --git a/changes/feature_4354_add-signature-header-to-incoming-email b/changes/feature_4354_add-signature-header-to-incoming-email
new file mode 100644
index 0000000..866b68e
--- /dev/null
+++ b/changes/feature_4354_add-signature-header-to-incoming-email
@@ -0,0 +1,2 @@
+  o Add a header to incoming emails that reflects if a valid signature was
+    found when decrypting. Closes #4354.
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 3422ed5..38612e1 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -22,8 +22,12 @@ import json
 import ssl
 import threading
 import time
+import copy
+from StringIO import StringIO
 
 from email.parser import Parser
+from email.generator import Generator
+from email.utils import parseaddr
 
 from twisted.python import log
 from twisted.internet.task import LoopingCall
@@ -65,6 +69,16 @@ class LeapIncomingMail(object):
     INCOMING_KEY = "incoming"
     CONTENT_KEY = "content"
 
+    LEAP_SIGNATURE_HEADER = 'X-Leap-Signature'
+    """
+    Header added to messages when they are decrypted by the IMAP fetcher,
+    which states the validity of an eventual signature that might be included
+    in the encrypted blob.
+    """
+    LEAP_SIGNATURE_VALID = 'valid'
+    LEAP_SIGNATURE_INVALID = 'invalid'
+    LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify'
+
     fetching_lock = threading.Lock()
 
     def __init__(self, keymanager, soledad, imap_account,
@@ -260,11 +274,9 @@ class LeapIncomingMail(object):
             if self._is_msg(keys):
                 # Ok, this looks like a legit msg.
                 # Let's process it!
-                encdata = doc.content[ENC_JSON_KEY]
-
                 # Deferred chain for individual messages
-                d = deferToThread(self._decrypt_msg, doc, encdata)
-                d.addCallback(self._process_decrypted)
+                d = deferToThread(self._decrypt_doc, doc)
+                d.addCallback(self._process_decrypted_doc)
                 d.addErrback(self._log_err)
                 d.addCallback(self._add_message_locally)
                 d.addErrback(self._log_err)
@@ -293,32 +305,40 @@ class LeapIncomingMail(object):
         """
         return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
 
-    def _decrypt_msg(self, doc, encdata):
+    def _decrypt_doc(self, doc):
+        """
+        Decrypt the contents of a document.
+
+        :param doc: A document containing an encrypted message.
+        :type doc: SoledadDocument
+
+        :return: A tuple containing the document and the decrypted message.
+        :rtype: (SoledadDocument, str)
+        """
         log.msg('decrypting msg')
-        key = self._pkey
+        success = False
         try:
-            decrdata = (self._keymanager.decrypt(
-                encdata, key,
-                passphrase=self._soledad.passphrase))
-            ok = True
+            decrdata = self._keymanager.decrypt(
+                doc.content[ENC_JSON_KEY],
+                self._pkey)
+            success = True
         except Exception as exc:
             # XXX move this to errback !!!
             logger.warning("Error while decrypting msg: %r" % (exc,))
             decrdata = ""
-            ok = False
-        leap_events.signal(IMAP_MSG_DECRYPTED, "1" if ok else "0")
+        leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0")
         return doc, decrdata
 
-    def _process_decrypted(self, msgtuple):
+    def _process_decrypted_doc(self, msgtuple):
         """
-        Process a successfully decrypted message.
+        Process a document containing a succesfully decrypted message.
 
         :param msgtuple: a tuple consisting of a SoledadDocument
                          instance containing the incoming message
                          and data, the json-encoded, decrypted content of the
                          incoming message
         :type msgtuple: (SoledadDocument, str)
-        :returns: a SoledadDocument and the processed data.
+        :return: a SoledadDocument and the processed data.
         :rtype: (doc, data)
         """
         doc, data = msgtuple
@@ -333,13 +353,13 @@ class LeapIncomingMail(object):
         if not rawmsg:
             return False
         try:
-            data = self._maybe_decrypt_gpg_msg(rawmsg)
+            data = self._maybe_decrypt_msg(rawmsg)
             return doc, data
         except keymanager_errors.EncryptionDecryptionFailed as exc:
             logger.error(exc)
             raise
 
-    def _maybe_decrypt_gpg_msg(self, data):
+    def _maybe_decrypt_msg(self, data):
         """
         Tries to decrypt a gpg message if data looks like one.
 
@@ -348,80 +368,168 @@ class LeapIncomingMail(object):
         :return: data, possibly descrypted.
         :rtype: str
         """
-        # TODO split this method
         leap_assert_type(data, unicode)
 
+        # parse the original message
         parser = Parser()
         encoding = get_email_charset(data)
         data = data.encode(encoding)
-        origmsg = parser.parsestr(data)
-
-        # handle multipart/encrypted messages
-        if origmsg.get_content_type() == 'multipart/encrypted':
-            # sanity check
-            payload = origmsg.get_payload()
-            if len(payload) != 2:
-                raise MalformedMessage(
-                    'Multipart/encrypted messages should have exactly 2 body '
-                    'parts (instead of %d).' % len(payload))
-            if payload[0].get_content_type() != 'application/pgp-encrypted':
-                raise MalformedMessage(
-                    "Multipart/encrypted messages' first body part should "
-                    "have content type equal to 'application/pgp-encrypted' "
-                    "(instead of %s)." % payload[0].get_content_type())
-            if payload[1].get_content_type() != 'application/octet-stream':
-                raise MalformedMessage(
-                    "Multipart/encrypted messages' second body part should "
-                    "have content type equal to 'octet-stream' (instead of "
-                    "%s)." % payload[1].get_content_type())
-
-            # parse message and get encrypted content
-            pgpencmsg = origmsg.get_payload()[1]
-            encdata = pgpencmsg.get_payload()
-
-            # decrypt and parse decrypted message
-            decrdata = self._keymanager.decrypt(
-                encdata, self._pkey,
-                passphrase=self._soledad.passphrase)
+        msg = parser.parsestr(data)
+
+        # try to obtain sender public key
+        senderPubkey = None
+        fromHeader = msg.get('from', None)
+        if fromHeader is not None:
+            _, senderAddress = parseaddr(fromHeader)
             try:
-                decrdata = decrdata.encode(encoding)
-            except (UnicodeEncodeError, UnicodeDecodeError) as e:
-                logger.error("Unicode error {0}".format(e))
-                decrdata = decrdata.encode(encoding, 'replace')
-
-            decrmsg = parser.parsestr(decrdata)
-            # remove original message's multipart/encrypted content-type
-            del(origmsg['content-type'])
-            # replace headers back in original message
-            for hkey, hval in decrmsg.items():
-                try:
-                    # this will raise KeyError if header is not present
-                    origmsg.replace_header(hkey, hval)
-                except KeyError:
-                    origmsg[hkey] = hval
-
-            # replace payload by unencrypted payload
-            origmsg.set_payload(decrmsg.get_payload())
-            return origmsg.as_string(unixfrom=False)
+                senderPubkey = self._keymanager.get_key(
+                    senderAddress, OpenPGPKey)
+            except keymanager_errors.KeyNotFound:
+                pass
+
+        valid_sig = False  # we will add a header saying if sig is valid
+        decrdata = ''
+        if msg.get_content_type() == 'multipart/encrypted':
+            decrmsg, valid_sig = self._decrypt_multipart_encrypted_msg(
+                msg, encoding, senderPubkey)
+        else:
+            decrmsg, valid_sig = self._maybe_decrypt_inline_encrypted_msg(
+                msg, encoding, senderPubkey)
+
+        # add x-leap-signature header
+        if senderPubkey is None:
+            decrmsg.add_header(
+                self.LEAP_SIGNATURE_HEADER,
+                self.LEAP_SIGNATURE_COULD_NOT_VERIFY)
         else:
-            PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
-            PGP_END = "-----END PGP MESSAGE-----"
-            # handle inline PGP messages
-            if PGP_BEGIN in data:
-                begin = data.find(PGP_BEGIN)
-                end = data.rfind(PGP_END)
-                pgp_message = data[begin:begin+end]
-                decrdata = (self._keymanager.decrypt(
-                    pgp_message, self._pkey,
-                    passphrase=self._soledad.passphrase))
-                # replace encrypted by decrypted content
-                data = data.replace(pgp_message, decrdata)
+            decrmsg.add_header(
+                self.LEAP_SIGNATURE_HEADER,
+                self.LEAP_SIGNATURE_VALID if valid_sig else \
+                self.LEAP_SIGNATURE_INVALID,
+                pubkey=senderPubkey.key_id)
+
+        return decrmsg.as_string()
+
+    def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey):
+        """
+        Decrypt a message with content-type 'multipart/encrypted'.
+
+        :param msg: The original encrypted message.
+        :type msg: Message
+        :param encoding: The encoding of the email message.
+        :type encoding: str
+        :param senderPubkey: The key of the sender of the message.
+        :type senderPubkey: OpenPGPKey
+
+        :return: A unitary tuple containing a decrypted message.
+        :rtype: (Message)
+        """
+        msg = copy.deepcopy(msg)
+        # sanity check
+        payload = msg.get_payload()
+        if len(payload) != 2:
+            raise MalformedMessage(
+                'Multipart/encrypted messages should have exactly 2 body '
+                'parts (instead of %d).' % len(payload))
+        if payload[0].get_content_type() != 'application/pgp-encrypted':
+            raise MalformedMessage(
+                "Multipart/encrypted messages' first body part should "
+                "have content type equal to 'application/pgp-encrypted' "
+                "(instead of %s)." % payload[0].get_content_type())
+        if payload[1].get_content_type() != 'application/octet-stream':
+            raise MalformedMessage(
+                "Multipart/encrypted messages' second body part should "
+                "have content type equal to 'octet-stream' (instead of "
+                "%s)." % payload[1].get_content_type())
+        # parse message and get encrypted content
+        pgpencmsg = msg.get_payload()[1]
+        encdata = pgpencmsg.get_payload()
+        # decrypt and parse decrypted message
+        decrdata, valid_sig = self._decrypt_and_verify_data(
+            encdata, senderPubkey)
+        try:
+            decrdata = decrdata.encode(encoding)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            decrdata = decrdata.encode(encoding, 'replace')
+        parser = Parser()
+        decrmsg = parser.parsestr(decrdata)
+        # remove original message's multipart/encrypted content-type
+        del(msg['content-type'])
+        # replace headers back in original message
+        for hkey, hval in decrmsg.items():
+            try:
+                # this will raise KeyError if header is not present
+                msg.replace_header(hkey, hval)
+            except KeyError:
+                msg[hkey] = hval
+        # replace payload by unencrypted payload
+        msg.set_payload(decrmsg.get_payload())
+        return msg, valid_sig
+
+    def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
+                                                senderPubkey):
+        """
+        Possibly decrypt an inline OpenPGP encrypted message.
+
+        :param origmsg: The original, possibly encrypted message.
+        :type origmsg: Message
+        :param encoding: The encoding of the email message.
+        :type encoding: str
+        :param senderPubkey: The key of the sender of the message.
+        :type senderPubkey: OpenPGPKey
+
+        :return: A unitary tuple containing a decrypted message.
+        :rtype: (Message)
+        """
+        # serialize the original message
+        buf = StringIO()
+        g = Generator(buf)
+        g.flatten(origmsg)
+        data = buf.getvalue()
+        # handle exactly one inline PGP message
+        PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
+        PGP_END = "-----END PGP MESSAGE-----"
+        valid_sig = False
+        if PGP_BEGIN in data:
+            begin = data.find(PGP_BEGIN)
+            end = data.find(PGP_END)
+            pgp_message = data[begin:end+len(PGP_END)]
+            decrdata, valid_sig = self._decrypt_and_verify_data(
+                pgp_message, senderPubkey)
+            # replace encrypted by decrypted content
+            data = data.replace(pgp_message, decrdata)
         # if message is not encrypted, return raw data
-
         if isinstance(data, unicode):
             data = data.encode(encoding, 'replace')
+        parser = Parser()
+        return parser.parsestr(data), valid_sig
+
+    def _decrypt_and_verify_data(self, data, senderPubkey):
+        """
+        Decrypt C{data} using our private key and attempt to verify a
+        signature using C{senderPubkey}.
 
-        return data
+        :param data: The text to be decrypted.
+        :type data: unicode
+        :param senderPubkey: The public key of the sender of the message.
+        :type senderPubkey: OpenPGPKey
+
+        :return: The decrypted data and a boolean stating whether the
+                 signature could be verified.
+        :rtype: (str, bool)
+        """
+        valid_sig = False
+        try:
+            decrdata = self._keymanager.decrypt(
+                data, self._pkey,
+                verify=senderPubkey)
+            if senderPubkey is not None:
+                valid_sig = True
+        except keymanager_errors.InvalidSignature:
+            decrdata = self._keymanager.decrypt(
+                data, self._pkey)
+        return decrdata, valid_sig
 
     def _add_message_locally(self, msgtuple):
         """
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index ad11315..ca73a11 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -923,7 +923,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         self.server.theAccount.addMailbox('test-mailbox-e',
                                           creation_ts=42)
-        #import ipdb; ipdb.set_trace()
 
         self.examinedArgs = None
 
@@ -1108,16 +1107,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         mb = SimpleLEAPServer.theAccount.getMailbox('ROOT/SUBTHING')
         self.assertEqual(1, len(mb.messages))
 
-        #import ipdb; ipdb.set_trace()
         self.assertEqual(
             ['\\SEEN', '\\DELETED'],
-            mb.messages[1]['flags'])
+            mb.messages[1].content['flags'])
 
         self.assertEqual(
             'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
-            mb.messages[1]['date'])
+            mb.messages[1].content['date'])
 
-        self.assertEqual(open(infile).read(), mb.messages[1]['raw'])
+        self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
 
     def testPartialAppend(self):
         """
@@ -1152,11 +1150,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertEqual(1, len(mb.messages))
         self.assertEqual(
             ['\\SEEN',],
-            mb.messages[1]['flags']
+            mb.messages[1].content['flags']
         )
         self.assertEqual(
-            'Right now', mb.messages[1]['date'])
-        self.assertEqual(open(infile).read(), mb.messages[1]['raw'])
+            'Right now', mb.messages[1].content['date'])
+        self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
 
     def testCheck(self):
         """
@@ -1214,7 +1212,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def _cbTestClose(self, ignored, m):
         self.assertEqual(len(m.messages), 1)
         self.assertEqual(
-            m.messages[1]['subject'],
+            m.messages[1].content['subject'],
             'Message 2')
 
         self.failUnless(m.closed)
@@ -1257,7 +1255,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def _cbTestExpunge(self, ignored, m):
         self.assertEqual(len(m.messages), 1)
         self.assertEqual(
-            m.messages[1]['subject'],
+            m.messages[1].content['subject'],
             'Message 2')
         self.assertEqual(self.results, [0, 1])
         # XXX fix this thing with the indexes...