Make SMTP relay RFC 3156 compliant.

2 files changed, 478 insertions, 41 deletions

diff --git a/mail/src/leap/mail/smtp/rfc3156.py b/mail/src/leap/mail/smtp/rfc3156.py
new file mode 100644
index 0000000..dd48475
--- /dev/null
+++ b/mail/src/leap/mail/smtp/rfc3156.py
@@ -0,0 +1,379 @@
+# -*- coding: utf-8 -*-
+# rfc3156.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Implements RFC 3156: MIME Security with OpenPGP.
+"""
+
+import re
+import base64
+from abc import ABCMeta, abstractmethod
+from StringIO import StringIO
+
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email import errors
+from email.generator import (
+    Generator,
+    fcre,
+    NL,
+    _make_boundary,
+)
+
+
+#
+# A generator that solves http://bugs.python.org/issue14983
+#
+
+class RFC3156CompliantGenerator(Generator):
+    """
+    An email generator that addresses Python's issue #14983 for multipart
+    messages.
+
+    This is just a copy of email.generator.Generator which fixes the following
+    bug: http://bugs.python.org/issue14983
+    """
+
+    def _handle_multipart(self, msg):
+        """
+        A multipart handling implementation that addresses issue #14983.
+
+        This is just a copy of the parent's method which fixes the following
+        bug: http://bugs.python.org/issue14983 (see the line marked with
+        "(***)").
+
+        :param msg: The multipart message to be handled.
+        :type msg: email.message.Message
+        """
+        # The trick here is to write out each part separately, merge them all
+        # together, and then make sure that the boundary we've chosen isn't
+        # present in the payload.
+        msgtexts = []
+        subparts = msg.get_payload()
+        if subparts is None:
+            subparts = []
+        elif isinstance(subparts, basestring):
+            # e.g. a non-strict parse of a message with no starting boundary.
+            self._fp.write(subparts)
+            return
+        elif not isinstance(subparts, list):
+            # Scalar payload
+            subparts = [subparts]
+        for part in subparts:
+            s = StringIO()
+            g = self.clone(s)
+            g.flatten(part, unixfrom=False)
+            msgtexts.append(s.getvalue())
+        # BAW: What about boundaries that are wrapped in double-quotes?
+        boundary = msg.get_boundary()
+        if not boundary:
+            # Create a boundary that doesn't appear in any of the
+            # message texts.
+            alltext = NL.join(msgtexts)
+            boundary = _make_boundary(alltext)
+            msg.set_boundary(boundary)
+        # If there's a preamble, write it out, with a trailing CRLF
+        if msg.preamble is not None:
+            preamble = msg.preamble
+            if self._mangle_from_:
+                preamble = fcre.sub('>From ', msg.preamble)
+            self._fp.write(preamble + '\n')
+        # dash-boundary transport-padding CRLF
+        self._fp.write('--' + boundary + '\n')
+        # body-part
+        if msgtexts:
+            self._fp.write(msgtexts.pop(0))
+        # *encapsulation
+        # --> delimiter transport-padding
+        # --> CRLF body-part
+        for body_part in msgtexts:
+            # delimiter transport-padding CRLF
+            self._fp.write('\n--' + boundary + '\n')
+            # body-part
+            self._fp.write(body_part)
+        # close-delimiter transport-padding
+        self._fp.write('\n--' + boundary + '--' + '\n')  # (***) Solve #14983
+        if msg.epilogue is not None:
+            self._fp.write('\n')
+            epilogue = msg.epilogue
+            if self._mangle_from_:
+                epilogue = fcre.sub('>From ', msg.epilogue)
+            self._fp.write(epilogue)
+
+
+#
+# Base64 encoding: these are almost the same as python's email.encoder
+# solution, but a bit modified.
+#
+
+def _bencode(s):
+    """
+    Encode C{s} in base64.
+
+    :param s: The string to be encoded.
+    :type s: str
+    """
+    # We can't quite use base64.encodestring() since it tacks on a "courtesy
+    # newline".  Blech!
+    if not s:
+        return s
+    value = base64.encodestring(s)
+    return value[:-1]
+
+
+def encode_base64(msg):
+    """
+    Encode a non-multipart message's payload in Base64 (in place).
+
+    This method modifies the message contents in place and adds or replaces an
+    appropriate Content-Transfer-Encoding header.
+
+    :param msg: The non-multipart message to be encoded.
+    :type msg: email.message.Message
+    """
+    orig = msg.get_payload()
+    encdata = _bencode(orig)
+    msg.set_payload(encdata)
+    # replace or set the Content-Transfer-Encoding header.
+    try:
+        msg.replace_header('Content-Transfer-Encoding', 'base64')
+    except KeyError:
+        msg['Content-Transfer-Encoding'] = 'base64'
+
+
+def encode_base64_rec(msg):
+    """
+    Encode (possibly multipart) messages in base64 (in place).
+
+    This method modifies the message contents in place.
+
+    :param msg: The non-multipart message to be encoded.
+    :type msg: email.message.Message
+    """
+    if not msg.is_multipart():
+        encode_base64(msg)
+    else:
+        for sub in msg.get_payload():
+            encode_base64_rec(sub)
+
+
+#
+# RFC 1847: multipart/signed and multipart/encrypted
+#
+
+class MultipartSigned(MIMEMultipart):
+    """
+    Multipart/Signed MIME message according to RFC 1847.
+
+    2.1. Definition of Multipart/Signed
+
+      (1)  MIME type name: multipart
+      (2)  MIME subtype name: signed
+      (3)  Required parameters: boundary, protocol, and micalg
+      (4)  Optional parameters: none
+      (5)  Security considerations: Must be treated as opaque while in
+           transit
+
+    The multipart/signed content type contains exactly two body parts.
+    The first body part is the body part over which the digital signature
+    was created, including its MIME headers.  The second body part
+    contains the control information necessary to verify the digital
+    signature.  The first body part may contain any valid MIME content
+    type, labeled accordingly.  The second body part is labeled according
+    to the value of the protocol parameter.
+
+    When the OpenPGP digital signature is generated:
+
+    (1)   The data to be signed MUST first be converted to its content-
+          type specific canonical form.  For text/plain, this means
+          conversion to an appropriate character set and conversion of
+          line endings to the canonical <CR><LF> sequence.
+
+    (2)   An appropriate Content-Transfer-Encoding is then applied; see
+          section 3.  In particular, line endings in the encoded data
+          MUST use the canonical <CR><LF> sequence where appropriate
+          (note that the canonical line ending may or may not be present
+          on the last line of encoded data and MUST NOT be included in
+          the signature if absent).
+
+    (3)   MIME content headers are then added to the body, each ending
+          with the canonical <CR><LF> sequence.
+
+    (4)   As described in section 3 of this document, any trailing
+          whitespace MUST then be removed from the signed material.
+
+    (5)   As described in [2], the digital signature MUST be calculated
+          over both the data to be signed and its set of content headers.
+
+    (6)   The signature MUST be generated detached from the signed data
+          so that the process does not alter the signed data in any way.
+    """
+
+    def __init__(self, protocol, micalg, boundary=None, _subparts=None):
+        """
+        Initialize the multipart/signed message.
+
+        :param boundary: the multipart boundary string. By default it is
+            calculated as needed.
+        :type boundary: str
+        :param _subparts: a sequence of initial subparts for the payload. It
+            must be an iterable object, such as a list. You can always
+            attach new subparts to the message by using the attach() method.
+        :type _subparts: iterable
+        """
+        MIMEMultipart.__init__(
+            self, _subtype='signed', boundary=boundary,
+            _subparts=_subparts)
+        self.set_param('protocol', protocol)
+        self.set_param('micalg', micalg)
+
+    def attach(self, payload):
+        """
+        Add the C{payload} to the current payload list.
+
+        Also prevent from adding payloads with wrong Content-Type and from
+        exceeding a maximum of 2 payloads.
+
+        :param payload: The payload to be attached.
+        :type payload: email.message.Message
+        """
+        # second payload's content type must be equal to the protocol
+        # parameter given on object creation
+        if len(self.get_payload()) == 1:
+            if payload.get_content_type() != self.get_param('protocol'):
+                raise errors.MultipartConversionError(
+                    'Wrong content type %s.' % payload.get_content_type)
+        # prevent from adding more payloads
+        if len(self._payload) == 2:
+            raise errors.MultipartConversionError(
+                'Cannot have more than two subparts.')
+        MIMEMultipart.attach(self, payload)
+
+
+class MultipartEncrypted(MIMEMultipart):
+    """
+    Multipart/encrypted MIME message according to RFC 1847.
+
+    2.2. Definition of Multipart/Encrypted
+
+      (1)  MIME type name: multipart
+      (2)  MIME subtype name: encrypted
+      (3)  Required parameters: boundary, protocol
+      (4)  Optional parameters: none
+      (5)  Security considerations: none
+
+    The multipart/encrypted content type contains exactly two body parts.
+    The first body part contains the control information necessary to
+    decrypt the data in the second body part and is labeled according to
+    the value of the protocol parameter.  The second body part contains
+    the data which was encrypted and is always labeled
+    application/octet-stream.
+    """
+
+    def __init__(self, protocol, boundary=None, _subparts=None):
+        """
+        :param protocol: The encryption protocol to be added as a parameter to
+            the Content-Type header.
+        :type protocol: str
+        :param boundary: the multipart boundary string. By default it is
+            calculated as needed.
+        :type boundary: str
+        :param _subparts: a sequence of initial subparts for the payload. It
+            must be an iterable object, such as a list. You can always
+            attach new subparts to the message by using the attach() method.
+        :type _subparts: iterable
+        """
+        MIMEMultipart.__init__(
+            self, _subtype='encrypted', boundary=boundary,
+            _subparts=_subparts)
+        self.set_param('protocol', protocol)
+
+    def attach(self, payload):
+        """
+        Add the C{payload} to the current payload list.
+
+        Also prevent from adding payloads with wrong Content-Type and from
+        exceeding a maximum of 2 payloads.
+
+        :param payload: The payload to be attached.
+        :type payload: email.message.Message
+        """
+        # first payload's content type must be equal to the protocol parameter
+        # given on object creation
+        if len(self._payload) == 0:
+            if payload.get_content_type() != self.get_param('protocol'):
+                raise errors.MultipartConversionError(
+                    'Wrong content type.')
+        # second payload is always application/octet-stream
+        if len(self._payload) == 1:
+            if payload.get_content_type() != 'application/octet-stream':
+                raise errors.MultipartConversionError(
+                    'Wrong content type %s.' % payload.get_content_type)
+        # prevent from adding more payloads
+        if len(self._payload) == 2:
+            raise errors.MultipartConversionError(
+                'Cannot have more than two subparts.')
+        MIMEMultipart.attach(self, payload)
+
+
+#
+# RFC 3156: application/pgp-encrypted, application/pgp-signed and
+# application-pgp-signature.
+#
+
+class PGPEncrypted(MIMEApplication):
+    """
+    Application/pgp-encrypted MIME media type according to RFC 3156.
+
+      * MIME media type name: application
+      * MIME subtype name: pgp-encrypted
+      * Required parameters: none
+      * Optional parameters: none
+    """
+
+    def __init__(self, version=1):
+        data = "Version: %d" % version
+        MIMEApplication.__init__(self, data, 'pgp-encrypted')
+
+
+class PGPSignature(MIMEApplication):
+    """
+    Application/pgp-signature MIME media type according to RFC 3156.
+
+      * MIME media type name: application
+      * MIME subtype name: pgp-signature
+      * Required parameters: none
+      * Optional parameters: none
+    """
+    def __init__(self, _data, name='signature.asc'):
+        MIMEApplication.__init__(self, _data, 'pgp-signature',
+            _encoder=lambda x: x, name=name)
+        self.add_header('Content-Description', 'OpenPGP Digital Signature')
+
+
+class PGPKeys(MIMEApplication):
+    """
+    Application/pgp-keys MIME media type according to RFC 3156.
+
+      * MIME media type name: application
+      * MIME subtype name: pgp-keys
+      * Required parameters: none
+      * Optional parameters: none
+    """
+
+    def __init__(self, _data):
+        MIMEApplication.__init__(self, _data, 'pgp-keys')
diff --git a/mail/src/leap/mail/smtp/smtprelay.py b/mail/src/leap/mail/smtp/smtprelay.py
index 96eaa31..d9bbbf9 100644
--- a/mail/src/leap/mail/smtp/smtprelay.py
+++ b/mail/src/leap/mail/smtp/smtprelay.py
@@ -19,24 +19,38 @@
 LEAP SMTP encrypted relay.
 """
 
-from zope.interface import implements
+import re
 from StringIO import StringIO
+from email.Header import Header
+from email.utils import parseaddr
+from email.parser import Parser
+from email.mime.application import MIMEApplication
+
+from zope.interface import implements
 from OpenSSL import SSL
 from twisted.mail import smtp
 from twisted.internet.protocol import ServerFactory
 from twisted.internet import reactor, ssl
 from twisted.internet import defer
 from twisted.python import log
-from email.Header import Header
-from email.utils import parseaddr
-from email.parser import Parser
-
 
 from leap.common.check import leap_assert, leap_assert_type
 from leap.common.events import proto, signal
 from leap.keymanager import KeyManager
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.keymanager.errors import KeyNotFound
+from leap.mail.smtp.rfc3156 import (
+    MultipartSigned,
+    MultipartEncrypted,
+    PGPEncrypted,
+    PGPSignature,
+    RFC3156CompliantGenerator,
+    encode_base64_rec,
+)
+
+# replace email generator with a RFC 3156 compliant one.
+from email import generator
+generator.Generator = RFC3156CompliantGenerator
 
 
 #
@@ -260,7 +274,8 @@ class SMTPDelivery(object):
                 raise smtp.SMTPBadRcpt(user.dest.addrstr)
             log.msg("Warning: will send an unencrypted message (because "
                     "encrypted_only' is set to False).")
-            signal(proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr)
+            signal(
+                proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr)
         return lambda: EncryptedMessage(
             self._origin, user, self._km, self._config)
 
@@ -303,6 +318,16 @@ class CtxFactory(ssl.ClientContextFactory):
         return ctx
 
 
+def move_headers(origmsg, newmsg):
+    headers = origmsg.items()
+    unwanted_headers = ['content-type', 'mime-version', 'content-disposition',
+                        'content-transfer-encoding']
+    headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers)
+    for hkey, hval in headers:
+        newmsg.add_header(hkey, hval)
+        del(origmsg[hkey])
+
+
 class EncryptedMessage(object):
     """
     Receive plaintext from client, encrypt it and send message to a
@@ -343,6 +368,10 @@ class EncryptedMessage(object):
         # initialize list for message's lines
         self.lines = []
 
+    #
+    # methods from smtp.IMessage
+    #
+
     def lineReceived(self, line):
         """
         Handle another line.
@@ -360,9 +389,8 @@ class EncryptedMessage(object):
         """
         log.msg("Message data complete.")
         self.lines.append('')  # add a trailing newline
-        self.parseMessage()
         try:
-            self._encrypt_and_sign()
+            self._maybe_encrypt_and_sign()
             return self.sendMessage()
         except KeyNotFound:
             return None
@@ -372,7 +400,7 @@ class EncryptedMessage(object):
         Separate message headers from body.
         """
         parser = Parser()
-        self._message = parser.parsestr('\r\n'.join(self.lines))
+        return parser.parsestr('\r\n'.join(self.lines))
 
     def connectionLost(self):
         """
@@ -416,7 +444,7 @@ class EncryptedMessage(object):
             message send.
         @rtype: twisted.internet.defer.Deferred
         """
-        msg = self._message.as_string(False)
+        msg = self._msg.as_string(False)
 
         log.msg("Connecting to SMTP server %s:%s" % (self._config[HOST_KEY],
                                                      self._config[PORT_KEY]))
@@ -442,46 +470,76 @@ class EncryptedMessage(object):
         d.addErrback(self.sendError)
         return d
 
-    def _encrypt_and_sign_payload_rec(self, message, pubkey, signkey):
+    #
+    # encryption methods
+    #
+
+    def _encrypt_and_sign(self, pubkey, signkey):
         """
-        Recursivelly descend in C{message}'s payload encrypting to C{pubkey}
-        and signing with C{signkey}.
+        Create an RFC 3156 compliang PGP encrypted and signed message using
+        C{pubkey} to encrypt and C{signkey} to sign.
 
-        @param message: The message whose payload we want to encrypt.
-        @type message: email.message.Message
         @param pubkey: The public key used to encrypt the message.
         @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey
         @param signkey: The private key used to sign the message.
         @type signkey: leap.common.keymanager.openpgp.OpenPGPKey
         """
-        if message.is_multipart() is False:
-            message.set_payload(
-                self._km.encrypt(
-                    message.get_payload(), pubkey, sign=signkey))
-        else:
-            for msg in message.get_payload():
-                self._encrypt_and_sign_payload_rec(msg, pubkey, signkey)
+        # parse original message from received lines
+        origmsg = self.parseMessage()
+        # create new multipart/encrypted message with 'pgp-encrypted' protocol
+        newmsg = MultipartEncrypted('application/pgp-encrypted')
+        # move (almost) all headers from original message to the new message
+        move_headers(origmsg, newmsg)
+        # create 'application/octet-stream' encrypted message
+        encmsg = MIMEApplication(
+            self._km.encrypt(origmsg.as_string(unixfrom=False), pubkey,
+                             sign=signkey),
+            _subtype='octet-stream', _encoder=lambda x: x)
+        encmsg.add_header('content-disposition', 'attachment',
+                          filename='msg.asc')
+        # create meta message
+        metamsg = PGPEncrypted()
+        metamsg.add_header('Content-Disposition', 'attachment')
+        # attach pgp message parts to new message
+        newmsg.attach(metamsg)
+        newmsg.attach(encmsg)
+        self._msg = newmsg
+
+    def _sign(self, signkey):
+        """
+        Create an RFC 3156 compliant PGP signed MIME message using C{signkey}.
 
-    def _sign_payload_rec(self, message, signkey):
-        """
-        Recursivelly descend in C{message}'s payload signing with C{signkey}.
-
-        @param message: The message whose payload we want to encrypt.
-        @type message: email.message.Message
-        @param pubkey: The public key used to encrypt the message.
-        @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey
         @param signkey: The private key used to sign the message.
         @type signkey: leap.common.keymanager.openpgp.OpenPGPKey
         """
-        if message.is_multipart() is False:
-            message.set_payload(
-                self._km.sign(
-                    message.get_payload(), signkey))
-        else:
-            for msg in message.get_payload():
-                self._sign_payload_rec(msg, signkey)
-
-    def _encrypt_and_sign(self):
+        # parse original message from received lines
+        origmsg = self.parseMessage()
+        # create new multipart/signed message
+        newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512')
+        # move (almost) all headers from original message to the new message
+        move_headers(origmsg, newmsg)
+        # apply base64 content-transfer-encoding
+        encode_base64_rec(origmsg)
+        # get message text with headers and replace \n for \r\n
+        fp = StringIO()
+        g = RFC3156CompliantGenerator(
+            fp, mangle_from_=False, maxheaderlen=76)
+        g.flatten(origmsg)
+        msgtext = re.sub('\r?\n', '\r\n', fp.getvalue())
+        # make sure signed message ends with \r\n as per OpenPGP stantard.
+        if origmsg.is_multipart():
+            if not msgtext.endswith("\r\n"):
+                msgtext += "\r\n"
+        # calculate signature
+        signature = self._km.sign(msgtext, signkey, digest_algo='SHA512',
+                                  clearsign=False, detach=True, binary=False)
+        sigmsg = PGPSignature(signature)
+        # attach original message and signature to new message
+        newmsg.attach(origmsg)
+        newmsg.attach(sigmsg)
+        self._msg = newmsg
+
+    def _maybe_encrypt_and_sign(self):
         """
         Encrypt the message body.
 
@@ -504,7 +562,7 @@ class EncryptedMessage(object):
             log.msg("Will encrypt the message to %s." % pubkey.fingerprint)
             signal(proto.SMTP_START_ENCRYPT_AND_SIGN,
                    "%s,%s" % (self._fromAddress.addrstr, to_address))
-            self._encrypt_and_sign_payload_rec(self._message, pubkey, signkey)
+            self._encrypt_and_sign(pubkey, signkey)
             signal(proto.SMTP_END_ENCRYPT_AND_SIGN,
                    "%s,%s" % (self._fromAddress.addrstr, to_address))
         except KeyNotFound:
@@ -513,5 +571,5 @@ class EncryptedMessage(object):
             # rejected in SMTPDelivery.validateTo().
             log.msg('Will send unencrypted message to %s.' % to_address)
             signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr)
-            self._sign_payload_rec(self._message, signkey)
+            self._sign(signkey)
             signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr)