diff options
| -rw-r--r-- | mail/changes/feature_4029-comply-with-rfc3159 | 1 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/fetch.py | 72 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/rfc3156.py | 379 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/smtprelay.py | 140 |
4 files changed, 540 insertions, 52 deletions
diff --git a/mail/changes/feature_4029-comply-with-rfc3159 b/mail/changes/feature_4029-comply-with-rfc3159 new file mode 100644 index 00000000..c2c32afe --- /dev/null +++ b/mail/changes/feature_4029-comply-with-rfc3159 @@ -0,0 +1 @@ + o Comply with RFC 3156. Closes #4029. diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py index 4fb39100..2a430a0a 100644 --- a/mail/src/leap/mail/imap/fetch.py +++ b/mail/src/leap/mail/imap/fetch.py @@ -23,6 +23,8 @@ import ssl import threading import time +from email.parser import Parser + from twisted.python import log from twisted.internet.task import LoopingCall from twisted.internet.threads import deferToThread @@ -43,6 +45,13 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL logger = logging.getLogger(__name__) +class MalformedMessage(Exception): + """ + Raised when a given message is not well formed. + """ + pass + + class LeapIncomingMail(object): """ Fetches mail from the incoming queue. @@ -292,17 +301,58 @@ class LeapIncomingMail(object): :return: data, possibly descrypted. :rtype: str """ - PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" - PGP_END = "-----END PGP MESSAGE-----" - if PGP_BEGIN in data: - begin = data.find(PGP_BEGIN) - end = data.rfind(PGP_END) - pgp_message = data[begin:begin+end] - - decrdata = (self._keymanager.decrypt( - pgp_message, self._pkey, - passphrase=self._soledad.passphrase)) - data = data.replace(pgp_message, decrdata) + parser = Parser() + origmsg = parser.parsestr(data) + # handle multipart/encrypted messages + if origmsg.get_content_type() == 'multipart/encrypted': + # sanity check + payload = origmsg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + # parse message and get encrypted content + pgpencmsg = origmsg.get_payload()[1] + encdata = pgpencmsg.get_payload() + # decrypt and parse decrypted message + decrdata = self._keymanager.decrypt( + encdata, self._pkey, + passphrase=self._soledad.passphrase) + decrmsg = parser.parsestr(decrdata) + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + origmsg.replace_header(hkey, hval) + except KeyError: + origmsg[hkey] = hval + # replace payload by unencrypted payload + origmsg.set_payload(decrmsg.get_payload()) + return origmsg.as_string(unixfrom=False) + else: + PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" + PGP_END = "-----END PGP MESSAGE-----" + # handle inline PGP messages + if PGP_BEGIN in data: + begin = data.find(PGP_BEGIN) + end = data.rfind(PGP_END) + pgp_message = data[begin:begin+end] + decrdata = (self._keymanager.decrypt( + pgp_message, self._pkey, + passphrase=self._soledad.passphrase)) + # replace encrypted by decrypted content + data = data.replace(pgp_message, decrdata) + # if message is not encrypted, return raw data return data def _add_message_locally(self, msgtuple): diff --git a/mail/src/leap/mail/smtp/rfc3156.py b/mail/src/leap/mail/smtp/rfc3156.py new file mode 100644 index 00000000..dd484753 --- /dev/null +++ b/mail/src/leap/mail/smtp/rfc3156.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +# rfc3156.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Implements RFC 3156: MIME Security with OpenPGP. +""" + +import re +import base64 +from abc import ABCMeta, abstractmethod +from StringIO import StringIO + +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email import errors +from email.generator import ( + Generator, + fcre, + NL, + _make_boundary, +) + + +# +# A generator that solves http://bugs.python.org/issue14983 +# + +class RFC3156CompliantGenerator(Generator): + """ + An email generator that addresses Python's issue #14983 for multipart + messages. + + This is just a copy of email.generator.Generator which fixes the following + bug: http://bugs.python.org/issue14983 + """ + + def _handle_multipart(self, msg): + """ + A multipart handling implementation that addresses issue #14983. + + This is just a copy of the parent's method which fixes the following + bug: http://bugs.python.org/issue14983 (see the line marked with + "(***)"). + + :param msg: The multipart message to be handled. + :type msg: email.message.Message + """ + # The trick here is to write out each part separately, merge them all + # together, and then make sure that the boundary we've chosen isn't + # present in the payload. + msgtexts = [] + subparts = msg.get_payload() + if subparts is None: + subparts = [] + elif isinstance(subparts, basestring): + # e.g. a non-strict parse of a message with no starting boundary. + self._fp.write(subparts) + return + elif not isinstance(subparts, list): + # Scalar payload + subparts = [subparts] + for part in subparts: + s = StringIO() + g = self.clone(s) + g.flatten(part, unixfrom=False) + msgtexts.append(s.getvalue()) + # BAW: What about boundaries that are wrapped in double-quotes? + boundary = msg.get_boundary() + if not boundary: + # Create a boundary that doesn't appear in any of the + # message texts. + alltext = NL.join(msgtexts) + boundary = _make_boundary(alltext) + msg.set_boundary(boundary) + # If there's a preamble, write it out, with a trailing CRLF + if msg.preamble is not None: + preamble = msg.preamble + if self._mangle_from_: + preamble = fcre.sub('>From ', msg.preamble) + self._fp.write(preamble + '\n') + # dash-boundary transport-padding CRLF + self._fp.write('--' + boundary + '\n') + # body-part + if msgtexts: + self._fp.write(msgtexts.pop(0)) + # *encapsulation + # --> delimiter transport-padding + # --> CRLF body-part + for body_part in msgtexts: + # delimiter transport-padding CRLF + self._fp.write('\n--' + boundary + '\n') + # body-part + self._fp.write(body_part) + # close-delimiter transport-padding + self._fp.write('\n--' + boundary + '--' + '\n') # (***) Solve #14983 + if msg.epilogue is not None: + self._fp.write('\n') + epilogue = msg.epilogue + if self._mangle_from_: + epilogue = fcre.sub('>From ', msg.epilogue) + self._fp.write(epilogue) + + +# +# Base64 encoding: these are almost the same as python's email.encoder +# solution, but a bit modified. +# + +def _bencode(s): + """ + Encode C{s} in base64. + + :param s: The string to be encoded. + :type s: str + """ + # We can't quite use base64.encodestring() since it tacks on a "courtesy + # newline". Blech! + if not s: + return s + value = base64.encodestring(s) + return value[:-1] + + +def encode_base64(msg): + """ + Encode a non-multipart message's payload in Base64 (in place). + + This method modifies the message contents in place and adds or replaces an + appropriate Content-Transfer-Encoding header. + + :param msg: The non-multipart message to be encoded. + :type msg: email.message.Message + """ + orig = msg.get_payload() + encdata = _bencode(orig) + msg.set_payload(encdata) + # replace or set the Content-Transfer-Encoding header. + try: + msg.replace_header('Content-Transfer-Encoding', 'base64') + except KeyError: + msg['Content-Transfer-Encoding'] = 'base64' + + +def encode_base64_rec(msg): + """ + Encode (possibly multipart) messages in base64 (in place). + + This method modifies the message contents in place. + + :param msg: The non-multipart message to be encoded. + :type msg: email.message.Message + """ + if not msg.is_multipart(): + encode_base64(msg) + else: + for sub in msg.get_payload(): + encode_base64_rec(sub) + + +# +# RFC 1847: multipart/signed and multipart/encrypted +# + +class MultipartSigned(MIMEMultipart): + """ + Multipart/Signed MIME message according to RFC 1847. + + 2.1. Definition of Multipart/Signed + + (1) MIME type name: multipart + (2) MIME subtype name: signed + (3) Required parameters: boundary, protocol, and micalg + (4) Optional parameters: none + (5) Security considerations: Must be treated as opaque while in + transit + + The multipart/signed content type contains exactly two body parts. + The first body part is the body part over which the digital signature + was created, including its MIME headers. The second body part + contains the control information necessary to verify the digital + signature. The first body part may contain any valid MIME content + type, labeled accordingly. The second body part is labeled according + to the value of the protocol parameter. + + When the OpenPGP digital signature is generated: + + (1) The data to be signed MUST first be converted to its content- + type specific canonical form. For text/plain, this means + conversion to an appropriate character set and conversion of + line endings to the canonical <CR><LF> sequence. + + (2) An appropriate Content-Transfer-Encoding is then applied; see + section 3. In particular, line endings in the encoded data + MUST use the canonical <CR><LF> sequence where appropriate + (note that the canonical line ending may or may not be present + on the last line of encoded data and MUST NOT be included in + the signature if absent). + + (3) MIME content headers are then added to the body, each ending + with the canonical <CR><LF> sequence. + + (4) As described in section 3 of this document, any trailing + whitespace MUST then be removed from the signed material. + + (5) As described in [2], the digital signature MUST be calculated + over both the data to be signed and its set of content headers. + + (6) The signature MUST be generated detached from the signed data + so that the process does not alter the signed data in any way. + """ + + def __init__(self, protocol, micalg, boundary=None, _subparts=None): + """ + Initialize the multipart/signed message. + + :param boundary: the multipart boundary string. By default it is + calculated as needed. + :type boundary: str + :param _subparts: a sequence of initial subparts for the payload. It + must be an iterable object, such as a list. You can always + attach new subparts to the message by using the attach() method. + :type _subparts: iterable + """ + MIMEMultipart.__init__( + self, _subtype='signed', boundary=boundary, + _subparts=_subparts) + self.set_param('protocol', protocol) + self.set_param('micalg', micalg) + + def attach(self, payload): + """ + Add the C{payload} to the current payload list. + + Also prevent from adding payloads with wrong Content-Type and from + exceeding a maximum of 2 payloads. + + :param payload: The payload to be attached. + :type payload: email.message.Message + """ + # second payload's content type must be equal to the protocol + # parameter given on object creation + if len(self.get_payload()) == 1: + if payload.get_content_type() != self.get_param('protocol'): + raise errors.MultipartConversionError( + 'Wrong content type %s.' % payload.get_content_type) + # prevent from adding more payloads + if len(self._payload) == 2: + raise errors.MultipartConversionError( + 'Cannot have more than two subparts.') + MIMEMultipart.attach(self, payload) + + +class MultipartEncrypted(MIMEMultipart): + """ + Multipart/encrypted MIME message according to RFC 1847. + + 2.2. Definition of Multipart/Encrypted + + (1) MIME type name: multipart + (2) MIME subtype name: encrypted + (3) Required parameters: boundary, protocol + (4) Optional parameters: none + (5) Security considerations: none + + The multipart/encrypted content type contains exactly two body parts. + The first body part contains the control information necessary to + decrypt the data in the second body part and is labeled according to + the value of the protocol parameter. The second body part contains + the data which was encrypted and is always labeled + application/octet-stream. + """ + + def __init__(self, protocol, boundary=None, _subparts=None): + """ + :param protocol: The encryption protocol to be added as a parameter to + the Content-Type header. + :type protocol: str + :param boundary: the multipart boundary string. By default it is + calculated as needed. + :type boundary: str + :param _subparts: a sequence of initial subparts for the payload. It + must be an iterable object, such as a list. You can always + attach new subparts to the message by using the attach() method. + :type _subparts: iterable + """ + MIMEMultipart.__init__( + self, _subtype='encrypted', boundary=boundary, + _subparts=_subparts) + self.set_param('protocol', protocol) + + def attach(self, payload): + """ + Add the C{payload} to the current payload list. + + Also prevent from adding payloads with wrong Content-Type and from + exceeding a maximum of 2 payloads. + + :param payload: The payload to be attached. + :type payload: email.message.Message + """ + # first payload's content type must be equal to the protocol parameter + # given on object creation + if len(self._payload) == 0: + if payload.get_content_type() != self.get_param('protocol'): + raise errors.MultipartConversionError( + 'Wrong content type.') + # second payload is always application/octet-stream + if len(self._payload) == 1: + if payload.get_content_type() != 'application/octet-stream': + raise errors.MultipartConversionError( + 'Wrong content type %s.' % payload.get_content_type) + # prevent from adding more payloads + if len(self._payload) == 2: + raise errors.MultipartConversionError( + 'Cannot have more than two subparts.') + MIMEMultipart.attach(self, payload) + + +# +# RFC 3156: application/pgp-encrypted, application/pgp-signed and +# application-pgp-signature. +# + +class PGPEncrypted(MIMEApplication): + """ + Application/pgp-encrypted MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-encrypted + * Required parameters: none + * Optional parameters: none + """ + + def __init__(self, version=1): + data = "Version: %d" % version + MIMEApplication.__init__(self, data, 'pgp-encrypted') + + +class PGPSignature(MIMEApplication): + """ + Application/pgp-signature MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-signature + * Required parameters: none + * Optional parameters: none + """ + def __init__(self, _data, name='signature.asc'): + MIMEApplication.__init__(self, _data, 'pgp-signature', + _encoder=lambda x: x, name=name) + self.add_header('Content-Description', 'OpenPGP Digital Signature') + + +class PGPKeys(MIMEApplication): + """ + Application/pgp-keys MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-keys + * Required parameters: none + * Optional parameters: none + """ + + def __init__(self, _data): + MIMEApplication.__init__(self, _data, 'pgp-keys') diff --git a/mail/src/leap/mail/smtp/smtprelay.py b/mail/src/leap/mail/smtp/smtprelay.py index 96eaa319..d9bbbf9b 100644 --- a/mail/src/leap/mail/smtp/smtprelay.py +++ b/mail/src/leap/mail/smtp/smtprelay.py @@ -19,24 +19,38 @@ LEAP SMTP encrypted relay. """ -from zope.interface import implements +import re from StringIO import StringIO +from email.Header import Header +from email.utils import parseaddr +from email.parser import Parser +from email.mime.application import MIMEApplication + +from zope.interface import implements from OpenSSL import SSL from twisted.mail import smtp from twisted.internet.protocol import ServerFactory from twisted.internet import reactor, ssl from twisted.internet import defer from twisted.python import log -from email.Header import Header -from email.utils import parseaddr -from email.parser import Parser - from leap.common.check import leap_assert, leap_assert_type from leap.common.events import proto, signal from leap.keymanager import KeyManager from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound +from leap.mail.smtp.rfc3156 import ( + MultipartSigned, + MultipartEncrypted, + PGPEncrypted, + PGPSignature, + RFC3156CompliantGenerator, + encode_base64_rec, +) + +# replace email generator with a RFC 3156 compliant one. +from email import generator +generator.Generator = RFC3156CompliantGenerator # @@ -260,7 +274,8 @@ class SMTPDelivery(object): raise smtp.SMTPBadRcpt(user.dest.addrstr) log.msg("Warning: will send an unencrypted message (because " "encrypted_only' is set to False).") - signal(proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) + signal( + proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) return lambda: EncryptedMessage( self._origin, user, self._km, self._config) @@ -303,6 +318,16 @@ class CtxFactory(ssl.ClientContextFactory): return ctx +def move_headers(origmsg, newmsg): + headers = origmsg.items() + unwanted_headers = ['content-type', 'mime-version', 'content-disposition', + 'content-transfer-encoding'] + headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del(origmsg[hkey]) + + class EncryptedMessage(object): """ Receive plaintext from client, encrypt it and send message to a @@ -343,6 +368,10 @@ class EncryptedMessage(object): # initialize list for message's lines self.lines = [] + # + # methods from smtp.IMessage + # + def lineReceived(self, line): """ Handle another line. @@ -360,9 +389,8 @@ class EncryptedMessage(object): """ log.msg("Message data complete.") self.lines.append('') # add a trailing newline - self.parseMessage() try: - self._encrypt_and_sign() + self._maybe_encrypt_and_sign() return self.sendMessage() except KeyNotFound: return None @@ -372,7 +400,7 @@ class EncryptedMessage(object): Separate message headers from body. """ parser = Parser() - self._message = parser.parsestr('\r\n'.join(self.lines)) + return parser.parsestr('\r\n'.join(self.lines)) def connectionLost(self): """ @@ -416,7 +444,7 @@ class EncryptedMessage(object): message send. @rtype: twisted.internet.defer.Deferred """ - msg = self._message.as_string(False) + msg = self._msg.as_string(False) log.msg("Connecting to SMTP server %s:%s" % (self._config[HOST_KEY], self._config[PORT_KEY])) @@ -442,46 +470,76 @@ class EncryptedMessage(object): d.addErrback(self.sendError) return d - def _encrypt_and_sign_payload_rec(self, message, pubkey, signkey): + # + # encryption methods + # + + def _encrypt_and_sign(self, pubkey, signkey): """ - Recursivelly descend in C{message}'s payload encrypting to C{pubkey} - and signing with C{signkey}. + Create an RFC 3156 compliang PGP encrypted and signed message using + C{pubkey} to encrypt and C{signkey} to sign. - @param message: The message whose payload we want to encrypt. - @type message: email.message.Message @param pubkey: The public key used to encrypt the message. @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey @param signkey: The private key used to sign the message. @type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ - if message.is_multipart() is False: - message.set_payload( - self._km.encrypt( - message.get_payload(), pubkey, sign=signkey)) - else: - for msg in message.get_payload(): - self._encrypt_and_sign_payload_rec(msg, pubkey, signkey) + # parse original message from received lines + origmsg = self.parseMessage() + # create new multipart/encrypted message with 'pgp-encrypted' protocol + newmsg = MultipartEncrypted('application/pgp-encrypted') + # move (almost) all headers from original message to the new message + move_headers(origmsg, newmsg) + # create 'application/octet-stream' encrypted message + encmsg = MIMEApplication( + self._km.encrypt(origmsg.as_string(unixfrom=False), pubkey, + sign=signkey), + _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + self._msg = newmsg + + def _sign(self, signkey): + """ + Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. - def _sign_payload_rec(self, message, signkey): - """ - Recursivelly descend in C{message}'s payload signing with C{signkey}. - - @param message: The message whose payload we want to encrypt. - @type message: email.message.Message - @param pubkey: The public key used to encrypt the message. - @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey @param signkey: The private key used to sign the message. @type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ - if message.is_multipart() is False: - message.set_payload( - self._km.sign( - message.get_payload(), signkey)) - else: - for msg in message.get_payload(): - self._sign_payload_rec(msg, signkey) - - def _encrypt_and_sign(self): + # parse original message from received lines + origmsg = self.parseMessage() + # create new multipart/signed message + newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') + # move (almost) all headers from original message to the new message + move_headers(origmsg, newmsg) + # apply base64 content-transfer-encoding + encode_base64_rec(origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + # calculate signature + signature = self._km.sign(msgtext, signkey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + newmsg.attach(origmsg) + newmsg.attach(sigmsg) + self._msg = newmsg + + def _maybe_encrypt_and_sign(self): """ Encrypt the message body. @@ -504,7 +562,7 @@ class EncryptedMessage(object): log.msg("Will encrypt the message to %s." % pubkey.fingerprint) signal(proto.SMTP_START_ENCRYPT_AND_SIGN, "%s,%s" % (self._fromAddress.addrstr, to_address)) - self._encrypt_and_sign_payload_rec(self._message, pubkey, signkey) + self._encrypt_and_sign(pubkey, signkey) signal(proto.SMTP_END_ENCRYPT_AND_SIGN, "%s,%s" % (self._fromAddress.addrstr, to_address)) except KeyNotFound: @@ -513,5 +571,5 @@ class EncryptedMessage(object): # rejected in SMTPDelivery.validateTo(). log.msg('Will send unencrypted message to %s.' % to_address) signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr) - self._sign_payload_rec(self._message, signkey) + self._sign(signkey) signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr) |