From dd21400a035e6064fe18e786b64fac7419a3c060 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 5 Nov 2013 17:13:22 -0200 Subject: add README.rst to manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 7f6148e..83264d4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include pkg/* include versioneer.py include LICENSE include CHANGELOG +include README.rst -- cgit v1.2.3 From 8369307bf67821e847ccfe607470380f43e72feb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 6 Nov 2013 11:09:19 -0200 Subject: Make the pkey a property so we can allow multiple accounts in the imap fetcher. --- changes/bug_4394-update-pkey | 1 + pkg/requirements.pip | 1 + src/leap/mail/imap/fetch.py | 19 +++++++++++++++---- src/leap/mail/imap/service/imap.py | 7 +++++-- 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 changes/bug_4394-update-pkey diff --git a/changes/bug_4394-update-pkey b/changes/bug_4394-update-pkey new file mode 100644 index 0000000..d0a60b1 --- /dev/null +++ b/changes/bug_4394-update-pkey @@ -0,0 +1 @@ + o Update pkey to allow multiple accounts. Solves: #4394 diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 4780b5c..ae1a38b 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -2,3 +2,4 @@ leap.soledad.client>=0.3.0 leap.common>=0.3.5 leap.keymanager>=0.3.4 twisted # >= 12.0.3 ?? +zope.proxy diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index dd65def..4d47408 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -28,6 +28,7 @@ from email.parser import Parser from twisted.python import log from twisted.internet.task import LoopingCall from twisted.internet.threads import deferToThread +from zope.proxy import sameProxiedObjects from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type @@ -39,6 +40,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors +from leap.keymanager.openpgp import OpenPGPKey from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -66,7 +68,7 @@ class LeapIncomingMail(object): fetching_lock = threading.Lock() def __init__(self, keymanager, soledad, imap_account, - check_period): + check_period, userid): """ Initialize LeapIMAP. @@ -88,14 +90,14 @@ class LeapIncomingMail(object): leap_assert_type(soledad, Soledad) leap_assert(check_period, "need a period to check incoming mail") leap_assert_type(check_period, int) + leap_assert(userid, "need a userid to initialize") self._keymanager = keymanager self._soledad = soledad self.imapAccount = imap_account self._inbox = self.imapAccount.getMailbox('inbox') + self._userid = userid - self._pkey = self._keymanager.get_all_keys_in_local_db( - private=True).pop() self._loop = None self._check_period = check_period @@ -107,6 +109,13 @@ class LeapIncomingMail(object): """ self._soledad.create_index("just-mail", "incoming") + @property + def _pkey(self): + if sameProxiedObjects(self._keymanager, None): + logger.warning('tried to get key, but null keymanager found') + return None + return self._keymanager.get_key(self._userid, OpenPGPKey, private=True) + # # Public API: fetch, start_loop, stop. # @@ -118,6 +127,8 @@ class LeapIncomingMail(object): Calls a deferred that will execute the fetch callback in a separate thread """ + logger.debug("fetching mail for: %s %s" % ( + self._soledad.uuid, self._userid)) if not self.fetching_lock.locked(): d = deferToThread(self._sync_soledad) d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error) @@ -334,6 +345,7 @@ class LeapIncomingMail(object): :return: data, possibly descrypted. :rtype: str """ + # TODO split this method leap_assert_type(data, unicode) parser = Parser() @@ -417,7 +429,6 @@ class LeapIncomingMail(object): incoming message :type msgtuple: (SoledadDocument, str) """ - print "adding message locally....." doc, data = msgtuple self._inbox.addMessage(data, (self.RECENT_FLAG,)) leap_events.signal(IMAP_MSG_SAVED_LOCALLY) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 5f7322a..984ad04 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -29,7 +29,7 @@ from twisted.python import log logger = logging.getLogger(__name__) from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type +from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager from leap.mail.imap.server import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail @@ -152,6 +152,8 @@ def run_service(*args, **kwargs): port = kwargs.get('port', IMAP_PORT) check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) + userid = kwargs.get('userid', None) + leap_check(userid is not None, "need an user id") uuid = soledad._get_uuid() factory = LeapIMAPFactory(uuid, soledad) @@ -165,7 +167,8 @@ def run_service(*args, **kwargs): keymanager, soledad, factory.theAccount, - check_period) + check_period, + userid) except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) -- cgit v1.2.3 From bf5f250f7ffe5a0c07f162635ee9e4bf14240327 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 6 Nov 2013 14:40:55 -0200 Subject: add missing zope.interface dep --- pkg/requirements.pip | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 4780b5c..13d04f9 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -1,3 +1,4 @@ +zope.interface leap.soledad.client>=0.3.0 leap.common>=0.3.5 leap.keymanager>=0.3.4 -- cgit v1.2.3 From 08c13654268e986d03341c564d7cc91db80d940d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Wed, 6 Nov 2013 11:51:33 -0300 Subject: Reject senders if they aren't the logged in user --- changes/bug_reject_bad_sender | 2 ++ src/leap/mail/smtp/__init__.py | 6 ++++-- src/leap/mail/smtp/smtprelay.py | 17 ++++++++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 changes/bug_reject_bad_sender diff --git a/changes/bug_reject_bad_sender b/changes/bug_reject_bad_sender new file mode 100644 index 0000000..0e46c28 --- /dev/null +++ b/changes/bug_reject_bad_sender @@ -0,0 +1,2 @@ + o Reject senders that aren't the user that is currently logged + in. Fixes #3952. \ No newline at end of file diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index b30cd20..be568b8 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -29,7 +29,7 @@ from leap.common.events import proto, signal from leap.mail.smtp.smtprelay import SMTPFactory -def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, +def setup_smtp_relay(port, userid, keymanager, smtp_host, smtp_port, smtp_cert, smtp_key, encrypted_only): """ Setup SMTP relay to run with Twisted. @@ -39,6 +39,8 @@ def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, :param port: The port in which to run the server. :type port: int + :param userid: The user currently logged in + :type userid: unicode :param keymanager: A Key Manager from where to get recipients' public keys. :type keymanager: leap.common.keymanager.KeyManager @@ -75,7 +77,7 @@ def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, } # configure the use of this service with twistd - factory = SMTPFactory(keymanager, config) + factory = SMTPFactory(userid, keymanager, config) try: tport = reactor.listenTCP(port, factory, interface="localhost") diff --git a/src/leap/mail/smtp/smtprelay.py b/src/leap/mail/smtp/smtprelay.py index fca66c0..92a9f0e 100644 --- a/src/leap/mail/smtp/smtprelay.py +++ b/src/leap/mail/smtp/smtprelay.py @@ -153,7 +153,7 @@ class SMTPFactory(ServerFactory): Factory for an SMTP server with encrypted relaying capabilities. """ - def __init__(self, keymanager, config): + def __init__(self, userid, keymanager, config): """ Initialize the SMTP factory. @@ -169,11 +169,14 @@ class SMTPFactory(ServerFactory): ENCRYPTED_ONLY_KEY: , } @type config: dict + @param userid: The user currently logged in + @type userid: unicode """ # assert params leap_assert_type(keymanager, KeyManager) assert_config_structure(config) # and store them + self._userid = userid self._km = keymanager self._config = config @@ -187,7 +190,8 @@ class SMTPFactory(ServerFactory): @return: The protocol. @rtype: SMTPDelivery """ - smtpProtocol = smtp.SMTP(SMTPDelivery(self._km, self._config)) + smtpProtocol = smtp.SMTP(SMTPDelivery(self._userid, self._km, + self._config)) smtpProtocol.factory = self return smtpProtocol @@ -203,7 +207,7 @@ class SMTPDelivery(object): implements(smtp.IMessageDelivery) - def __init__(self, keymanager, config): + def __init__(self, userid, keymanager, config): """ Initialize the SMTP delivery object. @@ -219,11 +223,14 @@ class SMTPDelivery(object): ENCRYPTED_ONLY_KEY: , } @type config: dict + @param userid: The user currently logged in + @type userid: unicode """ # assert params leap_assert_type(keymanager, KeyManager) assert_config_structure(config) # and store them + self._userid = userid self._km = keymanager self._config = config self._origin = None @@ -310,6 +317,10 @@ class SMTPDelivery(object): """ # accept mail from anywhere. To reject an address, raise # smtp.SMTPBadSender here. + if str(origin) != str(self._userid): + log.msg("Rejecting sender {0}, expected {1}".format(origin, + self._userid)) + raise smtp.SMTPBadSender(origin) self._origin = origin return origin -- cgit v1.2.3 From 9b8265b2a5f4a19d844b4d0d474e079f3338f68a Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Nov 2013 14:52:20 -0200 Subject: Do not encrypt already encrypted mails in SMTP relay. --- ...re-4324_prevent-double-encryption-when-relaying | 2 + src/leap/mail/smtp/smtprelay.py | 60 ++++++++++++++-------- 2 files changed, 42 insertions(+), 20 deletions(-) create mode 100644 changes/feature-4324_prevent-double-encryption-when-relaying diff --git a/changes/feature-4324_prevent-double-encryption-when-relaying b/changes/feature-4324_prevent-double-encryption-when-relaying new file mode 100644 index 0000000..a3d70a9 --- /dev/null +++ b/changes/feature-4324_prevent-double-encryption-when-relaying @@ -0,0 +1,2 @@ + o Prevent already encrypted outgoing messages from being encrypted again. + Closes #4324. diff --git a/src/leap/mail/smtp/smtprelay.py b/src/leap/mail/smtp/smtprelay.py index 92a9f0e..14de849 100644 --- a/src/leap/mail/smtp/smtprelay.py +++ b/src/leap/mail/smtp/smtprelay.py @@ -284,7 +284,8 @@ class SMTPDelivery(object): # try to find recipient's public key try: address = validate_address(user.dest.addrstr) - pubkey = self._km.get_key(address, OpenPGPKey) + # verify if recipient key is available in keyring + self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound log.msg("Accepting mail for %s..." % user.dest.addrstr) signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) except KeyNotFound: @@ -510,15 +511,13 @@ class EncryptedMessage(object): @param signkey: The private key used to sign the message. @type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ - # parse original message from received lines - origmsg = self.parseMessage() # create new multipart/encrypted message with 'pgp-encrypted' protocol newmsg = MultipartEncrypted('application/pgp-encrypted') # move (almost) all headers from original message to the new message - move_headers(origmsg, newmsg) + move_headers(self._origmsg, newmsg) # create 'application/octet-stream' encrypted message encmsg = MIMEApplication( - self._km.encrypt(origmsg.as_string(unixfrom=False), pubkey, + self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey, sign=signkey), _subtype='octet-stream', _encoder=lambda x: x) encmsg.add_header('content-disposition', 'attachment', @@ -538,22 +537,20 @@ class EncryptedMessage(object): @param signkey: The private key used to sign the message. @type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ - # parse original message from received lines - origmsg = self.parseMessage() # create new multipart/signed message newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') # move (almost) all headers from original message to the new message - move_headers(origmsg, newmsg) + move_headers(self._origmsg, newmsg) # apply base64 content-transfer-encoding - encode_base64_rec(origmsg) + encode_base64_rec(self._origmsg) # get message text with headers and replace \n for \r\n fp = StringIO() g = RFC3156CompliantGenerator( fp, mangle_from_=False, maxheaderlen=76) - g.flatten(origmsg) + g.flatten(self._origmsg) msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) # make sure signed message ends with \r\n as per OpenPGP stantard. - if origmsg.is_multipart(): + if self._origmsg.is_multipart(): if not msgtext.endswith("\r\n"): msgtext += "\r\n" # calculate signature @@ -561,23 +558,46 @@ class EncryptedMessage(object): clearsign=False, detach=True, binary=False) sigmsg = PGPSignature(signature) # attach original message and signature to new message - newmsg.attach(origmsg) + newmsg.attach(self._origmsg) newmsg.attach(sigmsg) self._msg = newmsg def _maybe_encrypt_and_sign(self): """ - Encrypt the message body. + Attempt to encrypt and sign the outgoing message. - Fetch the recipient key and encrypt the content to the - recipient. If a key is not found, then the behaviour depends on the - configuration parameter ENCRYPTED_ONLY_KEY. If it is False, the message - is sent unencrypted and a warning is logged. If it is True, the - encryption fails with a KeyNotFound exception. + The behaviour of this method depends on: - @raise KeyNotFound: Raised when the recipient key was not found and - the ENCRYPTED_ONLY_KEY configuration parameter is set to True. + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the relay: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ """ + # pass if the original message's content-type is "multipart/encrypted" + self._origmsg = self.parseMessage() + if self._origmsg.get_content_type() == 'multipart/encrypted': + self._msg = self._origmsg + return + from_address = validate_address(self._fromAddress.addrstr) signkey = self._km.get_key(from_address, OpenPGPKey, private=True) log.msg("Will sign the message with %s." % signkey.fingerprint) -- cgit v1.2.3 From 4df74038a89dccff213c07795795154831330ace Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Nov 2013 16:44:59 -0200 Subject: Cleanup code and fix tests. --- src/leap/mail/imap/tests/test_imap.py | 4 +- src/leap/mail/smtp/__init__.py | 33 +--- src/leap/mail/smtp/smtprelay.py | 260 +++++++++++++---------------- src/leap/mail/smtp/tests/__init__.py | 4 +- src/leap/mail/smtp/tests/test_smtprelay.py | 52 ++++-- 5 files changed, 159 insertions(+), 194 deletions(-) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 3411795..ad11315 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -91,7 +91,7 @@ def initialize_soledad(email, gnupg_home, tempdir): """ uuid = "foobar-uuid" - passphrase = "verysecretpassphrase" + passphrase = u"verysecretpassphrase" secret_path = os.path.join(tempdir, "secret.gpg") local_db_path = os.path.join(tempdir, "soledad.u1db") server_url = "http://provider" @@ -101,6 +101,8 @@ def initialize_soledad(email, gnupg_home, tempdir): get_doc = Mock(return_value=None) put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) def __call__(self): return self diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index be568b8..753ef34 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -46,7 +46,7 @@ def setup_smtp_relay(port, userid, keymanager, smtp_host, smtp_port, :type keymanager: leap.common.keymanager.KeyManager :param smtp_host: The hostname of the remote SMTP server. :type smtp_host: str - :param smtp_port: The port of the remote SMTP server. + :param smtp_port: The port of the remote SMTP server. :type smtp_port: int :param smtp_cert: The client certificate for authentication. :type smtp_cert: str @@ -58,36 +58,17 @@ def setup_smtp_relay(port, userid, keymanager, smtp_host, smtp_port, :returns: tuple of SMTPFactory, twisted.internet.tcp.Port """ - # The configuration for the SMTP relay is a dict with the following - # format: - # - # { - # 'host': '', - # 'port': , - # 'cert': '', - # 'key': '', - # 'encrypted_only': - # } - config = { - 'host': smtp_host, - 'port': smtp_port, - 'cert': smtp_cert, - 'key': smtp_key, - 'encrypted_only': encrypted_only - } - # configure the use of this service with twistd - factory = SMTPFactory(userid, keymanager, config) + factory = SMTPFactory(userid, keymanager, smtp_host, smtp_port, smtp_cert, + smtp_key, encrypted_only) try: - tport = reactor.listenTCP(port, factory, - interface="localhost") - signal(proto.SMTP_SERVICE_STARTED, str(smtp_port)) + tport = reactor.listenTCP(port, factory, interface="localhost") + signal(proto.SMTP_SERVICE_STARTED, str(port)) return factory, tport except CannotListenError: logger.error("STMP Service failed to start: " - "cannot listen in port %s" % ( - smtp_port,)) - signal(proto.SMTP_SERVICE_FAILED_TO_START, str(smtp_port)) + "cannot listen in port %s" % port) + signal(proto.SMTP_SERVICE_FAILED_TO_START, str(port)) except Exception as exc: logger.error("Unhandled error while launching smtp relay service") logger.exception(exc) diff --git a/src/leap/mail/smtp/smtprelay.py b/src/leap/mail/smtp/smtprelay.py index 14de849..474fc3b 100644 --- a/src/leap/mail/smtp/smtprelay.py +++ b/src/leap/mail/smtp/smtprelay.py @@ -67,68 +67,16 @@ from email import generator generator.Generator = RFC3156CompliantGenerator -# -# Exceptions -# - -class MalformedConfig(Exception): - """ - Raised when the configuration dictionary passed as parameter is malformed. - """ - pass - - # # Helper utilities # -HOST_KEY = 'host' -PORT_KEY = 'port' -CERT_KEY = 'cert' -KEY_KEY = 'key' -ENCRYPTED_ONLY_KEY = 'encrypted_only' - - -def assert_config_structure(config): - """ - Assert that C{config} is a dict with the following structure: - - { - HOST_KEY: '', - PORT_KEY: , - CERT_KEY: '', - KEY_KEY: '', - ENCRYPTED_ONLY_KEY: , - } - - @param config: The dictionary to check. - @type config: dict - """ - # assert smtp config structure is valid - leap_assert_type(config, dict) - leap_assert(HOST_KEY in config) - leap_assert_type(config[HOST_KEY], str) - leap_assert(PORT_KEY in config) - leap_assert_type(config[PORT_KEY], int) - leap_assert(CERT_KEY in config) - leap_assert_type(config[CERT_KEY], (str, unicode)) - leap_assert(KEY_KEY in config) - leap_assert_type(config[KEY_KEY], (str, unicode)) - leap_assert(ENCRYPTED_ONLY_KEY in config) - leap_assert_type(config[ENCRYPTED_ONLY_KEY], bool) - # assert received params are not empty - leap_assert(config[HOST_KEY] != '') - leap_assert(config[PORT_KEY] is not 0) - leap_assert(config[CERT_KEY] != '') - leap_assert(config[KEY_KEY] != '') - - def validate_address(address): """ Validate C{address} as defined in RFC 2822. - @param address: The address to be validated. - @type address: str + :param address: The address to be validated. + :type address: str @return: A valid address. @rtype: str @@ -153,45 +101,60 @@ class SMTPFactory(ServerFactory): Factory for an SMTP server with encrypted relaying capabilities. """ - def __init__(self, userid, keymanager, config): + def __init__(self, userid, keymanager, host, port, cert, key, + encrypted_only): """ Initialize the SMTP factory. - @param keymanager: A KeyManager for retrieving recipient's keys. - @type keymanager: leap.common.keymanager.KeyManager - @param config: A dictionary with smtp configuration. Should have - the following structure: - { - HOST_KEY: '', - PORT_KEY: , - CERT_KEY: '', - KEY_KEY: '', - ENCRYPTED_ONLY_KEY: , - } - @type config: dict - @param userid: The user currently logged in - @type userid: unicode + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + :param encrypted_only: Whether the SMTP relay should send unencrypted mail + or not. + :type encrypted_only: bool """ # assert params leap_assert_type(keymanager, KeyManager) - assert_config_structure(config) + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, str) + leap_assert(cert != '') + leap_assert_type(key, str) + leap_assert(key != '') + leap_assert_type(encrypted_only, bool) # and store them self._userid = userid self._km = keymanager - self._config = config + self._host = host + self._port = port + self._cert = cert + self._key = key + self._encrypted_only = encrypted_only def buildProtocol(self, addr): """ Return a protocol suitable for the job. - @param addr: An address, e.g. a TCP (host, port). - @type addr: twisted.internet.interfaces.IAddress + :param addr: An address, e.g. a TCP (host, port). + :type addr: twisted.internet.interfaces.IAddress @return: The protocol. @rtype: SMTPDelivery """ - smtpProtocol = smtp.SMTP(SMTPDelivery(self._userid, self._km, - self._config)) + smtpProtocol = smtp.SMTP(SMTPDelivery( + self._userid, self._km, self._host, self._port, self._cert, + self._key, self._encrypted_only)) smtpProtocol.factory = self return smtpProtocol @@ -207,49 +170,53 @@ class SMTPDelivery(object): implements(smtp.IMessageDelivery) - def __init__(self, userid, keymanager, config): + def __init__(self, userid, keymanager, host, port, cert, key, + encrypted_only): """ Initialize the SMTP delivery object. - @param keymanager: A KeyManager for retrieving recipient's keys. - @type keymanager: leap.common.keymanager.KeyManager - @param config: A dictionary with smtp configuration. Should have - the following structure: - { - HOST_KEY: '', - PORT_KEY: , - CERT_KEY: '', - KEY_KEY: '', - ENCRYPTED_ONLY_KEY: , - } - @type config: dict - @param userid: The user currently logged in - @type userid: unicode + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + :param encrypted_only: Whether the SMTP relay should send unencrypted mail + or not. + :type encrypted_only: bool """ - # assert params - leap_assert_type(keymanager, KeyManager) - assert_config_structure(config) - # and store them self._userid = userid self._km = keymanager - self._config = config + self._userid = userid + self._km = keymanager + self._host = host + self._port = port + self._cert = cert + self._key = key + self._encrypted_only = encrypted_only self._origin = None def receivedHeader(self, helo, origin, recipients): """ Generate the 'Received:' header for a message. - @param helo: The argument to the HELO command and the client's IP + :param helo: The argument to the HELO command and the client's IP address. - @type helo: (str, str) - @param origin: The address the message is from. - @type origin: twisted.mail.smtp.Address - @param recipients: A list of the addresses for which this message is + :type helo: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + :param recipients: A list of the addresses for which this message is bound. - @type: list of twisted.mail.smtp.User + :type: list of twisted.mail.smtp.User @return: The full "Received" header string. - @type: str + :type: str """ myHostname, clientIP = helo headerValue = "by %s from %s with ESMTP ; %s" % ( @@ -269,8 +236,8 @@ class SMTPDelivery(object): In the end, it returns an encrypted message object that is able to send itself to the C{user}'s address. - @param user: The user whose address we wish to validate. - @type: twisted.mail.smtp.User + :param user: The user whose address we wish to validate. + :type: twisted.mail.smtp.User @return: A Deferred which becomes, or a callable which takes no arguments and returns an object implementing IMessage. This will @@ -290,7 +257,7 @@ class SMTPDelivery(object): signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) except KeyNotFound: # if key was not found, check config to see if will send anyway. - if self._config[ENCRYPTED_ONLY_KEY]: + if self._encrypted_only: signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) raise smtp.SMTPBadRcpt(user.dest.addrstr) log.msg("Warning: will send an unencrypted message (because " @@ -298,17 +265,18 @@ class SMTPDelivery(object): signal( proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) return lambda: EncryptedMessage( - self._origin, user, self._km, self._config) + self._origin, user, self._km, self._host, self._port, self._cert, + self._key) def validateFrom(self, helo, origin): """ Validate the address from which the message originates. - @param helo: The argument to the HELO command and the client's IP + :param helo: The argument to the HELO command and the client's IP address. - @type: (str, str) - @param origin: The address the message is from. - @type origin: twisted.mail.smtp.Address + :type: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address @return: origin or a Deferred whose callback will be passed origin. @rtype: Deferred or Address @@ -360,36 +328,36 @@ class EncryptedMessage(object): """ implements(smtp.IMessage) - def __init__(self, fromAddress, user, keymanager, config): + def __init__(self, fromAddress, user, keymanager, host, port, cert, key): """ Initialize the encrypted message. - @param fromAddress: The address of the sender. - @type fromAddress: twisted.mail.smtp.Address - @param user: The recipient of this message. - @type user: twisted.mail.smtp.User - @param keymanager: A KeyManager for retrieving recipient's keys. - @type keymanager: leap.common.keymanager.KeyManager - @param config: A dictionary with smtp configuration. Should have - the following structure: - { - HOST_KEY: '', - PORT_KEY: , - CERT_KEY: '', - KEY_KEY: '', - ENCRYPTED_ONLY_KEY: , - } - @type config: dict + :param fromAddress: The address of the sender. + :type fromAddress: twisted.mail.smtp.Address + :param user: The recipient of this message. + :type user: twisted.mail.smtp.User + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str """ # assert params leap_assert_type(user, smtp.User) leap_assert_type(keymanager, KeyManager) - assert_config_structure(config) # and store them self._fromAddress = fromAddress self._user = user self._km = keymanager - self._config = config + self._host = host + self._port = port + self._cert = cert + self._key = key # initialize list for message's lines self.lines = [] @@ -401,8 +369,8 @@ class EncryptedMessage(object): """ Handle another line. - @param line: The received line. - @type line: str + :param line: The received line. + :type line: str """ self.lines.append(line) @@ -441,8 +409,8 @@ class EncryptedMessage(object): """ Callback for a successful send. - @param r: The result from the last previous callback in the chain. - @type r: anything + :param r: The result from the last previous callback in the chain. + :type r: anything """ log.msg(r) signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr) @@ -451,8 +419,8 @@ class EncryptedMessage(object): """ Callback for an unsuccessfull send. - @param e: The result from the last errback. - @type e: anything + :param e: The result from the last errback. + :type e: anything """ log.msg(e) log.err() @@ -471,8 +439,7 @@ class EncryptedMessage(object): """ msg = self._msg.as_string(False) - log.msg("Connecting to SMTP server %s:%s" % (self._config[HOST_KEY], - self._config[PORT_KEY])) + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) d = defer.Deferred() # we don't pass an ssl context factory to the ESMTPSenderFactory @@ -488,11 +455,8 @@ class EncryptedMessage(object): requireTransportSecurity=True) signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) reactor.connectSSL( - self._config[HOST_KEY], - self._config[PORT_KEY], - factory, - contextFactory=SSLContextFactory(self._config[CERT_KEY], - self._config[KEY_KEY])) + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) d.addCallback(self.sendSuccess) d.addErrback(self.sendError) return d @@ -506,10 +470,10 @@ class EncryptedMessage(object): Create an RFC 3156 compliang PGP encrypted and signed message using C{pubkey} to encrypt and C{signkey} to sign. - @param pubkey: The public key used to encrypt the message. - @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey - @param signkey: The private key used to sign the message. - @type signkey: leap.common.keymanager.openpgp.OpenPGPKey + :param pubkey: The public key used to encrypt the message. + :type pubkey: leap.common.keymanager.openpgp.OpenPGPKey + :param signkey: The private key used to sign the message. + :type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ # create new multipart/encrypted message with 'pgp-encrypted' protocol newmsg = MultipartEncrypted('application/pgp-encrypted') @@ -534,8 +498,8 @@ class EncryptedMessage(object): """ Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. - @param signkey: The private key used to sign the message. - @type signkey: leap.common.keymanager.openpgp.OpenPGPKey + :param signkey: The private key used to sign the message. + :type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ # create new multipart/signed message newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') diff --git a/src/leap/mail/smtp/tests/__init__.py b/src/leap/mail/smtp/tests/__init__.py index 9b54de3..ee6de9b 100644 --- a/src/leap/mail/smtp/tests/__init__.py +++ b/src/leap/mail/smtp/tests/__init__.py @@ -59,7 +59,7 @@ class TestCaseWithKeyManager(BaseLeapTest): # setup our own stuff address = 'leap@leap.se' # user's address in the form user@provider uuid = 'leap@leap.se' - passphrase = '123' + passphrase = u'123' secrets_path = os.path.join(self.tempdir, 'secret.gpg') local_db_path = os.path.join(self.tempdir, 'soledad.u1db') server_url = 'http://provider/' @@ -88,6 +88,8 @@ class TestCaseWithKeyManager(BaseLeapTest): get_doc = Mock(return_value=None) put_doc = Mock(side_effect=_put_doc_side_effect) + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) def __call__(self): return self diff --git a/src/leap/mail/smtp/tests/test_smtprelay.py b/src/leap/mail/smtp/tests/test_smtprelay.py index 7fefe77..25c780e 100644 --- a/src/leap/mail/smtp/tests/test_smtprelay.py +++ b/src/leap/mail/smtp/tests/test_smtprelay.py @@ -102,8 +102,10 @@ class TestSmtpRelay(TestCaseWithKeyManager): '250 Sender address accepted', '250 Recipient address accepted', '354 Continue'] - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) for i, line in enumerate(self.EMAIL_DATA): @@ -117,11 +119,15 @@ class TestSmtpRelay(TestCaseWithKeyManager): """ Test if message gets encrypted to destination email. """ - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) fromAddr = Address(ADDRESS_2) dest = User(ADDRESS, 'relay.leap.se', proto, ADDRESS) - m = EncryptedMessage(fromAddr, dest, self._km, self._config) + m = EncryptedMessage( + fromAddr, dest, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) m.eomReceived() @@ -149,11 +155,15 @@ class TestSmtpRelay(TestCaseWithKeyManager): Test if message gets encrypted to destination email and signed with sender key. """ - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) user = User(ADDRESS, 'relay.leap.se', proto, ADDRESS) fromAddr = Address(ADDRESS_2) - m = EncryptedMessage(fromAddr, user, self._km, self._config) + m = EncryptedMessage( + fromAddr, user, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) # trigger encryption and signing @@ -185,11 +195,15 @@ class TestSmtpRelay(TestCaseWithKeyManager): """ # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) user = User('ihavenopubkey@nonleap.se', 'relay.leap.se', proto, ADDRESS) fromAddr = Address(ADDRESS_2) - m = EncryptedMessage(fromAddr, user, self._km, self._config) + m = EncryptedMessage( + fromAddr, user, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) # trigger signing @@ -237,8 +251,10 @@ class TestSmtpRelay(TestCaseWithKeyManager): # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) # prepare the SMTP factory - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') @@ -263,11 +279,11 @@ class TestSmtpRelay(TestCaseWithKeyManager): pgp.delete_key(pubkey) # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) - # change the configuration - self._config['encrypted_only'] = False - # prepare the SMTP factory - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + # prepare the SMTP factory with encrypted only equal to false + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + False).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') -- cgit v1.2.3 From 275c435ee45463ed6011f8988d5c7f90cb42fd01 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Nov 2013 17:09:56 -0200 Subject: Change SMTP "relay" to "gateway". --- changes/bug_4416-change-smtp-relay-to-gateway | 1 + src/leap/mail/smtp/README.rst | 38 +- src/leap/mail/smtp/__init__.py | 14 +- src/leap/mail/smtp/gateway.py | 585 ++++++++++++++++++++++++++ src/leap/mail/smtp/smtprelay.py | 585 -------------------------- src/leap/mail/smtp/tests/__init__.py | 2 +- src/leap/mail/smtp/tests/test_gateway.py | 297 +++++++++++++ src/leap/mail/smtp/tests/test_smtprelay.py | 297 ------------- 8 files changed, 900 insertions(+), 919 deletions(-) create mode 100644 changes/bug_4416-change-smtp-relay-to-gateway create mode 100644 src/leap/mail/smtp/gateway.py delete mode 100644 src/leap/mail/smtp/smtprelay.py create mode 100644 src/leap/mail/smtp/tests/test_gateway.py delete mode 100644 src/leap/mail/smtp/tests/test_smtprelay.py diff --git a/changes/bug_4416-change-smtp-relay-to-gateway b/changes/bug_4416-change-smtp-relay-to-gateway new file mode 100644 index 0000000..08bead7 --- /dev/null +++ b/changes/bug_4416-change-smtp-relay-to-gateway @@ -0,0 +1 @@ + o Change SMTP service name from "relay" to "gateway". Closes #4416. diff --git a/src/leap/mail/smtp/README.rst b/src/leap/mail/smtp/README.rst index 2b2a118..f625441 100644 --- a/src/leap/mail/smtp/README.rst +++ b/src/leap/mail/smtp/README.rst @@ -1,43 +1,23 @@ -Leap SMTP Relay -=============== +Leap SMTP Gateway +================= Outgoing mail workflow: * LEAP client runs a thin SMTP proxy on the user's device, bound to localhost. - * User's MUA is configured outgoing SMTP to localhost - * When SMTP proxy receives an email from MUA + * User's MUA is configured outgoing SMTP to localhost. + * When SMTP proxy receives an email from MUA: * SMTP proxy queries Key Manager for the user's private key and public - keys of all recipients + keys of all recipients. * Message is signed by sender and encrypted to recipients. * If recipient's key is missing, email goes out in cleartext (unless - user has configured option to send only encrypted email) - * Finally, message is relayed to provider's SMTP relay - - -Dependencies ------------- - -Leap SMTP Relay depends on the following python libraries: - - * Twisted 12.3.0 [1] - * zope.interface 4.0.3 [2] - -[1] http://pypi.python.org/pypi/Twisted/12.3.0 -[2] http://pypi.python.org/pypi/zope.interface/4.0.3 - - -How to run ----------- - -To launch the SMTP relay, run the following command: - - twistd -y smtprelay.tac + user has configured option to send only encrypted email). + * Finally, message is gatewayed to provider's SMTP server. Running tests ------------- -Tests are run using Twisted's Trial API, like this: +Tests are run using Twisted's Trial API, like this:: - trial leap.email.smtp.tests + python setup.py test -s leap.mail.gateway.tests diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 753ef34..d3eb9e8 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -16,7 +16,7 @@ # along with this program. If not, see . """ -SMTP relay helper function. +SMTP gateway helper function. """ import logging @@ -26,15 +26,15 @@ from twisted.internet.error import CannotListenError logger = logging.getLogger(__name__) from leap.common.events import proto, signal -from leap.mail.smtp.smtprelay import SMTPFactory +from leap.mail.smtp.gateway import SMTPFactory -def setup_smtp_relay(port, userid, keymanager, smtp_host, smtp_port, +def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, smtp_cert, smtp_key, encrypted_only): """ - Setup SMTP relay to run with Twisted. + Setup SMTP gateway to run with Twisted. - This function sets up the SMTP relay configuration and the Twisted + This function sets up the SMTP gateway configuration and the Twisted reactor. :param port: The port in which to run the server. @@ -52,7 +52,7 @@ def setup_smtp_relay(port, userid, keymanager, smtp_host, smtp_port, :type smtp_cert: str :param smtp_key: The client key for authentication. :type smtp_key: str - :param encrypted_only: Whether the SMTP relay should send unencrypted mail + :param encrypted_only: Whether the SMTP gateway should send unencrypted mail or not. :type encrypted_only: bool @@ -70,5 +70,5 @@ def setup_smtp_relay(port, userid, keymanager, smtp_host, smtp_port, "cannot listen in port %s" % port) signal(proto.SMTP_SERVICE_FAILED_TO_START, str(port)) except Exception as exc: - logger.error("Unhandled error while launching smtp relay service") + logger.error("Unhandled error while launching smtp gateway service") logger.exception(exc) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py new file mode 100644 index 0000000..06405b4 --- /dev/null +++ b/src/leap/mail/smtp/gateway.py @@ -0,0 +1,585 @@ +# -*- coding: utf-8 -*- +# gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +LEAP SMTP encrypted gateway. + +The following classes comprise the SMTP gateway service: + + * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides + the SMTPDelivery protocol. + * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It + knows how to validate sender and receiver of messages and it generates + an EncryptedMessage for each recipient. + * SSLContextFactory - Contains the relevant ssl information for the + connection. + * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that + knows how to encrypt/sign itself before sending. + + +""" + +import re +from StringIO import StringIO +from email.Header import Header +from email.utils import parseaddr +from email.parser import Parser +from email.mime.application import MIMEApplication + +from zope.interface import implements +from OpenSSL import SSL +from twisted.mail import smtp +from twisted.internet.protocol import ServerFactory +from twisted.internet import reactor, ssl +from twisted.internet import defer +from twisted.python import log + +from leap.common.check import leap_assert, leap_assert_type +from leap.common.events import proto, signal +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey +from leap.keymanager.errors import KeyNotFound +from leap.mail.smtp.rfc3156 import ( + MultipartSigned, + MultipartEncrypted, + PGPEncrypted, + PGPSignature, + RFC3156CompliantGenerator, + encode_base64_rec, +) + +# replace email generator with a RFC 3156 compliant one. +from email import generator +generator.Generator = RFC3156CompliantGenerator + + +# +# Helper utilities +# + +def validate_address(address): + """ + Validate C{address} as defined in RFC 2822. + + :param address: The address to be validated. + :type address: str + + @return: A valid address. + @rtype: str + + @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. + """ + leap_assert_type(address, str) + # in the following, the address is parsed as described in RFC 2822 and + # ('', '') is returned if the parse fails. + _, address = parseaddr(address) + if address == '': + raise smtp.SMTPBadRcpt(address) + return address + + +# +# SMTPFactory +# + +class SMTPFactory(ServerFactory): + """ + Factory for an SMTP server with encrypted gatewaying capabilities. + """ + + def __init__(self, userid, keymanager, host, port, cert, key, + encrypted_only): + """ + Initialize the SMTP factory. + + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. + :type encrypted_only: bool + """ + # assert params + leap_assert_type(keymanager, KeyManager) + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + leap_assert_type(encrypted_only, bool) + # and store them + self._userid = userid + self._km = keymanager + self._host = host + self._port = port + self._cert = cert + self._key = key + self._encrypted_only = encrypted_only + + def buildProtocol(self, addr): + """ + Return a protocol suitable for the job. + + :param addr: An address, e.g. a TCP (host, port). + :type addr: twisted.internet.interfaces.IAddress + + @return: The protocol. + @rtype: SMTPDelivery + """ + smtpProtocol = smtp.SMTP(SMTPDelivery( + self._userid, self._km, self._host, self._port, self._cert, + self._key, self._encrypted_only)) + smtpProtocol.factory = self + return smtpProtocol + + +# +# SMTPDelivery +# + +class SMTPDelivery(object): + """ + Validate email addresses and handle message delivery. + """ + + implements(smtp.IMessageDelivery) + + def __init__(self, userid, keymanager, host, port, cert, key, + encrypted_only): + """ + Initialize the SMTP delivery object. + + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. + :type encrypted_only: bool + """ + self._userid = userid + self._km = keymanager + self._userid = userid + self._km = keymanager + self._host = host + self._port = port + self._cert = cert + self._key = key + self._encrypted_only = encrypted_only + self._origin = None + + def receivedHeader(self, helo, origin, recipients): + """ + Generate the 'Received:' header for a message. + + :param helo: The argument to the HELO command and the client's IP + address. + :type helo: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + :param recipients: A list of the addresses for which this message is + bound. + :type: list of twisted.mail.smtp.User + + @return: The full "Received" header string. + :type: str + """ + myHostname, clientIP = helo + headerValue = "by %s from %s with ESMTP ; %s" % ( + myHostname, clientIP, smtp.rfc822date()) + # email.Header.Header used for automatic wrapping of long lines + return "Received: %s" % Header(headerValue) + + def validateTo(self, user): + """ + Validate the address of C{user}, a recipient of the message. + + This method is called once for each recipient and validates the + C{user}'s address against the RFC 2822 definition. If the + configuration option ENCRYPTED_ONLY_KEY is True, it also asserts the + existence of the user's key. + + In the end, it returns an encrypted message object that is able to + send itself to the C{user}'s address. + + :param user: The user whose address we wish to validate. + :type: twisted.mail.smtp.User + + @return: A Deferred which becomes, or a callable which takes no + arguments and returns an object implementing IMessage. This will + be called and the returned object used to deliver the message when + it arrives. + @rtype: no-argument callable + + @raise SMTPBadRcpt: Raised if messages to the address are not to be + accepted. + """ + # try to find recipient's public key + try: + address = validate_address(user.dest.addrstr) + # verify if recipient key is available in keyring + self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound + log.msg("Accepting mail for %s..." % user.dest.addrstr) + signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) + except KeyNotFound: + # if key was not found, check config to see if will send anyway. + if self._encrypted_only: + signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + raise smtp.SMTPBadRcpt(user.dest.addrstr) + log.msg("Warning: will send an unencrypted message (because " + "encrypted_only' is set to False).") + signal( + proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) + return lambda: EncryptedMessage( + self._origin, user, self._km, self._host, self._port, self._cert, + self._key) + + def validateFrom(self, helo, origin): + """ + Validate the address from which the message originates. + + :param helo: The argument to the HELO command and the client's IP + address. + :type: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + + @return: origin or a Deferred whose callback will be passed origin. + @rtype: Deferred or Address + + @raise twisted.mail.smtp.SMTPBadSender: Raised if messages from this + address are not to be accepted. + """ + # accept mail from anywhere. To reject an address, raise + # smtp.SMTPBadSender here. + if str(origin) != str(self._userid): + log.msg("Rejecting sender {0}, expected {1}".format(origin, + self._userid)) + raise smtp.SMTPBadSender(origin) + self._origin = origin + return origin + + +# +# EncryptedMessage +# + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def getContext(self): + self.method = SSL.TLSv1_METHOD # SSLv23_METHOD + ctx = ssl.ClientContextFactory.getContext(self) + ctx.use_certificate_file(self.cert) + ctx.use_privatekey_file(self.key) + return ctx + + +def move_headers(origmsg, newmsg): + headers = origmsg.items() + unwanted_headers = ['content-type', 'mime-version', 'content-disposition', + 'content-transfer-encoding'] + headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del(origmsg[hkey]) + + +class EncryptedMessage(object): + """ + Receive plaintext from client, encrypt it and send message to a + recipient. + """ + implements(smtp.IMessage) + + def __init__(self, fromAddress, user, keymanager, host, port, cert, key): + """ + Initialize the encrypted message. + + :param fromAddress: The address of the sender. + :type fromAddress: twisted.mail.smtp.Address + :param user: The recipient of this message. + :type user: twisted.mail.smtp.User + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + """ + # assert params + leap_assert_type(user, smtp.User) + leap_assert_type(keymanager, KeyManager) + # and store them + self._fromAddress = fromAddress + self._user = user + self._km = keymanager + self._host = host + self._port = port + self._cert = cert + self._key = key + # initialize list for message's lines + self.lines = [] + + # + # methods from smtp.IMessage + # + + def lineReceived(self, line): + """ + Handle another line. + + :param line: The received line. + :type line: str + """ + self.lines.append(line) + + def eomReceived(self): + """ + Handle end of message. + + This method will encrypt and send the message. + """ + log.msg("Message data complete.") + self.lines.append('') # add a trailing newline + try: + self._maybe_encrypt_and_sign() + return self.sendMessage() + except KeyNotFound: + return None + + def parseMessage(self): + """ + Separate message headers from body. + """ + parser = Parser() + return parser.parsestr('\r\n'.join(self.lines)) + + def connectionLost(self): + """ + Log an error when the connection is lost. + """ + log.msg("Connection lost unexpectedly!") + log.err() + signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) + # unexpected loss of connection; don't save + self.lines = [] + + def sendSuccess(self, r): + """ + Callback for a successful send. + + :param r: The result from the last previous callback in the chain. + :type r: anything + """ + log.msg(r) + signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr) + + def sendError(self, e): + """ + Callback for an unsuccessfull send. + + :param e: The result from the last errback. + :type e: anything + """ + log.msg(e) + log.err() + signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + + def sendMessage(self): + """ + Send the message. + + This method will prepare the message (headers and possibly encrypted + body) and send it using the ESMTPSenderFactory. + + @return: A deferred with callbacks for error and success of this + message send. + @rtype: twisted.internet.defer.Deferred + """ + msg = self._msg.as_string(False) + + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + + d = defer.Deferred() + # we don't pass an ssl context factory to the ESMTPSenderFactory + # because ssl will be handled by reactor.connectSSL() below. + factory = smtp.ESMTPSenderFactory( + "", # username is blank because server does not use auth. + "", # password is blank because server does not use auth. + self._fromAddress.addrstr, + self._user.dest.addrstr, + StringIO(msg), + d, + requireAuthentication=False, + requireTransportSecurity=True) + signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) + reactor.connectSSL( + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) + d.addCallback(self.sendSuccess) + d.addErrback(self.sendError) + return d + + # + # encryption methods + # + + def _encrypt_and_sign(self, pubkey, signkey): + """ + Create an RFC 3156 compliang PGP encrypted and signed message using + C{pubkey} to encrypt and C{signkey} to sign. + + :param pubkey: The public key used to encrypt the message. + :type pubkey: leap.common.keymanager.openpgp.OpenPGPKey + :param signkey: The private key used to sign the message. + :type signkey: leap.common.keymanager.openpgp.OpenPGPKey + """ + # create new multipart/encrypted message with 'pgp-encrypted' protocol + newmsg = MultipartEncrypted('application/pgp-encrypted') + # move (almost) all headers from original message to the new message + move_headers(self._origmsg, newmsg) + # create 'application/octet-stream' encrypted message + encmsg = MIMEApplication( + self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey, + sign=signkey), + _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + self._msg = newmsg + + def _sign(self, signkey): + """ + Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. + + :param signkey: The private key used to sign the message. + :type signkey: leap.common.keymanager.openpgp.OpenPGPKey + """ + # create new multipart/signed message + newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') + # move (almost) all headers from original message to the new message + move_headers(self._origmsg, newmsg) + # apply base64 content-transfer-encoding + encode_base64_rec(self._origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(self._origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if self._origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + # calculate signature + signature = self._km.sign(msgtext, signkey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + newmsg.attach(self._origmsg) + newmsg.attach(sigmsg) + self._msg = newmsg + + def _maybe_encrypt_and_sign(self): + """ + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ + """ + # pass if the original message's content-type is "multipart/encrypted" + self._origmsg = self.parseMessage() + if self._origmsg.get_content_type() == 'multipart/encrypted': + self._msg = self._origmsg + return + + from_address = validate_address(self._fromAddress.addrstr) + signkey = self._km.get_key(from_address, OpenPGPKey, private=True) + log.msg("Will sign the message with %s." % signkey.fingerprint) + to_address = validate_address(self._user.dest.addrstr) + try: + # try to get the recipient pubkey + pubkey = self._km.get_key(to_address, OpenPGPKey) + log.msg("Will encrypt the message to %s." % pubkey.fingerprint) + signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + "%s,%s" % (self._fromAddress.addrstr, to_address)) + self._encrypt_and_sign(pubkey, signkey) + signal(proto.SMTP_END_ENCRYPT_AND_SIGN, + "%s,%s" % (self._fromAddress.addrstr, to_address)) + except KeyNotFound: + # at this point we _can_ send unencrypted mail, because if the + # configuration said the opposite the address would have been + # rejected in SMTPDelivery.validateTo(). + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr) + self._sign(signkey) + signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr) diff --git a/src/leap/mail/smtp/smtprelay.py b/src/leap/mail/smtp/smtprelay.py deleted file mode 100644 index 474fc3b..0000000 --- a/src/leap/mail/smtp/smtprelay.py +++ /dev/null @@ -1,585 +0,0 @@ -# -*- coding: utf-8 -*- -# smtprelay.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -LEAP SMTP encrypted relay. - -The following classes comprise the SMTP relay service: - - * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides - the SMTPDelivery protocol. - * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It - knows how to validate sender and receiver of messages and it generates - an EncryptedMessage for each recipient. - * SSLContextFactory - Contains the relevant ssl information for the - connection. - * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that - knows how to encrypt/sign itself before sending. - - -""" - -import re -from StringIO import StringIO -from email.Header import Header -from email.utils import parseaddr -from email.parser import Parser -from email.mime.application import MIMEApplication - -from zope.interface import implements -from OpenSSL import SSL -from twisted.mail import smtp -from twisted.internet.protocol import ServerFactory -from twisted.internet import reactor, ssl -from twisted.internet import defer -from twisted.python import log - -from leap.common.check import leap_assert, leap_assert_type -from leap.common.events import proto, signal -from leap.keymanager import KeyManager -from leap.keymanager.openpgp import OpenPGPKey -from leap.keymanager.errors import KeyNotFound -from leap.mail.smtp.rfc3156 import ( - MultipartSigned, - MultipartEncrypted, - PGPEncrypted, - PGPSignature, - RFC3156CompliantGenerator, - encode_base64_rec, -) - -# replace email generator with a RFC 3156 compliant one. -from email import generator -generator.Generator = RFC3156CompliantGenerator - - -# -# Helper utilities -# - -def validate_address(address): - """ - Validate C{address} as defined in RFC 2822. - - :param address: The address to be validated. - :type address: str - - @return: A valid address. - @rtype: str - - @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. - """ - leap_assert_type(address, str) - # in the following, the address is parsed as described in RFC 2822 and - # ('', '') is returned if the parse fails. - _, address = parseaddr(address) - if address == '': - raise smtp.SMTPBadRcpt(address) - return address - - -# -# SMTPFactory -# - -class SMTPFactory(ServerFactory): - """ - Factory for an SMTP server with encrypted relaying capabilities. - """ - - def __init__(self, userid, keymanager, host, port, cert, key, - encrypted_only): - """ - Initialize the SMTP factory. - - :param userid: The user currently logged in - :type userid: unicode - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - :param cert: The client certificate for authentication. - :type cert: str - :param key: The client key for authentication. - :type key: str - :param encrypted_only: Whether the SMTP relay should send unencrypted mail - or not. - :type encrypted_only: bool - """ - # assert params - leap_assert_type(keymanager, KeyManager) - leap_assert_type(host, str) - leap_assert(host != '') - leap_assert_type(port, int) - leap_assert(port is not 0) - leap_assert_type(cert, str) - leap_assert(cert != '') - leap_assert_type(key, str) - leap_assert(key != '') - leap_assert_type(encrypted_only, bool) - # and store them - self._userid = userid - self._km = keymanager - self._host = host - self._port = port - self._cert = cert - self._key = key - self._encrypted_only = encrypted_only - - def buildProtocol(self, addr): - """ - Return a protocol suitable for the job. - - :param addr: An address, e.g. a TCP (host, port). - :type addr: twisted.internet.interfaces.IAddress - - @return: The protocol. - @rtype: SMTPDelivery - """ - smtpProtocol = smtp.SMTP(SMTPDelivery( - self._userid, self._km, self._host, self._port, self._cert, - self._key, self._encrypted_only)) - smtpProtocol.factory = self - return smtpProtocol - - -# -# SMTPDelivery -# - -class SMTPDelivery(object): - """ - Validate email addresses and handle message delivery. - """ - - implements(smtp.IMessageDelivery) - - def __init__(self, userid, keymanager, host, port, cert, key, - encrypted_only): - """ - Initialize the SMTP delivery object. - - :param userid: The user currently logged in - :type userid: unicode - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - :param cert: The client certificate for authentication. - :type cert: str - :param key: The client key for authentication. - :type key: str - :param encrypted_only: Whether the SMTP relay should send unencrypted mail - or not. - :type encrypted_only: bool - """ - self._userid = userid - self._km = keymanager - self._userid = userid - self._km = keymanager - self._host = host - self._port = port - self._cert = cert - self._key = key - self._encrypted_only = encrypted_only - self._origin = None - - def receivedHeader(self, helo, origin, recipients): - """ - Generate the 'Received:' header for a message. - - :param helo: The argument to the HELO command and the client's IP - address. - :type helo: (str, str) - :param origin: The address the message is from. - :type origin: twisted.mail.smtp.Address - :param recipients: A list of the addresses for which this message is - bound. - :type: list of twisted.mail.smtp.User - - @return: The full "Received" header string. - :type: str - """ - myHostname, clientIP = helo - headerValue = "by %s from %s with ESMTP ; %s" % ( - myHostname, clientIP, smtp.rfc822date()) - # email.Header.Header used for automatic wrapping of long lines - return "Received: %s" % Header(headerValue) - - def validateTo(self, user): - """ - Validate the address of C{user}, a recipient of the message. - - This method is called once for each recipient and validates the - C{user}'s address against the RFC 2822 definition. If the - configuration option ENCRYPTED_ONLY_KEY is True, it also asserts the - existence of the user's key. - - In the end, it returns an encrypted message object that is able to - send itself to the C{user}'s address. - - :param user: The user whose address we wish to validate. - :type: twisted.mail.smtp.User - - @return: A Deferred which becomes, or a callable which takes no - arguments and returns an object implementing IMessage. This will - be called and the returned object used to deliver the message when - it arrives. - @rtype: no-argument callable - - @raise SMTPBadRcpt: Raised if messages to the address are not to be - accepted. - """ - # try to find recipient's public key - try: - address = validate_address(user.dest.addrstr) - # verify if recipient key is available in keyring - self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound - log.msg("Accepting mail for %s..." % user.dest.addrstr) - signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) - except KeyNotFound: - # if key was not found, check config to see if will send anyway. - if self._encrypted_only: - signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) - raise smtp.SMTPBadRcpt(user.dest.addrstr) - log.msg("Warning: will send an unencrypted message (because " - "encrypted_only' is set to False).") - signal( - proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) - return lambda: EncryptedMessage( - self._origin, user, self._km, self._host, self._port, self._cert, - self._key) - - def validateFrom(self, helo, origin): - """ - Validate the address from which the message originates. - - :param helo: The argument to the HELO command and the client's IP - address. - :type: (str, str) - :param origin: The address the message is from. - :type origin: twisted.mail.smtp.Address - - @return: origin or a Deferred whose callback will be passed origin. - @rtype: Deferred or Address - - @raise twisted.mail.smtp.SMTPBadSender: Raised if messages from this - address are not to be accepted. - """ - # accept mail from anywhere. To reject an address, raise - # smtp.SMTPBadSender here. - if str(origin) != str(self._userid): - log.msg("Rejecting sender {0}, expected {1}".format(origin, - self._userid)) - raise smtp.SMTPBadSender(origin) - self._origin = origin - return origin - - -# -# EncryptedMessage -# - -class SSLContextFactory(ssl.ClientContextFactory): - def __init__(self, cert, key): - self.cert = cert - self.key = key - - def getContext(self): - self.method = SSL.TLSv1_METHOD # SSLv23_METHOD - ctx = ssl.ClientContextFactory.getContext(self) - ctx.use_certificate_file(self.cert) - ctx.use_privatekey_file(self.key) - return ctx - - -def move_headers(origmsg, newmsg): - headers = origmsg.items() - unwanted_headers = ['content-type', 'mime-version', 'content-disposition', - 'content-transfer-encoding'] - headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del(origmsg[hkey]) - - -class EncryptedMessage(object): - """ - Receive plaintext from client, encrypt it and send message to a - recipient. - """ - implements(smtp.IMessage) - - def __init__(self, fromAddress, user, keymanager, host, port, cert, key): - """ - Initialize the encrypted message. - - :param fromAddress: The address of the sender. - :type fromAddress: twisted.mail.smtp.Address - :param user: The recipient of this message. - :type user: twisted.mail.smtp.User - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - :param cert: The client certificate for authentication. - :type cert: str - :param key: The client key for authentication. - :type key: str - """ - # assert params - leap_assert_type(user, smtp.User) - leap_assert_type(keymanager, KeyManager) - # and store them - self._fromAddress = fromAddress - self._user = user - self._km = keymanager - self._host = host - self._port = port - self._cert = cert - self._key = key - # initialize list for message's lines - self.lines = [] - - # - # methods from smtp.IMessage - # - - def lineReceived(self, line): - """ - Handle another line. - - :param line: The received line. - :type line: str - """ - self.lines.append(line) - - def eomReceived(self): - """ - Handle end of message. - - This method will encrypt and send the message. - """ - log.msg("Message data complete.") - self.lines.append('') # add a trailing newline - try: - self._maybe_encrypt_and_sign() - return self.sendMessage() - except KeyNotFound: - return None - - def parseMessage(self): - """ - Separate message headers from body. - """ - parser = Parser() - return parser.parsestr('\r\n'.join(self.lines)) - - def connectionLost(self): - """ - Log an error when the connection is lost. - """ - log.msg("Connection lost unexpectedly!") - log.err() - signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) - # unexpected loss of connection; don't save - self.lines = [] - - def sendSuccess(self, r): - """ - Callback for a successful send. - - :param r: The result from the last previous callback in the chain. - :type r: anything - """ - log.msg(r) - signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr) - - def sendError(self, e): - """ - Callback for an unsuccessfull send. - - :param e: The result from the last errback. - :type e: anything - """ - log.msg(e) - log.err() - signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) - - def sendMessage(self): - """ - Send the message. - - This method will prepare the message (headers and possibly encrypted - body) and send it using the ESMTPSenderFactory. - - @return: A deferred with callbacks for error and success of this - message send. - @rtype: twisted.internet.defer.Deferred - """ - msg = self._msg.as_string(False) - - log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) - - d = defer.Deferred() - # we don't pass an ssl context factory to the ESMTPSenderFactory - # because ssl will be handled by reactor.connectSSL() below. - factory = smtp.ESMTPSenderFactory( - "", # username is blank because server does not use auth. - "", # password is blank because server does not use auth. - self._fromAddress.addrstr, - self._user.dest.addrstr, - StringIO(msg), - d, - requireAuthentication=False, - requireTransportSecurity=True) - signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) - reactor.connectSSL( - self._host, self._port, factory, - contextFactory=SSLContextFactory(self._cert, self._key)) - d.addCallback(self.sendSuccess) - d.addErrback(self.sendError) - return d - - # - # encryption methods - # - - def _encrypt_and_sign(self, pubkey, signkey): - """ - Create an RFC 3156 compliang PGP encrypted and signed message using - C{pubkey} to encrypt and C{signkey} to sign. - - :param pubkey: The public key used to encrypt the message. - :type pubkey: leap.common.keymanager.openpgp.OpenPGPKey - :param signkey: The private key used to sign the message. - :type signkey: leap.common.keymanager.openpgp.OpenPGPKey - """ - # create new multipart/encrypted message with 'pgp-encrypted' protocol - newmsg = MultipartEncrypted('application/pgp-encrypted') - # move (almost) all headers from original message to the new message - move_headers(self._origmsg, newmsg) - # create 'application/octet-stream' encrypted message - encmsg = MIMEApplication( - self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey, - sign=signkey), - _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - self._msg = newmsg - - def _sign(self, signkey): - """ - Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. - - :param signkey: The private key used to sign the message. - :type signkey: leap.common.keymanager.openpgp.OpenPGPKey - """ - # create new multipart/signed message - newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') - # move (almost) all headers from original message to the new message - move_headers(self._origmsg, newmsg) - # apply base64 content-transfer-encoding - encode_base64_rec(self._origmsg) - # get message text with headers and replace \n for \r\n - fp = StringIO() - g = RFC3156CompliantGenerator( - fp, mangle_from_=False, maxheaderlen=76) - g.flatten(self._origmsg) - msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) - # make sure signed message ends with \r\n as per OpenPGP stantard. - if self._origmsg.is_multipart(): - if not msgtext.endswith("\r\n"): - msgtext += "\r\n" - # calculate signature - signature = self._km.sign(msgtext, signkey, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - newmsg.attach(self._origmsg) - newmsg.attach(sigmsg) - self._msg = newmsg - - def _maybe_encrypt_and_sign(self): - """ - Attempt to encrypt and sign the outgoing message. - - The behaviour of this method depends on: - - 1. the original message's content-type, and - 2. the availability of the recipient's public key. - - If the original message's content-type is "multipart/encrypted", then - the original message is not altered. For any other content-type, the - method attempts to fetch the recipient's public key. If the - recipient's public key is available, the message is encrypted and - signed; otherwise it is only signed. - - Note that, if the C{encrypted_only} configuration is set to True and - the recipient's public key is not available, then the recipient - address would have been rejected in SMTPDelivery.validateTo(). - - The following table summarizes the overall behaviour of the relay: - - +---------------------------------------------------+----------------+ - | content-type | rcpt pubkey | enforce encr. | action | - +---------------------+-------------+---------------+----------------+ - | multipart/encrypted | any | any | pass | - | other | available | any | encrypt + sign | - | other | unavailable | yes | reject | - | other | unavailable | no | sign | - +---------------------+-------------+---------------+----------------+ - """ - # pass if the original message's content-type is "multipart/encrypted" - self._origmsg = self.parseMessage() - if self._origmsg.get_content_type() == 'multipart/encrypted': - self._msg = self._origmsg - return - - from_address = validate_address(self._fromAddress.addrstr) - signkey = self._km.get_key(from_address, OpenPGPKey, private=True) - log.msg("Will sign the message with %s." % signkey.fingerprint) - to_address = validate_address(self._user.dest.addrstr) - try: - # try to get the recipient pubkey - pubkey = self._km.get_key(to_address, OpenPGPKey) - log.msg("Will encrypt the message to %s." % pubkey.fingerprint) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._fromAddress.addrstr, to_address)) - self._encrypt_and_sign(pubkey, signkey) - signal(proto.SMTP_END_ENCRYPT_AND_SIGN, - "%s,%s" % (self._fromAddress.addrstr, to_address)) - except KeyNotFound: - # at this point we _can_ send unencrypted mail, because if the - # configuration said the opposite the address would have been - # rejected in SMTPDelivery.validateTo(). - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr) - self._sign(signkey) - signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr) diff --git a/src/leap/mail/smtp/tests/__init__.py b/src/leap/mail/smtp/tests/__init__.py index ee6de9b..62b015f 100644 --- a/src/leap/mail/smtp/tests/__init__.py +++ b/src/leap/mail/smtp/tests/__init__.py @@ -17,7 +17,7 @@ """ -Base classes and keys for SMTP relay tests. +Base classes and keys for SMTP gateway tests. """ import os diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py new file mode 100644 index 0000000..f9ea027 --- /dev/null +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -0,0 +1,297 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +SMTP gateway tests. +""" + + +import re + +from datetime import datetime +from gnupg._util import _make_binary_stream +from twisted.test import proto_helpers +from twisted.mail.smtp import ( + User, + Address, + SMTPBadRcpt, +) +from mock import Mock + +from leap.mail.smtp.gateway import ( + SMTPFactory, + EncryptedMessage, +) +from leap.mail.smtp.tests import ( + TestCaseWithKeyManager, + ADDRESS, + ADDRESS_2, +) +from leap.keymanager import openpgp + +# some regexps +IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \ + "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" +HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ + "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" +IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' + + +class TestSmtpGateway(TestCaseWithKeyManager): + + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def assertMatch(self, string, pattern, msg=None): + if not re.match(pattern, string): + msg = self._formatMessage(msg, '"%s" does not match pattern "%s".' + % (string, pattern)) + raise self.failureException(msg) + + def test_openpgp_encrypt_decrypt(self): + "Test if openpgp can encrypt and decrypt." + text = "simple raw text" + pubkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=False) + encrypted = self._km.encrypt(text, pubkey) + self.assertNotEqual( + text, encrypted, "Ciphertext is equal to plaintext.") + privkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=True) + decrypted = self._km.decrypt(encrypted, privkey) + self.assertEqual(text, decrypted, + "Decrypted text differs from plaintext.") + + def test_gateway_accepts_valid_email(self): + """ + Test if SMTP server responds correctly for valid interaction. + """ + + SMTP_ANSWERS = ['220 ' + IP_OR_HOST_REGEX + + ' NO UCE NO UBE NO RELAY PROBES', + '250 ' + IP_OR_HOST_REGEX + ' Hello ' + + IP_OR_HOST_REGEX + ', nice to meet you', + '250 Sender address accepted', + '250 Recipient address accepted', + '354 Continue'] + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + for i, line in enumerate(self.EMAIL_DATA): + proto.lineReceived(line + '\r\n') + self.assertMatch(transport.value(), + '\r\n'.join(SMTP_ANSWERS[0:i + 1]), + 'Did not get expected answer from gateway.') + proto.setTimeout(None) + + def test_message_encrypt(self): + """ + Test if message gets encrypted to destination email. + """ + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + fromAddr = Address(ADDRESS_2) + dest = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) + m = EncryptedMessage( + fromAddr, dest, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) + for line in self.EMAIL_DATA[4:12]: + m.lineReceived(line) + m.eomReceived() + # assert structure of encrypted message + self.assertTrue('Content-Type' in m._msg) + self.assertEqual('multipart/encrypted', m._msg.get_content_type()) + self.assertEqual('application/pgp-encrypted', + m._msg.get_param('protocol')) + self.assertEqual(2, len(m._msg.get_payload())) + self.assertEqual('application/pgp-encrypted', + m._msg.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + m._msg.get_payload(1).get_content_type()) + privkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=True) + decrypted = self._km.decrypt( + m._msg.get_payload(1).get_payload(), privkey) + self.assertEqual( + '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n', + decrypted, + 'Decrypted text differs from plaintext.') + + def test_message_encrypt_sign(self): + """ + Test if message gets encrypted to destination email and signed with + sender key. + """ + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) + fromAddr = Address(ADDRESS_2) + m = EncryptedMessage( + fromAddr, user, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) + for line in self.EMAIL_DATA[4:12]: + m.lineReceived(line) + # trigger encryption and signing + m.eomReceived() + # assert structure of encrypted message + self.assertTrue('Content-Type' in m._msg) + self.assertEqual('multipart/encrypted', m._msg.get_content_type()) + self.assertEqual('application/pgp-encrypted', + m._msg.get_param('protocol')) + self.assertEqual(2, len(m._msg.get_payload())) + self.assertEqual('application/pgp-encrypted', + m._msg.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + m._msg.get_payload(1).get_content_type()) + # decrypt and verify + privkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=True) + pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) + decrypted = self._km.decrypt( + m._msg.get_payload(1).get_payload(), privkey, verify=pubkey) + self.assertEqual( + '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n', + decrypted, + 'Decrypted text differs from plaintext.') + + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self._km.fetch_keys_from_server = Mock(return_value=[]) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + user = User('ihavenopubkey@nonleap.se', 'gateway.leap.se', proto, ADDRESS) + fromAddr = Address(ADDRESS_2) + m = EncryptedMessage( + fromAddr, user, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) + for line in self.EMAIL_DATA[4:12]: + m.lineReceived(line) + # trigger signing + m.eomReceived() + # assert structure of signed message + self.assertTrue('Content-Type' in m._msg) + self.assertEqual('multipart/signed', m._msg.get_content_type()) + self.assertEqual('application/pgp-signature', + m._msg.get_param('protocol')) + self.assertEqual('pgp-sha512', m._msg.get_param('micalg')) + # assert content of message + self.assertEqual( + m._msg.get_payload(0).get_payload(decode=True), + '\r\n'.join(self.EMAIL_DATA[9:13])) + # assert content of signature + self.assertTrue( + m._msg.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + m._msg.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + # assert signature is valid + pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) + # replace EOL before verifying (according to rfc3156) + signed_text = re.sub('\r?\n', '\r\n', + m._msg.get_payload(0).as_string()) + self.assertTrue( + self._km.verify(signed_text, + pubkey, + detached_sig=m._msg.get_payload(1).get_payload()), + 'Signature could not be verified.') + + def test_missing_key_rejects_address(self): + """ + Test if server rejects to send unencrypted when 'encrypted_only' is + True. + """ + # remove key from key manager + pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.GPG_BINARY_PATH) + pgp.delete_key(pubkey) + # mock the key fetching + self._km.fetch_keys_from_server = Mock(return_value=[]) + # prepare the SMTP factory + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') + proto.lineReceived(self.EMAIL_DATA[1] + '\r\n') + proto.lineReceived(self.EMAIL_DATA[2] + '\r\n') + # ensure the address was rejected + lines = transport.value().rstrip().split('\n') + self.assertEqual( + '550 Cannot receive for specified address', + lines[-1], + 'Address should have been rejecetd with appropriate message.') + + def test_missing_key_accepts_address(self): + """ + Test if server accepts to send unencrypted when 'encrypted_only' is + False. + """ + # remove key from key manager + pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.GPG_BINARY_PATH) + pgp.delete_key(pubkey) + # mock the key fetching + self._km.fetch_keys_from_server = Mock(return_value=[]) + # prepare the SMTP factory with encrypted only equal to false + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + False).buildProtocol(('127.0.0.1', 0)) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') + proto.lineReceived(self.EMAIL_DATA[1] + '\r\n') + proto.lineReceived(self.EMAIL_DATA[2] + '\r\n') + # ensure the address was accepted + lines = transport.value().rstrip().split('\n') + self.assertEqual( + '250 Recipient address accepted', + lines[-1], + 'Address should have been accepted with appropriate message.') diff --git a/src/leap/mail/smtp/tests/test_smtprelay.py b/src/leap/mail/smtp/tests/test_smtprelay.py deleted file mode 100644 index 25c780e..0000000 --- a/src/leap/mail/smtp/tests/test_smtprelay.py +++ /dev/null @@ -1,297 +0,0 @@ -# -*- coding: utf-8 -*- -# test_smtprelay.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -SMTP relay tests. -""" - - -import re - -from datetime import datetime -from gnupg._util import _make_binary_stream -from twisted.test import proto_helpers -from twisted.mail.smtp import ( - User, - Address, - SMTPBadRcpt, -) -from mock import Mock - -from leap.mail.smtp.smtprelay import ( - SMTPFactory, - EncryptedMessage, -) -from leap.mail.smtp.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, -) -from leap.keymanager import openpgp - -# some regexps -IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \ - "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" -HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ - "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" -IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' - - -class TestSmtpRelay(TestCaseWithKeyManager): - - EMAIL_DATA = ['HELO relay.leap.se', - 'MAIL FROM: <%s>' % ADDRESS_2, - 'RCPT TO: <%s>' % ADDRESS, - 'DATA', - 'From: User <%s>' % ADDRESS_2, - 'To: Leap <%s>' % ADDRESS, - 'Date: ' + datetime.now().strftime('%c'), - 'Subject: test message', - '', - 'This is a secret message.', - 'Yours,', - 'A.', - '', - '.', - 'QUIT'] - - def assertMatch(self, string, pattern, msg=None): - if not re.match(pattern, string): - msg = self._formatMessage(msg, '"%s" does not match pattern "%s".' - % (string, pattern)) - raise self.failureException(msg) - - def test_openpgp_encrypt_decrypt(self): - "Test if openpgp can encrypt and decrypt." - text = "simple raw text" - pubkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=False) - encrypted = self._km.encrypt(text, pubkey) - self.assertNotEqual( - text, encrypted, "Ciphertext is equal to plaintext.") - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt(encrypted, privkey) - self.assertEqual(text, decrypted, - "Decrypted text differs from plaintext.") - - def test_relay_accepts_valid_email(self): - """ - Test if SMTP server responds correctly for valid interaction. - """ - - SMTP_ANSWERS = ['220 ' + IP_OR_HOST_REGEX + - ' NO UCE NO UBE NO RELAY PROBES', - '250 ' + IP_OR_HOST_REGEX + ' Hello ' + - IP_OR_HOST_REGEX + ', nice to meet you', - '250 Sender address accepted', - '250 Recipient address accepted', - '354 Continue'] - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - transport = proto_helpers.StringTransport() - proto.makeConnection(transport) - for i, line in enumerate(self.EMAIL_DATA): - proto.lineReceived(line + '\r\n') - self.assertMatch(transport.value(), - '\r\n'.join(SMTP_ANSWERS[0:i + 1]), - 'Did not get expected answer from relay.') - proto.setTimeout(None) - - def test_message_encrypt(self): - """ - Test if message gets encrypted to destination email. - """ - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - fromAddr = Address(ADDRESS_2) - dest = User(ADDRESS, 'relay.leap.se', proto, ADDRESS) - m = EncryptedMessage( - fromAddr, dest, self._km, self._config['host'], - self._config['port'], self._config['cert'], self._config['key']) - for line in self.EMAIL_DATA[4:12]: - m.lineReceived(line) - m.eomReceived() - # assert structure of encrypted message - self.assertTrue('Content-Type' in m._msg) - self.assertEqual('multipart/encrypted', m._msg.get_content_type()) - self.assertEqual('application/pgp-encrypted', - m._msg.get_param('protocol')) - self.assertEqual(2, len(m._msg.get_payload())) - self.assertEqual('application/pgp-encrypted', - m._msg.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - m._msg.get_payload(1).get_content_type()) - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt( - m._msg.get_payload(1).get_payload(), privkey) - self.assertEqual( - '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n', - decrypted, - 'Decrypted text differs from plaintext.') - - def test_message_encrypt_sign(self): - """ - Test if message gets encrypted to destination email and signed with - sender key. - """ - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - user = User(ADDRESS, 'relay.leap.se', proto, ADDRESS) - fromAddr = Address(ADDRESS_2) - m = EncryptedMessage( - fromAddr, user, self._km, self._config['host'], - self._config['port'], self._config['cert'], self._config['key']) - for line in self.EMAIL_DATA[4:12]: - m.lineReceived(line) - # trigger encryption and signing - m.eomReceived() - # assert structure of encrypted message - self.assertTrue('Content-Type' in m._msg) - self.assertEqual('multipart/encrypted', m._msg.get_content_type()) - self.assertEqual('application/pgp-encrypted', - m._msg.get_param('protocol')) - self.assertEqual(2, len(m._msg.get_payload())) - self.assertEqual('application/pgp-encrypted', - m._msg.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - m._msg.get_payload(1).get_content_type()) - # decrypt and verify - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - decrypted = self._km.decrypt( - m._msg.get_payload(1).get_payload(), privkey, verify=pubkey) - self.assertEqual( - '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n', - decrypted, - 'Decrypted text differs from plaintext.') - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - user = User('ihavenopubkey@nonleap.se', 'relay.leap.se', proto, ADDRESS) - fromAddr = Address(ADDRESS_2) - m = EncryptedMessage( - fromAddr, user, self._km, self._config['host'], - self._config['port'], self._config['cert'], self._config['key']) - for line in self.EMAIL_DATA[4:12]: - m.lineReceived(line) - # trigger signing - m.eomReceived() - # assert structure of signed message - self.assertTrue('Content-Type' in m._msg) - self.assertEqual('multipart/signed', m._msg.get_content_type()) - self.assertEqual('application/pgp-signature', - m._msg.get_param('protocol')) - self.assertEqual('pgp-sha512', m._msg.get_param('micalg')) - # assert content of message - self.assertEqual( - m._msg.get_payload(0).get_payload(decode=True), - '\r\n'.join(self.EMAIL_DATA[9:13])) - # assert content of signature - self.assertTrue( - m._msg.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - m._msg.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - # assert signature is valid - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - # replace EOL before verifying (according to rfc3156) - signed_text = re.sub('\r?\n', '\r\n', - m._msg.get_payload(0).as_string()) - self.assertTrue( - self._km.verify(signed_text, - pubkey, - detached_sig=m._msg.get_payload(1).get_payload()), - 'Signature could not be verified.') - - def test_missing_key_rejects_address(self): - """ - Test if server rejects to send unencrypted when 'encrypted_only' is - True. - """ - # remove key from key manager - pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.delete_key(pubkey) - # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) - # prepare the SMTP factory - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - transport = proto_helpers.StringTransport() - proto.makeConnection(transport) - proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[1] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[2] + '\r\n') - # ensure the address was rejected - lines = transport.value().rstrip().split('\n') - self.assertEqual( - '550 Cannot receive for specified address', - lines[-1], - 'Address should have been rejecetd with appropriate message.') - - def test_missing_key_accepts_address(self): - """ - Test if server accepts to send unencrypted when 'encrypted_only' is - False. - """ - # remove key from key manager - pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.delete_key(pubkey) - # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) - # prepare the SMTP factory with encrypted only equal to false - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], - self._config['cert'], self._config['key'], - False).buildProtocol(('127.0.0.1', 0)) - transport = proto_helpers.StringTransport() - proto.makeConnection(transport) - proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[1] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[2] + '\r\n') - # ensure the address was accepted - lines = transport.value().rstrip().split('\n') - self.assertEqual( - '250 Recipient address accepted', - lines[-1], - 'Address should have been accepted with appropriate message.') -- cgit v1.2.3 From a163ef77b0ac27228588cf4001dd0533e3f2f4cb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 11 Nov 2013 13:43:30 -0200 Subject: remove print --- src/leap/mail/imap/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 7a9f810..11f3ccf 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -1153,7 +1153,6 @@ class SoledadMailbox(WithMsgFields): # the server itself is a listener to the mailbox. # so we can notify it (and should!) after chanes in flags # and number of messages. - print "emptying the listeners" map(lambda i: self.listeners.remove(i), self.listeners) def addListener(self, listener): -- cgit v1.2.3 From 42f43f2eed79e6c0a9569a823d148ad1ea1ad3af Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 11 Nov 2013 13:44:54 -0200 Subject: add a fqdn as the local domain, always --- src/leap/mail/smtp/gateway.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 06405b4..6367c0d 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -71,6 +71,8 @@ generator.Generator = RFC3156CompliantGenerator # Helper utilities # +LOCAL_FQDN = "bitmask.local" + def validate_address(address): """ Validate C{address} as defined in RFC 2822. @@ -96,10 +98,18 @@ def validate_address(address): # SMTPFactory # +class SMTPHeloLocalhost(smtp.SMTP): + + def __init__(self, *args): + smtp.SMTP.__init__(self, *args) + self.host = LOCAL_FQDN + + class SMTPFactory(ServerFactory): """ Factory for an SMTP server with encrypted gatewaying capabilities. """ + domain = LOCAL_FQDN def __init__(self, userid, keymanager, host, port, cert, key, encrypted_only): @@ -152,7 +162,7 @@ class SMTPFactory(ServerFactory): @return: The protocol. @rtype: SMTPDelivery """ - smtpProtocol = smtp.SMTP(SMTPDelivery( + smtpProtocol = SMTPHeloLocalhost(SMTPDelivery( self._userid, self._km, self._host, self._port, self._cert, self._key, self._encrypted_only)) smtpProtocol.factory = self @@ -451,8 +461,10 @@ class EncryptedMessage(object): self._user.dest.addrstr, StringIO(msg), d, + heloFallback=True, requireAuthentication=False, requireTransportSecurity=True) + factory.domain = LOCAL_FQDN signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) reactor.connectSSL( self._host, self._port, factory, -- cgit v1.2.3 From 80f0e428065f84292f20954845500028a77a96e0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 11 Nov 2013 13:45:14 -0200 Subject: refactor callbacks so we properly catch remote errors --- src/leap/mail/smtp/gateway.py | 49 +++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 6367c0d..7e9e420 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -14,7 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - """ LEAP SMTP encrypted gateway. @@ -32,7 +31,6 @@ The following classes comprise the SMTP gateway service: """ - import re from StringIO import StringIO from email.Header import Header @@ -46,6 +44,7 @@ from twisted.mail import smtp from twisted.internet.protocol import ServerFactory from twisted.internet import reactor, ssl from twisted.internet import defer +from twisted.internet.threads import deferToThread from twisted.python import log from leap.common.check import leap_assert, leap_assert_type @@ -415,6 +414,15 @@ class EncryptedMessage(object): # unexpected loss of connection; don't save self.lines = [] + def sendQueued(self, r): + """ + Callback for the queued message. + + @param r: The result from the last previous callback in the chain. + @type r: anything + """ + log.msg(r) + def sendSuccess(self, r): """ Callback for a successful send. @@ -425,33 +433,51 @@ class EncryptedMessage(object): log.msg(r) signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr) - def sendError(self, e): + def sendError(self, failure): """ Callback for an unsuccessfull send. :param e: The result from the last errback. :type e: anything """ - log.msg(e) - log.err() signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + err = failure.value + log.err(err) + raise err def sendMessage(self): """ - Send the message. - - This method will prepare the message (headers and possibly encrypted - body) and send it using the ESMTPSenderFactory. - + Sends the message. @return: A deferred with callbacks for error and success of this message send. @rtype: twisted.internet.defer.Deferred """ + # FIXME this should not be blocking the main ui, since it returns + # a deferred and it has its own cb set. ???! + d = deferToThread(self._sendMessage) + d.addCallbacks(self._route_msg, self.sendError) + d.addCallbacks(self.sendQueued, self.sendError) + return d + + def _sendMessage(self): + """ + Send the message. + + This method will prepare the message (headers and possibly encrypted + body) + """ msg = self._msg.as_string(False) + return msg + def _route_msg(self, msg): + """ + Sends the msg using the ESMTPSenderFactory. + """ log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + # we construct a defer to pass to the ESMTPSenderFactory d = defer.Deferred() + d.addCallbacks(self.sendSuccess, self.sendError) # we don't pass an ssl context factory to the ESMTPSenderFactory # because ssl will be handled by reactor.connectSSL() below. factory = smtp.ESMTPSenderFactory( @@ -469,9 +495,6 @@ class EncryptedMessage(object): reactor.connectSSL( self._host, self._port, factory, contextFactory=SSLContextFactory(self._cert, self._key)) - d.addCallback(self.sendSuccess) - d.addErrback(self.sendError) - return d # # encryption methods -- cgit v1.2.3 From d15cd3f018001114c78b191085a84630b335c37e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 11 Nov 2013 13:46:09 -0200 Subject: add changes --- changes/bug_4441_fix-fqdn | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/bug_4441_fix-fqdn diff --git a/changes/bug_4441_fix-fqdn b/changes/bug_4441_fix-fqdn new file mode 100644 index 0000000..e758d65 --- /dev/null +++ b/changes/bug_4441_fix-fqdn @@ -0,0 +1 @@ + o Identify ourselves with a fqdn, always. Closes: #4441 -- cgit v1.2.3 From acd5c93f916be7c7509d49f3fa24f4ba0d1d52f7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 11 Nov 2013 18:01:13 -0200 Subject: use deferToThread in the sendMail. Closes: #3937 --- changes/bug_3937_fix_ui_freeze | 1 + src/leap/mail/smtp/gateway.py | 80 ++++++++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 34 deletions(-) create mode 100644 changes/bug_3937_fix_ui_freeze diff --git a/changes/bug_3937_fix_ui_freeze b/changes/bug_3937_fix_ui_freeze new file mode 100644 index 0000000..b91938c --- /dev/null +++ b/changes/bug_3937_fix_ui_freeze @@ -0,0 +1 @@ + o Uses deferToThread for sendMail. Closes #3937 diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 7e9e420..f6366af 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -72,6 +72,7 @@ generator.Generator = RFC3156CompliantGenerator LOCAL_FQDN = "bitmask.local" + def validate_address(address): """ Validate C{address} as defined in RFC 2822. @@ -98,6 +99,13 @@ def validate_address(address): # class SMTPHeloLocalhost(smtp.SMTP): + """ + An SMTP class that ensures a proper FQDN + for localhost. + + This avoids a problem in which unproperly configured providers + would complain about the helo not being a fqdn. + """ def __init__(self, *args): smtp.SMTP.__init__(self, *args) @@ -388,21 +396,14 @@ class EncryptedMessage(object): Handle end of message. This method will encrypt and send the message. + + :returns: a deferred """ log.msg("Message data complete.") self.lines.append('') # add a trailing newline - try: - self._maybe_encrypt_and_sign() - return self.sendMessage() - except KeyNotFound: - return None - - def parseMessage(self): - """ - Separate message headers from body. - """ - parser = Parser() - return parser.parsestr('\r\n'.join(self.lines)) + d = deferToThread(self._maybe_encrypt_and_sign) + d.addCallbacks(self.sendMessage, self.skipNoKeyErrBack) + return d def connectionLost(self): """ @@ -414,12 +415,34 @@ class EncryptedMessage(object): # unexpected loss of connection; don't save self.lines = [] + # ends IMessage implementation + + def skipNoKeyErrBack(self, failure): + """ + Errback that ignores a KeyNotFound + + :param failure: the failure + :type Failure: Failure + """ + err = failure.value + if failure.check(KeyNotFound): + pass + else: + raise err + + def parseMessage(self): + """ + Separate message headers from body. + """ + parser = Parser() + return parser.parsestr('\r\n'.join(self.lines)) + def sendQueued(self, r): """ Callback for the queued message. - @param r: The result from the last previous callback in the chain. - @type r: anything + :param r: The result from the last previous callback in the chain. + :type r: anything """ log.msg(r) @@ -445,35 +468,24 @@ class EncryptedMessage(object): log.err(err) raise err - def sendMessage(self): + def sendMessage(self, *args): """ Sends the message. - @return: A deferred with callbacks for error and success of this - message send. - @rtype: twisted.internet.defer.Deferred - """ - # FIXME this should not be blocking the main ui, since it returns - # a deferred and it has its own cb set. ???! - d = deferToThread(self._sendMessage) - d.addCallbacks(self._route_msg, self.sendError) - d.addCallbacks(self.sendQueued, self.sendError) - return d - def _sendMessage(self): + :return: A deferred with callbacks for error and success of this + #message send. + :rtype: twisted.internet.defer.Deferred """ - Send the message. - - This method will prepare the message (headers and possibly encrypted - body) - """ - msg = self._msg.as_string(False) - return msg + d = deferToThread(self._route_msg) + d.addCallbacks(self.sendQueued, self.sendError) + return - def _route_msg(self, msg): + def _route_msg(self): """ Sends the msg using the ESMTPSenderFactory. """ log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + msg = self._msg.as_string(False) # we construct a defer to pass to the ESMTPSenderFactory d = defer.Deferred() -- cgit v1.2.3 From 31ab19e74018999e64e8a0a73d3968504dd7cf12 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 11 Nov 2013 18:18:45 -0200 Subject: Correcly handle message headers when gatewaying. Closes #4322 and #4447. --- ..._4447-4322-fix-headers-when-gatewaying-messages | 2 + src/leap/mail/smtp/gateway.py | 83 +++++++++++++++++----- 2 files changed, 66 insertions(+), 19 deletions(-) create mode 100644 changes/feature_4447-4322-fix-headers-when-gatewaying-messages diff --git a/changes/feature_4447-4322-fix-headers-when-gatewaying-messages b/changes/feature_4447-4322-fix-headers-when-gatewaying-messages new file mode 100644 index 0000000..986937c --- /dev/null +++ b/changes/feature_4447-4322-fix-headers-when-gatewaying-messages @@ -0,0 +1,2 @@ + o Correctly handle email headers when gatewaying messages. Also add + OpenPGP header. Closes #4322 and #4447. diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index f6366af..f09ee14 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -210,8 +210,6 @@ class SMTPDelivery(object): """ self._userid = userid self._km = keymanager - self._userid = userid - self._km = keymanager self._host = host self._port = port self._cert = cert @@ -236,10 +234,10 @@ class SMTPDelivery(object): :type: str """ myHostname, clientIP = helo - headerValue = "by %s from %s with ESMTP ; %s" % ( - myHostname, clientIP, smtp.rfc822date()) + headerValue = "by bitmask.local from %s with ESMTP ; %s" % ( + clientIP, smtp.rfc822date()) # email.Header.Header used for automatic wrapping of long lines - return "Received: %s" % Header(headerValue) + return "Received: %s" % Header(s=headerValue, header_name='Received') def validateTo(self, user): """ @@ -328,16 +326,6 @@ class SSLContextFactory(ssl.ClientContextFactory): return ctx -def move_headers(origmsg, newmsg): - headers = origmsg.items() - unwanted_headers = ['content-type', 'mime-version', 'content-disposition', - 'content-transfer-encoding'] - headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del(origmsg[hkey]) - - class EncryptedMessage(object): """ Receive plaintext from client, encrypt it and send message to a @@ -518,14 +506,14 @@ class EncryptedMessage(object): C{pubkey} to encrypt and C{signkey} to sign. :param pubkey: The public key used to encrypt the message. - :type pubkey: leap.common.keymanager.openpgp.OpenPGPKey + :type pubkey: OpenPGPKey :param signkey: The private key used to sign the message. - :type signkey: leap.common.keymanager.openpgp.OpenPGPKey + :type signkey: OpenPGPKey """ # create new multipart/encrypted message with 'pgp-encrypted' protocol newmsg = MultipartEncrypted('application/pgp-encrypted') # move (almost) all headers from original message to the new message - move_headers(self._origmsg, newmsg) + self._fix_headers(self._origmsg, newmsg, signkey) # create 'application/octet-stream' encrypted message encmsg = MIMEApplication( self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey, @@ -551,7 +539,7 @@ class EncryptedMessage(object): # create new multipart/signed message newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') # move (almost) all headers from original message to the new message - move_headers(self._origmsg, newmsg) + self._fix_headers(self._origmsg, newmsg, signkey) # apply base64 content-transfer-encoding encode_base64_rec(self._origmsg) # get message text with headers and replace \n for \r\n @@ -630,3 +618,60 @@ class EncryptedMessage(object): signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr) self._sign(signkey) signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr) + + def _fix_headers(self, origmsg, newmsg, signkey): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param origmsg: The original message. + :type origmsg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param signkey: The key used to sign C{newmsg} + :type signkey: OpenPGPKey + """ + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del(origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # add openpgp header to newmsg + username, domain = signkey.address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/openpgp/%s' % (domain, username)) + # delete user-agent from origmsg + del(origmsg['user-agent']) -- cgit v1.2.3 From a63fb0de4f9660dac955c6669860ce35c72fbf9e Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 12 Nov 2013 11:49:43 -0200 Subject: Remove 'multipart/encrypted' header after decrypting incoming mail. Closes #4454. --- changes/bug_4454_remove-multipart-encrypted-header-after-decrypting | 2 ++ src/leap/mail/imap/fetch.py | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 changes/bug_4454_remove-multipart-encrypted-header-after-decrypting diff --git a/changes/bug_4454_remove-multipart-encrypted-header-after-decrypting b/changes/bug_4454_remove-multipart-encrypted-header-after-decrypting new file mode 100644 index 0000000..8aa0aaa --- /dev/null +++ b/changes/bug_4454_remove-multipart-encrypted-header-after-decrypting @@ -0,0 +1,2 @@ + o Remove 'multipart/encrypted' header after decrypting incoming mail. Closes + #4454. diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 4d47408..bc04bd1 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -387,6 +387,8 @@ class LeapIncomingMail(object): decrdata = decrdata.encode(encoding, 'replace') decrmsg = parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(origmsg['content-type']) # replace headers back in original message for hkey, hval in decrmsg.items(): try: -- cgit v1.2.3 From 76ca78e7f5a5e7b24ceadc10569d49955c58dc05 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 12 Nov 2013 10:43:54 -0200 Subject: check username on imap authentication --- changes/bug_imap-user-check | 1 + src/leap/mail/imap/service/imap.py | 50 +++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 changes/bug_imap-user-check diff --git a/changes/bug_imap-user-check b/changes/bug_imap-user-check new file mode 100644 index 0000000..678871d --- /dev/null +++ b/changes/bug_imap-user-check @@ -0,0 +1 @@ + o Check username in authentications. Closes: #4299 diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 984ad04..feb2593 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -25,6 +25,7 @@ from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError from twisted.mail import imap4 from twisted.python import log +from twisted import cred logger = logging.getLogger(__name__) @@ -54,10 +55,13 @@ class LeapIMAPServer(imap4.IMAP4Server): def __init__(self, *args, **kwargs): # pop extraneous arguments soledad = kwargs.pop('soledad', None) - user = kwargs.pop('user', None) + uuid = kwargs.pop('uuid', None) + userid = kwargs.pop('userid', None) leap_assert(soledad, "need a soledad instance") leap_assert_type(soledad, Soledad) - leap_assert(user, "need a user in the initialization") + leap_assert(uuid, "need a user in the initialization") + + self._userid = userid # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -77,6 +81,12 @@ class LeapIMAPServer(imap4.IMAP4Server): #self.theAccount = theAccount def lineReceived(self, line): + """ + Attempt to parse a single line from the server. + + :param line: the line from the server, without the line delimiter. + :type line: str + """ if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. @@ -87,7 +97,21 @@ class LeapIMAPServer(imap4.IMAP4Server): imap4.IMAP4Server.lineReceived(self, line) def authenticateLogin(self, username, password): - # all is allowed so far. use realm instead + """ + Lookup the account with the given parameters, and deny + the improper combinations. + + :param username: the username that is attempting authentication. + :type username: str + :param password: the password to authenticate with. + :type password: str + """ + # XXX this should use portal: + # return portal.login(cred.credentials.UsernamePassword(user, pass) + if username != self._userid: + # bad username, reject. + raise cred.error.UnauthorizedLogin() + # any dummy password is allowed so far. use realm instead! leap_events.signal(IMAP_CLIENT_LOGIN, "1") return imap4.IAccount, self.theAccount, lambda: None @@ -108,28 +132,32 @@ class LeapIMAPFactory(ServerFactory): capabilities. """ - def __init__(self, user, soledad): + def __init__(self, uuid, userid, soledad): """ Initializes the server factory. - :param user: user ID. **right now it's uuid** - this might change! - :type user: str + :param uuid: user uuid + :type uuid: str + + :param userid: user id (user@provider.org) + :type userid: str :param soledad: soledad instance :type soledad: Soledad """ - self._user = user + self._uuid = uuid + self._userid = userid self._soledad = soledad theAccount = SoledadBackedAccount( - user, soledad=soledad) + uuid, soledad=soledad) self.theAccount = theAccount def buildProtocol(self, addr): "Return a protocol suitable for the job." imapProtocol = LeapIMAPServer( - user=self._user, + uuid=self._uuid, + userid=self._userid, soledad=self._soledad) imapProtocol.theAccount = self.theAccount imapProtocol.factory = self @@ -156,7 +184,7 @@ def run_service(*args, **kwargs): leap_check(userid is not None, "need an user id") uuid = soledad._get_uuid() - factory = LeapIMAPFactory(uuid, soledad) + factory = LeapIMAPFactory(uuid, userid, soledad) from twisted.internet import reactor -- cgit v1.2.3 From 9d2da209f6371b38d73248490eea9fd3bc803ec9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 12 Nov 2013 23:21:34 -0200 Subject: fix mail UID indexing for non-sequential uids --- changes/bug_4461_fix-uid-indexing | 4 + src/leap/mail/imap/fetch.py | 11 ++- src/leap/mail/imap/server.py | 150 ++++++++++++++++++++++++++++++-------- 3 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 changes/bug_4461_fix-uid-indexing diff --git a/changes/bug_4461_fix-uid-indexing b/changes/bug_4461_fix-uid-indexing new file mode 100644 index 0000000..881bb24 --- /dev/null +++ b/changes/bug_4461_fix-uid-indexing @@ -0,0 +1,4 @@ + o Fix several bugs with imap mailbox getUIDNext and notifiers that were breaking + the mail indexing after message deletion. This solves also the perceived + mismatch between the number of unread mails reported by bitmask_client and + the number reported by MUAs. Closes: #4461 diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index bc04bd1..3422ed5 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -141,16 +141,19 @@ class LeapIncomingMail(object): """ Starts a loop to fetch mail. """ - self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) + if self._loop is None: + self._loop = LoopingCall(self.fetch) + self._loop.start(self._check_period) + else: + logger.warning("Tried to start an already running fetching loop.") def stop(self): """ Stops the loop that fetches mail. """ - # XXX should cancel ongoing fetches too. if self._loop and self._loop.running is True: self._loop.stop() + self._loop = None # # Private methods. @@ -214,7 +217,7 @@ class LeapIncomingMail(object): Generic errback """ err = failure.value - logger.error("error!: %r" % (err,)) + logger.exception("error!: %r" % (err,)) def _decryption_error(self, failure): """ diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 11f3ccf..bb2830d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -23,6 +23,7 @@ import StringIO import cStringIO import time +from collections import defaultdict from email.parser import Parser from zope.interface import implements @@ -241,6 +242,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :rtype: SoledadDocument """ + # XXX only upper for INBOX --- name = name.upper() doc = self._soledad.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, name) @@ -274,6 +276,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ + # XXX only upper for INBOX name = name.upper() if name not in self.mailboxes: raise imap4.MailboxException("No such mailbox") @@ -299,6 +302,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :returns: True if successful :rtype: bool """ + # XXX only upper for INBOX name = name.upper() # XXX should check mailbox name for RFC-compliant form @@ -360,6 +364,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :rtype: bool """ + # XXX only upper for INBOX name = name.upper() if name not in self.mailboxes: @@ -385,6 +390,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): names. use with care. :type force: bool """ + # XXX only upper for INBOX name = name.upper() if not name in self.mailboxes: raise imap4.MailboxException("No such mailbox") @@ -422,6 +428,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :param newname: new name of the mailbox :type newname: str """ + # XXX only upper for INBOX oldname = oldname.upper() newname = newname.upper() @@ -487,7 +494,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): # maybe we should store subscriptions in another # document... if not name in self.mailboxes: - print "not this mbox" self.addMailbox(name) mbox = self._get_mailbox_by_name(name) @@ -785,6 +791,7 @@ class LeapMessage(WithMsgFields): return dict(filter_by_cond) # --- no multipart for now + # XXX Fix MULTIPART SUPPORT! def isMultipart(self): return False @@ -967,6 +974,7 @@ class MessageCollection(WithMsgFields, IndexedDB): docs = self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_UID_IDX, self.TYPE_MESSAGE_VAL, self.mbox, str(uid)) + return docs[0] if docs else None def get_msg_by_uid(self, uid): @@ -984,6 +992,47 @@ class MessageCollection(WithMsgFields, IndexedDB): if doc: return LeapMessage(doc) + def get_by_index(self, index): + """ + Retrieves a mesage document by mailbox index. + + :param index: the index of the sequence (zero-indexed) + :type index: int + """ + try: + return self.get_all()[index] + except IndexError: + return None + + def get_msg_by_index(self, index): + """ + Retrieves a LeapMessage by sequence index. + + :param index: the index of the sequence (zero-indexed) + :type index: int + """ + doc = self.get_by_index(index) + if doc: + return LeapMessage(doc) + + def is_deleted(self, doc): + """ + Returns whether a given doc is deleted or not. + + :param doc: the document to check + :rtype: bool + """ + return self.DELETED_FLAG in doc.content[self.FLAGS_KEY] + + def get_last(self): + """ + Gets the last LeapMessage + """ + _all = self.get_all() + if not _all: + return None + return LeapMessage(_all[-1]) + def get_all(self): """ Get all message documents for the selected mailbox. @@ -993,9 +1042,13 @@ class MessageCollection(WithMsgFields, IndexedDB): :rtype: list of SoledadDocument """ # XXX this should return LeapMessage instances - return self._soledad.get_from_index( + all_docs = [doc for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox) + self.TYPE_MESSAGE_VAL, self.mbox)] + #if not self.is_deleted(doc)] + # highly inneficient, but first let's grok it and then + # let's worry about efficiency. + return sorted(all_docs, key=lambda item: item.content['uid']) def unseen_iter(self): """ @@ -1075,8 +1128,11 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: LeapMessage or None if not found. :rtype: LeapMessage """ + #try: + #return self.get_msg_by_uid(uid) try: - return self.get_msg_by_uid(uid) + return [doc + for doc in self.get_all()][uid - 1] except IndexError: return None @@ -1116,7 +1172,7 @@ class SoledadMailbox(WithMsgFields): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" - listeners = [] + _listeners = defaultdict(set) def __init__(self, mbox, soledad=None, rw=1): """ @@ -1150,10 +1206,18 @@ class SoledadMailbox(WithMsgFields): if not self.getFlags(): self.setFlags(self.INIT_FLAGS) - # the server itself is a listener to the mailbox. - # so we can notify it (and should!) after chanes in flags - # and number of messages. - map(lambda i: self.listeners.remove(i), self.listeners) + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox] def addListener(self, listener): """ @@ -1163,7 +1227,7 @@ class SoledadMailbox(WithMsgFields): :type listener: an object that implements IMailboxListener """ logger.debug('adding mailbox listener: %s' % listener) - self.listeners.append(listener) + self.listeners.add(listener) def removeListener(self, listener): """ @@ -1172,25 +1236,24 @@ class SoledadMailbox(WithMsgFields): :param listener: listener to remove :type listener: an object that implements IMailboxListener """ - logger.debug('removing mailbox listener: %s' % listener) - try: - self.listeners.remove(listener) - except ValueError: - logger.error( - "removeListener: cannot remove listener %s" % listener) + self.listeners.remove(listener) def _get_mbox(self): """ Returns mailbox document. - :return: A SoledadDocument containing this mailbox. - :rtype: SoledadDocument + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. """ - query = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() + try: + query = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + self.TYPE_MBOX_VAL, self.mbox) + if query: + return query.pop() + except Exception as exc: + logger.error("Unhandled error %r" % exc) def getFlags(self): """ @@ -1287,8 +1350,12 @@ class SoledadMailbox(WithMsgFields): :rtype: int """ - # XXX reimplement with proper index - return self.messages.count() + 1 + last = self.messages.get_last() + if last: + nextuid = last.getUID() + 1 + else: + nextuid = 1 + return nextuid def getMessageCount(self): """ @@ -1375,6 +1442,8 @@ class SoledadMailbox(WithMsgFields): self.messages.add_msg(message, flags=flags, date=date, uid=uid_next) + + # XXX recent should not include deleted...?? exists = len(self.messages) recent = len(self.messages.get_recent()) for listener in self.listeners: @@ -1434,16 +1503,35 @@ class SoledadMailbox(WithMsgFields): :rtype: A tuple of two-tuples of message sequence numbers and LeapMessage """ - # XXX implement sequence numbers (uid = 0) result = [] + sequence = True if uid == 0 else False if not messages.last: - messages.last = self.messages.count() + try: + iter(messages) + except TypeError: + # looks like we cannot iterate + last = self.messages.get_last() + uid_last = last.getUID() + messages.last = uid_last + + # for sequence numbers (uid = 0) + if sequence: + for msg_id in messages: + msg = self.messages.get_msg_by_index(msg_id - 1) + if msg: + result.append((msg.getUID(), msg)) + else: + print "fetch %s, no msg found!!!" % msg_id + + else: + for msg_id in messages: + msg = self.messages.get_msg_by_uid(msg_id) + if msg: + result.append((msg_id, msg)) + else: + print "fetch %s, no msg found!!!" % msg_id - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) return tuple(result) def _signal_unread_to_ui(self): -- cgit v1.2.3 From 297dfda10617cb7c30aca11c55771c1d60948c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 15 Nov 2013 10:13:45 -0300 Subject: Fold in changes --- CHANGELOG | 23 +++++++++++++++++++++- changes/bug_3937_fix_ui_freeze | 1 - changes/bug_4394-update-pkey | 1 - changes/bug_4416-change-smtp-relay-to-gateway | 1 - changes/bug_4441_fix-fqdn | 1 - ...ove-multipart-encrypted-header-after-decrypting | 2 -- changes/bug_4461_fix-uid-indexing | 4 ---- changes/bug_imap-user-check | 1 - changes/bug_reject_bad_sender | 2 -- ...re-4324_prevent-double-encryption-when-relaying | 2 -- ..._4447-4322-fix-headers-when-gatewaying-messages | 2 -- 11 files changed, 22 insertions(+), 18 deletions(-) delete mode 100644 changes/bug_3937_fix_ui_freeze delete mode 100644 changes/bug_4394-update-pkey delete mode 100644 changes/bug_4416-change-smtp-relay-to-gateway delete mode 100644 changes/bug_4441_fix-fqdn delete mode 100644 changes/bug_4454_remove-multipart-encrypted-header-after-decrypting delete mode 100644 changes/bug_4461_fix-uid-indexing delete mode 100644 changes/bug_imap-user-check delete mode 100644 changes/bug_reject_bad_sender delete mode 100644 changes/feature-4324_prevent-double-encryption-when-relaying delete mode 100644 changes/feature_4447-4322-fix-headers-when-gatewaying-messages diff --git a/CHANGELOG b/CHANGELOG index 5755e59..f15482c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,23 @@ +0.3.7 Nov 15: + o Uses deferToThread for sendMail. Closes #3937 + o Update pkey to allow multiple accounts. Solves: #4394 + o Change SMTP service name from "relay" to "gateway". Closes #4416. + o Identify ourselves with a fqdn, always. Closes: #4441 + o Remove 'multipart/encrypted' header after decrypting incoming + mail. Closes #4454. + o Fix several bugs with imap mailbox getUIDNext and notifiers that + were breaking the mail indexing after message deletion. This + solves also the perceived mismatch between the number of unread + mails reported by bitmask_client and the number reported by + MUAs. Closes: #4461 + o Check username in authentications. Closes: #4299 + o Reject senders that aren't the user that is currently logged + in. Fixes #3952. + o Prevent already encrypted outgoing messages from being encrypted + again. Closes #4324. + o Correctly handle email headers when gatewaying messages. Also add + OpenPGP header. Closes #4322 and #4447. + 0.3.6 Nov 1: o Add support for non-ascii characters in emails. Closes #4000. o Default to UTF-8 when there is no charset parsed from the mail @@ -23,7 +43,8 @@ 0.3.2 Sep 6: o Make mail services bind to 127.0.0.1. Closes: #3627. - o Signal unread message to UI when message is saved locally. Closes: #3654. + o Signal unread message to UI when message is saved locally. Closes: + #3654. o Signal unread to UI when flag in message change. Closes: #3662. o Use dirspec instead of plain xdg. Closes #3574. o SMTP service invocation returns factory instance. diff --git a/changes/bug_3937_fix_ui_freeze b/changes/bug_3937_fix_ui_freeze deleted file mode 100644 index b91938c..0000000 --- a/changes/bug_3937_fix_ui_freeze +++ /dev/null @@ -1 +0,0 @@ - o Uses deferToThread for sendMail. Closes #3937 diff --git a/changes/bug_4394-update-pkey b/changes/bug_4394-update-pkey deleted file mode 100644 index d0a60b1..0000000 --- a/changes/bug_4394-update-pkey +++ /dev/null @@ -1 +0,0 @@ - o Update pkey to allow multiple accounts. Solves: #4394 diff --git a/changes/bug_4416-change-smtp-relay-to-gateway b/changes/bug_4416-change-smtp-relay-to-gateway deleted file mode 100644 index 08bead7..0000000 --- a/changes/bug_4416-change-smtp-relay-to-gateway +++ /dev/null @@ -1 +0,0 @@ - o Change SMTP service name from "relay" to "gateway". Closes #4416. diff --git a/changes/bug_4441_fix-fqdn b/changes/bug_4441_fix-fqdn deleted file mode 100644 index e758d65..0000000 --- a/changes/bug_4441_fix-fqdn +++ /dev/null @@ -1 +0,0 @@ - o Identify ourselves with a fqdn, always. Closes: #4441 diff --git a/changes/bug_4454_remove-multipart-encrypted-header-after-decrypting b/changes/bug_4454_remove-multipart-encrypted-header-after-decrypting deleted file mode 100644 index 8aa0aaa..0000000 --- a/changes/bug_4454_remove-multipart-encrypted-header-after-decrypting +++ /dev/null @@ -1,2 +0,0 @@ - o Remove 'multipart/encrypted' header after decrypting incoming mail. Closes - #4454. diff --git a/changes/bug_4461_fix-uid-indexing b/changes/bug_4461_fix-uid-indexing deleted file mode 100644 index 881bb24..0000000 --- a/changes/bug_4461_fix-uid-indexing +++ /dev/null @@ -1,4 +0,0 @@ - o Fix several bugs with imap mailbox getUIDNext and notifiers that were breaking - the mail indexing after message deletion. This solves also the perceived - mismatch between the number of unread mails reported by bitmask_client and - the number reported by MUAs. Closes: #4461 diff --git a/changes/bug_imap-user-check b/changes/bug_imap-user-check deleted file mode 100644 index 678871d..0000000 --- a/changes/bug_imap-user-check +++ /dev/null @@ -1 +0,0 @@ - o Check username in authentications. Closes: #4299 diff --git a/changes/bug_reject_bad_sender b/changes/bug_reject_bad_sender deleted file mode 100644 index 0e46c28..0000000 --- a/changes/bug_reject_bad_sender +++ /dev/null @@ -1,2 +0,0 @@ - o Reject senders that aren't the user that is currently logged - in. Fixes #3952. \ No newline at end of file diff --git a/changes/feature-4324_prevent-double-encryption-when-relaying b/changes/feature-4324_prevent-double-encryption-when-relaying deleted file mode 100644 index a3d70a9..0000000 --- a/changes/feature-4324_prevent-double-encryption-when-relaying +++ /dev/null @@ -1,2 +0,0 @@ - o Prevent already encrypted outgoing messages from being encrypted again. - Closes #4324. diff --git a/changes/feature_4447-4322-fix-headers-when-gatewaying-messages b/changes/feature_4447-4322-fix-headers-when-gatewaying-messages deleted file mode 100644 index 986937c..0000000 --- a/changes/feature_4447-4322-fix-headers-when-gatewaying-messages +++ /dev/null @@ -1,2 +0,0 @@ - o Correctly handle email headers when gatewaying messages. Also add - OpenPGP header. Closes #4322 and #4447. -- cgit v1.2.3