diff options
Diffstat (limited to 'mail')
| -rw-r--r-- | mail/CHANGELOG | 23 | ||||
| -rw-r--r-- | mail/MANIFEST.in | 1 | ||||
| -rw-r--r-- | mail/pkg/requirements.pip | 2 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/fetch.py | 32 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/server.py | 151 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/service/imap.py | 57 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/tests/test_imap.py | 4 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/README.rst | 38 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/__init__.py | 49 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/gateway.py (renamed from mail/src/leap/mail/smtp/smtprelay.py) | 501 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/tests/__init__.py | 6 | ||||
| -rw-r--r-- | mail/src/leap/mail/smtp/tests/test_gateway.py (renamed from mail/src/leap/mail/smtp/tests/test_smtprelay.py) | 72 |
12 files changed, 582 insertions, 354 deletions
diff --git a/mail/CHANGELOG b/mail/CHANGELOG index 5755e593..f15482c4 100644 --- a/mail/CHANGELOG +++ b/mail/CHANGELOG @@ -1,3 +1,23 @@ +0.3.7 Nov 15: + o Uses deferToThread for sendMail. Closes #3937 + o Update pkey to allow multiple accounts. Solves: #4394 + o Change SMTP service name from "relay" to "gateway". Closes #4416. + o Identify ourselves with a fqdn, always. Closes: #4441 + o Remove 'multipart/encrypted' header after decrypting incoming + mail. Closes #4454. + o Fix several bugs with imap mailbox getUIDNext and notifiers that + were breaking the mail indexing after message deletion. This + solves also the perceived mismatch between the number of unread + mails reported by bitmask_client and the number reported by + MUAs. Closes: #4461 + o Check username in authentications. Closes: #4299 + o Reject senders that aren't the user that is currently logged + in. Fixes #3952. + o Prevent already encrypted outgoing messages from being encrypted + again. Closes #4324. + o Correctly handle email headers when gatewaying messages. Also add + OpenPGP header. Closes #4322 and #4447. + 0.3.6 Nov 1: o Add support for non-ascii characters in emails. Closes #4000. o Default to UTF-8 when there is no charset parsed from the mail @@ -23,7 +43,8 @@ 0.3.2 Sep 6: o Make mail services bind to 127.0.0.1. Closes: #3627. - o Signal unread message to UI when message is saved locally. Closes: #3654. + o Signal unread message to UI when message is saved locally. Closes: + #3654. o Signal unread to UI when flag in message change. Closes: #3662. o Use dirspec instead of plain xdg. Closes #3574. o SMTP service invocation returns factory instance. diff --git a/mail/MANIFEST.in b/mail/MANIFEST.in index 7f6148ef..83264d46 100644 --- a/mail/MANIFEST.in +++ b/mail/MANIFEST.in @@ -2,3 +2,4 @@ include pkg/* include versioneer.py include LICENSE include CHANGELOG +include README.rst diff --git a/mail/pkg/requirements.pip b/mail/pkg/requirements.pip index 4780b5c7..7ed50878 100644 --- a/mail/pkg/requirements.pip +++ b/mail/pkg/requirements.pip @@ -1,4 +1,6 @@ +zope.interface leap.soledad.client>=0.3.0 leap.common>=0.3.5 leap.keymanager>=0.3.4 twisted # >= 12.0.3 ?? +zope.proxy diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py index dd65def2..3422ed50 100644 --- a/mail/src/leap/mail/imap/fetch.py +++ b/mail/src/leap/mail/imap/fetch.py @@ -28,6 +28,7 @@ from email.parser import Parser from twisted.python import log from twisted.internet.task import LoopingCall from twisted.internet.threads import deferToThread +from zope.proxy import sameProxiedObjects from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type @@ -39,6 +40,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors +from leap.keymanager.openpgp import OpenPGPKey from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -66,7 +68,7 @@ class LeapIncomingMail(object): fetching_lock = threading.Lock() def __init__(self, keymanager, soledad, imap_account, - check_period): + check_period, userid): """ Initialize LeapIMAP. @@ -88,14 +90,14 @@ class LeapIncomingMail(object): leap_assert_type(soledad, Soledad) leap_assert(check_period, "need a period to check incoming mail") leap_assert_type(check_period, int) + leap_assert(userid, "need a userid to initialize") self._keymanager = keymanager self._soledad = soledad self.imapAccount = imap_account self._inbox = self.imapAccount.getMailbox('inbox') + self._userid = userid - self._pkey = self._keymanager.get_all_keys_in_local_db( - private=True).pop() self._loop = None self._check_period = check_period @@ -107,6 +109,13 @@ class LeapIncomingMail(object): """ self._soledad.create_index("just-mail", "incoming") + @property + def _pkey(self): + if sameProxiedObjects(self._keymanager, None): + logger.warning('tried to get key, but null keymanager found') + return None + return self._keymanager.get_key(self._userid, OpenPGPKey, private=True) + # # Public API: fetch, start_loop, stop. # @@ -118,6 +127,8 @@ class LeapIncomingMail(object): Calls a deferred that will execute the fetch callback in a separate thread """ + logger.debug("fetching mail for: %s %s" % ( + self._soledad.uuid, self._userid)) if not self.fetching_lock.locked(): d = deferToThread(self._sync_soledad) d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error) @@ -130,16 +141,19 @@ class LeapIncomingMail(object): """ Starts a loop to fetch mail. """ - self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) + if self._loop is None: + self._loop = LoopingCall(self.fetch) + self._loop.start(self._check_period) + else: + logger.warning("Tried to start an already running fetching loop.") def stop(self): """ Stops the loop that fetches mail. """ - # XXX should cancel ongoing fetches too. if self._loop and self._loop.running is True: self._loop.stop() + self._loop = None # # Private methods. @@ -203,7 +217,7 @@ class LeapIncomingMail(object): Generic errback """ err = failure.value - logger.error("error!: %r" % (err,)) + logger.exception("error!: %r" % (err,)) def _decryption_error(self, failure): """ @@ -334,6 +348,7 @@ class LeapIncomingMail(object): :return: data, possibly descrypted. :rtype: str """ + # TODO split this method leap_assert_type(data, unicode) parser = Parser() @@ -375,6 +390,8 @@ class LeapIncomingMail(object): decrdata = decrdata.encode(encoding, 'replace') decrmsg = parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(origmsg['content-type']) # replace headers back in original message for hkey, hval in decrmsg.items(): try: @@ -417,7 +434,6 @@ class LeapIncomingMail(object): incoming message :type msgtuple: (SoledadDocument, str) """ - print "adding message locally....." doc, data = msgtuple self._inbox.addMessage(data, (self.RECENT_FLAG,)) leap_events.signal(IMAP_MSG_SAVED_LOCALLY) diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py index 7a9f8107..bb2830d9 100644 --- a/mail/src/leap/mail/imap/server.py +++ b/mail/src/leap/mail/imap/server.py @@ -23,6 +23,7 @@ import StringIO import cStringIO import time +from collections import defaultdict from email.parser import Parser from zope.interface import implements @@ -241,6 +242,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :rtype: SoledadDocument """ + # XXX only upper for INBOX --- name = name.upper() doc = self._soledad.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, name) @@ -274,6 +276,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ + # XXX only upper for INBOX name = name.upper() if name not in self.mailboxes: raise imap4.MailboxException("No such mailbox") @@ -299,6 +302,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :returns: True if successful :rtype: bool """ + # XXX only upper for INBOX name = name.upper() # XXX should check mailbox name for RFC-compliant form @@ -360,6 +364,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :rtype: bool """ + # XXX only upper for INBOX name = name.upper() if name not in self.mailboxes: @@ -385,6 +390,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): names. use with care. :type force: bool """ + # XXX only upper for INBOX name = name.upper() if not name in self.mailboxes: raise imap4.MailboxException("No such mailbox") @@ -422,6 +428,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :param newname: new name of the mailbox :type newname: str """ + # XXX only upper for INBOX oldname = oldname.upper() newname = newname.upper() @@ -487,7 +494,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): # maybe we should store subscriptions in another # document... if not name in self.mailboxes: - print "not this mbox" self.addMailbox(name) mbox = self._get_mailbox_by_name(name) @@ -785,6 +791,7 @@ class LeapMessage(WithMsgFields): return dict(filter_by_cond) # --- no multipart for now + # XXX Fix MULTIPART SUPPORT! def isMultipart(self): return False @@ -967,6 +974,7 @@ class MessageCollection(WithMsgFields, IndexedDB): docs = self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_UID_IDX, self.TYPE_MESSAGE_VAL, self.mbox, str(uid)) + return docs[0] if docs else None def get_msg_by_uid(self, uid): @@ -984,6 +992,47 @@ class MessageCollection(WithMsgFields, IndexedDB): if doc: return LeapMessage(doc) + def get_by_index(self, index): + """ + Retrieves a mesage document by mailbox index. + + :param index: the index of the sequence (zero-indexed) + :type index: int + """ + try: + return self.get_all()[index] + except IndexError: + return None + + def get_msg_by_index(self, index): + """ + Retrieves a LeapMessage by sequence index. + + :param index: the index of the sequence (zero-indexed) + :type index: int + """ + doc = self.get_by_index(index) + if doc: + return LeapMessage(doc) + + def is_deleted(self, doc): + """ + Returns whether a given doc is deleted or not. + + :param doc: the document to check + :rtype: bool + """ + return self.DELETED_FLAG in doc.content[self.FLAGS_KEY] + + def get_last(self): + """ + Gets the last LeapMessage + """ + _all = self.get_all() + if not _all: + return None + return LeapMessage(_all[-1]) + def get_all(self): """ Get all message documents for the selected mailbox. @@ -993,9 +1042,13 @@ class MessageCollection(WithMsgFields, IndexedDB): :rtype: list of SoledadDocument """ # XXX this should return LeapMessage instances - return self._soledad.get_from_index( + all_docs = [doc for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox) + self.TYPE_MESSAGE_VAL, self.mbox)] + #if not self.is_deleted(doc)] + # highly inneficient, but first let's grok it and then + # let's worry about efficiency. + return sorted(all_docs, key=lambda item: item.content['uid']) def unseen_iter(self): """ @@ -1075,8 +1128,11 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: LeapMessage or None if not found. :rtype: LeapMessage """ + #try: + #return self.get_msg_by_uid(uid) try: - return self.get_msg_by_uid(uid) + return [doc + for doc in self.get_all()][uid - 1] except IndexError: return None @@ -1116,7 +1172,7 @@ class SoledadMailbox(WithMsgFields): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" - listeners = [] + _listeners = defaultdict(set) def __init__(self, mbox, soledad=None, rw=1): """ @@ -1150,11 +1206,18 @@ class SoledadMailbox(WithMsgFields): if not self.getFlags(): self.setFlags(self.INIT_FLAGS) - # the server itself is a listener to the mailbox. - # so we can notify it (and should!) after chanes in flags - # and number of messages. - print "emptying the listeners" - map(lambda i: self.listeners.remove(i), self.listeners) + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox] def addListener(self, listener): """ @@ -1164,7 +1227,7 @@ class SoledadMailbox(WithMsgFields): :type listener: an object that implements IMailboxListener """ logger.debug('adding mailbox listener: %s' % listener) - self.listeners.append(listener) + self.listeners.add(listener) def removeListener(self, listener): """ @@ -1173,25 +1236,24 @@ class SoledadMailbox(WithMsgFields): :param listener: listener to remove :type listener: an object that implements IMailboxListener """ - logger.debug('removing mailbox listener: %s' % listener) - try: - self.listeners.remove(listener) - except ValueError: - logger.error( - "removeListener: cannot remove listener %s" % listener) + self.listeners.remove(listener) def _get_mbox(self): """ Returns mailbox document. - :return: A SoledadDocument containing this mailbox. - :rtype: SoledadDocument + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. """ - query = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() + try: + query = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + self.TYPE_MBOX_VAL, self.mbox) + if query: + return query.pop() + except Exception as exc: + logger.error("Unhandled error %r" % exc) def getFlags(self): """ @@ -1288,8 +1350,12 @@ class SoledadMailbox(WithMsgFields): :rtype: int """ - # XXX reimplement with proper index - return self.messages.count() + 1 + last = self.messages.get_last() + if last: + nextuid = last.getUID() + 1 + else: + nextuid = 1 + return nextuid def getMessageCount(self): """ @@ -1376,6 +1442,8 @@ class SoledadMailbox(WithMsgFields): self.messages.add_msg(message, flags=flags, date=date, uid=uid_next) + + # XXX recent should not include deleted...?? exists = len(self.messages) recent = len(self.messages.get_recent()) for listener in self.listeners: @@ -1435,16 +1503,35 @@ class SoledadMailbox(WithMsgFields): :rtype: A tuple of two-tuples of message sequence numbers and LeapMessage """ - # XXX implement sequence numbers (uid = 0) result = [] + sequence = True if uid == 0 else False if not messages.last: - messages.last = self.messages.count() + try: + iter(messages) + except TypeError: + # looks like we cannot iterate + last = self.messages.get_last() + uid_last = last.getUID() + messages.last = uid_last + + # for sequence numbers (uid = 0) + if sequence: + for msg_id in messages: + msg = self.messages.get_msg_by_index(msg_id - 1) + if msg: + result.append((msg.getUID(), msg)) + else: + print "fetch %s, no msg found!!!" % msg_id + + else: + for msg_id in messages: + msg = self.messages.get_msg_by_uid(msg_id) + if msg: + result.append((msg_id, msg)) + else: + print "fetch %s, no msg found!!!" % msg_id - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) return tuple(result) def _signal_unread_to_ui(self): diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py index 5f7322a9..feb2593a 100644 --- a/mail/src/leap/mail/imap/service/imap.py +++ b/mail/src/leap/mail/imap/service/imap.py @@ -25,11 +25,12 @@ from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError from twisted.mail import imap4 from twisted.python import log +from twisted import cred logger = logging.getLogger(__name__) from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type +from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager from leap.mail.imap.server import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail @@ -54,10 +55,13 @@ class LeapIMAPServer(imap4.IMAP4Server): def __init__(self, *args, **kwargs): # pop extraneous arguments soledad = kwargs.pop('soledad', None) - user = kwargs.pop('user', None) + uuid = kwargs.pop('uuid', None) + userid = kwargs.pop('userid', None) leap_assert(soledad, "need a soledad instance") leap_assert_type(soledad, Soledad) - leap_assert(user, "need a user in the initialization") + leap_assert(uuid, "need a user in the initialization") + + self._userid = userid # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -77,6 +81,12 @@ class LeapIMAPServer(imap4.IMAP4Server): #self.theAccount = theAccount def lineReceived(self, line): + """ + Attempt to parse a single line from the server. + + :param line: the line from the server, without the line delimiter. + :type line: str + """ if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. @@ -87,7 +97,21 @@ class LeapIMAPServer(imap4.IMAP4Server): imap4.IMAP4Server.lineReceived(self, line) def authenticateLogin(self, username, password): - # all is allowed so far. use realm instead + """ + Lookup the account with the given parameters, and deny + the improper combinations. + + :param username: the username that is attempting authentication. + :type username: str + :param password: the password to authenticate with. + :type password: str + """ + # XXX this should use portal: + # return portal.login(cred.credentials.UsernamePassword(user, pass) + if username != self._userid: + # bad username, reject. + raise cred.error.UnauthorizedLogin() + # any dummy password is allowed so far. use realm instead! leap_events.signal(IMAP_CLIENT_LOGIN, "1") return imap4.IAccount, self.theAccount, lambda: None @@ -108,28 +132,32 @@ class LeapIMAPFactory(ServerFactory): capabilities. """ - def __init__(self, user, soledad): + def __init__(self, uuid, userid, soledad): """ Initializes the server factory. - :param user: user ID. **right now it's uuid** - this might change! - :type user: str + :param uuid: user uuid + :type uuid: str + + :param userid: user id (user@provider.org) + :type userid: str :param soledad: soledad instance :type soledad: Soledad """ - self._user = user + self._uuid = uuid + self._userid = userid self._soledad = soledad theAccount = SoledadBackedAccount( - user, soledad=soledad) + uuid, soledad=soledad) self.theAccount = theAccount def buildProtocol(self, addr): "Return a protocol suitable for the job." imapProtocol = LeapIMAPServer( - user=self._user, + uuid=self._uuid, + userid=self._userid, soledad=self._soledad) imapProtocol.theAccount = self.theAccount imapProtocol.factory = self @@ -152,9 +180,11 @@ def run_service(*args, **kwargs): port = kwargs.get('port', IMAP_PORT) check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) + userid = kwargs.get('userid', None) + leap_check(userid is not None, "need an user id") uuid = soledad._get_uuid() - factory = LeapIMAPFactory(uuid, soledad) + factory = LeapIMAPFactory(uuid, userid, soledad) from twisted.internet import reactor @@ -165,7 +195,8 @@ def run_service(*args, **kwargs): keymanager, soledad, factory.theAccount, - check_period) + check_period, + userid) except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py index 34117956..ad11315d 100644 --- a/mail/src/leap/mail/imap/tests/test_imap.py +++ b/mail/src/leap/mail/imap/tests/test_imap.py @@ -91,7 +91,7 @@ def initialize_soledad(email, gnupg_home, tempdir): """ uuid = "foobar-uuid" - passphrase = "verysecretpassphrase" + passphrase = u"verysecretpassphrase" secret_path = os.path.join(tempdir, "secret.gpg") local_db_path = os.path.join(tempdir, "soledad.u1db") server_url = "http://provider" @@ -101,6 +101,8 @@ def initialize_soledad(email, gnupg_home, tempdir): get_doc = Mock(return_value=None) put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) def __call__(self): return self diff --git a/mail/src/leap/mail/smtp/README.rst b/mail/src/leap/mail/smtp/README.rst index 2b2a1180..f6254419 100644 --- a/mail/src/leap/mail/smtp/README.rst +++ b/mail/src/leap/mail/smtp/README.rst @@ -1,43 +1,23 @@ -Leap SMTP Relay -=============== +Leap SMTP Gateway +================= Outgoing mail workflow: * LEAP client runs a thin SMTP proxy on the user's device, bound to localhost. - * User's MUA is configured outgoing SMTP to localhost - * When SMTP proxy receives an email from MUA + * User's MUA is configured outgoing SMTP to localhost. + * When SMTP proxy receives an email from MUA: * SMTP proxy queries Key Manager for the user's private key and public - keys of all recipients + keys of all recipients. * Message is signed by sender and encrypted to recipients. * If recipient's key is missing, email goes out in cleartext (unless - user has configured option to send only encrypted email) - * Finally, message is relayed to provider's SMTP relay - - -Dependencies ------------- - -Leap SMTP Relay depends on the following python libraries: - - * Twisted 12.3.0 [1] - * zope.interface 4.0.3 [2] - -[1] http://pypi.python.org/pypi/Twisted/12.3.0 -[2] http://pypi.python.org/pypi/zope.interface/4.0.3 - - -How to run ----------- - -To launch the SMTP relay, run the following command: - - twistd -y smtprelay.tac + user has configured option to send only encrypted email). + * Finally, message is gatewayed to provider's SMTP server. Running tests ------------- -Tests are run using Twisted's Trial API, like this: +Tests are run using Twisted's Trial API, like this:: - trial leap.email.smtp.tests + python setup.py test -s leap.mail.gateway.tests diff --git a/mail/src/leap/mail/smtp/__init__.py b/mail/src/leap/mail/smtp/__init__.py index b30cd20c..d3eb9e85 100644 --- a/mail/src/leap/mail/smtp/__init__.py +++ b/mail/src/leap/mail/smtp/__init__.py @@ -16,7 +16,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -SMTP relay helper function. +SMTP gateway helper function. """ import logging @@ -26,66 +26,49 @@ from twisted.internet.error import CannotListenError logger = logging.getLogger(__name__) from leap.common.events import proto, signal -from leap.mail.smtp.smtprelay import SMTPFactory +from leap.mail.smtp.gateway import SMTPFactory -def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, +def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, smtp_cert, smtp_key, encrypted_only): """ - Setup SMTP relay to run with Twisted. + Setup SMTP gateway to run with Twisted. - This function sets up the SMTP relay configuration and the Twisted + This function sets up the SMTP gateway configuration and the Twisted reactor. :param port: The port in which to run the server. :type port: int + :param userid: The user currently logged in + :type userid: unicode :param keymanager: A Key Manager from where to get recipients' public keys. :type keymanager: leap.common.keymanager.KeyManager :param smtp_host: The hostname of the remote SMTP server. :type smtp_host: str - :param smtp_port: The port of the remote SMTP server. + :param smtp_port: The port of the remote SMTP server. :type smtp_port: int :param smtp_cert: The client certificate for authentication. :type smtp_cert: str :param smtp_key: The client key for authentication. :type smtp_key: str - :param encrypted_only: Whether the SMTP relay should send unencrypted mail + :param encrypted_only: Whether the SMTP gateway should send unencrypted mail or not. :type encrypted_only: bool :returns: tuple of SMTPFactory, twisted.internet.tcp.Port """ - # The configuration for the SMTP relay is a dict with the following - # format: - # - # { - # 'host': '<host>', - # 'port': <int>, - # 'cert': '<cert path>', - # 'key': '<key path>', - # 'encrypted_only': <True/False> - # } - config = { - 'host': smtp_host, - 'port': smtp_port, - 'cert': smtp_cert, - 'key': smtp_key, - 'encrypted_only': encrypted_only - } - # configure the use of this service with twistd - factory = SMTPFactory(keymanager, config) + factory = SMTPFactory(userid, keymanager, smtp_host, smtp_port, smtp_cert, + smtp_key, encrypted_only) try: - tport = reactor.listenTCP(port, factory, - interface="localhost") - signal(proto.SMTP_SERVICE_STARTED, str(smtp_port)) + tport = reactor.listenTCP(port, factory, interface="localhost") + signal(proto.SMTP_SERVICE_STARTED, str(port)) return factory, tport except CannotListenError: logger.error("STMP Service failed to start: " - "cannot listen in port %s" % ( - smtp_port,)) - signal(proto.SMTP_SERVICE_FAILED_TO_START, str(smtp_port)) + "cannot listen in port %s" % port) + signal(proto.SMTP_SERVICE_FAILED_TO_START, str(port)) except Exception as exc: - logger.error("Unhandled error while launching smtp relay service") + logger.error("Unhandled error while launching smtp gateway service") logger.exception(exc) diff --git a/mail/src/leap/mail/smtp/smtprelay.py b/mail/src/leap/mail/smtp/gateway.py index fca66c0b..f09ee141 100644 --- a/mail/src/leap/mail/smtp/smtprelay.py +++ b/mail/src/leap/mail/smtp/gateway.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# smtprelay.py +# gateway.py # Copyright (C) 2013 LEAP # # This program is free software: you can redistribute it and/or modify @@ -14,11 +14,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - """ -LEAP SMTP encrypted relay. +LEAP SMTP encrypted gateway. -The following classes comprise the SMTP relay service: +The following classes comprise the SMTP gateway service: * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides the SMTPDelivery protocol. @@ -32,7 +31,6 @@ The following classes comprise the SMTP relay service: """ - import re from StringIO import StringIO from email.Header import Header @@ -46,6 +44,7 @@ from twisted.mail import smtp from twisted.internet.protocol import ServerFactory from twisted.internet import reactor, ssl from twisted.internet import defer +from twisted.internet.threads import deferToThread from twisted.python import log from leap.common.check import leap_assert, leap_assert_type @@ -68,67 +67,18 @@ generator.Generator = RFC3156CompliantGenerator # -# Exceptions -# - -class MalformedConfig(Exception): - """ - Raised when the configuration dictionary passed as parameter is malformed. - """ - pass - - -# # Helper utilities # -HOST_KEY = 'host' -PORT_KEY = 'port' -CERT_KEY = 'cert' -KEY_KEY = 'key' -ENCRYPTED_ONLY_KEY = 'encrypted_only' - - -def assert_config_structure(config): - """ - Assert that C{config} is a dict with the following structure: - - { - HOST_KEY: '<str>', - PORT_KEY: <int>, - CERT_KEY: '<str>', - KEY_KEY: '<str>', - ENCRYPTED_ONLY_KEY: <bool>, - } - - @param config: The dictionary to check. - @type config: dict - """ - # assert smtp config structure is valid - leap_assert_type(config, dict) - leap_assert(HOST_KEY in config) - leap_assert_type(config[HOST_KEY], str) - leap_assert(PORT_KEY in config) - leap_assert_type(config[PORT_KEY], int) - leap_assert(CERT_KEY in config) - leap_assert_type(config[CERT_KEY], (str, unicode)) - leap_assert(KEY_KEY in config) - leap_assert_type(config[KEY_KEY], (str, unicode)) - leap_assert(ENCRYPTED_ONLY_KEY in config) - leap_assert_type(config[ENCRYPTED_ONLY_KEY], bool) - # assert received params are not empty - leap_assert(config[HOST_KEY] != '') - leap_assert(config[PORT_KEY] is not 0) - leap_assert(config[CERT_KEY] != '') - leap_assert(config[KEY_KEY] != '') +LOCAL_FQDN = "bitmask.local" def validate_address(address): """ Validate C{address} as defined in RFC 2822. - @param address: The address to be validated. - @type address: str + :param address: The address to be validated. + :type address: str @return: A valid address. @rtype: str @@ -148,46 +98,80 @@ def validate_address(address): # SMTPFactory # +class SMTPHeloLocalhost(smtp.SMTP): + """ + An SMTP class that ensures a proper FQDN + for localhost. + + This avoids a problem in which unproperly configured providers + would complain about the helo not being a fqdn. + """ + + def __init__(self, *args): + smtp.SMTP.__init__(self, *args) + self.host = LOCAL_FQDN + + class SMTPFactory(ServerFactory): """ - Factory for an SMTP server with encrypted relaying capabilities. + Factory for an SMTP server with encrypted gatewaying capabilities. """ + domain = LOCAL_FQDN - def __init__(self, keymanager, config): + def __init__(self, userid, keymanager, host, port, cert, key, + encrypted_only): """ Initialize the SMTP factory. - @param keymanager: A KeyManager for retrieving recipient's keys. - @type keymanager: leap.common.keymanager.KeyManager - @param config: A dictionary with smtp configuration. Should have - the following structure: - { - HOST_KEY: '<str>', - PORT_KEY: <int>, - CERT_KEY: '<str>', - KEY_KEY: '<str>', - ENCRYPTED_ONLY_KEY: <bool>, - } - @type config: dict + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. + :type encrypted_only: bool """ # assert params leap_assert_type(keymanager, KeyManager) - assert_config_structure(config) + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + leap_assert_type(encrypted_only, bool) # and store them + self._userid = userid self._km = keymanager - self._config = config + self._host = host + self._port = port + self._cert = cert + self._key = key + self._encrypted_only = encrypted_only def buildProtocol(self, addr): """ Return a protocol suitable for the job. - @param addr: An address, e.g. a TCP (host, port). - @type addr: twisted.internet.interfaces.IAddress + :param addr: An address, e.g. a TCP (host, port). + :type addr: twisted.internet.interfaces.IAddress @return: The protocol. @rtype: SMTPDelivery """ - smtpProtocol = smtp.SMTP(SMTPDelivery(self._km, self._config)) + smtpProtocol = SMTPHeloLocalhost(SMTPDelivery( + self._userid, self._km, self._host, self._port, self._cert, + self._key, self._encrypted_only)) smtpProtocol.factory = self return smtpProtocol @@ -203,52 +187,57 @@ class SMTPDelivery(object): implements(smtp.IMessageDelivery) - def __init__(self, keymanager, config): + def __init__(self, userid, keymanager, host, port, cert, key, + encrypted_only): """ Initialize the SMTP delivery object. - @param keymanager: A KeyManager for retrieving recipient's keys. - @type keymanager: leap.common.keymanager.KeyManager - @param config: A dictionary with smtp configuration. Should have - the following structure: - { - HOST_KEY: '<str>', - PORT_KEY: <int>, - CERT_KEY: '<str>', - KEY_KEY: '<str>', - ENCRYPTED_ONLY_KEY: <bool>, - } - @type config: dict - """ - # assert params - leap_assert_type(keymanager, KeyManager) - assert_config_structure(config) - # and store them + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. + :type encrypted_only: bool + """ + self._userid = userid self._km = keymanager - self._config = config + self._host = host + self._port = port + self._cert = cert + self._key = key + self._encrypted_only = encrypted_only self._origin = None def receivedHeader(self, helo, origin, recipients): """ Generate the 'Received:' header for a message. - @param helo: The argument to the HELO command and the client's IP + :param helo: The argument to the HELO command and the client's IP address. - @type helo: (str, str) - @param origin: The address the message is from. - @type origin: twisted.mail.smtp.Address - @param recipients: A list of the addresses for which this message is + :type helo: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + :param recipients: A list of the addresses for which this message is bound. - @type: list of twisted.mail.smtp.User + :type: list of twisted.mail.smtp.User @return: The full "Received" header string. - @type: str + :type: str """ myHostname, clientIP = helo - headerValue = "by %s from %s with ESMTP ; %s" % ( - myHostname, clientIP, smtp.rfc822date()) + headerValue = "by bitmask.local from %s with ESMTP ; %s" % ( + clientIP, smtp.rfc822date()) # email.Header.Header used for automatic wrapping of long lines - return "Received: %s" % Header(headerValue) + return "Received: %s" % Header(s=headerValue, header_name='Received') def validateTo(self, user): """ @@ -262,8 +251,8 @@ class SMTPDelivery(object): In the end, it returns an encrypted message object that is able to send itself to the C{user}'s address. - @param user: The user whose address we wish to validate. - @type: twisted.mail.smtp.User + :param user: The user whose address we wish to validate. + :type: twisted.mail.smtp.User @return: A Deferred which becomes, or a callable which takes no arguments and returns an object implementing IMessage. This will @@ -277,12 +266,13 @@ class SMTPDelivery(object): # try to find recipient's public key try: address = validate_address(user.dest.addrstr) - pubkey = self._km.get_key(address, OpenPGPKey) + # verify if recipient key is available in keyring + self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound log.msg("Accepting mail for %s..." % user.dest.addrstr) signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) except KeyNotFound: # if key was not found, check config to see if will send anyway. - if self._config[ENCRYPTED_ONLY_KEY]: + if self._encrypted_only: signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) raise smtp.SMTPBadRcpt(user.dest.addrstr) log.msg("Warning: will send an unencrypted message (because " @@ -290,17 +280,18 @@ class SMTPDelivery(object): signal( proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) return lambda: EncryptedMessage( - self._origin, user, self._km, self._config) + self._origin, user, self._km, self._host, self._port, self._cert, + self._key) def validateFrom(self, helo, origin): """ Validate the address from which the message originates. - @param helo: The argument to the HELO command and the client's IP + :param helo: The argument to the HELO command and the client's IP address. - @type: (str, str) - @param origin: The address the message is from. - @type origin: twisted.mail.smtp.Address + :type: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address @return: origin or a Deferred whose callback will be passed origin. @rtype: Deferred or Address @@ -310,6 +301,10 @@ class SMTPDelivery(object): """ # accept mail from anywhere. To reject an address, raise # smtp.SMTPBadSender here. + if str(origin) != str(self._userid): + log.msg("Rejecting sender {0}, expected {1}".format(origin, + self._userid)) + raise smtp.SMTPBadSender(origin) self._origin = origin return origin @@ -331,16 +326,6 @@ class SSLContextFactory(ssl.ClientContextFactory): return ctx -def move_headers(origmsg, newmsg): - headers = origmsg.items() - unwanted_headers = ['content-type', 'mime-version', 'content-disposition', - 'content-transfer-encoding'] - headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del(origmsg[hkey]) - - class EncryptedMessage(object): """ Receive plaintext from client, encrypt it and send message to a @@ -348,36 +333,36 @@ class EncryptedMessage(object): """ implements(smtp.IMessage) - def __init__(self, fromAddress, user, keymanager, config): + def __init__(self, fromAddress, user, keymanager, host, port, cert, key): """ Initialize the encrypted message. - @param fromAddress: The address of the sender. - @type fromAddress: twisted.mail.smtp.Address - @param user: The recipient of this message. - @type user: twisted.mail.smtp.User - @param keymanager: A KeyManager for retrieving recipient's keys. - @type keymanager: leap.common.keymanager.KeyManager - @param config: A dictionary with smtp configuration. Should have - the following structure: - { - HOST_KEY: '<str>', - PORT_KEY: <int>, - CERT_KEY: '<str>', - KEY_KEY: '<str>', - ENCRYPTED_ONLY_KEY: <bool>, - } - @type config: dict + :param fromAddress: The address of the sender. + :type fromAddress: twisted.mail.smtp.Address + :param user: The recipient of this message. + :type user: twisted.mail.smtp.User + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + :param cert: The client certificate for authentication. + :type cert: str + :param key: The client key for authentication. + :type key: str """ # assert params leap_assert_type(user, smtp.User) leap_assert_type(keymanager, KeyManager) - assert_config_structure(config) # and store them self._fromAddress = fromAddress self._user = user self._km = keymanager - self._config = config + self._host = host + self._port = port + self._cert = cert + self._key = key # initialize list for message's lines self.lines = [] @@ -389,8 +374,8 @@ class EncryptedMessage(object): """ Handle another line. - @param line: The received line. - @type line: str + :param line: The received line. + :type line: str """ self.lines.append(line) @@ -399,21 +384,14 @@ class EncryptedMessage(object): Handle end of message. This method will encrypt and send the message. + + :returns: a deferred """ log.msg("Message data complete.") self.lines.append('') # add a trailing newline - try: - self._maybe_encrypt_and_sign() - return self.sendMessage() - except KeyNotFound: - return None - - def parseMessage(self): - """ - Separate message headers from body. - """ - parser = Parser() - return parser.parsestr('\r\n'.join(self.lines)) + d = deferToThread(self._maybe_encrypt_and_sign) + d.addCallbacks(self.sendMessage, self.skipNoKeyErrBack) + return d def connectionLost(self): """ @@ -425,44 +403,81 @@ class EncryptedMessage(object): # unexpected loss of connection; don't save self.lines = [] + # ends IMessage implementation + + def skipNoKeyErrBack(self, failure): + """ + Errback that ignores a KeyNotFound + + :param failure: the failure + :type Failure: Failure + """ + err = failure.value + if failure.check(KeyNotFound): + pass + else: + raise err + + def parseMessage(self): + """ + Separate message headers from body. + """ + parser = Parser() + return parser.parsestr('\r\n'.join(self.lines)) + + def sendQueued(self, r): + """ + Callback for the queued message. + + :param r: The result from the last previous callback in the chain. + :type r: anything + """ + log.msg(r) + def sendSuccess(self, r): """ Callback for a successful send. - @param r: The result from the last previous callback in the chain. - @type r: anything + :param r: The result from the last previous callback in the chain. + :type r: anything """ log.msg(r) signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr) - def sendError(self, e): + def sendError(self, failure): """ Callback for an unsuccessfull send. - @param e: The result from the last errback. - @type e: anything + :param e: The result from the last errback. + :type e: anything """ - log.msg(e) - log.err() signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + err = failure.value + log.err(err) + raise err - def sendMessage(self): + def sendMessage(self, *args): """ - Send the message. + Sends the message. - This method will prepare the message (headers and possibly encrypted - body) and send it using the ESMTPSenderFactory. + :return: A deferred with callbacks for error and success of this + #message send. + :rtype: twisted.internet.defer.Deferred + """ + d = deferToThread(self._route_msg) + d.addCallbacks(self.sendQueued, self.sendError) + return - @return: A deferred with callbacks for error and success of this - message send. - @rtype: twisted.internet.defer.Deferred + def _route_msg(self): + """ + Sends the msg using the ESMTPSenderFactory. """ + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) msg = self._msg.as_string(False) - log.msg("Connecting to SMTP server %s:%s" % (self._config[HOST_KEY], - self._config[PORT_KEY])) - + # we construct a defer to pass to the ESMTPSenderFactory d = defer.Deferred() + d.addCallbacks(self.sendSuccess, self.sendError) # we don't pass an ssl context factory to the ESMTPSenderFactory # because ssl will be handled by reactor.connectSSL() below. factory = smtp.ESMTPSenderFactory( @@ -472,18 +487,14 @@ class EncryptedMessage(object): self._user.dest.addrstr, StringIO(msg), d, + heloFallback=True, requireAuthentication=False, requireTransportSecurity=True) + factory.domain = LOCAL_FQDN signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) reactor.connectSSL( - self._config[HOST_KEY], - self._config[PORT_KEY], - factory, - contextFactory=SSLContextFactory(self._config[CERT_KEY], - self._config[KEY_KEY])) - d.addCallback(self.sendSuccess) - d.addErrback(self.sendError) - return d + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) # # encryption methods @@ -494,20 +505,18 @@ class EncryptedMessage(object): Create an RFC 3156 compliang PGP encrypted and signed message using C{pubkey} to encrypt and C{signkey} to sign. - @param pubkey: The public key used to encrypt the message. - @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey - @param signkey: The private key used to sign the message. - @type signkey: leap.common.keymanager.openpgp.OpenPGPKey + :param pubkey: The public key used to encrypt the message. + :type pubkey: OpenPGPKey + :param signkey: The private key used to sign the message. + :type signkey: OpenPGPKey """ - # parse original message from received lines - origmsg = self.parseMessage() # create new multipart/encrypted message with 'pgp-encrypted' protocol newmsg = MultipartEncrypted('application/pgp-encrypted') # move (almost) all headers from original message to the new message - move_headers(origmsg, newmsg) + self._fix_headers(self._origmsg, newmsg, signkey) # create 'application/octet-stream' encrypted message encmsg = MIMEApplication( - self._km.encrypt(origmsg.as_string(unixfrom=False), pubkey, + self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey, sign=signkey), _subtype='octet-stream', _encoder=lambda x: x) encmsg.add_header('content-disposition', 'attachment', @@ -524,25 +533,23 @@ class EncryptedMessage(object): """ Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. - @param signkey: The private key used to sign the message. - @type signkey: leap.common.keymanager.openpgp.OpenPGPKey + :param signkey: The private key used to sign the message. + :type signkey: leap.common.keymanager.openpgp.OpenPGPKey """ - # parse original message from received lines - origmsg = self.parseMessage() # create new multipart/signed message newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') # move (almost) all headers from original message to the new message - move_headers(origmsg, newmsg) + self._fix_headers(self._origmsg, newmsg, signkey) # apply base64 content-transfer-encoding - encode_base64_rec(origmsg) + encode_base64_rec(self._origmsg) # get message text with headers and replace \n for \r\n fp = StringIO() g = RFC3156CompliantGenerator( fp, mangle_from_=False, maxheaderlen=76) - g.flatten(origmsg) + g.flatten(self._origmsg) msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) # make sure signed message ends with \r\n as per OpenPGP stantard. - if origmsg.is_multipart(): + if self._origmsg.is_multipart(): if not msgtext.endswith("\r\n"): msgtext += "\r\n" # calculate signature @@ -550,23 +557,46 @@ class EncryptedMessage(object): clearsign=False, detach=True, binary=False) sigmsg = PGPSignature(signature) # attach original message and signature to new message - newmsg.attach(origmsg) + newmsg.attach(self._origmsg) newmsg.attach(sigmsg) self._msg = newmsg def _maybe_encrypt_and_sign(self): """ - Encrypt the message body. + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. - Fetch the recipient key and encrypt the content to the - recipient. If a key is not found, then the behaviour depends on the - configuration parameter ENCRYPTED_ONLY_KEY. If it is False, the message - is sent unencrypted and a warning is logged. If it is True, the - encryption fails with a KeyNotFound exception. + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. - @raise KeyNotFound: Raised when the recipient key was not found and - the ENCRYPTED_ONLY_KEY configuration parameter is set to True. + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ """ + # pass if the original message's content-type is "multipart/encrypted" + self._origmsg = self.parseMessage() + if self._origmsg.get_content_type() == 'multipart/encrypted': + self._msg = self._origmsg + return + from_address = validate_address(self._fromAddress.addrstr) signkey = self._km.get_key(from_address, OpenPGPKey, private=True) log.msg("Will sign the message with %s." % signkey.fingerprint) @@ -588,3 +618,60 @@ class EncryptedMessage(object): signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr) self._sign(signkey) signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr) + + def _fix_headers(self, origmsg, newmsg, signkey): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param origmsg: The original message. + :type origmsg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param signkey: The key used to sign C{newmsg} + :type signkey: OpenPGPKey + """ + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del(origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # add openpgp header to newmsg + username, domain = signkey.address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/openpgp/%s' % (domain, username)) + # delete user-agent from origmsg + del(origmsg['user-agent']) diff --git a/mail/src/leap/mail/smtp/tests/__init__.py b/mail/src/leap/mail/smtp/tests/__init__.py index 9b54de34..62b015f0 100644 --- a/mail/src/leap/mail/smtp/tests/__init__.py +++ b/mail/src/leap/mail/smtp/tests/__init__.py @@ -17,7 +17,7 @@ """ -Base classes and keys for SMTP relay tests. +Base classes and keys for SMTP gateway tests. """ import os @@ -59,7 +59,7 @@ class TestCaseWithKeyManager(BaseLeapTest): # setup our own stuff address = 'leap@leap.se' # user's address in the form user@provider uuid = 'leap@leap.se' - passphrase = '123' + passphrase = u'123' secrets_path = os.path.join(self.tempdir, 'secret.gpg') local_db_path = os.path.join(self.tempdir, 'soledad.u1db') server_url = 'http://provider/' @@ -88,6 +88,8 @@ class TestCaseWithKeyManager(BaseLeapTest): get_doc = Mock(return_value=None) put_doc = Mock(side_effect=_put_doc_side_effect) + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) def __call__(self): return self diff --git a/mail/src/leap/mail/smtp/tests/test_smtprelay.py b/mail/src/leap/mail/smtp/tests/test_gateway.py index 7fefe772..f9ea027c 100644 --- a/mail/src/leap/mail/smtp/tests/test_smtprelay.py +++ b/mail/src/leap/mail/smtp/tests/test_gateway.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# test_smtprelay.py +# test_gateway.py # Copyright (C) 2013 LEAP # # This program is free software: you can redistribute it and/or modify @@ -17,7 +17,7 @@ """ -SMTP relay tests. +SMTP gateway tests. """ @@ -33,7 +33,7 @@ from twisted.mail.smtp import ( ) from mock import Mock -from leap.mail.smtp.smtprelay import ( +from leap.mail.smtp.gateway import ( SMTPFactory, EncryptedMessage, ) @@ -52,9 +52,9 @@ HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' -class TestSmtpRelay(TestCaseWithKeyManager): +class TestSmtpGateway(TestCaseWithKeyManager): - EMAIL_DATA = ['HELO relay.leap.se', + EMAIL_DATA = ['HELO gateway.leap.se', 'MAIL FROM: <%s>' % ADDRESS_2, 'RCPT TO: <%s>' % ADDRESS, 'DATA', @@ -90,7 +90,7 @@ class TestSmtpRelay(TestCaseWithKeyManager): self.assertEqual(text, decrypted, "Decrypted text differs from plaintext.") - def test_relay_accepts_valid_email(self): + def test_gateway_accepts_valid_email(self): """ Test if SMTP server responds correctly for valid interaction. """ @@ -102,26 +102,32 @@ class TestSmtpRelay(TestCaseWithKeyManager): '250 Sender address accepted', '250 Recipient address accepted', '354 Continue'] - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) for i, line in enumerate(self.EMAIL_DATA): proto.lineReceived(line + '\r\n') self.assertMatch(transport.value(), '\r\n'.join(SMTP_ANSWERS[0:i + 1]), - 'Did not get expected answer from relay.') + 'Did not get expected answer from gateway.') proto.setTimeout(None) def test_message_encrypt(self): """ Test if message gets encrypted to destination email. """ - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) fromAddr = Address(ADDRESS_2) - dest = User(ADDRESS, 'relay.leap.se', proto, ADDRESS) - m = EncryptedMessage(fromAddr, dest, self._km, self._config) + dest = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) + m = EncryptedMessage( + fromAddr, dest, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) m.eomReceived() @@ -149,11 +155,15 @@ class TestSmtpRelay(TestCaseWithKeyManager): Test if message gets encrypted to destination email and signed with sender key. """ - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) - user = User(ADDRESS, 'relay.leap.se', proto, ADDRESS) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) fromAddr = Address(ADDRESS_2) - m = EncryptedMessage(fromAddr, user, self._km, self._config) + m = EncryptedMessage( + fromAddr, user, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) # trigger encryption and signing @@ -185,11 +195,15 @@ class TestSmtpRelay(TestCaseWithKeyManager): """ # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) - user = User('ihavenopubkey@nonleap.se', 'relay.leap.se', proto, ADDRESS) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + user = User('ihavenopubkey@nonleap.se', 'gateway.leap.se', proto, ADDRESS) fromAddr = Address(ADDRESS_2) - m = EncryptedMessage(fromAddr, user, self._km, self._config) + m = EncryptedMessage( + fromAddr, user, self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) # trigger signing @@ -237,8 +251,10 @@ class TestSmtpRelay(TestCaseWithKeyManager): # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) # prepare the SMTP factory - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') @@ -263,11 +279,11 @@ class TestSmtpRelay(TestCaseWithKeyManager): pgp.delete_key(pubkey) # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) - # change the configuration - self._config['encrypted_only'] = False - # prepare the SMTP factory - proto = SMTPFactory( - self._km, self._config).buildProtocol(('127.0.0.1', 0)) + # prepare the SMTP factory with encrypted only equal to false + proto = SMTPFactory(u'anotheruser@leap.se', + self._km, self._config['host'], self._config['port'], + self._config['cert'], self._config['key'], + False).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') |