[feature] SMTP delivery bounces

We catch any error on SMTP delivery and format it as a bounce message
delivered to the user Inbox.

this doesn't comply with the bounce format, but it's a nice first start.
leaving proper structuring of the delivery failure report for future
iterations.

- Resolves: #7263

4 files changed, 160 insertions, 21 deletions

diff --git a/mail/changes/next-changelog.rst b/mail/changes/next-changelog.rst
index f89af89..9b2a9d6 100644
--- a/mail/changes/next-changelog.rst
+++ b/mail/changes/next-changelog.rst
@@ -13,6 +13,7 @@ Features
 - `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events.
 - `#4008 <https://leap.se/code/issues/4008>`_: Add token-based authentication to local IMAP/SMTP services.
 - `#7889 <https://leap.se/code/issues/7889>`_: Use cryptography instead of pycryptopp to reduce dependencies.
+- `#7263 <https://leap.se/code/issues/7263>`_: Implement local bounces to notify user of SMTP delivery errors.
 - Use twisted.cred to authenticate IMAP users.
 
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
diff --git a/mail/src/leap/mail/outgoing/service.py b/mail/src/leap/mail/outgoing/service.py
index eeb5d32..335cae4 100644
--- a/mail/src/leap/mail/outgoing/service.py
+++ b/mail/src/leap/mail/outgoing/service.py
@@ -73,7 +73,7 @@ class SSLContextFactory(ssl.ClientContextFactory):
         return ctx
 
 
-def outgoingFactory(userid, keymanager, opts, check_cert=True):
+def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None):
 
     cert = unicode(opts.cert)
     key = unicode(opts.key)
@@ -85,7 +85,9 @@ def outgoingFactory(userid, keymanager, opts, check_cert=True):
             raise errors.ConfigurationError(
                 'No valid SMTP certificate could be found for %s!' % userid)
 
-    return OutgoingMail(str(userid), keymanager, cert, key, hostname, port)
+    return OutgoingMail(
+        str(userid), keymanager, cert, key, hostname, port,
+        bouncer)
 
 
 class OutgoingMail(object):
@@ -93,7 +95,8 @@ class OutgoingMail(object):
     Sends Outgoing Mail, encrypting and signing if needed.
     """
 
-    def __init__(self, from_address, keymanager, cert, key, host, port):
+    def __init__(self, from_address, keymanager, cert, key, host, port,
+                 bouncer=None):
         """
         Initialize the outgoing mail service.
 
@@ -133,6 +136,7 @@ class OutgoingMail(object):
         self._cert = cert
         self._from_address = from_address
         self._keymanager = keymanager
+        self._bouncer = bouncer
 
     def send_message(self, raw, recipient):
         """
@@ -145,8 +149,8 @@ class OutgoingMail(object):
         :return: a deferred which delivers the message when fired
         """
         d = self._maybe_encrypt_and_sign(raw, recipient)
-        d.addCallback(self._route_msg)
-        d.addErrback(self.sendError)
+        d.addCallback(self._route_msg, raw)
+        d.addErrback(self.sendError, raw)
         return d
 
     def sendSuccess(self, smtp_sender_result):
@@ -163,21 +167,36 @@ class OutgoingMail(object):
         emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS,
                    fromaddr, dest_addrstr)
 
-    def sendError(self, failure):
+    def sendError(self, failure, origmsg):
         """
         Callback for an unsuccessfull send.
 
-        :param e: The result from the last errback.
-        :type e: anything
+        :param failure: The result from the last errback.
+        :type failure: anything
+        :param origmsg: the original, unencrypted, raw message, to be passed to
+                        the bouncer.
+        :type origmsg: str
         """
-        # XXX: need to get the address from the exception to send signal
+        # XXX: need to get the address from the original message to send signal
         # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address,
         #   self._user.dest.addrstr)
+
+        # TODO when we implement outgoing queues/long-term-retries, we could
+        # examine the error *here* and delay the notification if it's just a
+        # temporal error. We might want to notify the permanent errors
+        # differently.
+
         err = failure.value
         log.err(err)
-        raise err
 
-    def _route_msg(self, encrypt_and_sign_result):
+        if self._bouncer:
+            self._bouncer.bounce_message(
+                err.message, to=self._from_address,
+                orig=origmsg)
+        else:
+            raise err
+
+    def _route_msg(self, encrypt_and_sign_result, raw):
         """
         Sends the msg using the ESMTPSenderFactory.
 
@@ -191,7 +210,8 @@ class OutgoingMail(object):
 
         # we construct a defer to pass to the ESMTPSenderFactory
         d = defer.Deferred()
-        d.addCallbacks(self.sendSuccess, self.sendError)
+        d.addCallback(self.sendSuccess)
+        d.addErrback(self.sendError, raw)
         # we don't pass an ssl context factory to the ESMTPSenderFactory
         # because ssl will be handled by reactor.connectSSL() below.
         factory = smtp.ESMTPSenderFactory(
diff --git a/mail/src/leap/mail/smtp/bounces.py b/mail/src/leap/mail/smtp/bounces.py
new file mode 100644
index 0000000..64f2dd7
--- /dev/null
+++ b/mail/src/leap/mail/smtp/bounces.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# bounces.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Deliver bounces to the user Inbox.
+"""
+import time
+from email.message import Message
+from email.utils import formatdate
+
+from leap.mail.constants import INBOX_NAME
+from leap.mail.mail import Account
+
+
+# TODO implement localization for this template.
+
+BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost.
+
+I'm sorry to have to inform you that your message could not be delivered to one
+or more recipients.
+
+The reasons I got for the error are:
+
+{raw_error}
+
+If the problem persists and it's not a network connectivity issue, you might
+want to contact your provider ({provider}) with this information (remove any
+sensitive data before).
+
+--- Original message (*before* it was encrypted by bitmask) below ----:
+
+{orig}"""
+
+
+class Bouncer(object):
+    """
+    Implements a mechanism to deliver bounces to user inbox.
+    """
+    # TODO this should follow RFC 6522, and compose a correct multipart
+    # attaching the report and the original message. Leaving this for a future
+    # iteration.
+
+    def __init__(self, inbox_collection):
+        self._inbox_collection = inbox_collection
+
+    def bounce_message(self, error_data, to, date=None, orig=''):
+        if not date:
+            date = formatdate(time.time())
+
+        raw_data = self._format_msg(error_data, to, date, orig)
+        d = self._inbox_collection.add_msg(
+            raw_data, ('\\Recent',), date=date)
+        return d
+
+    def _format_msg(self, error_data, to, date, orig):
+        provider = to.split('@')[1]
+
+        msg = Message()
+        msg.add_header(
+            'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)')
+        msg.add_header('To', to)
+        msg.add_header('Subject', 'Undelivered Message')
+        msg.add_header('Date', date)
+        msg.set_payload(BOUNCE_TEMPLATE.format(
+            raw_error=error_data,
+            provider=provider,
+            orig=orig))
+
+        return msg.as_string()
+
+
+def bouncerFactory(soledad):
+    acc = Account(soledad)
+    d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME))
+    d.addCallback(lambda inbox: Bouncer(inbox))
+    return d
diff --git a/mail/src/leap/mail/smtp/gateway.py b/mail/src/leap/mail/smtp/gateway.py
index 7ff6b14..cb1b060 100644
--- a/mail/src/leap/mail/smtp/gateway.py
+++ b/mail/src/leap/mail/smtp/gateway.py
@@ -29,7 +29,6 @@ The following classes comprise the SMTP gateway service:
     * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that
       knows how to encrypt/sign itself before sending.
 """
-
 from email.Header import Header
 
 from zope.interface import implements
@@ -48,6 +47,7 @@ from leap.mail.cred import LocalSoledadTokenChecker
 from leap.mail.utils import validate_address
 from leap.mail.rfc3156 import RFC3156CompliantGenerator
 from leap.mail.outgoing.service import outgoingFactory
+from leap.mail.smtp.bounces import bouncerFactory
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.keymanager.errors import KeyNotFound
 
@@ -65,7 +65,7 @@ class LocalSMTPRealm(object):
 
     _encoding = 'utf-8'
 
-    def __init__(self, keymanager_sessions, sendmail_opts,
+    def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts,
                  encrypted_only=False):
         """
         :param keymanager_sessions: a dict-like object, containing instances
@@ -73,21 +73,31 @@ class LocalSMTPRealm(object):
                                  userid.
         """
         self._keymanager_sessions = keymanager_sessions
+        self._soledad_sessions = soledad_sessions
         self._sendmail_opts = sendmail_opts
         self.encrypted_only = encrypted_only
 
     def requestAvatar(self, avatarId, mind, *interfaces):
+
         if isinstance(avatarId, str):
             avatarId = avatarId.decode(self._encoding)
 
-        def gotKeymanager(keymanager):
+        def gotKeymanagerAndSoledad(result):
+            keymanager, soledad = result
+            d = bouncerFactory(soledad)
+            d.addCallback(lambda bouncer: (keymanager, soledad, bouncer))
+            return d
 
+        def getMessageDelivery(result):
+            keymanager, soledad, bouncer = result
             # TODO use IMessageDeliveryFactory instead ?
             # it could reuse the connections.
             if smtp.IMessageDelivery in interfaces:
                 userid = avatarId
                 opts = self.getSendingOpts(userid)
-                outgoing = outgoingFactory(userid, keymanager, opts)
+
+                outgoing = outgoingFactory(
+                    userid, keymanager, opts, bouncer=bouncer)
                 avatar = SMTPDelivery(userid, keymanager, self.encrypted_only,
                                       outgoing)
 
@@ -96,10 +106,15 @@ class LocalSMTPRealm(object):
 
             raise NotImplementedError(self, interfaces)
 
-        return self.lookupKeymanagerInstance(avatarId).addCallback(
-            gotKeymanager)
+        d1 = self.lookupKeymanagerInstance(avatarId)
+        d2 = self.lookupSoledadInstance(avatarId)
+        d = defer.gatherResults([d1, d2])
+        d.addCallback(gotKeymanagerAndSoledad)
+        d.addCallback(getMessageDelivery)
+        return d
 
     def lookupKeymanagerInstance(self, userid):
+        print 'getting KM INSTNACE>>>'
         try:
             keymanager = self._keymanager_sessions[userid]
         except:
@@ -109,6 +124,16 @@ class LocalSMTPRealm(object):
         # XXX this should return the instance after whenReady callback
         return defer.succeed(keymanager)
 
+    def lookupSoledadInstance(self, userid):
+        try:
+            soledad = self._soledad_sessions[userid]
+        except:
+            raise errors.AuthenticationError(
+                'No soledad session found for user %s. Is it authenticated?'
+                % userid)
+        # XXX this should return the instance after whenReady callback
+        return defer.succeed(soledad)
+
     def getSendingOpts(self, userid):
         try:
             opts = self._sendmail_opts[userid]
@@ -134,8 +159,9 @@ class LEAPInitMixin(object):
     """
     def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts,
                  encrypted_only=False):
-        realm = LocalSMTPRealm(keymanager_sessions, sendmail_opts,
-                               encrypted_only)
+        realm = LocalSMTPRealm(
+            keymanager_sessions, soledad_sessions, sendmail_opts,
+            encrypted_only)
         portal = Portal(realm)
 
         checker = SMTPTokenChecker(soledad_sessions)
@@ -161,6 +187,7 @@ class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin):
         smtp.ESMTP.__init__(self, *args, **kw)
 
 
+# TODO implement retries -- see smtp.SenderMixin
 class SMTPFactory(protocol.ServerFactory):
     """
     Factory for an SMTP server with encrypted gatewaying capabilities.
@@ -171,7 +198,9 @@ class SMTPFactory(protocol.ServerFactory):
     timeout = 600
     encrypted_only = False
 
-    def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts):
+    def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts,
+                 deferred=None, retries=3):
+
         self._soledad_sessions = soledad_sessions
         self._keymanager_sessions = keymanager_sessions
         self._sendmail_opts = sendmail_opts