Merge tag '0.6.0' into debian/experimental

Tag leap.mx version 0.6.0

6 files changed, 195 insertions, 17 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 04833b4..87f6ef1 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,21 +1,29 @@
-0.3.6 Apr 4:
+0.6.0 Sept 26, 2014:
+  o Bounce mails when there's a problematic situation for an email,
+    such as no public key. Fixes #4803.
+  o Properly log tracebacks for exceptions occuring in the mail
+    processing loop.
+  o Add errdecr key defaulting to each mail Soledad Document. Closes
+    #6072.
+  o Any inconsistent user would make the mx stop decrypting mails for
+    all subsequent users.
+
+0.5.0 Apr 4, 2014:
   o Use CouchDocument to comply with new Soledad couch backend. Fixes
     #4475.
   o Some emails are multipart and each part has its own encoding.
 
--- 2014 --
-
-0.3.5 Dec 10:
+0.3.5 Dec 10, 2013:
   o Add X-Leap-Provenance header. Closes #4356.
   o Add tester script to ease testing problematic emails offline.
 
-0.3.4 Nov 15:
+0.3.4 Nov 15, 2013:
   o Some mail may be skipped at processing because of possible
     problems (like connectivity issues to our couch nodes), MX now
     looks for unprocessed mails every half hour and tries to process
     them. Fixes #3628.
 
-0.3.3 Nov 1:
+0.3.3 Nov 1, 2013:
   o Fix return codes for check recipient access. Fixes #3356.
   o Improve logging in general and support possible unicode parameters
     without breaking.
@@ -27,19 +35,19 @@
     mail headers, which improves performance.
   o Look for public keys based on uuid instead of mail address.
 
-0.3.2 Sep 6:
+0.3.2 Sep 6, 2013:
   o Keep file watcher in memory to prevent losing file events.
   o Properly save the incoming mail as a doc in couch.
   o Properly parse mail address of the form "Name <user@domain>". Fixes #3653.
 
-0.3.1 Aug 23:
+0.3.1 Aug 23, 2013:
   o Migrate mx functions to work on the new couchdb structure and
     views. Fixes #3502.
   o Update to new soledad package scheme (common, client and
     server). Closes #3487.
   o Add versioneer.
 
-0.3.0 Aug 9:
+0.3.0 Aug 9, 2013:
   o Give a return code for bare usernames too. Closes: #3405
   o Adapt to Soledad 0.2.1 API.
   o Fix broken pip install
diff --git a/README.md b/README.md
index c467496..03b0ade 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,9 @@ $ git clone https://github.com/leapcode/leap_mx.git
 ~~~
 Although, **it is advised** to install inside a python virtualenv.
 
+## [configuration](#configuration) ##
+A sample config file can be found in pkg/mx.conf.sample
+
 ## [running](#running) ##
 =========================
 
@@ -78,4 +81,3 @@ Our bugtracker is [here](https://leap.se/code/projects/eip/issue/new).
 Please use that for bug reports and feature requests instead of github's
 tracker. We're using github for code commenting and review between
 collaborators.
-
diff --git a/pkg/leap_mx.tac b/pkg/leap_mx.tac
index c101de9..75d2405 100644
--- a/pkg/leap_mx.tac
+++ b/pkg/leap_mx.tac
@@ -46,6 +46,15 @@ password = config.get("couchdb", "password")
 server = config.get("couchdb", "server")
 port = config.get("couchdb", "port")
 
+bounce_from = "bounce"
+bounce_subject = "Delivery failure"
+
+try:
+    bounce_from = config.get("bounce", "from")
+    bounce_subject = config.get("bounce", "subject")
+except ConfigParser.NoSectionError:
+    pass  # we use the defaults above
+
 alias_port = config.getint("alias map", "port")
 check_recipient_port = config.getint("check recipient", "port")
 
@@ -74,11 +83,12 @@ mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user,
                                                 port)
 directories = []
 for section in config.sections():
-    if section in ("couchdb", "alias map", "check recipient"):
+    if section in ("couchdb", "alias map", "check recipient", "bounce"):
         continue
     to_watch = config.get(section, "path")
     recursive = config.getboolean(section, "recursive")
     directories.append([to_watch, recursive])
 
-mr = MailReceiver(mail_couch_url_prefix, cdb, directories)
+mr = MailReceiver(mail_couch_url_prefix, cdb, directories, bounce_from,
+                  bounce_subject)
 mr.setServiceParent(application)
diff --git a/pkg/mx.conf.sample b/pkg/mx.conf.sample
new file mode 100644
index 0000000..c9ad0f8
--- /dev/null
+++ b/pkg/mx.conf.sample
@@ -0,0 +1,19 @@
+[mail1]
+path=/path/to/Maildir/
+recursive=<whether to analyze the above path recursively or not (True/False)>
+
+[couchdb]
+user=<couch user>
+password=<password>
+server=localhost
+port=6666
+
+[alias map]
+port=4242
+
+[check recipient]
+port=2244
+
+[bounce]
+from=<address for the From: of the bounce email without domain>
+subject=Delivery failure
\ No newline at end of file
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index 41604ba..f20f1dd 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -171,7 +171,11 @@ class ConnectedCouchDB(client.CouchDB):
         :rtype: str or None
         """
         for row in result["rows"]:
-            if row["doc"]["user_id"] == uuid:
+            user_id = row["doc"].get("user_id")
+            if not user_id:
+                print("User %s is in an inconsistent state")
+                continue
+            if user_id == uuid:
                 return row["value"]
         return None
 
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 3561d33..dd76f08 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -18,6 +18,16 @@
 
 """
 MailReceiver service definition
+
+If there's a user facing problem when processing an email, it will be
+bounced back to the sender.
+
+User facing problems could be:
+- Unknown user (missing uuid)
+- Public key not found
+
+Any other problem is a bug, which will be logged. Until the bug is
+fixed, the email will stay in there waiting.
 """
 
 import os
@@ -28,9 +38,13 @@ import email.utils
 import socket
 
 from email import message_from_string
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEText import MIMEText
+from email.Utils import formatdate
+from email.header import decode_header
 
 from twisted.application.service import Service
-from twisted.internet import inotify, defer, task
+from twisted.internet import inotify, defer, task, reactor
 from twisted.python import filepath, log
 
 from leap.soledad.common.crypto import (
@@ -41,6 +55,77 @@ from leap.soledad.common.crypto import (
 from leap.soledad.common.couch import CouchDatabase, CouchDocument
 from leap.keymanager import openpgp
 
+BOUNCE_TEMPLATE = """
+Delivery to the following recipient failed:
+    {0}
+
+Reasons:
+    {1}
+
+Original message:
+    {2}
+""".strip()
+
+
+from twisted.internet import protocol
+from twisted.internet.error import ProcessDone
+
+
+class BouncerSubprocessProtocol(protocol.ProcessProtocol):
+    """
+    Bouncer subprocess protocol that will feed the msg contents to be
+    bounced through stdin
+    """
+
+    def __init__(self, msg):
+        """
+        Constructor for the BouncerSubprocessProtocol
+
+        :param msg: Message to send to stdin when the process has
+                    launched
+        :type msg: str
+        """
+        self._msg = msg
+        self._outBuffer = ""
+        self._errBuffer = ""
+        self._d = None
+
+    def connectionMade(self):
+        self._d = defer.Deferred()
+
+        self.transport.write(self._msg)
+        self.transport.closeStdin()
+
+    def outReceived(self, data):
+        self._outBuffer += data
+
+    def errReceived(self, data):
+        self._errBuffer += data
+
+    def processEnded(self, reason):
+        if reason.check(ProcessDone):
+            self._d.callback(self._outBuffer)
+        else:
+            self._d.errback(reason)
+
+
+def async_check_output(args, msg):
+    """
+    Async spawn a process and return a defer to be able to check the
+    output with a callback/errback
+
+    :param args: the command to execute along with the params for it
+    :type args: list of str
+    :param msg: string that will be send to stdin of the process once
+                it's spawned
+    :type msg: str
+
+    :rtype: defer.Deferred
+    """
+    pprotocol = BouncerSubprocessProtocol(msg)
+    reactor.spawnProcess(pprotocol, args[0], args)
+    return pprotocol.d
+
 
 class MailReceiver(Service):
     """
@@ -48,8 +133,10 @@ class MailReceiver(Service):
     """
 
     INCOMING_KEY = 'incoming'
+    ERROR_DECRYPTING_KEY = "errdecr"
 
-    def __init__(self, mail_couch_url, users_cdb, directories):
+    def __init__(self, mail_couch_url, users_cdb, directories, bounce_from,
+                 bounce_subject):
         """
         Constructor
 
@@ -61,6 +148,10 @@ class MailReceiver(Service):
         :type users_cdb: ConnectedCouchDB
         :param directories: list of directories to monitor
         :type directories: list of tuples (path: str, recursive: bool)
+        :param bounce_from: Email address of the bouncer
+        :type bounce_from: str
+        :param bounce_subject: Subject line used in the bounced mail
+        :type bounce_subject: str
         """
         # Service doesn't define an __init__
         self._mail_couch_url = mail_couch_url
@@ -69,6 +160,9 @@ class MailReceiver(Service):
         self._domain = socket.gethostbyaddr(socket.gethostname())[0]
         self._processing_skipped = False
 
+        self._bounce_from = bounce_from
+        self._bounce_subject = bounce_subject
+
     def startService(self):
         """
         Starts the MailReceiver service
@@ -119,6 +213,7 @@ class MailReceiver(Service):
         if pubkey is None or len(pubkey) == 0:
             doc.content = {
                 self.INCOMING_KEY: True,
+                self.ERROR_DECRYPTING_KEY: False,
                 ENC_SCHEME_KEY: EncryptionSchemes.NONE,
                 ENC_JSON_KEY: json.dumps(data,
                                          ensure_ascii=False)
@@ -145,6 +240,7 @@ class MailReceiver(Service):
                 data = {'incoming': True, 'content': message.as_string()}
             doc.content = {
                 self.INCOMING_KEY: True,
+                self.ERROR_DECRYPTING_KEY: False,
                 ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
                 ENC_JSON_KEY: str(gpg.encrypt(
                     json.dumps(data, ensure_ascii=False),
@@ -228,6 +324,37 @@ class MailReceiver(Service):
 
         return uuid
 
+    @defer.inlineCallbacks
+    def _bounce_mail(self, orig_msg, filepath, reason):
+        """
+        Bounces the email contained in orig_msg to it's sender and
+        removes it from the queue.
+
+        :param orig_msg: Message that is going to be bounced
+        :type orig_msg: email.message.Message
+        :param filepath: Path for that message
+        :type filepath: twisted.python.filepath.FilePath
+        :param reason: Brief explanation about why it's being bounced
+        :type reason: str
+        """
+        to = orig_msg.get("From")
+
+        msg = MIMEMultipart()
+        msg['From'] = self._bounce_from
+        msg['To'] = to
+        msg['Date'] = formatdate(localtime=True)
+        msg['Subject'] = self._bounce_subject
+
+        decoded_to = " ".join([x[0] for x in decode_header(to)])
+        text = BOUNCE_TEMPLATE.format(decoded_to,
+                                      reason,
+                                      orig_msg.as_string())
+
+        msg.attach(MIMEText(text))
+
+        yield async_check_output(["/usr/sbin/sendmail", "-t"], msg.as_string())
+        yield self._conditional_remove(True, filepath)
+
     def sleep(self, secs):
         """
         Async sleep for a defer. Use this when you want to wait for
@@ -266,13 +393,13 @@ class MailReceiver(Service):
                             fullpath = os.path.join(root, fname)
                             fpath = filepath.FilePath(fullpath)
                             yield self._step_process_mail_backend(fpath)
-                        except Exception as e:
+                        except Exception:
                             log.msg("Error processing skipped mail: %r" % \
                                     (fullpath,))
                             log.err()
                     if not recursive:
                         break
-        except Exception as e:
+        except Exception:
             log.msg("Error processing skipped mail")
             log.err()
         finally:
@@ -299,6 +426,9 @@ class MailReceiver(Service):
             if uuid is None:
                 log.msg("Don't know how to deliver mail %r, skipping..." %
                         (filepath.path,))
+                bounce_reason = "Missing UUID: There was a problem " \
+                                "locating the user in our database."
+                yield self._bounce_mail(msg, filepath, bounce_reason)
                 defer.returnValue(None)
             log.msg("Mail owner: %s" % (uuid,))
 
@@ -309,6 +439,10 @@ class MailReceiver(Service):
             pubkey = yield self._users_cdb.getPubKey(uuid)
             if pubkey is None or len(pubkey) == 0:
                 log.msg("No public key, stopping the processing chain")
+                bounce_reason = "Missing PubKey: There was a problem " \
+                                "locating the user's public key in our " \
+                                "database."
+                yield self._bounce_mail(msg, filepath, bounce_reason)
                 defer.returnValue(None)
 
             log.msg("Encrypting message to %s's pubkey" % (uuid,))
@@ -340,3 +474,4 @@ class MailReceiver(Service):
         except Exception as e:
             log.msg("Something went wrong while processing {0!r}: {1!r}"
                     .format(filepath, e))
+            log.err()