Merge branch 'release-0.3.5'0.3.5

3 files changed, 96 insertions, 25 deletions

diff --git a/CHANGELOG b/CHANGELOG
index cbd9c74..9de1ed7 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,7 @@
+0.3.5 Dec 10:
+  o Add X-Leap-Provenance header. Closes #4356.
+  o Add tester script to ease testing problematic emails offline.
+
 0.3.4 Nov 15:
   o Some mail may be skipped at processing because of possible
     problems (like connectivity issues to our couch nodes), MX now
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 3890eb2..c90dda9 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -105,7 +105,7 @@ class MailReceiver(Service):
         :param pubkey: public key for the owner of the message
         :type pubkey: str
         :param message: message contents
-        :type message: str
+        :type message: email.message.Message
 
         :return: doc to sync with Soledad or None, None if something
                  went wrong.
@@ -116,16 +116,19 @@ class MailReceiver(Service):
                     "I know: %r" % (pubkey,))
             return None
 
-        doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
-
-        encoding = get_email_charset(message.decode("utf8", "replace"),
-                                     default=None)
+        # find message's encoding
+        message_as_string = message.as_string()
+        encoding = get_email_charset(
+            message_as_string.decode("utf8", "replace"),
+            default=None)
         if encoding is None:
-            result = chardet.detect(message)
+            result = chardet.detect(message_as_string)
             encoding = result["encoding"]
 
-        data = {'incoming': True, 'content': message}
+        doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
 
+        # store plain text if pubkey is not available
+        data = {'incoming': True, 'content': message_as_string}
         if pubkey is None or len(pubkey) == 0:
             doc.content = {
                 self.INCOMING_KEY: True,
@@ -134,7 +137,7 @@ class MailReceiver(Service):
             }
             return doc
 
-        openpgp_key = None
+        # otherwise, encrypt
         with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
             gpg.import_keys(pubkey)
             key = gpg.list_keys().pop()
@@ -142,6 +145,15 @@ class MailReceiver(Service):
             # dummy one, we just care about the import of the pubkey
             openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey)
 
+            # add X-Leap-Provenance header if message is not encrypted
+            if message.get_content_type() != 'multipart/encrypted' and \
+                    '-----BEGIN PGP MESSAGE-----' not in \
+                    message_as_string:
+                message.add_header(
+                    'X-Leap-Provenance',
+                    email.utils.formatdate(),
+                    pubkey=openpgp_key.key_id)
+                data = {'incoming': True, 'content': message.as_string()}
             doc.content = {
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
@@ -254,20 +266,29 @@ class MailReceiver(Service):
             defer.returnValue(None)
 
         self._processing_skipped = True
-        log.msg("Starting processing skipped mail...")
-        log.msg("-"*50)
-
-        for directory, recursive in self._directories:
-            for root, dirs, files in os.walk(directory):
-                for fname in files:
-                    fullpath = os.path.join(root, fname)
-                    fpath = filepath.FilePath(fullpath)
-                    yield self._step_process_mail_backend(fpath)
-
-                if not recursive:
-                    break
+        try:
+            log.msg("Starting processing skipped mail...")
+            log.msg("-"*50)
+
+            for directory, recursive in self._directories:
+                for root, dirs, files in os.walk(directory):
+                    for fname in files:
+                        try:
+                            fullpath = os.path.join(root, fname)
+                            fpath = filepath.FilePath(fullpath)
+                            yield self._step_process_mail_backend(fpath)
+                        except Exception as e:
+                            log.msg("Error processing skipped mail: %r" % \
+                                    (fullpath,))
+                            log.err()
+                    if not recursive:
+                        break
+        except Exception as e:
+            log.msg("Error processing skipped mail")
+            log.err()
+        finally:
+            self._processing_skipped = False
 
-        self._processing_skipped = False
         log.msg("+"*50)
         log.msg("Done processing skipped mail")
 
@@ -284,8 +305,8 @@ class MailReceiver(Service):
         log.msg("Processing new mail at %r" % (filepath.path,))
         with filepath.open("r") as f:
             mail_data = f.read()
-            mail = message_from_string(mail_data)
-            uuid = self._get_owner(mail)
+            msg = message_from_string(mail_data)
+            uuid = self._get_owner(msg)
             if uuid is None:
                 log.msg("Don't know how to deliver mail %r, skipping..." %
                         (filepath.path,))
@@ -297,12 +318,12 @@ class MailReceiver(Service):
                 defer.returnValue(None)
 
             pubkey = yield self._users_cdb.getPubKey(uuid)
-            if pubkey is None or len(pubkey):
+            if pubkey is None or len(pubkey) == 0:
                 log.msg("No public key, stopping the processing chain")
                 defer.returnValue(None)
 
             log.msg("Encrypting message to %s's pubkey" % (uuid,))
-            doc = yield self._encrypt_message(pubkey, mail_data)
+            doc = yield self._encrypt_message(pubkey, msg)
 
             do_remove = yield self._export_message(uuid, doc)
             yield self._conditional_remove(do_remove, filepath)
diff --git a/src/leap/mx/tester.py b/src/leap/mx/tester.py
new file mode 100644
index 0000000..05d2d05
--- /dev/null
+++ b/src/leap/mx/tester.py
@@ -0,0 +1,46 @@
+import ConfigParser
+import sys
+import os
+
+from twisted.internet import reactor, defer
+from twisted.python import filepath, log
+
+from leap.mx import couchdbhelper
+from leap.mx.mail_receiver import MailReceiver
+
+if __name__ == "__main__":
+    log.startLogging(sys.stdout)
+    fullpath = os.path.realpath(sys.argv[1])
+
+    log.msg("Starting test for %s..." % (fullpath,))
+
+    config_file = "/etc/leap/mx.conf"
+
+    config = ConfigParser.ConfigParser()
+    config.read(config_file)
+
+    user = config.get("couchdb", "user")
+    password = config.get("couchdb", "password")
+
+    server = config.get("couchdb", "server")
+    port = config.get("couchdb", "port")
+
+    cdb = couchdbhelper.ConnectedCouchDB(server,
+                                         port=port,
+                                         dbName="identities",
+                                         username=user,
+                                         password=password)
+
+    # Mail receiver
+    mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user,
+                                                    password,
+                                                    server,
+                                                    port)
+
+    mr = MailReceiver(mail_couch_url_prefix, cdb, [])
+    fpath = filepath.FilePath(fullpath)
+
+    d = mr._process_incoming_email(None, fpath, 0)
+    d.addCallback(lambda x: reactor.stop())
+
+    reactor.run()