summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKali Kaneko <kali@futeisha.org>2014-11-11 21:53:25 +0100
committerKali Kaneko <kali@futeisha.org>2014-11-11 21:53:25 +0100
commit20b7b701508bf6e2c9a9cee38db9757dd07e52fe (patch)
tree1101ed78a557bbf41c29e2d6ff00849a45cbc568
parent0420989d62c7508e31f946ca7b52e98b87cc2bcb (diff)
parentd8893584886a593dd6ddcfa1a6930288c7fd8ecc (diff)
Merge tag '0.6.0' into debian/experimental
Tag leap.mx version 0.6.0
-rw-r--r--CHANGELOG26
-rw-r--r--README.md4
-rw-r--r--pkg/leap_mx.tac14
-rw-r--r--pkg/mx.conf.sample19
-rw-r--r--src/leap/mx/couchdbhelper.py6
-rw-r--r--src/leap/mx/mail_receiver.py143
6 files changed, 195 insertions, 17 deletions
diff --git a/CHANGELOG b/CHANGELOG
index 04833b4..87f6ef1 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,21 +1,29 @@
-0.3.6 Apr 4:
+0.6.0 Sept 26, 2014:
+ o Bounce mails when there's a problematic situation for an email,
+ such as no public key. Fixes #4803.
+ o Properly log tracebacks for exceptions occuring in the mail
+ processing loop.
+ o Add errdecr key defaulting to each mail Soledad Document. Closes
+ #6072.
+ o Any inconsistent user would make the mx stop decrypting mails for
+ all subsequent users.
+
+0.5.0 Apr 4, 2014:
o Use CouchDocument to comply with new Soledad couch backend. Fixes
#4475.
o Some emails are multipart and each part has its own encoding.
--- 2014 --
-
-0.3.5 Dec 10:
+0.3.5 Dec 10, 2013:
o Add X-Leap-Provenance header. Closes #4356.
o Add tester script to ease testing problematic emails offline.
-0.3.4 Nov 15:
+0.3.4 Nov 15, 2013:
o Some mail may be skipped at processing because of possible
problems (like connectivity issues to our couch nodes), MX now
looks for unprocessed mails every half hour and tries to process
them. Fixes #3628.
-0.3.3 Nov 1:
+0.3.3 Nov 1, 2013:
o Fix return codes for check recipient access. Fixes #3356.
o Improve logging in general and support possible unicode parameters
without breaking.
@@ -27,19 +35,19 @@
mail headers, which improves performance.
o Look for public keys based on uuid instead of mail address.
-0.3.2 Sep 6:
+0.3.2 Sep 6, 2013:
o Keep file watcher in memory to prevent losing file events.
o Properly save the incoming mail as a doc in couch.
o Properly parse mail address of the form "Name <user@domain>". Fixes #3653.
-0.3.1 Aug 23:
+0.3.1 Aug 23, 2013:
o Migrate mx functions to work on the new couchdb structure and
views. Fixes #3502.
o Update to new soledad package scheme (common, client and
server). Closes #3487.
o Add versioneer.
-0.3.0 Aug 9:
+0.3.0 Aug 9, 2013:
o Give a return code for bare usernames too. Closes: #3405
o Adapt to Soledad 0.2.1 API.
o Fix broken pip install
diff --git a/README.md b/README.md
index c467496..03b0ade 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,9 @@ $ git clone https://github.com/leapcode/leap_mx.git
~~~
Although, **it is advised** to install inside a python virtualenv.
+## [configuration](#configuration) ##
+A sample config file can be found in pkg/mx.conf.sample
+
## [running](#running) ##
=========================
@@ -78,4 +81,3 @@ Our bugtracker is [here](https://leap.se/code/projects/eip/issue/new).
Please use that for bug reports and feature requests instead of github's
tracker. We're using github for code commenting and review between
collaborators.
-
diff --git a/pkg/leap_mx.tac b/pkg/leap_mx.tac
index c101de9..75d2405 100644
--- a/pkg/leap_mx.tac
+++ b/pkg/leap_mx.tac
@@ -46,6 +46,15 @@ password = config.get("couchdb", "password")
server = config.get("couchdb", "server")
port = config.get("couchdb", "port")
+bounce_from = "bounce"
+bounce_subject = "Delivery failure"
+
+try:
+ bounce_from = config.get("bounce", "from")
+ bounce_subject = config.get("bounce", "subject")
+except ConfigParser.NoSectionError:
+ pass # we use the defaults above
+
alias_port = config.getint("alias map", "port")
check_recipient_port = config.getint("check recipient", "port")
@@ -74,11 +83,12 @@ mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user,
port)
directories = []
for section in config.sections():
- if section in ("couchdb", "alias map", "check recipient"):
+ if section in ("couchdb", "alias map", "check recipient", "bounce"):
continue
to_watch = config.get(section, "path")
recursive = config.getboolean(section, "recursive")
directories.append([to_watch, recursive])
-mr = MailReceiver(mail_couch_url_prefix, cdb, directories)
+mr = MailReceiver(mail_couch_url_prefix, cdb, directories, bounce_from,
+ bounce_subject)
mr.setServiceParent(application)
diff --git a/pkg/mx.conf.sample b/pkg/mx.conf.sample
new file mode 100644
index 0000000..c9ad0f8
--- /dev/null
+++ b/pkg/mx.conf.sample
@@ -0,0 +1,19 @@
+[mail1]
+path=/path/to/Maildir/
+recursive=<whether to analyze the above path recursively or not (True/False)>
+
+[couchdb]
+user=<couch user>
+password=<password>
+server=localhost
+port=6666
+
+[alias map]
+port=4242
+
+[check recipient]
+port=2244
+
+[bounce]
+from=<address for the From: of the bounce email without domain>
+subject=Delivery failure \ No newline at end of file
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index 41604ba..f20f1dd 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -171,7 +171,11 @@ class ConnectedCouchDB(client.CouchDB):
:rtype: str or None
"""
for row in result["rows"]:
- if row["doc"]["user_id"] == uuid:
+ user_id = row["doc"].get("user_id")
+ if not user_id:
+ print("User %s is in an inconsistent state")
+ continue
+ if user_id == uuid:
return row["value"]
return None
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 3561d33..dd76f08 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -18,6 +18,16 @@
"""
MailReceiver service definition
+
+If there's a user facing problem when processing an email, it will be
+bounced back to the sender.
+
+User facing problems could be:
+- Unknown user (missing uuid)
+- Public key not found
+
+Any other problem is a bug, which will be logged. Until the bug is
+fixed, the email will stay in there waiting.
"""
import os
@@ -28,9 +38,13 @@ import email.utils
import socket
from email import message_from_string
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEText import MIMEText
+from email.Utils import formatdate
+from email.header import decode_header
from twisted.application.service import Service
-from twisted.internet import inotify, defer, task
+from twisted.internet import inotify, defer, task, reactor
from twisted.python import filepath, log
from leap.soledad.common.crypto import (
@@ -41,6 +55,77 @@ from leap.soledad.common.crypto import (
from leap.soledad.common.couch import CouchDatabase, CouchDocument
from leap.keymanager import openpgp
+BOUNCE_TEMPLATE = """
+Delivery to the following recipient failed:
+ {0}
+
+Reasons:
+ {1}
+
+Original message:
+ {2}
+""".strip()
+
+
+from twisted.internet import protocol
+from twisted.internet.error import ProcessDone
+
+
+class BouncerSubprocessProtocol(protocol.ProcessProtocol):
+ """
+ Bouncer subprocess protocol that will feed the msg contents to be
+ bounced through stdin
+ """
+
+ def __init__(self, msg):
+ """
+ Constructor for the BouncerSubprocessProtocol
+
+ :param msg: Message to send to stdin when the process has
+ launched
+ :type msg: str
+ """
+ self._msg = msg
+ self._outBuffer = ""
+ self._errBuffer = ""
+ self._d = None
+
+ def connectionMade(self):
+ self._d = defer.Deferred()
+
+ self.transport.write(self._msg)
+ self.transport.closeStdin()
+
+ def outReceived(self, data):
+ self._outBuffer += data
+
+ def errReceived(self, data):
+ self._errBuffer += data
+
+ def processEnded(self, reason):
+ if reason.check(ProcessDone):
+ self._d.callback(self._outBuffer)
+ else:
+ self._d.errback(reason)
+
+
+def async_check_output(args, msg):
+ """
+ Async spawn a process and return a defer to be able to check the
+ output with a callback/errback
+
+ :param args: the command to execute along with the params for it
+ :type args: list of str
+ :param msg: string that will be send to stdin of the process once
+ it's spawned
+ :type msg: str
+
+ :rtype: defer.Deferred
+ """
+ pprotocol = BouncerSubprocessProtocol(msg)
+ reactor.spawnProcess(pprotocol, args[0], args)
+ return pprotocol.d
+
class MailReceiver(Service):
"""
@@ -48,8 +133,10 @@ class MailReceiver(Service):
"""
INCOMING_KEY = 'incoming'
+ ERROR_DECRYPTING_KEY = "errdecr"
- def __init__(self, mail_couch_url, users_cdb, directories):
+ def __init__(self, mail_couch_url, users_cdb, directories, bounce_from,
+ bounce_subject):
"""
Constructor
@@ -61,6 +148,10 @@ class MailReceiver(Service):
:type users_cdb: ConnectedCouchDB
:param directories: list of directories to monitor
:type directories: list of tuples (path: str, recursive: bool)
+ :param bounce_from: Email address of the bouncer
+ :type bounce_from: str
+ :param bounce_subject: Subject line used in the bounced mail
+ :type bounce_subject: str
"""
# Service doesn't define an __init__
self._mail_couch_url = mail_couch_url
@@ -69,6 +160,9 @@ class MailReceiver(Service):
self._domain = socket.gethostbyaddr(socket.gethostname())[0]
self._processing_skipped = False
+ self._bounce_from = bounce_from
+ self._bounce_subject = bounce_subject
+
def startService(self):
"""
Starts the MailReceiver service
@@ -119,6 +213,7 @@ class MailReceiver(Service):
if pubkey is None or len(pubkey) == 0:
doc.content = {
self.INCOMING_KEY: True,
+ self.ERROR_DECRYPTING_KEY: False,
ENC_SCHEME_KEY: EncryptionSchemes.NONE,
ENC_JSON_KEY: json.dumps(data,
ensure_ascii=False)
@@ -145,6 +240,7 @@ class MailReceiver(Service):
data = {'incoming': True, 'content': message.as_string()}
doc.content = {
self.INCOMING_KEY: True,
+ self.ERROR_DECRYPTING_KEY: False,
ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
ENC_JSON_KEY: str(gpg.encrypt(
json.dumps(data, ensure_ascii=False),
@@ -228,6 +324,37 @@ class MailReceiver(Service):
return uuid
+ @defer.inlineCallbacks
+ def _bounce_mail(self, orig_msg, filepath, reason):
+ """
+ Bounces the email contained in orig_msg to it's sender and
+ removes it from the queue.
+
+ :param orig_msg: Message that is going to be bounced
+ :type orig_msg: email.message.Message
+ :param filepath: Path for that message
+ :type filepath: twisted.python.filepath.FilePath
+ :param reason: Brief explanation about why it's being bounced
+ :type reason: str
+ """
+ to = orig_msg.get("From")
+
+ msg = MIMEMultipart()
+ msg['From'] = self._bounce_from
+ msg['To'] = to
+ msg['Date'] = formatdate(localtime=True)
+ msg['Subject'] = self._bounce_subject
+
+ decoded_to = " ".join([x[0] for x in decode_header(to)])
+ text = BOUNCE_TEMPLATE.format(decoded_to,
+ reason,
+ orig_msg.as_string())
+
+ msg.attach(MIMEText(text))
+
+ yield async_check_output(["/usr/sbin/sendmail", "-t"], msg.as_string())
+ yield self._conditional_remove(True, filepath)
+
def sleep(self, secs):
"""
Async sleep for a defer. Use this when you want to wait for
@@ -266,13 +393,13 @@ class MailReceiver(Service):
fullpath = os.path.join(root, fname)
fpath = filepath.FilePath(fullpath)
yield self._step_process_mail_backend(fpath)
- except Exception as e:
+ except Exception:
log.msg("Error processing skipped mail: %r" % \
(fullpath,))
log.err()
if not recursive:
break
- except Exception as e:
+ except Exception:
log.msg("Error processing skipped mail")
log.err()
finally:
@@ -299,6 +426,9 @@ class MailReceiver(Service):
if uuid is None:
log.msg("Don't know how to deliver mail %r, skipping..." %
(filepath.path,))
+ bounce_reason = "Missing UUID: There was a problem " \
+ "locating the user in our database."
+ yield self._bounce_mail(msg, filepath, bounce_reason)
defer.returnValue(None)
log.msg("Mail owner: %s" % (uuid,))
@@ -309,6 +439,10 @@ class MailReceiver(Service):
pubkey = yield self._users_cdb.getPubKey(uuid)
if pubkey is None or len(pubkey) == 0:
log.msg("No public key, stopping the processing chain")
+ bounce_reason = "Missing PubKey: There was a problem " \
+ "locating the user's public key in our " \
+ "database."
+ yield self._bounce_mail(msg, filepath, bounce_reason)
defer.returnValue(None)
log.msg("Encrypting message to %s's pubkey" % (uuid,))
@@ -340,3 +474,4 @@ class MailReceiver(Service):
except Exception as e:
log.msg("Something went wrong while processing {0!r}: {1!r}"
.format(filepath, e))
+ log.err()