From 5f2f8a65aba2672ae6661052af2f735ba17ad7a5 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 11 Sep 2015 11:43:31 +0200 Subject: [feat] Don't add any X-Leap-Provenance header - Resolves: #7439 --- src/leap/mx/mail_receiver.py | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 446fd38..697bd13 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -203,16 +203,6 @@ class MailReceiver(Service): with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: gpg.import_keys(pubkey) key = gpg.list_keys().pop() - - # add X-Leap-Provenance header if message is not encrypted - if message.get_content_type() != 'multipart/encrypted' and \ - '-----BEGIN PGP MESSAGE-----' not in \ - message_as_string: - message.add_header( - 'X-Leap-Provenance', - email.utils.formatdate(), - pubkey=key["keyid"]) - data = {'incoming': True, 'content': message.as_string()} doc.content = { self.INCOMING_KEY: True, self.ERROR_DECRYPTING_KEY: False, -- cgit v1.2.3 From 8384b006f575ac0b769f9f6e9ce6b2c623ec9fa1 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 11 Sep 2015 11:45:17 +0200 Subject: [test] add test infrastructure Some refactor on the couchdb usage was needed to be able to mock couchdb. - Resolves: #7435 --- src/leap/mx/mail_receiver.py | 55 +++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 36 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 697bd13..4554624 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -51,7 +51,7 @@ from zope.interface import implements from leap.soledad.common.crypto import EncryptionSchemes from leap.soledad.common.crypto import ENC_JSON_KEY from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.couch import CouchDatabase, CouchDocument +from leap.soledad.common.couch import CouchDocument from leap.keymanager import openpgp @@ -75,15 +75,11 @@ class MailReceiver(Service): """ RETRY_DIR_WATCH_DELAY = 60 * 5 # 5 minutes - def __init__(self, mail_couch_url, users_cdb, directories, bounce_from, + def __init__(self, users_cdb, directories, bounce_from, bounce_subject): """ Constructor - :param mail_couch_url: URL prefix for the couchdb where mail - should be stored - :type mail_couch_url: str - :param users_cdb: CouchDB instance from where to get the uuid and pubkey for a user :type users_cdb: ConnectedCouchDB @@ -98,7 +94,6 @@ class MailReceiver(Service): :type bounce_subject: str """ # IService doesn't define an __init__ - self._mail_couch_url = mail_couch_url self._users_cdb = users_cdb self._directories = directories self._bounce_from = bounce_from @@ -215,6 +210,7 @@ class MailReceiver(Service): return doc + @defer.inlineCallbacks def _export_message(self, uuid, doc): """ Given a UUID and a CouchDocument, it saves it directly in the @@ -226,44 +222,32 @@ class MailReceiver(Service): :param doc: CouchDocument that represents the email :type doc: CouchDocument - :return: True if it's ok to remove the message, False - otherwise - :rtype: bool + :return: A Deferred which fires if it's ok to remove the message, + or fails otherwise + :rtype: Deferred """ if uuid is None or doc is None: log.msg("_export_message: Something went wrong, here's all " "I know: %r | %r" % (uuid, doc)) - return False + raise Exception("No uuid or doc") log.msg("Exporting message for %s" % (uuid,)) - - db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,)) - db.put_doc(doc) - + yield self._users_cdb.put_doc(uuid, doc) log.msg("Done exporting") - return True - - def _conditional_remove(self, do_remove, filepath): + def _remove(self, filepath): """ - Removes the message if do_remove is True. + Removes the message. - :param do_remove: True if the message should be removed, False - otherwise - :type do_remove: bool :param filepath: path to the mail :type filepath: twisted.python.filepath.FilePath """ - if do_remove: - # remove the original mail - try: - log.msg("Removing %r" % (filepath.path,)) - filepath.remove() - log.msg("Done removing") - except Exception: - log.err() - else: - log.msg("Not removing %r" % (filepath.path,)) + try: + log.msg("Removing %r" % (filepath.path,)) + filepath.remove() + log.msg("Done removing") + except Exception: + log.err() def _get_owner(self, mail): """ @@ -307,7 +291,7 @@ class MailReceiver(Service): except InvalidReturnPathError: # give up bouncing this message! log.msg("Will not bounce message because of invalid return path.") - yield self._conditional_remove(True, filepath) + yield self._remove(filepath) def sleep(self, secs): """ @@ -403,8 +387,8 @@ class MailReceiver(Service): log.msg("Encrypting message to %s's pubkey" % (uuid,)) doc = yield self._encrypt_message(pubkey, msg) - do_remove = yield self._export_message(uuid, doc) - yield self._conditional_remove(do_remove, filepath) + yield self._export_message(uuid, doc) + yield self._remove(filepath) @defer.inlineCallbacks def _process_incoming_email(self, otherself, filepath, mask): @@ -430,4 +414,3 @@ class MailReceiver(Service): except Exception as e: log.msg("Something went wrong while processing {0!r}: {1!r}" .format(filepath, e)) - log.err() -- cgit v1.2.3 From 17545618c79478a104ca7be5dd601020f8749780 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Tue, 3 Nov 2015 19:38:23 +0100 Subject: [feat] update leap.mx usage of soledad CouchDatabase - Related: #7565 --- src/leap/mx/mail_receiver.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 4554624..a9344c3 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -51,7 +51,7 @@ from zope.interface import implements from leap.soledad.common.crypto import EncryptionSchemes from leap.soledad.common.crypto import ENC_JSON_KEY from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.couch import CouchDocument +from leap.soledad.common.document import ServerDocument from leap.keymanager import openpgp @@ -170,7 +170,7 @@ class MailReceiver(Service): :return: doc to sync with Soledad or None, None if something went wrong. - :rtype: CouchDocument + :rtype: ServerDocument """ if pubkey is None or len(pubkey) == 0: log.msg("_encrypt_message: Something went wrong, here's all " @@ -180,7 +180,7 @@ class MailReceiver(Service): # find message's encoding message_as_string = message.as_string() - doc = CouchDocument(doc_id=str(pyuuid.uuid4())) + doc = ServerDocument(doc_id=str(pyuuid.uuid4())) # store plain text if pubkey is not available data = {'incoming': True, 'content': message_as_string} @@ -213,14 +213,14 @@ class MailReceiver(Service): @defer.inlineCallbacks def _export_message(self, uuid, doc): """ - Given a UUID and a CouchDocument, it saves it directly in the + Given a UUID and a ServerDocument, it saves it directly in the couchdb that serves as a backend for Soledad, in a db accessible to the recipient of the mail. :param uuid: the mail owner's uuid :type uuid: str - :param doc: CouchDocument that represents the email - :type doc: CouchDocument + :param doc: ServerDocument that represents the email + :type doc: ServerDocument :return: A Deferred which fires if it's ok to remove the message, or fails otherwise -- cgit v1.2.3 From 6fd862abc9e8711bd4aa117697c5df85cf6f560a Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 28 Oct 2015 18:52:34 +0100 Subject: [feat] add '@deliver.local' to uuid - Resolves: #5959 --- src/leap/mx/mail_receiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index a9344c3..ea13658 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- # mail_receiver.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -269,7 +269,7 @@ class MailReceiver(Service): return None final_address = delivereds.pop(0) _, addr = email.utils.parseaddr(final_address) - uuid, _ = addr.split("@") + uuid = addr.split("@")[0] return uuid @defer.inlineCallbacks -- cgit v1.2.3 From b26643ea9848869ece6ead2ba6c0991333d3c6f4 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 25 Nov 2015 19:15:33 +0100 Subject: [feat] use the original message for encryption Do not reconstruct the message from the parsed one. Added test for missleading encoding. - Resolves: #7253 --- src/leap/mx/mail_receiver.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index ea13658..4d82849 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -166,7 +166,7 @@ class MailReceiver(Service): :param pubkey: public key for the owner of the message :type pubkey: str :param message: message contents - :type message: email.message.Message + :type message: str :return: doc to sync with Soledad or None, None if something went wrong. @@ -177,13 +177,10 @@ class MailReceiver(Service): "I know: %r" % (pubkey,)) return None - # find message's encoding - message_as_string = message.as_string() - doc = ServerDocument(doc_id=str(pyuuid.uuid4())) # store plain text if pubkey is not available - data = {'incoming': True, 'content': message_as_string} + data = {'incoming': True, 'content': message} if pubkey is None or len(pubkey) == 0: doc.content = { self.INCOMING_KEY: True, @@ -385,7 +382,7 @@ class MailReceiver(Service): defer.returnValue(None) log.msg("Encrypting message to %s's pubkey" % (uuid,)) - doc = yield self._encrypt_message(pubkey, msg) + doc = yield self._encrypt_message(pubkey, mail_data) yield self._export_message(uuid, doc) yield self._remove(filepath) -- cgit v1.2.3 From 8d45bc5b692801fc3df425d497a7210f84dae123 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 7 Apr 2016 18:41:20 +0200 Subject: [style] remove unused code --- src/leap/mx/mail_receiver.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 4d82849..69079be 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -366,10 +366,6 @@ class MailReceiver(Service): defer.returnValue(None) log.msg("Mail owner: %s" % (uuid,)) - if uuid is None: - log.msg("BUG: There was no uuid!") - defer.returnValue(None) - pubkey = yield self._users_cdb.getPubkey(uuid) if pubkey is None or len(pubkey) == 0: log.msg( -- cgit v1.2.3 From db3b1369ce12efea2c3d06d0671bfa655ef49571 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 7 Apr 2016 19:22:31 +0200 Subject: [feature] Bounce stalled emails after a timeout. * Resolves: #7998 --- src/leap/mx/mail_receiver.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'src/leap/mx/mail_receiver.py') diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 69079be..7c5a368 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -40,6 +40,7 @@ import signal import json import email.utils +from datetime import datetime, timedelta from email import message_from_string from twisted.application.service import Service, IService @@ -75,6 +76,11 @@ class MailReceiver(Service): """ RETRY_DIR_WATCH_DELAY = 60 * 5 # 5 minutes + """ + Time delta to keep stalled emails + """ + MAX_BOUNCE_DELTA = timedelta(days=5) + def __init__(self, users_cdb, directories, bounce_from, bounce_subject): """ @@ -98,6 +104,7 @@ class MailReceiver(Service): self._directories = directories self._bounce_from = bounce_from self._bounce_subject = bounce_subject + self._bounce_timestamp = {} self._processing_skipped = False def startService(self): @@ -378,10 +385,31 @@ class MailReceiver(Service): defer.returnValue(None) log.msg("Encrypting message to %s's pubkey" % (uuid,)) - doc = yield self._encrypt_message(pubkey, mail_data) + try: + doc = yield self._encrypt_message(pubkey, mail_data) + + yield self._export_message(uuid, doc) + yield self._remove(filepath) + except Exception as e: + yield self._bounce_with_timeout(filepath, msg, e) + + @defer.inlineCallbacks + def _bounce_with_timeout(self, filepath, msg, error): + if filepath not in self._bounce_timestamp: + self._bounce_timestamp[filepath] = datetime.now() + log.msg("New stalled email {0!r}: {1!r}".format(filepath, error)) + defer.returnValue(None) - yield self._export_message(uuid, doc) - yield self._remove(filepath) + current_delta = datetime.now() - self._bounce_timestamp[filepath] + if current_delta > self.MAX_BOUNCE_DELTA: + log.msg("Bouncing stalled email {0!r}: {1!r}" + .format(filepath, error)) + bounce_reason = "There was a problem in the server and the " \ + "email could not be delivered." + yield self._bounce_message(msg, filepath, bounce_reason) + else: + log.msg("Still stalled email {0!r} for the last {1}: {2!r}" + .format(filepath, str(current_delta), error)) @defer.inlineCallbacks def _process_incoming_email(self, otherself, filepath, mask): -- cgit v1.2.3