From 33bbf32cc90e05f63f4c68ea2feff90855d2f7e5 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 25 Nov 2013 11:25:27 -0200 Subject: Add X-Leap-Provenance header. Closes #4356. --- changes/feature_4356_add-x-leap-provenance-header | 1 + src/leap/mx/mail_receiver.py | 36 +++++++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 changes/feature_4356_add-x-leap-provenance-header diff --git a/changes/feature_4356_add-x-leap-provenance-header b/changes/feature_4356_add-x-leap-provenance-header new file mode 100644 index 0000000..b826ec4 --- /dev/null +++ b/changes/feature_4356_add-x-leap-provenance-header @@ -0,0 +1 @@ + o Add X-Leap-Provenance header. Closes #4356. diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 3890eb2..c06754e 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -105,7 +105,7 @@ class MailReceiver(Service): :param pubkey: public key for the owner of the message :type pubkey: str :param message: message contents - :type message: str + :type message: email.message.Message :return: doc to sync with Soledad or None, None if something went wrong. @@ -116,16 +116,19 @@ class MailReceiver(Service): "I know: %r" % (pubkey,)) return None - doc = SoledadDocument(doc_id=str(pyuuid.uuid4())) - - encoding = get_email_charset(message.decode("utf8", "replace"), - default=None) + # find message's encoding + message_as_string = message.as_string() + encoding = get_email_charset( + message_as_string.decode("utf8", "replace"), + default=None) if encoding is None: - result = chardet.detect(message) + result = chardet.detect(message_as_string) encoding = result["encoding"] - data = {'incoming': True, 'content': message} + doc = SoledadDocument(doc_id=str(pyuuid.uuid4())) + # store plain text if pubkey is not available + data = {'incoming': True, 'content': message_as_string} if pubkey is None or len(pubkey) == 0: doc.content = { self.INCOMING_KEY: True, @@ -134,7 +137,7 @@ class MailReceiver(Service): } return doc - openpgp_key = None + # otherwise, encrypt with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: gpg.import_keys(pubkey) key = gpg.list_keys().pop() @@ -142,6 +145,15 @@ class MailReceiver(Service): # dummy one, we just care about the import of the pubkey openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey) + # add X-Leap-Provenance header if message is not encrypted + if message.get_content_type() != 'multipart/encrypted' and \ + '-----BEGIN PGP MESSAGE-----' not in \ + message_as_string: + message.add_header( + 'X-Leap-Provenance', + email.utils.formatdate(), + pubkey=openpgp_key.key_id) + data = {'incoming': True, 'content': message.as_string()} doc.content = { self.INCOMING_KEY: True, ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, @@ -284,8 +296,8 @@ class MailReceiver(Service): log.msg("Processing new mail at %r" % (filepath.path,)) with filepath.open("r") as f: mail_data = f.read() - mail = message_from_string(mail_data) - uuid = self._get_owner(mail) + msg = message_from_string(mail_data) + uuid = self._get_owner(msg) if uuid is None: log.msg("Don't know how to deliver mail %r, skipping..." % (filepath.path,)) @@ -297,12 +309,12 @@ class MailReceiver(Service): defer.returnValue(None) pubkey = yield self._users_cdb.getPubKey(uuid) - if pubkey is None or len(pubkey): + if pubkey is None or len(pubkey) == 0: log.msg("No public key, stopping the processing chain") defer.returnValue(None) log.msg("Encrypting message to %s's pubkey" % (uuid,)) - doc = yield self._encrypt_message(pubkey, mail_data) + doc = yield self._encrypt_message(pubkey, msg) do_remove = yield self._export_message(uuid, doc) yield self._conditional_remove(do_remove, filepath) -- cgit v1.2.3 From d2a6fd962a608e72a1367eada8fd791183da4cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Tue, 26 Nov 2013 15:46:05 -0300 Subject: Add tester script to ease debugging problematic mails. --- changes/feature_add_tester_script | 1 + src/leap/mx/tester.py | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 changes/feature_add_tester_script create mode 100644 src/leap/mx/tester.py diff --git a/changes/feature_add_tester_script b/changes/feature_add_tester_script new file mode 100644 index 0000000..7042254 --- /dev/null +++ b/changes/feature_add_tester_script @@ -0,0 +1 @@ + o Add tester script to ease testing problematic emails offline. \ No newline at end of file diff --git a/src/leap/mx/tester.py b/src/leap/mx/tester.py new file mode 100644 index 0000000..05d2d05 --- /dev/null +++ b/src/leap/mx/tester.py @@ -0,0 +1,46 @@ +import ConfigParser +import sys +import os + +from twisted.internet import reactor, defer +from twisted.python import filepath, log + +from leap.mx import couchdbhelper +from leap.mx.mail_receiver import MailReceiver + +if __name__ == "__main__": + log.startLogging(sys.stdout) + fullpath = os.path.realpath(sys.argv[1]) + + log.msg("Starting test for %s..." % (fullpath,)) + + config_file = "/etc/leap/mx.conf" + + config = ConfigParser.ConfigParser() + config.read(config_file) + + user = config.get("couchdb", "user") + password = config.get("couchdb", "password") + + server = config.get("couchdb", "server") + port = config.get("couchdb", "port") + + cdb = couchdbhelper.ConnectedCouchDB(server, + port=port, + dbName="identities", + username=user, + password=password) + + # Mail receiver + mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, + password, + server, + port) + + mr = MailReceiver(mail_couch_url_prefix, cdb, []) + fpath = filepath.FilePath(fullpath) + + d = mr._process_incoming_email(None, fpath, 0) + d.addCallback(lambda x: reactor.stop()) + + reactor.run() -- cgit v1.2.3 From 8a4f63baadf8f28209d5fab95383634e8c813948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 5 Dec 2013 09:16:24 -0300 Subject: Fix deadlock situation if adding a skipped mail fails --- src/leap/mx/mail_receiver.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index c06754e..c90dda9 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -266,20 +266,29 @@ class MailReceiver(Service): defer.returnValue(None) self._processing_skipped = True - log.msg("Starting processing skipped mail...") - log.msg("-"*50) - - for directory, recursive in self._directories: - for root, dirs, files in os.walk(directory): - for fname in files: - fullpath = os.path.join(root, fname) - fpath = filepath.FilePath(fullpath) - yield self._step_process_mail_backend(fpath) - - if not recursive: - break + try: + log.msg("Starting processing skipped mail...") + log.msg("-"*50) + + for directory, recursive in self._directories: + for root, dirs, files in os.walk(directory): + for fname in files: + try: + fullpath = os.path.join(root, fname) + fpath = filepath.FilePath(fullpath) + yield self._step_process_mail_backend(fpath) + except Exception as e: + log.msg("Error processing skipped mail: %r" % \ + (fullpath,)) + log.err() + if not recursive: + break + except Exception as e: + log.msg("Error processing skipped mail") + log.err() + finally: + self._processing_skipped = False - self._processing_skipped = False log.msg("+"*50) log.msg("Done processing skipped mail") -- cgit v1.2.3 From c0c66ed40b4953fdac2aea0f19ffa8748e2e8ccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Mon, 9 Dec 2013 21:47:02 -0300 Subject: Fold in changes --- CHANGELOG | 4 ++++ changes/feature_4356_add-x-leap-provenance-header | 1 - changes/feature_add_tester_script | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 changes/feature_4356_add-x-leap-provenance-header delete mode 100644 changes/feature_add_tester_script diff --git a/CHANGELOG b/CHANGELOG index cbd9c74..9de1ed7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +0.3.5 Dec 10: + o Add X-Leap-Provenance header. Closes #4356. + o Add tester script to ease testing problematic emails offline. + 0.3.4 Nov 15: o Some mail may be skipped at processing because of possible problems (like connectivity issues to our couch nodes), MX now diff --git a/changes/feature_4356_add-x-leap-provenance-header b/changes/feature_4356_add-x-leap-provenance-header deleted file mode 100644 index b826ec4..0000000 --- a/changes/feature_4356_add-x-leap-provenance-header +++ /dev/null @@ -1 +0,0 @@ - o Add X-Leap-Provenance header. Closes #4356. diff --git a/changes/feature_add_tester_script b/changes/feature_add_tester_script deleted file mode 100644 index 7042254..0000000 --- a/changes/feature_add_tester_script +++ /dev/null @@ -1 +0,0 @@ - o Add tester script to ease testing problematic emails offline. \ No newline at end of file -- cgit v1.2.3