diff options
Diffstat (limited to 'src/leap/mail/imap')
| -rw-r--r-- | src/leap/mail/imap/fetch.py | 89 | ||||
| -rw-r--r-- | src/leap/mail/imap/messages.py | 4 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/test_imap.py | 7 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/test_incoming_mail.py | 158 |
4 files changed, 232 insertions, 26 deletions
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index d78b4ee..863f5fe 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -19,9 +19,9 @@ Incoming mail fetcher. """ import copy import logging +import shlex import threading import time -import sys import traceback import warnings @@ -29,6 +29,7 @@ from email.parser import Parser from email.generator import Generator from email.utils import parseaddr from StringIO import StringIO +from urlparse import urlparse from twisted.python import log from twisted.internet import defer, reactor @@ -169,7 +170,7 @@ class LeapIncomingMail(object): DeprecationWarning) doclist = self._soledad.get_from_index( fields.JUST_MAIL_COMPAT_IDX, "*") - self._process_doclist(doclist) + return self._process_doclist(doclist) logger.debug("fetching mail for: %s %s" % ( self._soledad.uuid, self._userid)) @@ -209,7 +210,7 @@ class LeapIncomingMail(object): def _errback(self, failure): logger.exception(failure.value) - traceback.print_tb(*sys.exc_info()) + traceback.print_exc() @deferred_to_thread def _sync_soledad(self): @@ -273,6 +274,7 @@ class LeapIncomingMail(object): return num_mails = len(doclist) + deferreds = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) leap_events.signal( @@ -288,19 +290,15 @@ class LeapIncomingMail(object): if has_errors is None: warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", DeprecationWarning) + if has_errors: logger.debug("skipping msg with decrypting errors...") - - if self._is_msg(keys) and not has_errors: - # Evaluating to bool of has_errors is intentional here. - # We don't mind at this point if it's None or False. - - # Ok, this looks like a legit msg, and with no errors. - # Let's process it! - - d1 = self._decrypt_doc(doc) - d = defer.gatherResults([d1], consumeErrors=True) + elif self._is_msg(keys): + d = self._decrypt_doc(doc) + d.addCallback(self._extract_keys) d.addCallbacks(self._add_message_locally, self._errback) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) # # operations on individual messages @@ -581,20 +579,77 @@ class LeapIncomingMail(object): data, self._pkey) return (decrdata, valid_sig) - def _add_message_locally(self, result): + def _extract_keys(self, msgtuple): + """ + Parse message headers for an *OpenPGP* header as described on the + `IETF draft + <http://tools.ietf.org/html/draft-josefsson-openpgp-mailnews-header-06>` + only urls with https and the same hostname than the email are supported + for security reasons. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + """ + OpenPGP_HEADER = 'OpenPGP' + doc, data = msgtuple + + # XXX the parsing of the message is done in mailbox.addMessage, maybe + # we should do it in this module so we don't need to parse it again + # here + msg = self._parser.parsestr(data) + header = msg.get(OpenPGP_HEADER, None) + if header is not None: + self._extract_openpgp_header(msg, header) + + return msgtuple + + def _extract_openpgp_header(self, msg, header): + """ + Import keys from the OpenPGP header + + :param msg: parsed email + :type msg: email.Message + :param header: OpenPGP header string + :type header: str + """ + fields = dict([f.strip(' ').split('=') for f in header.split(';')]) + if 'url' in fields: + url = shlex.split(fields['url'])[0] # remove quotations + _, fromAddress = parseaddr(msg['from']) + urlparts = urlparse(url) + fromHostname = fromAddress.split('@')[1] + if (urlparts.scheme == 'https' + and urlparts.hostname == fromHostname): + try: + self._keymanager.fetch_key(fromAddress, url, OpenPGPKey) + logger.info("Imported key from header %s" % (url,)) + except keymanager_errors.KeyNotFound: + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + except keymanager_errors.KeyAttributesDiffer: + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, fromAddress)) + else: + logger.debug("No valid url on OpenPGP header %s" % (url,)) + else: + logger.debug("There is no url on the OpenPGP header: %s" + % (header,)) + + def _add_message_locally(self, msgtuple): """ Adds a message to local inbox and delete it from the incoming db in soledad. - # XXX this comes from a gatherresult... :param msgtuple: a tuple consisting of a SoledadDocument instance containing the incoming message and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) """ - msgtuple = first(result) - doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 0356600..e8d64d1 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -29,7 +29,7 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.mail import imap4 -from twisted.internet import defer +from twisted.internet import defer, reactor from zope.interface import implements from zope.proxy import sameProxiedObjects @@ -134,7 +134,6 @@ class LeapMessage(fields, MBoxParser): self.__chash = None self.__bdoc = None - from twisted.internet import reactor self.reactor = reactor # XXX make these properties public @@ -740,7 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): else: self._initialized[mbox] = True - from twisted.internet import reactor self.reactor = reactor def _get_empty_doc(self, _type=FLAGS_DOC): diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 631a2c1..7837aaa 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -26,11 +26,6 @@ XXX add authors from the original twisted tests. """ # XXX review license of the original tests!!! -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - import os import types @@ -218,7 +213,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): #mc._soledad.create_doc(newmsg) #self.assertEqual(mc.count(), 3) #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py new file mode 100644 index 0000000..5b72fac --- /dev/null +++ b/src/leap/mail/imap/tests/test_incoming_mail.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +# test_imap.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Test case for leap.email.imap.fetch + +@authors: Ruben Pollan, <meskio@sindominio.net> + +@license: GPLv3, see included LICENSE file +""" + +import json + +from email.parser import Parser +from mock import Mock +from twisted.trial import unittest + +from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.fetch import LeapIncomingMail +from leap.mail.imap.fields import fields +from leap.mail.imap.memorystore import MemoryStore +from leap.mail.imap.service.imap import INCOMING_CHECK_PERIOD +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, +) +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + + +class LeapIncomingMailTestCase(TestCaseWithKeyManager, unittest.TestCase): + """ + Tests for the incoming mail parser + """ + NICKSERVER = "http://domain" + FROM_ADDRESS = "test@somedomain.com" + BODY = """ +Governments of the Industrial World, you weary giants of flesh and steel, I +come from Cyberspace, the new home of Mind. On behalf of the future, I ask +you of the past to leave us alone. You are not welcome among us. You have +no sovereignty where we gather. + """ + EMAIL = """from: Test from SomeDomain <%(from)s> +to: %(to)s +subject: independence of cyberspace + +%(body)s + """ % { + "from": FROM_ADDRESS, + "to": ADDRESS, + "body": BODY + } + + def setUp(self): + super(LeapIncomingMailTestCase, self).setUp() + + # Soledad sync makes trial block forever. The sync it's mocked to fix + # this problem. _mock_soledad_get_from_index can be used from the tests + # to provide documents. + self._soledad.sync = Mock() + + memstore = MemoryStore() + theAccount = SoledadBackedAccount( + ADDRESS, + soledad=self._soledad, + memstore=memstore) + self.fetcher = LeapIncomingMail( + self._km, + self._soledad, + theAccount, + INCOMING_CHECK_PERIOD, + ADDRESS) + + def tearDown(self): + del self.fetcher + super(LeapIncomingMailTestCase, self).tearDown() + + def testExtractOpenPGPHeader(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://somedomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + email = self._create_incoming_email(message.as_string()) + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) + self.fetcher._keymanager.fetch_key = Mock() + d = self.fetcher.fetch() + + def fetch_key_called(ret): + self.fetcher._keymanager.fetch_key.assert_called_once_with( + self.FROM_ADDRESS, KEYURL, OpenPGPKey) + d.addCallback(fetch_key_called) + + return d + + def testExtractOpenPGPHeaderInvalidUrl(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://someotherdomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + email = self._create_incoming_email(message.as_string()) + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) + self.fetcher._keymanager.fetch_key = Mock() + d = self.fetcher.fetch() + + def fetch_key_called(ret): + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + d.addCallback(fetch_key_called) + + return d + + def _create_incoming_email(self, email_str): + email = SoledadDocument() + pubkey = self._km.get_key(ADDRESS, OpenPGPKey) + data = json.dumps( + {"incoming": True, "content": email_str}, + ensure_ascii=False) + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) + } + return email + + def _mock_soledad_get_from_index(self, index_name, value): + get_from_index = self._soledad.get_from_index + + def soledad_mock(idx_name, *key_values): + if index_name == idx_name: + return value + return get_from_index(idx_name, *key_values) + self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) |