Get keys from OpenPGP email header

7 files changed, 234 insertions, 27 deletions

diff --git a/mail/changes/feature-3879_openpgp_header b/mail/changes/feature-3879_openpgp_header
new file mode 100644
index 00000000..e04c9252
--- /dev/null
+++ b/mail/changes/feature-3879_openpgp_header
@@ -0,0 +1 @@
+- Parse OpenPGP header and import keys from it (Closes: #3879)
diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index d78b4eed..863f5fe7 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -19,9 +19,9 @@ Incoming mail fetcher.
 """
 import copy
 import logging
+import shlex
 import threading
 import time
-import sys
 import traceback
 import warnings
 
@@ -29,6 +29,7 @@ from email.parser import Parser
 from email.generator import Generator
 from email.utils import parseaddr
 from StringIO import StringIO
+from urlparse import urlparse
 
 from twisted.python import log
 from twisted.internet import defer, reactor
@@ -169,7 +170,7 @@ class LeapIncomingMail(object):
                               DeprecationWarning)
                 doclist = self._soledad.get_from_index(
                     fields.JUST_MAIL_COMPAT_IDX, "*")
-            self._process_doclist(doclist)
+            return self._process_doclist(doclist)
 
         logger.debug("fetching mail for: %s %s" % (
             self._soledad.uuid, self._userid))
@@ -209,7 +210,7 @@ class LeapIncomingMail(object):
 
     def _errback(self, failure):
         logger.exception(failure.value)
-        traceback.print_tb(*sys.exc_info())
+        traceback.print_exc()
 
     @deferred_to_thread
     def _sync_soledad(self):
@@ -273,6 +274,7 @@ class LeapIncomingMail(object):
             return
         num_mails = len(doclist)
 
+        deferreds = []
         for index, doc in enumerate(doclist):
             logger.debug("processing doc %d of %d" % (index + 1, num_mails))
             leap_events.signal(
@@ -288,19 +290,15 @@ class LeapIncomingMail(object):
             if has_errors is None:
                 warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
                               DeprecationWarning)
+
             if has_errors:
                 logger.debug("skipping msg with decrypting errors...")
-
-            if self._is_msg(keys) and not has_errors:
-                # Evaluating to bool of has_errors is intentional here.
-                # We don't mind at this point if it's None or False.
-
-                # Ok, this looks like a legit msg, and with no errors.
-                # Let's process it!
-
-                d1 = self._decrypt_doc(doc)
-                d = defer.gatherResults([d1], consumeErrors=True)
+            elif self._is_msg(keys):
+                d = self._decrypt_doc(doc)
+                d.addCallback(self._extract_keys)
                 d.addCallbacks(self._add_message_locally, self._errback)
+                deferreds.append(d)
+        return defer.gatherResults(deferreds, consumeErrors=True)
 
     #
     # operations on individual messages
@@ -581,20 +579,77 @@ class LeapIncomingMail(object):
                 data, self._pkey)
         return (decrdata, valid_sig)
 
-    def _add_message_locally(self, result):
+    def _extract_keys(self, msgtuple):
+        """
+        Parse message headers for an *OpenPGP* header as described on the
+        `IETF draft
+        <http://tools.ietf.org/html/draft-josefsson-openpgp-mailnews-header-06>`
+        only urls with https and the same hostname than the email are supported
+        for security reasons.
+
+        :param msgtuple: a tuple consisting of a SoledadDocument
+                         instance containing the incoming message
+                         and data, the json-encoded, decrypted content of the
+                         incoming message
+        :type msgtuple: (SoledadDocument, str)
+        """
+        OpenPGP_HEADER = 'OpenPGP'
+        doc, data = msgtuple
+
+        # XXX the parsing of the message is done in mailbox.addMessage, maybe
+        #     we should do it in this module so we don't need to parse it again
+        #     here
+        msg = self._parser.parsestr(data)
+        header = msg.get(OpenPGP_HEADER, None)
+        if header is not None:
+            self._extract_openpgp_header(msg, header)
+
+        return msgtuple
+
+    def _extract_openpgp_header(self, msg, header):
+        """
+        Import keys from the OpenPGP header
+
+        :param msg: parsed email
+        :type msg: email.Message
+        :param header: OpenPGP header string
+        :type header: str
+        """
+        fields = dict([f.strip(' ').split('=') for f in header.split(';')])
+        if 'url' in fields:
+            url = shlex.split(fields['url'])[0]  # remove quotations
+            _, fromAddress = parseaddr(msg['from'])
+            urlparts = urlparse(url)
+            fromHostname = fromAddress.split('@')[1]
+            if (urlparts.scheme == 'https'
+                    and urlparts.hostname == fromHostname):
+                try:
+                    self._keymanager.fetch_key(fromAddress, url, OpenPGPKey)
+                    logger.info("Imported key from header %s" % (url,))
+                except keymanager_errors.KeyNotFound:
+                    logger.warning("Url from OpenPGP header %s failed"
+                                   % (url,))
+                except keymanager_errors.KeyAttributesDiffer:
+                    logger.warning("Key from OpenPGP header url %s didn't "
+                                   "match the from address %s"
+                                   % (url, fromAddress))
+            else:
+                logger.debug("No valid url on OpenPGP header %s" % (url,))
+        else:
+            logger.debug("There is no url on the OpenPGP header: %s"
+                         % (header,))
+
+    def _add_message_locally(self, msgtuple):
         """
         Adds a message to local inbox and delete it from the incoming db
         in soledad.
 
-        # XXX this comes from a gatherresult...
         :param msgtuple: a tuple consisting of a SoledadDocument
                          instance containing the incoming message
                          and data, the json-encoded, decrypted content of the
                          incoming message
         :type msgtuple: (SoledadDocument, str)
         """
-        msgtuple = first(result)
-
         doc, data = msgtuple
         log.msg('adding message %s to local db' % (doc.doc_id,))
 
diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py
index 0356600a..e8d64d1c 100644
--- a/mail/src/leap/mail/imap/messages.py
+++ b/mail/src/leap/mail/imap/messages.py
@@ -29,7 +29,7 @@ from functools import partial
 
 from pycryptopp.hash import sha256
 from twisted.mail import imap4
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 from zope.interface import implements
 from zope.proxy import sameProxiedObjects
 
@@ -134,7 +134,6 @@ class LeapMessage(fields, MBoxParser):
         self.__chash = None
         self.__bdoc = None
 
-        from twisted.internet import reactor
         self.reactor = reactor
 
     # XXX make these properties public
@@ -740,7 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             else:
                 self._initialized[mbox] = True
 
-        from twisted.internet import reactor
         self.reactor = reactor
 
     def _get_empty_doc(self, _type=FLAGS_DOC):
diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py
index 631a2c1b..7837aaa3 100644
--- a/mail/src/leap/mail/imap/tests/test_imap.py
+++ b/mail/src/leap/mail/imap/tests/test_imap.py
@@ -26,11 +26,6 @@ XXX add authors from the original twisted tests.
 """
 # XXX review license of the original tests!!!
 
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
 import os
 import types
 
@@ -218,7 +213,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         #mc._soledad.create_doc(newmsg)
         #self.assertEqual(mc.count(), 3)
         #self.assertEqual(
-            #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
+        #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
 
 
 class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
diff --git a/mail/src/leap/mail/imap/tests/test_incoming_mail.py b/mail/src/leap/mail/imap/tests/test_incoming_mail.py
new file mode 100644
index 00000000..5b72facb
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/test_incoming_mail.py
@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+# test_imap.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Test case for leap.email.imap.fetch
+
+@authors: Ruben Pollan, <meskio@sindominio.net>
+
+@license: GPLv3, see included LICENSE file
+"""
+
+import json
+
+from email.parser import Parser
+from mock import Mock
+from twisted.trial import unittest
+
+from leap.keymanager.openpgp import OpenPGPKey
+from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.fetch import LeapIncomingMail
+from leap.mail.imap.fields import fields
+from leap.mail.imap.memorystore import MemoryStore
+from leap.mail.imap.service.imap import INCOMING_CHECK_PERIOD
+from leap.mail.tests import (
+    TestCaseWithKeyManager,
+    ADDRESS,
+)
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.crypto import (
+    EncryptionSchemes,
+    ENC_JSON_KEY,
+    ENC_SCHEME_KEY,
+)
+
+
+class LeapIncomingMailTestCase(TestCaseWithKeyManager, unittest.TestCase):
+    """
+    Tests for the incoming mail parser
+    """
+    NICKSERVER = "http://domain"
+    FROM_ADDRESS = "test@somedomain.com"
+    BODY = """
+Governments of the Industrial World, you weary giants of flesh and steel, I
+come from Cyberspace, the new home of Mind. On behalf of the future, I ask
+you of the past to leave us alone. You are not welcome among us. You have
+no sovereignty where we gather.
+    """
+    EMAIL = """from: Test from SomeDomain <%(from)s>
+to: %(to)s
+subject: independence of cyberspace
+
+%(body)s
+    """ % {
+        "from": FROM_ADDRESS,
+        "to": ADDRESS,
+        "body": BODY
+    }
+
+    def setUp(self):
+        super(LeapIncomingMailTestCase, self).setUp()
+
+        # Soledad sync makes trial block forever. The sync it's mocked to fix
+        # this problem. _mock_soledad_get_from_index can be used from the tests
+        # to provide documents.
+        self._soledad.sync = Mock()
+
+        memstore = MemoryStore()
+        theAccount = SoledadBackedAccount(
+            ADDRESS,
+            soledad=self._soledad,
+            memstore=memstore)
+        self.fetcher = LeapIncomingMail(
+            self._km,
+            self._soledad,
+            theAccount,
+            INCOMING_CHECK_PERIOD,
+            ADDRESS)
+
+    def tearDown(self):
+        del self.fetcher
+        super(LeapIncomingMailTestCase, self).tearDown()
+
+    def testExtractOpenPGPHeader(self):
+        """
+        Test the OpenPGP header key extraction
+        """
+        KEYURL = "https://somedomain.com/key.txt"
+        OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+        message = Parser().parsestr(self.EMAIL)
+        message.add_header("OpenPGP", OpenPGP)
+        email = self._create_incoming_email(message.as_string())
+        self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])
+        self.fetcher._keymanager.fetch_key = Mock()
+        d = self.fetcher.fetch()
+
+        def fetch_key_called(ret):
+            self.fetcher._keymanager.fetch_key.assert_called_once_with(
+                self.FROM_ADDRESS, KEYURL, OpenPGPKey)
+        d.addCallback(fetch_key_called)
+
+        return d
+
+    def testExtractOpenPGPHeaderInvalidUrl(self):
+        """
+        Test the OpenPGP header key extraction
+        """
+        KEYURL = "https://someotherdomain.com/key.txt"
+        OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+        message = Parser().parsestr(self.EMAIL)
+        message.add_header("OpenPGP", OpenPGP)
+        email = self._create_incoming_email(message.as_string())
+        self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])
+        self.fetcher._keymanager.fetch_key = Mock()
+        d = self.fetcher.fetch()
+
+        def fetch_key_called(ret):
+            self.assertFalse(self.fetcher._keymanager.fetch_key.called)
+        d.addCallback(fetch_key_called)
+
+        return d
+
+    def _create_incoming_email(self, email_str):
+        email = SoledadDocument()
+        pubkey = self._km.get_key(ADDRESS, OpenPGPKey)
+        data = json.dumps(
+            {"incoming": True, "content": email_str},
+            ensure_ascii=False)
+        email.content = {
+            fields.INCOMING_KEY: True,
+            fields.ERROR_DECRYPTING_KEY: False,
+            ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
+            ENC_JSON_KEY: str(self._km.encrypt(data, pubkey))
+        }
+        return email
+
+    def _mock_soledad_get_from_index(self, index_name, value):
+        get_from_index = self._soledad.get_from_index
+
+        def soledad_mock(idx_name, *key_values):
+            if index_name == idx_name:
+                return value
+            return get_from_index(idx_name, *key_values)
+        self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock)
diff --git a/mail/src/leap/mail/smtp/tests/test_gateway.py b/mail/src/leap/mail/smtp/tests/test_gateway.py
index 466677f2..3635a9f5 100644
--- a/mail/src/leap/mail/smtp/tests/test_gateway.py
+++ b/mail/src/leap/mail/smtp/tests/test_gateway.py
@@ -32,7 +32,7 @@ from leap.mail.smtp.gateway import (
     SMTPFactory,
     EncryptedMessage,
 )
-from leap.mail.smtp.tests import (
+from leap.mail.tests import (
     TestCaseWithKeyManager,
     ADDRESS,
     ADDRESS_2,
diff --git a/mail/src/leap/mail/smtp/tests/__init__.py b/mail/src/leap/mail/tests/__init__.py
index dc24293f..dc24293f 100644
--- a/mail/src/leap/mail/smtp/tests/__init__.py
+++ b/mail/src/leap/mail/tests/__init__.py