Merge branch 'release-0.3.6'0.3.6

13 files changed, 388 insertions, 91 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 319fda5..5755e59 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,13 @@
+0.3.6 Nov 1:
+  o Add support for non-ascii characters in emails. Closes #4000.
+  o Default to UTF-8 when there is no charset parsed from the mail
+    contents.
+  o Refactor get_email_charset to leap.common.
+  o Return the necessary references (factory, port) from IMAP4 launch
+    in order to be able to properly stop it. Related to #4199.
+  o Notify MUA of new mail, using IDLE as advertised. Closes: #3671
+  o Use TLS wrapper mode instead of STARTTLS. Closes #3637.
+
 0.3.5 Oct 18:
   o Do not log mail doc contents.
   o Comply with RFC 3156. Closes #4029.
diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index 6fa0df4..4780b5c 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -1,4 +1,4 @@
 leap.soledad.client>=0.3.0
-leap.common>=0.3.0
+leap.common>=0.3.5
 leap.keymanager>=0.3.4
 twisted  # >= 12.0.3 ??
diff --git a/setup.py b/setup.py
index f423f7b..499a9ee 100644
--- a/setup.py
+++ b/setup.py
@@ -17,6 +17,7 @@
 """
 Setup file for leap.mail
 """
+import re
 from setuptools import setup
 from setuptools import find_packages
 
@@ -46,25 +47,89 @@ trove_classifiers = [
     'Topic :: Software Development :: Libraries',
 ]
 
+DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mail/'
+                 'archive/%s.tar.gz')
+_versions = versioneer.get_versions()
+VERSION = _versions['version']
+VERSION_FULL = _versions['full']
+DOWNLOAD_URL = ""
+
+# get the short version for the download url
+_version_short = re.findall('\d+\.\d+\.\d+', VERSION)
+if len(_version_short) > 0:
+    VERSION_SHORT = _version_short[0]
+    DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
+
+cmdclass = versioneer.get_cmdclass()
+
+
+from setuptools import Command
+
+
+class freeze_debianver(Command):
+    """
+    Freezes the version in a debian branch.
+    To be used after merging the development branch onto the debian one.
+    """
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        proceed = str(raw_input(
+            "This will overwrite the file _version.py. Continue? [y/N] "))
+        if proceed != "y":
+            print("He. You scared. Aborting.")
+            return
+        template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.7+) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+version_full = '{version_full}'
+"""
+        templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+        return {'version': version_version, 'full': version_full}
+"""
+        subst_template = template.format(
+            version=VERSION_SHORT,
+            version_full=VERSION_FULL) + templatefun
+        with open(versioneer.versionfile_source, 'w') as f:
+            f.write(subst_template)
+
+
+cmdclass["freeze_debianver"] = freeze_debianver
+
 # XXX add ref to docs
 
 setup(
     name='leap.mail',
-    version=versioneer.get_version(),
-    cmdclass=versioneer.get_cmdclass(),
+    version=VERSION,
+    cmdclass=cmdclass,
     url='https://leap.se/',
+    download_url=DOWNLOAD_URL,
     license='GPLv3+',
     author='The LEAP Encryption Access Project',
     author_email='info@leap.se',
+    maintainer='Kali Kaneko',
+    maintainer_email='kali@leap.se',
     description='Mail Services provided by Bitmask, the LEAP Client.',
-    long_description=(
-        "Mail Services provided by Bitmask, the LEAP Client."
-    ),
+    long_description=open('README.rst').read() + '\n\n\n' +
+    open('CHANGELOG').read(),
     classifiers=trove_classifiers,
     namespace_packages=["leap"],
     package_dir={'': 'src'},
     packages=find_packages('src'),
-    test_suite='leap.mail.load_tests',
+    test_suite='leap.mail.load_tests.load_tests',
     install_requires=utils.parse_requirements(),
     tests_require=utils.parse_requirements(
         reqfiles=['pkg/requirements-testing.pip']),
diff --git a/src/leap/mail/__init__.py b/src/leap/mail/__init__.py
index 5b5ba9b..4b25fe6 100644
--- a/src/leap/mail/__init__.py
+++ b/src/leap/mail/__init__.py
@@ -17,17 +17,10 @@
 
 
 """
-Provide function for loading tests.
+Client mail bits.
 """
 
 
-# Do not force the unittest dependency
-# import unittest
-
-
-# def load_tests():
-#     return unittest.defaultTestLoader.discover('./src/leap/mail')
-
 from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 0a71f53..dd65def 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -31,15 +31,16 @@ from twisted.internet.threads import deferToThread
 
 from leap.common import events as leap_events
 from leap.common.check import leap_assert, leap_assert_type
-from leap.soledad.client import Soledad
-from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
-
 from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING
 from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING
 from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED
 from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY
 from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
+from leap.common.mail import get_email_charset
+from leap.keymanager import errors as keymanager_errors
+from leap.soledad.client import Soledad
+from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
 
 logger = logging.getLogger(__name__)
@@ -197,6 +198,29 @@ class LeapIncomingMail(object):
             logger.warning('Unknown error while '
                            'syncing soledad: %r' % (err,))
 
+    def _log_err(self, failure):
+        """
+        Generic errback
+        """
+        err = failure.value
+        logger.error("error!: %r" % (err,))
+
+    def _decryption_error(self, failure):
+        """
+        Errback for decryption errors.
+        """
+        # XXX should signal unrecoverable maybe.
+        err = failure.value
+        logger.error("error decrypting msg: %s" % (err,))
+
+    def _saving_error(self, failure):
+        """
+        Errback for local save errors.
+        """
+        # XXX should signal unrecoverable maybe.
+        err = failure.value
+        logger.error("error saving msg locally: %s" % (err,))
+
     def _process_doclist(self, doclist):
         """
         Iterates through the doclist, checks if each doc
@@ -227,7 +251,13 @@ class LeapIncomingMail(object):
                 # Deferred chain for individual messages
                 d = deferToThread(self._decrypt_msg, doc, encdata)
                 d.addCallback(self._process_decrypted)
+                d.addErrback(self._log_err)
                 d.addCallback(self._add_message_locally)
+                d.addErrback(self._log_err)
+                # XXX check this, add_locally should not get called if we
+                # get an error in process
+                #d.addCallbacks(self._process_decrypted, self._decryption_error)
+                #d.addCallbacks(self._add_message_locally, self._saving_error)
                 docs_cb.append(d)
             else:
                 # Ooops, this does not.
@@ -288,20 +318,29 @@ class LeapIncomingMail(object):
         rawmsg = msg.get(self.CONTENT_KEY, None)
         if not rawmsg:
             return False
-        data = self._maybe_decrypt_gpg_msg(rawmsg)
-        return doc, data
+        try:
+            data = self._maybe_decrypt_gpg_msg(rawmsg)
+            return doc, data
+        except keymanager_errors.EncryptionDecryptionFailed as exc:
+            logger.error(exc)
+            raise
 
     def _maybe_decrypt_gpg_msg(self, data):
         """
         Tries to decrypt a gpg message if data looks like one.
 
         :param data: the text to be decrypted.
-        :type data: str
+        :type data: unicode
         :return: data, possibly descrypted.
         :rtype: str
         """
+        leap_assert_type(data, unicode)
+
         parser = Parser()
+        encoding = get_email_charset(data)
+        data = data.encode(encoding)
         origmsg = parser.parsestr(data)
+
         # handle multipart/encrypted messages
         if origmsg.get_content_type() == 'multipart/encrypted':
             # sanity check
@@ -320,13 +359,21 @@ class LeapIncomingMail(object):
                     "Multipart/encrypted messages' second body part should "
                     "have content type equal to 'octet-stream' (instead of "
                     "%s)." % payload[1].get_content_type())
+
             # parse message and get encrypted content
             pgpencmsg = origmsg.get_payload()[1]
             encdata = pgpencmsg.get_payload()
+
             # decrypt and parse decrypted message
             decrdata = self._keymanager.decrypt(
                 encdata, self._pkey,
                 passphrase=self._soledad.passphrase)
+            try:
+                decrdata = decrdata.encode(encoding)
+            except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                logger.error("Unicode error {0}".format(e))
+                decrdata = decrdata.encode(encoding, 'replace')
+
             decrmsg = parser.parsestr(decrdata)
             # replace headers back in original message
             for hkey, hval in decrmsg.items():
@@ -335,6 +382,7 @@ class LeapIncomingMail(object):
                     origmsg.replace_header(hkey, hval)
                 except KeyError:
                     origmsg[hkey] = hval
+
             # replace payload by unencrypted payload
             origmsg.set_payload(decrmsg.get_payload())
             return origmsg.as_string(unixfrom=False)
@@ -352,6 +400,10 @@ class LeapIncomingMail(object):
                 # replace encrypted by decrypted content
                 data = data.replace(pgp_message, decrdata)
         # if message is not encrypted, return raw data
+
+        if isinstance(data, unicode):
+            data = data.encode(encoding, 'replace')
+
         return data
 
     def _add_message_locally(self, msgtuple):
@@ -365,6 +417,7 @@ class LeapIncomingMail(object):
                          incoming message
         :type msgtuple: (SoledadDocument, str)
         """
+        print "adding message locally....."
         doc, data = msgtuple
         self._inbox.addMessage(data, (self.RECENT_FLAG,))
         leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 10d338a..7a9f810 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -18,9 +18,7 @@
 Soledad-backed IMAP Server.
 """
 import copy
-import email
 import logging
-import re
 import StringIO
 import cStringIO
 import time
@@ -33,13 +31,10 @@ from twisted.mail import imap4
 from twisted.internet import defer
 from twisted.python import log
 
-#from twisted import cred
-
-#import u1db
-
 from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
+from leap.common.mail import get_email_charset
 from leap.soledad.client import Soledad
 
 logger = logging.getLogger(__name__)
@@ -184,7 +179,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         # messages
         TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
         TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
-        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(recent)', 'bool(seen)'],
+        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
+                                  'bool(recent)', 'bool(seen)'],
     }
 
     INBOX_NAME = "INBOX"
@@ -577,7 +573,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         return "<SoledadBackedAccount (%s)>" % self._account_name
 
 #######################################
-# Soledad Message, MessageCollection
+# LeapMessage, MessageCollection
 # and Mailbox
 #######################################
 
@@ -695,26 +691,6 @@ class LeapMessage(WithMsgFields):
     the more complex MIME-based interface.
     """
 
-    def _get_charset(self, content):
-        """
-        Mini parser to retrieve the charset of an email
-
-        :param content: mail contents
-        :type content: unicode
-
-        :returns: the charset as parsed from the contents
-        :rtype: str
-        """
-        charset = "UTF-8"
-        try:
-            em = email.message_from_string(content.encode("utf-8"))
-            # Miniparser for: Content-Type: <something>; charset=<charset>
-            charset_re = r'''charset=(?P<charset>[\w|\d|-]*)'''
-            charset = re.findall(charset_re, em["Content-Type"])[0]
-        except Exception:
-            pass
-        return charset
-
     def open(self):
         """
         Return an file-like object opened for reading.
@@ -726,8 +702,14 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = cStringIO.StringIO()
-        charset = self._get_charset(self._doc.content.get(self.RAW_KEY, ''))
-        fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset))
+        charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
+        content = self._doc.content.get(self.RAW_KEY, '')
+        try:
+            content = content.encode(charset)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            content = content.encode(charset, 'replace')
+        fd.write(content)
         fd.seek(0)
         return fd
 
@@ -746,8 +728,14 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        charset = self._get_charset(self._doc.content.get(self.RAW_KEY, ''))
-        fd.write(self._doc.content.get(self.RAW_KEY, '').encode(charset))
+        charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
+        content = self._doc.content.get(self.RAW_KEY, '')
+        try:
+            content = content.encode(charset)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            content = content.encode(charset, 'replace')
+        fd.write(content)
         # SHOULD use a separate BODY FIELD ...
         fd.seek(0)
         return fd
@@ -1111,6 +1099,7 @@ class SoledadMailbox(WithMsgFields):
     which we instantiate and make accessible in the `messages` attribute.
     """
     implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
+    # XXX should finish the implementation of IMailboxListener
 
     messages = None
     _closed = False
@@ -1127,6 +1116,8 @@ class SoledadMailbox(WithMsgFields):
     CMD_UIDVALIDITY = "UIDVALIDITY"
     CMD_UNSEEN = "UNSEEN"
 
+    listeners = []
+
     def __init__(self, mbox, soledad=None, rw=1):
         """
         SoledadMailbox constructor. Needs to get passed a name, plus a
@@ -1159,15 +1150,35 @@ class SoledadMailbox(WithMsgFields):
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
 
-        # XXX what is/was this used for? --------
-        # ---> mail/imap4.py +1155,
-        #      _cbSelectWork makes use of this
-        # probably should implement hooks here
-        # using leap.common.events
-        self.listeners = []
-        self.addListener = self.listeners.append
-        self.removeListener = self.listeners.remove
-        #------------------------------------------
+        # the server itself is a listener to the mailbox.
+        # so we can notify it (and should!) after chanes in flags
+        # and number of messages.
+        print "emptying the listeners"
+        map(lambda i: self.listeners.remove(i), self.listeners)
+
+    def addListener(self, listener):
+        """
+        Rdds a listener to the listeners queue.
+
+        :param listener: listener to add
+        :type listener: an object that implements IMailboxListener
+        """
+        logger.debug('adding mailbox listener: %s' % listener)
+        self.listeners.append(listener)
+
+    def removeListener(self, listener):
+        """
+        Removes a listener from the listeners queue.
+
+        :param listener: listener to remove
+        :type listener: an object that implements IMailboxListener
+        """
+        logger.debug('removing mailbox listener: %s' % listener)
+        try:
+            self.listeners.remove(listener)
+        except ValueError:
+            logger.error(
+                "removeListener: cannot remove listener %s" % listener)
 
     def _get_mbox(self):
         """
@@ -1192,6 +1203,7 @@ class SoledadMailbox(WithMsgFields):
         #return map(str, self.INIT_FLAGS)
 
         # XXX CHECK against thunderbird XXX
+        # XXX I think this is slightly broken.. :/
 
         mbox = self._get_mbox()
         if not mbox:
@@ -1364,6 +1376,10 @@ class SoledadMailbox(WithMsgFields):
 
         self.messages.add_msg(message, flags=flags, date=date,
                               uid=uid_next)
+        exists = len(self.messages)
+        recent = len(self.messages.get_recent())
+        for listener in self.listeners:
+            listener.newMessages(exists, recent)
         return defer.succeed(None)
 
     # commands, do not rename methods
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index b840e86..5f7322a 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -77,7 +77,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         #self.theAccount = theAccount
 
     def lineReceived(self, line):
-        if "login" in line:
+        if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
             msg = line[:7] + " [...]"
@@ -141,7 +141,9 @@ def run_service(*args, **kwargs):
     Main entry point to run the service from the client.
 
     :returns: the LoopingCall instance that will have to be stoppped
-              before shutting down the client.
+              before shutting down the client, the port as returned by
+              the reactor when starts listening, and the factory for
+              the protocol.
     """
     leap_assert(len(args) == 2)
     soledad, keymanager = args
@@ -157,8 +159,8 @@ def run_service(*args, **kwargs):
     from twisted.internet import reactor
 
     try:
-        reactor.listenTCP(port, factory,
-                          interface="localhost")
+        tport = reactor.listenTCP(port, factory,
+                                  interface="localhost")
         fetcher = LeapIncomingMail(
             keymanager,
             soledad,
@@ -174,7 +176,7 @@ def run_service(*args, **kwargs):
         fetcher.start_loop()
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
         leap_events.signal(IMAP_SERVICE_STARTED, str(port))
-        return fetcher
+        return fetcher, tport, factory
 
     # not ok, signal error.
     leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port))
diff --git a/src/leap/mail/load_tests.py b/src/leap/mail/load_tests.py
new file mode 100644
index 0000000..ee89fcc
--- /dev/null
+++ b/src/leap/mail/load_tests.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# tests.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Provide a function for loading tests.
+"""
+
+import unittest
+
+
+def load_tests():
+    suite = unittest.TestSuite()
+    for test in unittest.defaultTestLoader.discover(
+            './src/leap/mail/',
+            top_level_dir='./src/'):
+        suite.addTest(test)
+    return suite
diff --git a/src/leap/mail/smtp/smtprelay.py b/src/leap/mail/smtp/smtprelay.py
index d9bbbf9..fca66c0 100644
--- a/src/leap/mail/smtp/smtprelay.py
+++ b/src/leap/mail/smtp/smtprelay.py
@@ -17,6 +17,20 @@
 
 """
 LEAP SMTP encrypted relay.
+
+The following classes comprise the SMTP relay service:
+
+    * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides
+      the SMTPDelivery protocol.
+    * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It
+      knows how to validate sender and receiver of messages and it generates
+      an EncryptedMessage for each recipient.
+    * SSLContextFactory - Contains the relevant ssl information for the
+      connection.
+    * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that
+      knows how to encrypt/sign itself before sending.
+
+
 """
 
 import re
@@ -173,7 +187,6 @@ class SMTPFactory(ServerFactory):
         @return: The protocol.
         @rtype: SMTPDelivery
         """
-        # If needed, we might use ESMTPDelivery here instead.
         smtpProtocol = smtp.SMTP(SMTPDelivery(self._km, self._config))
         smtpProtocol.factory = self
         return smtpProtocol
@@ -305,7 +318,7 @@ class SMTPDelivery(object):
 # EncryptedMessage
 #
 
-class CtxFactory(ssl.ClientContextFactory):
+class SSLContextFactory(ssl.ClientContextFactory):
     def __init__(self, cert, key):
         self.cert = cert
         self.key = key
@@ -450,6 +463,8 @@ class EncryptedMessage(object):
                                                      self._config[PORT_KEY]))
 
         d = defer.Deferred()
+        # we don't pass an ssl context factory to the ESMTPSenderFactory
+        # because ssl will be handled by reactor.connectSSL() below.
         factory = smtp.ESMTPSenderFactory(
             "",  # username is blank because server does not use auth.
             "",  # password is blank because server does not use auth.
@@ -457,15 +472,15 @@ class EncryptedMessage(object):
             self._user.dest.addrstr,
             StringIO(msg),
             d,
-            contextFactory=CtxFactory(self._config[CERT_KEY],
-                                      self._config[KEY_KEY]),
-            requireAuthentication=False)
+            requireAuthentication=False,
+            requireTransportSecurity=True)
         signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr)
-        reactor.connectTCP(
+        reactor.connectSSL(
             self._config[HOST_KEY],
             self._config[PORT_KEY],
-            factory
-        )
+            factory,
+            contextFactory=SSLContextFactory(self._config[CERT_KEY],
+                                             self._config[KEY_KEY]))
         d.addCallback(self.sendSuccess)
         d.addErrback(self.sendError)
         return d
diff --git a/src/leap/mail/smtp/tests/__init__.py b/src/leap/mail/smtp/tests/__init__.py
index 7fed7da..9b54de3 100644
--- a/src/leap/mail/smtp/tests/__init__.py
+++ b/src/leap/mail/smtp/tests/__init__.py
@@ -113,8 +113,8 @@ class TestCaseWithKeyManager(BaseLeapTest):
             'username': address,
             'password': '<password>',
             'encrypted_only': True,
-            'cert': 'blah',
-            'key': 'bleh',
+            'cert': 'src/leap/mail/smtp/tests/cert/server.crt',
+            'key': 'src/leap/mail/smtp/tests/cert/server.key',
         }
 
         class Response(object):
diff --git a/src/leap/mail/smtp/tests/cert/server.crt b/src/leap/mail/smtp/tests/cert/server.crt
new file mode 100644
index 0000000..a27391c
--- /dev/null
+++ b/src/leap/mail/smtp/tests/cert/server.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFBjCCAu4CCQCWn3oMoQrDJTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJV
+UzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMB4XDTEzMTAyMzE0NDUwNFoXDTE2MDcxOTE0NDUwNFowRTELMAkG
+A1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0
+IFdpZGdpdHMgUHR5IEx0ZDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
+APexTvEvG7cSmZdAERHt9TB11cSor54Y/F7NmYMdSOJNi4Y0kwkSslpdfipi+mt/
+NFg/uGKi1mcgvuXdVbVPZ9rCgVpIzMncO8RAP7a5+I2zKUzqMCCbLH16sYpo/rDk
+VQ5V15TwLsTzOFGG8Cgp68TR8zHuZ4Edf2zMGC1IaiJ6W38LTnJgsowYOCFDAF3z
+L36kxMO5gNGEUYV6tjltx+rAcXka3po+xiAgvW6q65UUgDHcIdEGG2dc9bkxxPl7
+RkprF2RwwADNzYS7Tn+Hpmjy06pfYZHNME+Iw515bCRF3GQFUU4BpGnY7EO+h4P9
+Kb1h948gUT9/oswXG+q2Kwk8AoggMJkUOWDFiCa5UjW1GBoxxb7VtZ+QTJXxlFWc
+M2VzT7M/HX+P4b05vY4MXJjxPAFKrAGS7J8DKW8WJNUnXa9XSDBHg5qijDzZ/zGm
+HTdG6iADnJLmOHBQgFQ12a/n9mYV2GPVC6FlgDzG9f0/SUPBUCafyWYz1LwKY4VM
+2NLx/iwYMQsNIMSZQfNmufNDBr70+BShe3ZpbmKB/J33d87AuJd2HjnsThTEAAr+
+6CejyYmwFutoDUCF8IaKGJEp7OGP2//ub4nt5WwW8DYLRi8EqtzEnxPo5ZiayHMY
+GHR1jpX1O5JVJFUE79bZCFFHKmtJc4kVZS4m4rTLsk83AgMBAAEwDQYJKoZIhvcN
+AQEFBQADggIBAEt4PIRqVuALQSdgZ+GiZYuvEVjxoDVtMSc/ym93Gi8R7DDivFH9
+4suQc5QUiuEF8lpEtkmh+PZ+oFdQkjhBH80h7p4BUSyBy5Yi6dy7ATTlBAqwzCYZ
+4wzHeJzu1SI6FinZLksoULbcw04n410aGHkLa6I9O3vCC4kXSnBlwU1sUsJphxM2
+3pkHBpvv79XYf5kFqZPzF16aO7rxFuVvqgXLyzwuyP9kH5zMA21Kioxs/pNyg1lm
+5h0VinpHLPse+4tYih1L1WLMpEZiSwZgFhoRtlcdIVXokZPaX4G2EkdrMmSQruWg
+Uz8Av6LEYHmRfbYwYM2kEX/+AF8thpTQDbvxjqYk5oyGX4wpKGpih1ac/jYu3O8B
+VLhbxZlBYcLxCqqNsGJrWaiHj2Jf4GhUB0O9hXfaZDMqEGXT9GzOz0yF6b3pDQVy
+H0lKIBb+kQzB/jhZKu4vrTAowXtt/av5d7D+rpAU1SxfUhBOPNSRoJUI5NSBbokp
+a7u4azdB2IQETX3d2rhDk09EbG1XmMi5Vg1oa8nxfMOWXZnDMusJoZClKjrthmwd
+rtR5et44XYhX6p217RBkYMDOVFT7aZpu4SaFeqZIuarVYodSmgXToOFXPsrLppRQ
+adOT0FpU64RPNrQz5NF1bSIjqrHSaRVacf8yr7qqxNnpMsrtkDJzsMBz
+-----END CERTIFICATE-----
diff --git a/src/leap/mail/smtp/tests/cert/server.key b/src/leap/mail/smtp/tests/cert/server.key
new file mode 100644
index 0000000..197a449
--- /dev/null
+++ b/src/leap/mail/smtp/tests/cert/server.key
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEA97FO8S8btxKZl0AREe31MHXVxKivnhj8Xs2Zgx1I4k2LhjST
+CRKyWl1+KmL6a380WD+4YqLWZyC+5d1VtU9n2sKBWkjMydw7xEA/trn4jbMpTOow
+IJssfXqximj+sORVDlXXlPAuxPM4UYbwKCnrxNHzMe5ngR1/bMwYLUhqInpbfwtO
+cmCyjBg4IUMAXfMvfqTEw7mA0YRRhXq2OW3H6sBxeRremj7GICC9bqrrlRSAMdwh
+0QYbZ1z1uTHE+XtGSmsXZHDAAM3NhLtOf4emaPLTql9hkc0wT4jDnXlsJEXcZAVR
+TgGkadjsQ76Hg/0pvWH3jyBRP3+izBcb6rYrCTwCiCAwmRQ5YMWIJrlSNbUYGjHF
+vtW1n5BMlfGUVZwzZXNPsz8df4/hvTm9jgxcmPE8AUqsAZLsnwMpbxYk1Sddr1dI
+MEeDmqKMPNn/MaYdN0bqIAOckuY4cFCAVDXZr+f2ZhXYY9ULoWWAPMb1/T9JQ8FQ
+Jp/JZjPUvApjhUzY0vH+LBgxCw0gxJlB82a580MGvvT4FKF7dmluYoH8nfd3zsC4
+l3YeOexOFMQACv7oJ6PJibAW62gNQIXwhooYkSns4Y/b/+5vie3lbBbwNgtGLwSq
+3MSfE+jlmJrIcxgYdHWOlfU7klUkVQTv1tkIUUcqa0lziRVlLibitMuyTzcCAwEA
+AQKCAgAFQdcqGVTeQt/NrQdvuPw+RhH+dZIcqe0ZWgXLGaEFZJ30gEMqqyHr9xYJ
+ckZcZ7vFr7yLI2enkrNaj6MVERVkOEKkluz5z9nY5YA0safL4iSbRFE3L/P2ydbg
+2C+ns4D2p+3GdH6ZoYvtdw6723/skoQ16Bh8ThL5TS+qLmJKTwyIGsZUeSbxAEaY
+tiJY3COC7Z5bhSFt0QAl9B/QAjt/CQyfhGl7Hp/36Jn8slYDuQariD+TfyyvufJh
+NuQ2Y15vj+xULmx01+lnys30uP1YNuc1M4cPoCpJVd7JBd28u1rdKJu8Kx7BPGBv
+Y6jerU3ofh7SA96VmXDsIgVuquUo51Oklspe6a9VaDmzLvjYqJsBKQ7BH3J2f07x
+NiOob56CGXykX51Ig3WBK1wKn+pA69FL62DbkEa6SykGCqdZPdgBF/kiMc0TESsl
+867Em63Yx/2hq+mG3Dknnq8jWXf+Es/zZSSak6N4154IxPOD3m1hzuUq73PP7Ptt
+KFe6NfU0DmAuTJL3FqNli8F8lFfvJfuwmW2qk5iTMfwPxybSd8FPbGxi7aRgoZdh
+7fIbTFJ0X2f83/SO+9rCzV+B091+d7TM8AaOJ4dEoS74rlRZg53EgmAU0phVnE+l
+taMNKGHy2kpJrv9IHX3w5Gm6CjNJj5t4ccS0J18NFFJ+j077eQKCAQEA/RJNRUBS
+mI5l0eirl78Q9uDPh1usChZpQiLsvscIJITWQ1vtXSRCvP0hVQRRv8+4CtrZr2rX
+v0afkzg/3HNFaNsjYT6aHjgnombFqfpyS/NZN/p3gOzi2h+1Sujzz5fBUGhNLVgZ
+F2GLnJbiIHnM1BmKA6597pHpXcRMh1E3DSjDMQAEEsBgF6MyS+MT9WfNwHvJukii
+k028tNzR4wRq3Xo3WTfvXZRjbX54Ew9Zy3+TFiu19j2FmuOoqyj+ZvMic4EYmTaY
+BWm7viDff4dW34dR9sYCuTWWehLtMJGroA38e7lTLfNOHNDGaUZWkfxs4uJCsxvP
+0fPp3xlbU3NUGwKCAQEA+o8SeHwEN+VN2dZvC3wFvbnRvWLc1aLnNcndRE9QLVcC
+B4LMRuQMpxaNYRiSQPppoPTNq6zWbo6FEjUO5Md7R8I8dbg1vHo4PzuHOu2wXNcm
+DEicocCpSKShSS27NCK6uoSsTqTIlG4u+1x9/R2gJEjlTqjeIkOQkPv7PbWhrUyt
+XqvzPy4bewOz9Brmd6ryi8ZLtNbUSNwMyd64s9b1V4A6JRlYZrMDOQ6kXEZo+mbL
+ynet0vuj7lYxsAZvxoPIq+Gi5i0CrDYtze6JCg+kGahjMX0zXRjXrYh/YID8NWYT
+0GXr2+a0V5pXg86YCDp/jpr3lq75HJJ+vIvm2VHLFQKCAQATEm0GWgmfe6PKxPkh
+j4GsyVZ6gfseK4A1PsKOwhsn/WbUXrotuczZx03axV+P0AyzrLiZErk9rgnao3OU
+no9Njq5E5t3ghyTdhVdCLyCr/qPrpxGYgsG55IfaJGIzc+FauPGQCEKj03MdEvXp
+sqQwG9id3GmbMB3hNij6TbGTaU4EhFbKPvs+7Mqek3dumCsWZX3Xbx/pcANXsgiT
+TkLrfAltzNxaNhOkLdLIxPBkeLHSCutEqnBGMwAEHivGAG7JO6Jp8YZVahl/A6U0
+TDPM1rrjmRqdcJ9thb2gWmoPvt4XSOku3lY1r7o0NtvRVq+yDZEvRFpOHU6zxIpw
+aJGfAoIBAQDiTvvF62379pc8nJwr6VdeKEozHuqL49mmEbBTFLg8W4wvsIpFtZFg
+EdSc0I65NfTWNobV+wSrUvsKmPXc2fiVtfDZ+wo+NL49Ds10Al/7WzC4g5VF3DiK
+rngnGrEtw/iYo2Dmn5uzxVmWG9KIHowYeeb0Bz6sAA7BhXdGI5nmZ41oJzNL659S
+muOdJfboO3Vbnj2fFzMio+7BHvQBK7Tp1Z2vCJd6G1Jb5Me7uLT1BognVbWhDTzh
+9uRmM0oeKcXEycZS1HDHjyAMEtmgRsRXkGoXtxf/jIKx8MnsJlSm/o4C+yvvsQ9O
+2M8W9DEJrZys93eNmHjUv9TNBCf8Pg6JAoIBAQDDItnQPLntCUgd7dy0dDjQYBGN
+4wVRJNINpgjqwJj0hVjB/dmvrcxkXcOG4VAH+iNH8A25qLU+RTDcNipuL3uEFKbF
+O4DSjFih3qL1Y8otTXSrPeqZOMvYpY8dXS5uyI7DSWQQZyZ9bMpeWbxgx4LHqPPH
+rdcVJy9Egw1ZIOA7JBFM02uGn9TVwFzNUJk0G/3xwVHzDxYNbJ98vDfflc2vD4CH
+OAN6un0pOuol2h200F6zFgc5mbETWHCPIom+ZMXIX3bq7g341c/cgqIELPTk8DLS
+s+AgrZ4qYmskrFaD0PHakWsQNHGC8yOh80lgE3Gl4nxSGAvkcR7dkSmsIQFL
+-----END RSA PRIVATE KEY-----
diff --git a/src/leap/mail/smtp/tests/test_smtprelay.py b/src/leap/mail/smtp/tests/test_smtprelay.py
index a529c93..7fefe77 100644
--- a/src/leap/mail/smtp/tests/test_smtprelay.py
+++ b/src/leap/mail/smtp/tests/test_smtprelay.py
@@ -23,8 +23,8 @@ SMTP relay tests.
 
 import re
 
-
 from datetime import datetime
+from gnupg._util import _make_binary_stream
 from twisted.test import proto_helpers
 from twisted.mail.smtp import (
     User,
@@ -33,7 +33,6 @@ from twisted.mail.smtp import (
 )
 from mock import Mock
 
-
 from leap.mail.smtp.smtprelay import (
     SMTPFactory,
     EncryptedMessage,
@@ -45,7 +44,6 @@ from leap.mail.smtp.tests import (
 )
 from leap.keymanager import openpgp
 
-
 # some regexps
 IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \
     "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"
@@ -127,11 +125,22 @@ class TestSmtpRelay(TestCaseWithKeyManager):
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
         m.eomReceived()
+        # assert structure of encrypted message
+        self.assertTrue('Content-Type' in m._msg)
+        self.assertEqual('multipart/encrypted', m._msg.get_content_type())
+        self.assertEqual('application/pgp-encrypted',
+                         m._msg.get_param('protocol'))
+        self.assertEqual(2, len(m._msg.get_payload()))
+        self.assertEqual('application/pgp-encrypted',
+                         m._msg.get_payload(0).get_content_type())
+        self.assertEqual('application/octet-stream',
+                         m._msg.get_payload(1).get_content_type())
         privkey = self._km.get_key(
             ADDRESS, openpgp.OpenPGPKey, private=True)
-        decrypted = self._km.decrypt(m._message.get_payload(), privkey)
+        decrypted = self._km.decrypt(
+            m._msg.get_payload(1).get_payload(), privkey)
         self.assertEqual(
-            '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n',
+            '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n',
             decrypted,
             'Decrypted text differs from plaintext.')
 
@@ -149,14 +158,24 @@ class TestSmtpRelay(TestCaseWithKeyManager):
             m.lineReceived(line)
         # trigger encryption and signing
         m.eomReceived()
+        # assert structure of encrypted message
+        self.assertTrue('Content-Type' in m._msg)
+        self.assertEqual('multipart/encrypted', m._msg.get_content_type())
+        self.assertEqual('application/pgp-encrypted',
+                         m._msg.get_param('protocol'))
+        self.assertEqual(2, len(m._msg.get_payload()))
+        self.assertEqual('application/pgp-encrypted',
+                         m._msg.get_payload(0).get_content_type())
+        self.assertEqual('application/octet-stream',
+                         m._msg.get_payload(1).get_content_type())
         # decrypt and verify
         privkey = self._km.get_key(
             ADDRESS, openpgp.OpenPGPKey, private=True)
         pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey)
         decrypted = self._km.decrypt(
-            m._message.get_payload(), privkey, verify=pubkey)
+            m._msg.get_payload(1).get_payload(), privkey, verify=pubkey)
         self.assertEqual(
-            '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n',
+            '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n',
             decrypted,
             'Decrypted text differs from plaintext.')
 
@@ -175,22 +194,34 @@ class TestSmtpRelay(TestCaseWithKeyManager):
             m.lineReceived(line)
         # trigger signing
         m.eomReceived()
+        # assert structure of signed message
+        self.assertTrue('Content-Type' in m._msg)
+        self.assertEqual('multipart/signed', m._msg.get_content_type())
+        self.assertEqual('application/pgp-signature',
+                         m._msg.get_param('protocol'))
+        self.assertEqual('pgp-sha512', m._msg.get_param('micalg'))
         # assert content of message
+        self.assertEqual(
+            m._msg.get_payload(0).get_payload(decode=True),
+            '\r\n'.join(self.EMAIL_DATA[9:13]))
+        # assert content of signature
         self.assertTrue(
-            m._message.get_payload().startswith(
-                '-----BEGIN PGP SIGNED MESSAGE-----\n' +
-                'Hash: SHA1\n\n' + 
-                ('\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n' +
-                '-----BEGIN PGP SIGNATURE-----\n')),
+            m._msg.get_payload(1).get_payload().startswith(
+                '-----BEGIN PGP SIGNATURE-----\n'),
             'Message does not start with signature header.')
         self.assertTrue(
-            m._message.get_payload().endswith(
+            m._msg.get_payload(1).get_payload().endswith(
                 '-----END PGP SIGNATURE-----\n'),
             'Message does not end with signature footer.')
         # assert signature is valid
         pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey)
+        # replace EOL before verifying (according to rfc3156)
+        signed_text = re.sub('\r?\n', '\r\n',
+                             m._msg.get_payload(0).as_string())
         self.assertTrue(
-            self._km.verify(m._message.get_payload(), pubkey),
+            self._km.verify(signed_text,
+                            pubkey,
+                            detached_sig=m._msg.get_payload(1).get_payload()),
             'Signature could not be verified.')
 
     def test_missing_key_rejects_address(self):