Merge branch 'release-0.3.7'

12 files changed, 582 insertions, 354 deletions

diff --git a/mail/CHANGELOG b/mail/CHANGELOG
index 5755e59..f15482c 100644
--- a/mail/CHANGELOG
+++ b/mail/CHANGELOG
@@ -1,3 +1,23 @@
+0.3.7 Nov 15:
+  o Uses deferToThread for sendMail. Closes #3937
+  o Update pkey to allow multiple accounts. Solves: #4394
+  o Change SMTP service name from "relay" to "gateway". Closes #4416.
+  o Identify ourselves with a fqdn, always. Closes: #4441
+  o Remove 'multipart/encrypted' header after decrypting incoming
+    mail. Closes #4454.
+  o Fix several bugs with imap mailbox getUIDNext and notifiers that
+    were breaking the mail indexing after message deletion. This
+    solves also the perceived mismatch between the number of unread
+    mails reported by bitmask_client and the number reported by
+    MUAs. Closes: #4461
+  o Check username in authentications. Closes: #4299
+  o Reject senders that aren't the user that is currently logged
+    in. Fixes #3952.
+  o Prevent already encrypted outgoing messages from being encrypted
+    again. Closes #4324.
+  o Correctly handle email headers when gatewaying messages. Also add
+    OpenPGP header. Closes #4322 and #4447.
+
 0.3.6 Nov 1:
   o Add support for non-ascii characters in emails. Closes #4000.
   o Default to UTF-8 when there is no charset parsed from the mail
@@ -23,7 +43,8 @@
 
 0.3.2 Sep 6:
   o Make mail services bind to 127.0.0.1. Closes: #3627.
-  o Signal unread message to UI when message is saved locally. Closes: #3654.
+  o Signal unread message to UI when message is saved locally. Closes:
+    #3654.
   o Signal unread to UI when flag in message change. Closes: #3662.
   o Use dirspec instead of plain xdg. Closes #3574.
   o SMTP service invocation returns factory instance.
diff --git a/mail/MANIFEST.in b/mail/MANIFEST.in
index 7f6148e..83264d4 100644
--- a/mail/MANIFEST.in
+++ b/mail/MANIFEST.in
@@ -2,3 +2,4 @@ include pkg/*
 include versioneer.py
 include LICENSE
 include CHANGELOG
+include README.rst
diff --git a/mail/pkg/requirements.pip b/mail/pkg/requirements.pip
index 4780b5c..7ed5087 100644
--- a/mail/pkg/requirements.pip
+++ b/mail/pkg/requirements.pip
@@ -1,4 +1,6 @@
+zope.interface
 leap.soledad.client>=0.3.0
 leap.common>=0.3.5
 leap.keymanager>=0.3.4
 twisted  # >= 12.0.3 ??
+zope.proxy
diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index dd65def..3422ed5 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -28,6 +28,7 @@ from email.parser import Parser
 from twisted.python import log
 from twisted.internet.task import LoopingCall
 from twisted.internet.threads import deferToThread
+from zope.proxy import sameProxiedObjects
 
 from leap.common import events as leap_events
 from leap.common.check import leap_assert, leap_assert_type
@@ -39,6 +40,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
+from leap.keymanager.openpgp import OpenPGPKey
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
@@ -66,7 +68,7 @@ class LeapIncomingMail(object):
     fetching_lock = threading.Lock()
 
     def __init__(self, keymanager, soledad, imap_account,
-                 check_period):
+                 check_period, userid):
 
         """
         Initialize LeapIMAP.
@@ -88,14 +90,14 @@ class LeapIncomingMail(object):
         leap_assert_type(soledad, Soledad)
         leap_assert(check_period, "need a period to check incoming mail")
         leap_assert_type(check_period, int)
+        leap_assert(userid, "need a userid to initialize")
 
         self._keymanager = keymanager
         self._soledad = soledad
         self.imapAccount = imap_account
         self._inbox = self.imapAccount.getMailbox('inbox')
+        self._userid = userid
 
-        self._pkey = self._keymanager.get_all_keys_in_local_db(
-            private=True).pop()
         self._loop = None
         self._check_period = check_period
 
@@ -107,6 +109,13 @@ class LeapIncomingMail(object):
         """
         self._soledad.create_index("just-mail", "incoming")
 
+    @property
+    def _pkey(self):
+        if sameProxiedObjects(self._keymanager, None):
+            logger.warning('tried to get key, but null keymanager found')
+            return None
+        return self._keymanager.get_key(self._userid, OpenPGPKey, private=True)
+
     #
     # Public API: fetch, start_loop, stop.
     #
@@ -118,6 +127,8 @@ class LeapIncomingMail(object):
         Calls a deferred that will execute the fetch callback
         in a separate thread
         """
+        logger.debug("fetching mail for: %s %s" % (
+            self._soledad.uuid, self._userid))
         if not self.fetching_lock.locked():
             d = deferToThread(self._sync_soledad)
             d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error)
@@ -130,16 +141,19 @@ class LeapIncomingMail(object):
         """
         Starts a loop to fetch mail.
         """
-        self._loop = LoopingCall(self.fetch)
-        self._loop.start(self._check_period)
+        if self._loop is None:
+            self._loop = LoopingCall(self.fetch)
+            self._loop.start(self._check_period)
+        else:
+            logger.warning("Tried to start an already running fetching loop.")
 
     def stop(self):
         """
         Stops the loop that fetches mail.
         """
-        # XXX should cancel ongoing fetches too.
         if self._loop and self._loop.running is True:
             self._loop.stop()
+            self._loop = None
 
     #
     # Private methods.
@@ -203,7 +217,7 @@ class LeapIncomingMail(object):
         Generic errback
         """
         err = failure.value
-        logger.error("error!: %r" % (err,))
+        logger.exception("error!: %r" % (err,))
 
     def _decryption_error(self, failure):
         """
@@ -334,6 +348,7 @@ class LeapIncomingMail(object):
         :return: data, possibly descrypted.
         :rtype: str
         """
+        # TODO split this method
         leap_assert_type(data, unicode)
 
         parser = Parser()
@@ -375,6 +390,8 @@ class LeapIncomingMail(object):
                 decrdata = decrdata.encode(encoding, 'replace')
 
             decrmsg = parser.parsestr(decrdata)
+            # remove original message's multipart/encrypted content-type
+            del(origmsg['content-type'])
             # replace headers back in original message
             for hkey, hval in decrmsg.items():
                 try:
@@ -417,7 +434,6 @@ class LeapIncomingMail(object):
                          incoming message
         :type msgtuple: (SoledadDocument, str)
         """
-        print "adding message locally....."
         doc, data = msgtuple
         self._inbox.addMessage(data, (self.RECENT_FLAG,))
         leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py
index 7a9f810..bb2830d 100644
--- a/mail/src/leap/mail/imap/server.py
+++ b/mail/src/leap/mail/imap/server.py
@@ -23,6 +23,7 @@ import StringIO
 import cStringIO
 import time
 
+from collections import defaultdict
 from email.parser import Parser
 
 from zope.interface import implements
@@ -241,6 +242,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 
         :rtype: SoledadDocument
         """
+        # XXX only upper for INBOX ---
         name = name.upper()
         doc = self._soledad.get_from_index(
             self.TYPE_MBOX_IDX, self.MBOX_KEY, name)
@@ -274,6 +276,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :returns: a a SoledadMailbox instance
         :rtype: SoledadMailbox
         """
+        # XXX only upper for INBOX
         name = name.upper()
         if name not in self.mailboxes:
             raise imap4.MailboxException("No such mailbox")
@@ -299,6 +302,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :returns: True if successful
         :rtype: bool
         """
+        # XXX only upper for INBOX
         name = name.upper()
         # XXX should check mailbox name for RFC-compliant form
 
@@ -360,6 +364,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 
         :rtype: bool
         """
+        # XXX only upper for INBOX
         name = name.upper()
 
         if name not in self.mailboxes:
@@ -385,6 +390,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
                       names. use with care.
         :type force: bool
         """
+        # XXX only upper for INBOX
         name = name.upper()
         if not name in self.mailboxes:
             raise imap4.MailboxException("No such mailbox")
@@ -422,6 +428,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :param newname: new name of the mailbox
         :type newname: str
         """
+        # XXX only upper for INBOX
         oldname = oldname.upper()
         newname = newname.upper()
 
@@ -487,7 +494,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         # maybe we should store subscriptions in another
         # document...
         if not name in self.mailboxes:
-            print "not this mbox"
             self.addMailbox(name)
         mbox = self._get_mailbox_by_name(name)
 
@@ -785,6 +791,7 @@ class LeapMessage(WithMsgFields):
         return dict(filter_by_cond)
 
     # --- no multipart for now
+    # XXX Fix MULTIPART SUPPORT!
 
     def isMultipart(self):
         return False
@@ -967,6 +974,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         docs = self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_UID_IDX,
             self.TYPE_MESSAGE_VAL, self.mbox, str(uid))
+
         return docs[0] if docs else None
 
     def get_msg_by_uid(self, uid):
@@ -984,6 +992,47 @@ class MessageCollection(WithMsgFields, IndexedDB):
         if doc:
             return LeapMessage(doc)
 
+    def get_by_index(self, index):
+        """
+        Retrieves a mesage document by mailbox index.
+
+        :param index: the index of the sequence (zero-indexed)
+        :type index: int
+        """
+        try:
+            return self.get_all()[index]
+        except IndexError:
+            return None
+
+    def get_msg_by_index(self, index):
+        """
+        Retrieves a LeapMessage by sequence index.
+
+        :param index: the index of the sequence (zero-indexed)
+        :type index: int
+        """
+        doc = self.get_by_index(index)
+        if doc:
+            return LeapMessage(doc)
+
+    def is_deleted(self, doc):
+        """
+        Returns whether a given doc is deleted or not.
+
+        :param doc: the document to check
+        :rtype: bool
+        """
+        return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
+
+    def get_last(self):
+        """
+        Gets the last LeapMessage
+        """
+        _all = self.get_all()
+        if not _all:
+            return None
+        return LeapMessage(_all[-1])
+
     def get_all(self):
         """
         Get all message documents for the selected mailbox.
@@ -993,9 +1042,13 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :rtype: list of SoledadDocument
         """
         # XXX this should return LeapMessage instances
-        return self._soledad.get_from_index(
+        all_docs = [doc for doc in self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox)
+            self.TYPE_MESSAGE_VAL, self.mbox)]
+            #if not self.is_deleted(doc)]
+        # highly inneficient, but first let's grok it and then
+        # let's worry about efficiency.
+        return sorted(all_docs, key=lambda item: item.content['uid'])
 
     def unseen_iter(self):
         """
@@ -1075,8 +1128,11 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :return: LeapMessage or None if not found.
         :rtype: LeapMessage
         """
+        #try:
+            #return self.get_msg_by_uid(uid)
         try:
-            return self.get_msg_by_uid(uid)
+            return [doc
+                    for doc in self.get_all()][uid - 1]
         except IndexError:
             return None
 
@@ -1116,7 +1172,7 @@ class SoledadMailbox(WithMsgFields):
     CMD_UIDVALIDITY = "UIDVALIDITY"
     CMD_UNSEEN = "UNSEEN"
 
-    listeners = []
+    _listeners = defaultdict(set)
 
     def __init__(self, mbox, soledad=None, rw=1):
         """
@@ -1150,11 +1206,18 @@ class SoledadMailbox(WithMsgFields):
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
 
-        # the server itself is a listener to the mailbox.
-        # so we can notify it (and should!) after chanes in flags
-        # and number of messages.
-        print "emptying the listeners"
-        map(lambda i: self.listeners.remove(i), self.listeners)
+    @property
+    def listeners(self):
+        """
+        Returns listeners for this mbox.
+
+        The server itself is a listener to the mailbox.
+        so we can notify it (and should!) after changes in flags
+        and number of messages.
+
+        :rtype: set
+        """
+        return self._listeners[self.mbox]
 
     def addListener(self, listener):
         """
@@ -1164,7 +1227,7 @@ class SoledadMailbox(WithMsgFields):
         :type listener: an object that implements IMailboxListener
         """
         logger.debug('adding mailbox listener: %s' % listener)
-        self.listeners.append(listener)
+        self.listeners.add(listener)
 
     def removeListener(self, listener):
         """
@@ -1173,25 +1236,24 @@ class SoledadMailbox(WithMsgFields):
         :param listener: listener to remove
         :type listener: an object that implements IMailboxListener
         """
-        logger.debug('removing mailbox listener: %s' % listener)
-        try:
-            self.listeners.remove(listener)
-        except ValueError:
-            logger.error(
-                "removeListener: cannot remove listener %s" % listener)
+        self.listeners.remove(listener)
 
     def _get_mbox(self):
         """
         Returns mailbox document.
 
-        :return: A SoledadDocument containing this mailbox.
-        :rtype: SoledadDocument
+        :return: A SoledadDocument containing this mailbox, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
         """
-        query = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MBOX_VAL, self.mbox)
-        if query:
-            return query.pop()
+        try:
+            query = self._soledad.get_from_index(
+                SoledadBackedAccount.TYPE_MBOX_IDX,
+                self.TYPE_MBOX_VAL, self.mbox)
+            if query:
+                return query.pop()
+        except Exception as exc:
+            logger.error("Unhandled error %r" % exc)
 
     def getFlags(self):
         """
@@ -1288,8 +1350,12 @@ class SoledadMailbox(WithMsgFields):
 
         :rtype: int
         """
-        # XXX reimplement with proper index
-        return self.messages.count() + 1
+        last = self.messages.get_last()
+        if last:
+            nextuid = last.getUID() + 1
+        else:
+            nextuid = 1
+        return nextuid
 
     def getMessageCount(self):
         """
@@ -1376,6 +1442,8 @@ class SoledadMailbox(WithMsgFields):
 
         self.messages.add_msg(message, flags=flags, date=date,
                               uid=uid_next)
+
+        # XXX recent should not include deleted...??
         exists = len(self.messages)
         recent = len(self.messages.get_recent())
         for listener in self.listeners:
@@ -1435,16 +1503,35 @@ class SoledadMailbox(WithMsgFields):
         :rtype: A tuple of two-tuples of message sequence numbers and
                 LeapMessage
         """
-        # XXX implement sequence numbers (uid = 0)
         result = []
+        sequence = True if uid == 0 else False
 
         if not messages.last:
-            messages.last = self.messages.count()
+            try:
+                iter(messages)
+            except TypeError:
+                # looks like we cannot iterate
+                last = self.messages.get_last()
+                uid_last = last.getUID()
+                messages.last = uid_last
+
+        # for sequence numbers (uid = 0)
+        if sequence:
+            for msg_id in messages:
+                msg = self.messages.get_msg_by_index(msg_id - 1)
+                if msg:
+                    result.append((msg.getUID(), msg))
+                else:
+                    print "fetch %s, no msg found!!!" % msg_id
+
+        else:
+            for msg_id in messages:
+                msg = self.messages.get_msg_by_uid(msg_id)
+                if msg:
+                    result.append((msg_id, msg))
+                else:
+                    print "fetch %s, no msg found!!!" % msg_id
 
-        for msg_id in messages:
-            msg = self.messages.get_msg_by_uid(msg_id)
-            if msg:
-                result.append((msg_id, msg))
         return tuple(result)
 
     def _signal_unread_to_ui(self):
diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py
index 5f7322a..feb2593 100644
--- a/mail/src/leap/mail/imap/service/imap.py
+++ b/mail/src/leap/mail/imap/service/imap.py
@@ -25,11 +25,12 @@ from twisted.internet.protocol import ServerFactory
 from twisted.internet.error import CannotListenError
 from twisted.mail import imap4
 from twisted.python import log
+from twisted import cred
 
 logger = logging.getLogger(__name__)
 
 from leap.common import events as leap_events
-from leap.common.check import leap_assert, leap_assert_type
+from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import KeyManager
 from leap.mail.imap.server import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
@@ -54,10 +55,13 @@ class LeapIMAPServer(imap4.IMAP4Server):
     def __init__(self, *args, **kwargs):
         # pop extraneous arguments
         soledad = kwargs.pop('soledad', None)
-        user = kwargs.pop('user', None)
+        uuid = kwargs.pop('uuid', None)
+        userid = kwargs.pop('userid', None)
         leap_assert(soledad, "need a soledad instance")
         leap_assert_type(soledad, Soledad)
-        leap_assert(user, "need a user in the initialization")
+        leap_assert(uuid, "need a user in the initialization")
+
+        self._userid = userid
 
         # initialize imap server!
         imap4.IMAP4Server.__init__(self, *args, **kwargs)
@@ -77,6 +81,12 @@ class LeapIMAPServer(imap4.IMAP4Server):
         #self.theAccount = theAccount
 
     def lineReceived(self, line):
+        """
+        Attempt to parse a single line from the server.
+
+        :param line: the line from the server, without the line delimiter.
+        :type line: str
+        """
         if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
@@ -87,7 +97,21 @@ class LeapIMAPServer(imap4.IMAP4Server):
         imap4.IMAP4Server.lineReceived(self, line)
 
     def authenticateLogin(self, username, password):
-        # all is allowed so far. use realm instead
+        """
+        Lookup the account with the given parameters, and deny
+        the improper combinations.
+
+        :param username: the username that is attempting authentication.
+        :type username: str
+        :param password: the password to authenticate with.
+        :type password: str
+        """
+        # XXX this should use portal:
+        # return portal.login(cred.credentials.UsernamePassword(user, pass)
+        if username != self._userid:
+            # bad username, reject.
+            raise cred.error.UnauthorizedLogin()
+        # any dummy password is allowed so far. use realm instead!
         leap_events.signal(IMAP_CLIENT_LOGIN, "1")
         return imap4.IAccount, self.theAccount, lambda: None
 
@@ -108,28 +132,32 @@ class LeapIMAPFactory(ServerFactory):
     capabilities.
     """
 
-    def __init__(self, user, soledad):
+    def __init__(self, uuid, userid, soledad):
         """
         Initializes the server factory.
 
-        :param user: user ID. **right now it's uuid**
-                     this might change!
-        :type user: str
+        :param uuid: user uuid
+        :type uuid: str
+
+        :param userid: user id (user@provider.org)
+        :type userid: str
 
         :param soledad: soledad instance
         :type soledad: Soledad
         """
-        self._user = user
+        self._uuid = uuid
+        self._userid = userid
         self._soledad = soledad
 
         theAccount = SoledadBackedAccount(
-            user, soledad=soledad)
+            uuid, soledad=soledad)
         self.theAccount = theAccount
 
     def buildProtocol(self, addr):
         "Return a protocol suitable for the job."
         imapProtocol = LeapIMAPServer(
-            user=self._user,
+            uuid=self._uuid,
+            userid=self._userid,
             soledad=self._soledad)
         imapProtocol.theAccount = self.theAccount
         imapProtocol.factory = self
@@ -152,9 +180,11 @@ def run_service(*args, **kwargs):
 
     port = kwargs.get('port', IMAP_PORT)
     check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD)
+    userid = kwargs.get('userid', None)
+    leap_check(userid is not None, "need an user id")
 
     uuid = soledad._get_uuid()
-    factory = LeapIMAPFactory(uuid, soledad)
+    factory = LeapIMAPFactory(uuid, userid, soledad)
 
     from twisted.internet import reactor
 
@@ -165,7 +195,8 @@ def run_service(*args, **kwargs):
             keymanager,
             soledad,
             factory.theAccount,
-            check_period)
+            check_period,
+            userid)
     except CannotListenError:
         logger.error("IMAP Service failed to start: "
                      "cannot listen in port %s" % (port,))
diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py
index 3411795..ad11315 100644
--- a/mail/src/leap/mail/imap/tests/test_imap.py
+++ b/mail/src/leap/mail/imap/tests/test_imap.py
@@ -91,7 +91,7 @@ def initialize_soledad(email, gnupg_home, tempdir):
     """
 
     uuid = "foobar-uuid"
-    passphrase = "verysecretpassphrase"
+    passphrase = u"verysecretpassphrase"
     secret_path = os.path.join(tempdir, "secret.gpg")
     local_db_path = os.path.join(tempdir, "soledad.u1db")
     server_url = "http://provider"
@@ -101,6 +101,8 @@ def initialize_soledad(email, gnupg_home, tempdir):
 
         get_doc = Mock(return_value=None)
         put_doc = Mock()
+        lock = Mock(return_value=('atoken', 300))
+        unlock = Mock(return_value=True)
 
         def __call__(self):
             return self
diff --git a/mail/src/leap/mail/smtp/README.rst b/mail/src/leap/mail/smtp/README.rst
index 2b2a118..f625441 100644
--- a/mail/src/leap/mail/smtp/README.rst
+++ b/mail/src/leap/mail/smtp/README.rst
@@ -1,43 +1,23 @@
-Leap SMTP Relay
-===============
+Leap SMTP Gateway
+=================
 
 Outgoing mail workflow:
 
     * LEAP client runs a thin SMTP proxy on the user's device, bound to
       localhost.
-    * User's MUA is configured outgoing SMTP to localhost
-    * When SMTP proxy receives an email from MUA
+    * User's MUA is configured outgoing SMTP to localhost.
+    * When SMTP proxy receives an email from MUA:
         * SMTP proxy queries Key Manager for the user's private key and public
-          keys of all recipients
+          keys of all recipients.
         * Message is signed by sender and encrypted to recipients.
         * If recipient's key is missing, email goes out in cleartext (unless
-          user has configured option to send only encrypted email)
-        * Finally, message is relayed to provider's SMTP relay
-
-
-Dependencies
-------------
-
-Leap SMTP Relay depends on the following python libraries:
-
-  * Twisted 12.3.0 [1]
-  * zope.interface 4.0.3 [2]
-
-[1] http://pypi.python.org/pypi/Twisted/12.3.0
-[2] http://pypi.python.org/pypi/zope.interface/4.0.3
-
-
-How to run
-----------
-
-To launch the SMTP relay, run the following command:
-
-  twistd -y smtprelay.tac
+          user has configured option to send only encrypted email).
+        * Finally, message is gatewayed to provider's SMTP server.
 
 
 Running tests
 -------------
 
-Tests are run using Twisted's Trial API, like this:
+Tests are run using Twisted's Trial API, like this::
 
-  trial leap.email.smtp.tests
+    python setup.py test -s leap.mail.gateway.tests
diff --git a/mail/src/leap/mail/smtp/__init__.py b/mail/src/leap/mail/smtp/__init__.py
index b30cd20..d3eb9e8 100644
--- a/mail/src/leap/mail/smtp/__init__.py
+++ b/mail/src/leap/mail/smtp/__init__.py
@@ -16,7 +16,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 """
-SMTP relay helper function.
+SMTP gateway helper function.
 """
 import logging
 
@@ -26,66 +26,49 @@ from twisted.internet.error import CannotListenError
 logger = logging.getLogger(__name__)
 
 from leap.common.events import proto, signal
-from leap.mail.smtp.smtprelay import SMTPFactory
+from leap.mail.smtp.gateway import SMTPFactory
 
 
-def setup_smtp_relay(port, keymanager, smtp_host, smtp_port,
+def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
                      smtp_cert, smtp_key, encrypted_only):
     """
-    Setup SMTP relay to run with Twisted.
+    Setup SMTP gateway to run with Twisted.
 
-    This function sets up the SMTP relay configuration and the Twisted
+    This function sets up the SMTP gateway configuration and the Twisted
     reactor.
 
     :param port: The port in which to run the server.
     :type port: int
+    :param userid: The user currently logged in
+    :type userid: unicode
     :param keymanager: A Key Manager from where to get recipients' public
                        keys.
     :type keymanager: leap.common.keymanager.KeyManager
     :param smtp_host: The hostname of the remote SMTP server.
     :type smtp_host: str
-    :param smtp_port:  The port of the remote SMTP server.
+    :param smtp_port: The port of the remote SMTP server.
     :type smtp_port: int
     :param smtp_cert: The client certificate for authentication.
     :type smtp_cert: str
     :param smtp_key: The client key for authentication.
     :type smtp_key: str
-    :param encrypted_only: Whether the SMTP relay should send unencrypted mail
+    :param encrypted_only: Whether the SMTP gateway should send unencrypted mail
                            or not.
     :type encrypted_only: bool
 
     :returns: tuple of SMTPFactory, twisted.internet.tcp.Port
     """
-    # The configuration for the SMTP relay is a dict with the following
-    # format:
-    #
-    # {
-    #     'host': '<host>',
-    #     'port': <int>,
-    #     'cert': '<cert path>',
-    #     'key': '<key path>',
-    #     'encrypted_only': <True/False>
-    # }
-    config = {
-        'host': smtp_host,
-        'port': smtp_port,
-        'cert': smtp_cert,
-        'key': smtp_key,
-        'encrypted_only': encrypted_only
-    }
-
     # configure the use of this service with twistd
-    factory = SMTPFactory(keymanager, config)
+    factory = SMTPFactory(userid, keymanager, smtp_host, smtp_port, smtp_cert,
+                          smtp_key, encrypted_only)
     try:
-        tport = reactor.listenTCP(port, factory,
-                                  interface="localhost")
-        signal(proto.SMTP_SERVICE_STARTED, str(smtp_port))
+        tport = reactor.listenTCP(port, factory, interface="localhost")
+        signal(proto.SMTP_SERVICE_STARTED, str(port))
         return factory, tport
     except CannotListenError:
         logger.error("STMP Service failed to start: "
-                     "cannot listen in port %s" % (
-                         smtp_port,))
-        signal(proto.SMTP_SERVICE_FAILED_TO_START, str(smtp_port))
+                     "cannot listen in port %s" % port)
+        signal(proto.SMTP_SERVICE_FAILED_TO_START, str(port))
     except Exception as exc:
-        logger.error("Unhandled error while launching smtp relay service")
+        logger.error("Unhandled error while launching smtp gateway service")
         logger.exception(exc)
diff --git a/mail/src/leap/mail/smtp/smtprelay.py b/mail/src/leap/mail/smtp/gateway.py
index fca66c0..f09ee14 100644
--- a/mail/src/leap/mail/smtp/smtprelay.py
+++ b/mail/src/leap/mail/smtp/gateway.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# smtprelay.py
+# gateway.py
 # Copyright (C) 2013 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
@@ -14,11 +14,10 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
 """
-LEAP SMTP encrypted relay.
+LEAP SMTP encrypted gateway.
 
-The following classes comprise the SMTP relay service:
+The following classes comprise the SMTP gateway service:
 
     * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides
       the SMTPDelivery protocol.
@@ -32,7 +31,6 @@ The following classes comprise the SMTP relay service:
 
 
 """
-
 import re
 from StringIO import StringIO
 from email.Header import Header
@@ -46,6 +44,7 @@ from twisted.mail import smtp
 from twisted.internet.protocol import ServerFactory
 from twisted.internet import reactor, ssl
 from twisted.internet import defer
+from twisted.internet.threads import deferToThread
 from twisted.python import log
 
 from leap.common.check import leap_assert, leap_assert_type
@@ -68,67 +67,18 @@ generator.Generator = RFC3156CompliantGenerator
 
 
 #
-# Exceptions
-#
-
-class MalformedConfig(Exception):
-    """
-    Raised when the configuration dictionary passed as parameter is malformed.
-    """
-    pass
-
-
-#
 # Helper utilities
 #
 
-HOST_KEY = 'host'
-PORT_KEY = 'port'
-CERT_KEY = 'cert'
-KEY_KEY = 'key'
-ENCRYPTED_ONLY_KEY = 'encrypted_only'
-
-
-def assert_config_structure(config):
-    """
-    Assert that C{config} is a dict with the following structure:
-
-        {
-            HOST_KEY: '<str>',
-            PORT_KEY: <int>,
-            CERT_KEY: '<str>',
-            KEY_KEY: '<str>',
-            ENCRYPTED_ONLY_KEY: <bool>,
-        }
-
-    @param config: The dictionary to check.
-    @type config: dict
-    """
-    # assert smtp config structure is valid
-    leap_assert_type(config, dict)
-    leap_assert(HOST_KEY in config)
-    leap_assert_type(config[HOST_KEY], str)
-    leap_assert(PORT_KEY in config)
-    leap_assert_type(config[PORT_KEY], int)
-    leap_assert(CERT_KEY in config)
-    leap_assert_type(config[CERT_KEY], (str, unicode))
-    leap_assert(KEY_KEY in config)
-    leap_assert_type(config[KEY_KEY], (str, unicode))
-    leap_assert(ENCRYPTED_ONLY_KEY in config)
-    leap_assert_type(config[ENCRYPTED_ONLY_KEY], bool)
-    # assert received params are not empty
-    leap_assert(config[HOST_KEY] != '')
-    leap_assert(config[PORT_KEY] is not 0)
-    leap_assert(config[CERT_KEY] != '')
-    leap_assert(config[KEY_KEY] != '')
+LOCAL_FQDN = "bitmask.local"
 
 
 def validate_address(address):
     """
     Validate C{address} as defined in RFC 2822.
 
-    @param address: The address to be validated.
-    @type address: str
+    :param address: The address to be validated.
+    :type address: str
 
     @return: A valid address.
     @rtype: str
@@ -148,46 +98,80 @@ def validate_address(address):
 # SMTPFactory
 #
 
+class SMTPHeloLocalhost(smtp.SMTP):
+    """
+    An SMTP class that ensures a proper FQDN
+    for localhost.
+
+    This avoids a problem in which unproperly configured providers
+    would complain about the helo not being a fqdn.
+    """
+
+    def __init__(self, *args):
+        smtp.SMTP.__init__(self, *args)
+        self.host = LOCAL_FQDN
+
+
 class SMTPFactory(ServerFactory):
     """
-    Factory for an SMTP server with encrypted relaying capabilities.
+    Factory for an SMTP server with encrypted gatewaying capabilities.
     """
+    domain = LOCAL_FQDN
 
-    def __init__(self, keymanager, config):
+    def __init__(self, userid, keymanager, host, port, cert, key,
+                 encrypted_only):
         """
         Initialize the SMTP factory.
 
-        @param keymanager: A KeyManager for retrieving recipient's keys.
-        @type keymanager: leap.common.keymanager.KeyManager
-        @param config: A dictionary with smtp configuration. Should have
-            the following structure:
-                {
-                    HOST_KEY: '<str>',
-                    PORT_KEY: <int>,
-                    CERT_KEY: '<str>',
-                    KEY_KEY: '<str>',
-                    ENCRYPTED_ONLY_KEY: <bool>,
-                }
-        @type config: dict
+        :param userid: The user currently logged in
+        :type userid: unicode
+        :param keymanager: A KeyManager for retrieving recipient's keys.
+        :type keymanager: leap.common.keymanager.KeyManager
+        :param host: The hostname of the remote SMTP server.
+        :type host: str
+        :param port: The port of the remote SMTP server.
+        :type port: int
+        :param cert: The client certificate for authentication.
+        :type cert: str
+        :param key: The client key for authentication.
+        :type key: str
+        :param encrypted_only: Whether the SMTP gateway should send unencrypted
+                               mail or not.
+        :type encrypted_only: bool
         """
         # assert params
         leap_assert_type(keymanager, KeyManager)
-        assert_config_structure(config)
+        leap_assert_type(host, str)
+        leap_assert(host != '')
+        leap_assert_type(port, int)
+        leap_assert(port is not 0)
+        leap_assert_type(cert, unicode)
+        leap_assert(cert != '')
+        leap_assert_type(key, unicode)
+        leap_assert(key != '')
+        leap_assert_type(encrypted_only, bool)
         # and store them
+        self._userid = userid
         self._km = keymanager
-        self._config = config
+        self._host = host
+        self._port = port
+        self._cert = cert
+        self._key = key
+        self._encrypted_only = encrypted_only
 
     def buildProtocol(self, addr):
         """
         Return a protocol suitable for the job.
 
-        @param addr: An address, e.g. a TCP (host, port).
-        @type addr:  twisted.internet.interfaces.IAddress
+        :param addr: An address, e.g. a TCP (host, port).
+        :type addr:  twisted.internet.interfaces.IAddress
 
         @return: The protocol.
         @rtype: SMTPDelivery
         """
-        smtpProtocol = smtp.SMTP(SMTPDelivery(self._km, self._config))
+        smtpProtocol = SMTPHeloLocalhost(SMTPDelivery(
+            self._userid, self._km, self._host, self._port, self._cert,
+            self._key, self._encrypted_only))
         smtpProtocol.factory = self
         return smtpProtocol
 
@@ -203,52 +187,57 @@ class SMTPDelivery(object):
 
     implements(smtp.IMessageDelivery)
 
-    def __init__(self, keymanager, config):
+    def __init__(self, userid, keymanager, host, port, cert, key,
+                 encrypted_only):
         """
         Initialize the SMTP delivery object.
 
-        @param keymanager: A KeyManager for retrieving recipient's keys.
-        @type keymanager: leap.common.keymanager.KeyManager
-        @param config: A dictionary with smtp configuration. Should have
-            the following structure:
-                {
-                    HOST_KEY: '<str>',
-                    PORT_KEY: <int>,
-                    CERT_KEY: '<str>',
-                    KEY_KEY: '<str>',
-                    ENCRYPTED_ONLY_KEY: <bool>,
-                }
-        @type config: dict
-        """
-        # assert params
-        leap_assert_type(keymanager, KeyManager)
-        assert_config_structure(config)
-        # and store them
+        :param userid: The user currently logged in
+        :type userid: unicode
+        :param keymanager: A KeyManager for retrieving recipient's keys.
+        :type keymanager: leap.common.keymanager.KeyManager
+        :param host: The hostname of the remote SMTP server.
+        :type host: str
+        :param port: The port of the remote SMTP server.
+        :type port: int
+        :param cert: The client certificate for authentication.
+        :type cert: str
+        :param key: The client key for authentication.
+        :type key: str
+        :param encrypted_only: Whether the SMTP gateway should send unencrypted
+                               mail or not.
+        :type encrypted_only: bool
+        """
+        self._userid = userid
         self._km = keymanager
-        self._config = config
+        self._host = host
+        self._port = port
+        self._cert = cert
+        self._key = key
+        self._encrypted_only = encrypted_only
         self._origin = None
 
     def receivedHeader(self, helo, origin, recipients):
         """
         Generate the 'Received:' header for a message.
 
-        @param helo: The argument to the HELO command and the client's IP
+        :param helo: The argument to the HELO command and the client's IP
             address.
-        @type helo: (str, str)
-        @param origin: The address the message is from.
-        @type origin: twisted.mail.smtp.Address
-        @param recipients: A list of the addresses for which this message is
+        :type helo: (str, str)
+        :param origin: The address the message is from.
+        :type origin: twisted.mail.smtp.Address
+        :param recipients: A list of the addresses for which this message is
             bound.
-        @type: list of twisted.mail.smtp.User
+        :type: list of twisted.mail.smtp.User
 
         @return: The full "Received" header string.
-        @type: str
+        :type: str
         """
         myHostname, clientIP = helo
-        headerValue = "by %s from %s with ESMTP ; %s" % (
-            myHostname, clientIP, smtp.rfc822date())
+        headerValue = "by bitmask.local from %s with ESMTP ; %s" % (
+            clientIP, smtp.rfc822date())
         # email.Header.Header used for automatic wrapping of long lines
-        return "Received: %s" % Header(headerValue)
+        return "Received: %s" % Header(s=headerValue, header_name='Received')
 
     def validateTo(self, user):
         """
@@ -262,8 +251,8 @@ class SMTPDelivery(object):
         In the end, it returns an encrypted message object that is able to
         send itself to the C{user}'s address.
 
-        @param user: The user whose address we wish to validate.
-        @type: twisted.mail.smtp.User
+        :param user: The user whose address we wish to validate.
+        :type: twisted.mail.smtp.User
 
         @return: A Deferred which becomes, or a callable which takes no
             arguments and returns an object implementing IMessage. This will
@@ -277,12 +266,13 @@ class SMTPDelivery(object):
         # try to find recipient's public key
         try:
             address = validate_address(user.dest.addrstr)
-            pubkey = self._km.get_key(address, OpenPGPKey)
+            # verify if recipient key is available in keyring
+            self._km.get_key(address, OpenPGPKey)  # might raise KeyNotFound
             log.msg("Accepting mail for %s..." % user.dest.addrstr)
             signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr)
         except KeyNotFound:
             # if key was not found, check config to see if will send anyway.
-            if self._config[ENCRYPTED_ONLY_KEY]:
+            if self._encrypted_only:
                 signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr)
                 raise smtp.SMTPBadRcpt(user.dest.addrstr)
             log.msg("Warning: will send an unencrypted message (because "
@@ -290,17 +280,18 @@ class SMTPDelivery(object):
             signal(
                 proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr)
         return lambda: EncryptedMessage(
-            self._origin, user, self._km, self._config)
+            self._origin, user, self._km, self._host, self._port, self._cert,
+            self._key)
 
     def validateFrom(self, helo, origin):
         """
         Validate the address from which the message originates.
 
-        @param helo: The argument to the HELO command and the client's IP
+        :param helo: The argument to the HELO command and the client's IP
             address.
-        @type: (str, str)
-        @param origin: The address the message is from.
-        @type origin: twisted.mail.smtp.Address
+        :type: (str, str)
+        :param origin: The address the message is from.
+        :type origin: twisted.mail.smtp.Address
 
         @return: origin or a Deferred whose callback will be passed origin.
         @rtype: Deferred or Address
@@ -310,6 +301,10 @@ class SMTPDelivery(object):
         """
         # accept mail from anywhere. To reject an address, raise
         # smtp.SMTPBadSender here.
+        if str(origin) != str(self._userid):
+            log.msg("Rejecting sender {0}, expected {1}".format(origin,
+                                                                self._userid))
+            raise smtp.SMTPBadSender(origin)
         self._origin = origin
         return origin
 
@@ -331,16 +326,6 @@ class SSLContextFactory(ssl.ClientContextFactory):
         return ctx
 
 
-def move_headers(origmsg, newmsg):
-    headers = origmsg.items()
-    unwanted_headers = ['content-type', 'mime-version', 'content-disposition',
-                        'content-transfer-encoding']
-    headers = filter(lambda x: x[0].lower() not in unwanted_headers, headers)
-    for hkey, hval in headers:
-        newmsg.add_header(hkey, hval)
-        del(origmsg[hkey])
-
-
 class EncryptedMessage(object):
     """
     Receive plaintext from client, encrypt it and send message to a
@@ -348,36 +333,36 @@ class EncryptedMessage(object):
     """
     implements(smtp.IMessage)
 
-    def __init__(self, fromAddress, user, keymanager, config):
+    def __init__(self, fromAddress, user, keymanager, host, port, cert, key):
         """
         Initialize the encrypted message.
 
-        @param fromAddress: The address of the sender.
-        @type fromAddress: twisted.mail.smtp.Address
-        @param user: The recipient of this message.
-        @type user: twisted.mail.smtp.User
-        @param keymanager: A KeyManager for retrieving recipient's keys.
-        @type keymanager: leap.common.keymanager.KeyManager
-        @param config: A dictionary with smtp configuration. Should have
-            the following structure:
-                {
-                    HOST_KEY: '<str>',
-                    PORT_KEY: <int>,
-                    CERT_KEY: '<str>',
-                    KEY_KEY: '<str>',
-                    ENCRYPTED_ONLY_KEY: <bool>,
-                }
-        @type config: dict
+        :param fromAddress: The address of the sender.
+        :type fromAddress: twisted.mail.smtp.Address
+        :param user: The recipient of this message.
+        :type user: twisted.mail.smtp.User
+        :param keymanager: A KeyManager for retrieving recipient's keys.
+        :type keymanager: leap.common.keymanager.KeyManager
+        :param host: The hostname of the remote SMTP server.
+        :type host: str
+        :param port: The port of the remote SMTP server.
+        :type port: int
+        :param cert: The client certificate for authentication.
+        :type cert: str
+        :param key: The client key for authentication.
+        :type key: str
         """
         # assert params
         leap_assert_type(user, smtp.User)
         leap_assert_type(keymanager, KeyManager)
-        assert_config_structure(config)
         # and store them
         self._fromAddress = fromAddress
         self._user = user
         self._km = keymanager
-        self._config = config
+        self._host = host
+        self._port = port
+        self._cert = cert
+        self._key = key
         # initialize list for message's lines
         self.lines = []
 
@@ -389,8 +374,8 @@ class EncryptedMessage(object):
         """
         Handle another line.
 
-        @param line: The received line.
-        @type line: str
+        :param line: The received line.
+        :type line: str
         """
         self.lines.append(line)
 
@@ -399,21 +384,14 @@ class EncryptedMessage(object):
         Handle end of message.
 
         This method will encrypt and send the message.
+
+        :returns: a deferred
         """
         log.msg("Message data complete.")
         self.lines.append('')  # add a trailing newline
-        try:
-            self._maybe_encrypt_and_sign()
-            return self.sendMessage()
-        except KeyNotFound:
-            return None
-
-    def parseMessage(self):
-        """
-        Separate message headers from body.
-        """
-        parser = Parser()
-        return parser.parsestr('\r\n'.join(self.lines))
+        d = deferToThread(self._maybe_encrypt_and_sign)
+        d.addCallbacks(self.sendMessage, self.skipNoKeyErrBack)
+        return d
 
     def connectionLost(self):
         """
@@ -425,44 +403,81 @@ class EncryptedMessage(object):
         # unexpected loss of connection; don't save
         self.lines = []
 
+    # ends IMessage implementation
+
+    def skipNoKeyErrBack(self, failure):
+        """
+        Errback that ignores a KeyNotFound
+
+        :param failure: the failure
+        :type Failure: Failure
+        """
+        err = failure.value
+        if failure.check(KeyNotFound):
+            pass
+        else:
+            raise err
+
+    def parseMessage(self):
+        """
+        Separate message headers from body.
+        """
+        parser = Parser()
+        return parser.parsestr('\r\n'.join(self.lines))
+
+    def sendQueued(self, r):
+        """
+        Callback for the queued message.
+
+        :param r: The result from the last previous callback in the chain.
+        :type r: anything
+        """
+        log.msg(r)
+
     def sendSuccess(self, r):
         """
         Callback for a successful send.
 
-        @param r: The result from the last previous callback in the chain.
-        @type r: anything
+        :param r: The result from the last previous callback in the chain.
+        :type r: anything
         """
         log.msg(r)
         signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr)
 
-    def sendError(self, e):
+    def sendError(self, failure):
         """
         Callback for an unsuccessfull send.
 
-        @param e: The result from the last errback.
-        @type e: anything
+        :param e: The result from the last errback.
+        :type e: anything
         """
-        log.msg(e)
-        log.err()
         signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr)
+        err = failure.value
+        log.err(err)
+        raise err
 
-    def sendMessage(self):
+    def sendMessage(self, *args):
         """
-        Send the message.
+        Sends the message.
 
-        This method will prepare the message (headers and possibly encrypted
-        body) and send it using the ESMTPSenderFactory.
+        :return: A deferred with callbacks for error and success of this
+                 #message send.
+        :rtype: twisted.internet.defer.Deferred
+        """
+        d = deferToThread(self._route_msg)
+        d.addCallbacks(self.sendQueued, self.sendError)
+        return
 
-        @return: A deferred with callbacks for error and success of this
-            message send.
-        @rtype: twisted.internet.defer.Deferred
+    def _route_msg(self):
+        """
+        Sends the msg using the ESMTPSenderFactory.
         """
+        log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port))
         msg = self._msg.as_string(False)
 
-        log.msg("Connecting to SMTP server %s:%s" % (self._config[HOST_KEY],
-                                                     self._config[PORT_KEY]))
-
+        # we construct a defer to pass to the ESMTPSenderFactory
         d = defer.Deferred()
+        d.addCallbacks(self.sendSuccess, self.sendError)
         # we don't pass an ssl context factory to the ESMTPSenderFactory
         # because ssl will be handled by reactor.connectSSL() below.
         factory = smtp.ESMTPSenderFactory(
@@ -472,18 +487,14 @@ class EncryptedMessage(object):
             self._user.dest.addrstr,
             StringIO(msg),
             d,
+            heloFallback=True,
             requireAuthentication=False,
             requireTransportSecurity=True)
+        factory.domain = LOCAL_FQDN
         signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr)
         reactor.connectSSL(
-            self._config[HOST_KEY],
-            self._config[PORT_KEY],
-            factory,
-            contextFactory=SSLContextFactory(self._config[CERT_KEY],
-                                             self._config[KEY_KEY]))
-        d.addCallback(self.sendSuccess)
-        d.addErrback(self.sendError)
-        return d
+            self._host, self._port, factory,
+            contextFactory=SSLContextFactory(self._cert, self._key))
 
     #
     # encryption methods
@@ -494,20 +505,18 @@ class EncryptedMessage(object):
         Create an RFC 3156 compliang PGP encrypted and signed message using
         C{pubkey} to encrypt and C{signkey} to sign.
 
-        @param pubkey: The public key used to encrypt the message.
-        @type pubkey: leap.common.keymanager.openpgp.OpenPGPKey
-        @param signkey: The private key used to sign the message.
-        @type signkey: leap.common.keymanager.openpgp.OpenPGPKey
+        :param pubkey: The public key used to encrypt the message.
+        :type pubkey: OpenPGPKey
+        :param signkey: The private key used to sign the message.
+        :type signkey: OpenPGPKey
         """
-        # parse original message from received lines
-        origmsg = self.parseMessage()
         # create new multipart/encrypted message with 'pgp-encrypted' protocol
         newmsg = MultipartEncrypted('application/pgp-encrypted')
         # move (almost) all headers from original message to the new message
-        move_headers(origmsg, newmsg)
+        self._fix_headers(self._origmsg, newmsg, signkey)
         # create 'application/octet-stream' encrypted message
         encmsg = MIMEApplication(
-            self._km.encrypt(origmsg.as_string(unixfrom=False), pubkey,
+            self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey,
                              sign=signkey),
             _subtype='octet-stream', _encoder=lambda x: x)
         encmsg.add_header('content-disposition', 'attachment',
@@ -524,25 +533,23 @@ class EncryptedMessage(object):
         """
         Create an RFC 3156 compliant PGP signed MIME message using C{signkey}.
 
-        @param signkey: The private key used to sign the message.
-        @type signkey: leap.common.keymanager.openpgp.OpenPGPKey
+        :param signkey: The private key used to sign the message.
+        :type signkey: leap.common.keymanager.openpgp.OpenPGPKey
         """
-        # parse original message from received lines
-        origmsg = self.parseMessage()
         # create new multipart/signed message
         newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512')
         # move (almost) all headers from original message to the new message
-        move_headers(origmsg, newmsg)
+        self._fix_headers(self._origmsg, newmsg, signkey)
         # apply base64 content-transfer-encoding
-        encode_base64_rec(origmsg)
+        encode_base64_rec(self._origmsg)
         # get message text with headers and replace \n for \r\n
         fp = StringIO()
         g = RFC3156CompliantGenerator(
             fp, mangle_from_=False, maxheaderlen=76)
-        g.flatten(origmsg)
+        g.flatten(self._origmsg)
         msgtext = re.sub('\r?\n', '\r\n', fp.getvalue())
         # make sure signed message ends with \r\n as per OpenPGP stantard.
-        if origmsg.is_multipart():
+        if self._origmsg.is_multipart():
             if not msgtext.endswith("\r\n"):
                 msgtext += "\r\n"
         # calculate signature
@@ -550,23 +557,46 @@ class EncryptedMessage(object):
                                   clearsign=False, detach=True, binary=False)
         sigmsg = PGPSignature(signature)
         # attach original message and signature to new message
-        newmsg.attach(origmsg)
+        newmsg.attach(self._origmsg)
         newmsg.attach(sigmsg)
         self._msg = newmsg
 
     def _maybe_encrypt_and_sign(self):
         """
-        Encrypt the message body.
+        Attempt to encrypt and sign the outgoing message.
+
+        The behaviour of this method depends on:
+
+            1. the original message's content-type, and
+            2. the availability of the recipient's public key.
 
-        Fetch the recipient key and encrypt the content to the
-        recipient. If a key is not found, then the behaviour depends on the
-        configuration parameter ENCRYPTED_ONLY_KEY. If it is False, the message
-        is sent unencrypted and a warning is logged. If it is True, the
-        encryption fails with a KeyNotFound exception.
+        If the original message's content-type is "multipart/encrypted", then
+        the original message is not altered. For any other content-type, the
+        method attempts to fetch the recipient's public key. If the
+        recipient's public key is available, the message is encrypted and
+        signed; otherwise it is only signed.
 
-        @raise KeyNotFound: Raised when the recipient key was not found and
-            the ENCRYPTED_ONLY_KEY configuration parameter is set to True.
+        Note that, if the C{encrypted_only} configuration is set to True and
+        the recipient's public key is not available, then the recipient
+        address would have been rejected in SMTPDelivery.validateTo().
+
+        The following table summarizes the overall behaviour of the gateway:
+
+        +---------------------------------------------------+----------------+
+        | content-type        | rcpt pubkey | enforce encr. | action         |
+        +---------------------+-------------+---------------+----------------+
+        | multipart/encrypted | any         | any           | pass           |
+        | other               | available   | any           | encrypt + sign |
+        | other               | unavailable | yes           | reject         |
+        | other               | unavailable | no            | sign           |
+        +---------------------+-------------+---------------+----------------+
         """
+        # pass if the original message's content-type is "multipart/encrypted"
+        self._origmsg = self.parseMessage()
+        if self._origmsg.get_content_type() == 'multipart/encrypted':
+            self._msg = self._origmsg
+            return
+
         from_address = validate_address(self._fromAddress.addrstr)
         signkey = self._km.get_key(from_address, OpenPGPKey, private=True)
         log.msg("Will sign the message with %s." % signkey.fingerprint)
@@ -588,3 +618,60 @@ class EncryptedMessage(object):
             signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr)
             self._sign(signkey)
             signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr)
+
+    def _fix_headers(self, origmsg, newmsg, signkey):
+        """
+        Move some headers from C{origmsg} to C{newmsg}, delete unwanted
+        headers from C{origmsg} and add new headers to C{newms}.
+
+        Outgoing messages are either encrypted and signed or just signed
+        before being sent. Because of that, they are packed inside new
+        messages and some manipulation has to be made on their headers.
+
+        Allowed headers for passing through:
+
+            - From
+            - Date
+            - To
+            - Subject
+            - Reply-To
+            - References
+            - In-Reply-To
+            - Cc
+
+        Headers to be added:
+
+            - Message-ID (i.e. should not use origmsg's Message-Id)
+            - Received (this is added automatically by twisted smtp API)
+            - OpenPGP (see #4447)
+
+        Headers to be deleted:
+
+            - User-Agent
+
+        :param origmsg: The original message.
+        :type origmsg: email.message.Message
+        :param newmsg: The new message being created.
+        :type newmsg: email.message.Message
+        :param signkey: The key used to sign C{newmsg}
+        :type signkey: OpenPGPKey
+        """
+        # move headers from origmsg to newmsg
+        headers = origmsg.items()
+        passthrough = [
+            'from', 'date', 'to', 'subject', 'reply-to', 'references',
+            'in-reply-to', 'cc'
+        ]
+        headers = filter(lambda x: x[0].lower() in passthrough, headers)
+        for hkey, hval in headers:
+            newmsg.add_header(hkey, hval)
+            del(origmsg[hkey])
+        # add a new message-id to newmsg
+        newmsg.add_header('Message-Id', smtp.messageid())
+        # add openpgp header to newmsg
+        username, domain = signkey.address.split('@')
+        newmsg.add_header(
+            'OpenPGP', 'id=%s' % signkey.key_id,
+            url='https://%s/openpgp/%s' % (domain, username))
+        # delete user-agent from origmsg
+        del(origmsg['user-agent'])
diff --git a/mail/src/leap/mail/smtp/tests/__init__.py b/mail/src/leap/mail/smtp/tests/__init__.py
index 9b54de3..62b015f 100644
--- a/mail/src/leap/mail/smtp/tests/__init__.py
+++ b/mail/src/leap/mail/smtp/tests/__init__.py
@@ -17,7 +17,7 @@
 
 
 """
-Base classes and keys for SMTP relay tests.
+Base classes and keys for SMTP gateway tests.
 """
 
 import os
@@ -59,7 +59,7 @@ class TestCaseWithKeyManager(BaseLeapTest):
         # setup our own stuff
         address = 'leap@leap.se'  # user's address in the form user@provider
         uuid = 'leap@leap.se'
-        passphrase = '123'
+        passphrase = u'123'
         secrets_path = os.path.join(self.tempdir, 'secret.gpg')
         local_db_path = os.path.join(self.tempdir, 'soledad.u1db')
         server_url = 'http://provider/'
@@ -88,6 +88,8 @@ class TestCaseWithKeyManager(BaseLeapTest):
 
             get_doc = Mock(return_value=None)
             put_doc = Mock(side_effect=_put_doc_side_effect)
+            lock = Mock(return_value=('atoken', 300))
+            unlock = Mock(return_value=True)
 
             def __call__(self):
                 return self
diff --git a/mail/src/leap/mail/smtp/tests/test_smtprelay.py b/mail/src/leap/mail/smtp/tests/test_gateway.py
index 7fefe77..f9ea027 100644
--- a/mail/src/leap/mail/smtp/tests/test_smtprelay.py
+++ b/mail/src/leap/mail/smtp/tests/test_gateway.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# test_smtprelay.py
+# test_gateway.py
 # Copyright (C) 2013 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
 
 
 """
-SMTP relay tests.
+SMTP gateway tests.
 """
 
 
@@ -33,7 +33,7 @@ from twisted.mail.smtp import (
 )
 from mock import Mock
 
-from leap.mail.smtp.smtprelay import (
+from leap.mail.smtp.gateway import (
     SMTPFactory,
     EncryptedMessage,
 )
@@ -52,9 +52,9 @@ HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \
 IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')'
 
 
-class TestSmtpRelay(TestCaseWithKeyManager):
+class TestSmtpGateway(TestCaseWithKeyManager):
 
-    EMAIL_DATA = ['HELO relay.leap.se',
+    EMAIL_DATA = ['HELO gateway.leap.se',
                   'MAIL FROM: <%s>' % ADDRESS_2,
                   'RCPT TO: <%s>' % ADDRESS,
                   'DATA',
@@ -90,7 +90,7 @@ class TestSmtpRelay(TestCaseWithKeyManager):
         self.assertEqual(text, decrypted,
                          "Decrypted text differs from plaintext.")
 
-    def test_relay_accepts_valid_email(self):
+    def test_gateway_accepts_valid_email(self):
         """
         Test if SMTP server responds correctly for valid interaction.
         """
@@ -102,26 +102,32 @@ class TestSmtpRelay(TestCaseWithKeyManager):
                         '250 Sender address accepted',
                         '250 Recipient address accepted',
                         '354 Continue']
-        proto = SMTPFactory(
-            self._km, self._config).buildProtocol(('127.0.0.1', 0))
+        proto = SMTPFactory(u'anotheruser@leap.se',
+            self._km, self._config['host'], self._config['port'],
+            self._config['cert'], self._config['key'],
+            self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
         transport = proto_helpers.StringTransport()
         proto.makeConnection(transport)
         for i, line in enumerate(self.EMAIL_DATA):
             proto.lineReceived(line + '\r\n')
             self.assertMatch(transport.value(),
                              '\r\n'.join(SMTP_ANSWERS[0:i + 1]),
-                             'Did not get expected answer from relay.')
+                             'Did not get expected answer from gateway.')
         proto.setTimeout(None)
 
     def test_message_encrypt(self):
         """
         Test if message gets encrypted to destination email.
         """
-        proto = SMTPFactory(
-            self._km, self._config).buildProtocol(('127.0.0.1', 0))
+        proto = SMTPFactory(u'anotheruser@leap.se',
+            self._km, self._config['host'], self._config['port'],
+            self._config['cert'], self._config['key'],
+            self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
         fromAddr = Address(ADDRESS_2)
-        dest = User(ADDRESS, 'relay.leap.se', proto, ADDRESS)
-        m = EncryptedMessage(fromAddr, dest, self._km, self._config)
+        dest = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS)
+        m = EncryptedMessage(
+            fromAddr, dest, self._km, self._config['host'],
+            self._config['port'], self._config['cert'], self._config['key'])
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
         m.eomReceived()
@@ -149,11 +155,15 @@ class TestSmtpRelay(TestCaseWithKeyManager):
         Test if message gets encrypted to destination email and signed with
         sender key.
         """
-        proto = SMTPFactory(
-            self._km, self._config).buildProtocol(('127.0.0.1', 0))
-        user = User(ADDRESS, 'relay.leap.se', proto, ADDRESS)
+        proto = SMTPFactory(u'anotheruser@leap.se',
+            self._km, self._config['host'], self._config['port'],
+            self._config['cert'], self._config['key'],
+            self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
+        user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS)
         fromAddr = Address(ADDRESS_2)
-        m = EncryptedMessage(fromAddr, user, self._km, self._config)
+        m = EncryptedMessage(
+            fromAddr, user, self._km, self._config['host'],
+            self._config['port'], self._config['cert'], self._config['key'])
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
         # trigger encryption and signing
@@ -185,11 +195,15 @@ class TestSmtpRelay(TestCaseWithKeyManager):
         """
         # mock the key fetching
         self._km.fetch_keys_from_server = Mock(return_value=[])
-        proto = SMTPFactory(
-            self._km, self._config).buildProtocol(('127.0.0.1', 0))
-        user = User('ihavenopubkey@nonleap.se', 'relay.leap.se', proto, ADDRESS)
+        proto = SMTPFactory(u'anotheruser@leap.se',
+            self._km, self._config['host'], self._config['port'],
+            self._config['cert'], self._config['key'],
+            self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
+        user = User('ihavenopubkey@nonleap.se', 'gateway.leap.se', proto, ADDRESS)
         fromAddr = Address(ADDRESS_2)
-        m = EncryptedMessage(fromAddr, user, self._km, self._config)
+        m = EncryptedMessage(
+            fromAddr, user, self._km, self._config['host'],
+            self._config['port'], self._config['cert'], self._config['key'])
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
         # trigger signing
@@ -237,8 +251,10 @@ class TestSmtpRelay(TestCaseWithKeyManager):
         # mock the key fetching
         self._km.fetch_keys_from_server = Mock(return_value=[])
         # prepare the SMTP factory
-        proto = SMTPFactory(
-            self._km, self._config).buildProtocol(('127.0.0.1', 0))
+        proto = SMTPFactory(u'anotheruser@leap.se',
+            self._km, self._config['host'], self._config['port'],
+            self._config['cert'], self._config['key'],
+            self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
         transport = proto_helpers.StringTransport()
         proto.makeConnection(transport)
         proto.lineReceived(self.EMAIL_DATA[0] + '\r\n')
@@ -263,11 +279,11 @@ class TestSmtpRelay(TestCaseWithKeyManager):
         pgp.delete_key(pubkey)
         # mock the key fetching
         self._km.fetch_keys_from_server = Mock(return_value=[])
-        # change the configuration
-        self._config['encrypted_only'] = False
-        # prepare the SMTP factory
-        proto = SMTPFactory(
-            self._km, self._config).buildProtocol(('127.0.0.1', 0))
+        # prepare the SMTP factory with encrypted only equal to false
+        proto = SMTPFactory(u'anotheruser@leap.se',
+            self._km, self._config['host'], self._config['port'],
+            self._config['cert'], self._config['key'],
+            False).buildProtocol(('127.0.0.1', 0))
         transport = proto_helpers.StringTransport()
         proto.makeConnection(transport)
         proto.lineReceived(self.EMAIL_DATA[0] + '\r\n')