Merge tag '0.4.1' into debian/experimental

Tag version 0.4.1

7 files changed, 176 insertions, 140 deletions

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index cc56fff..459b0ba 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -49,6 +49,7 @@ if PROFILE_CMD:
 # Soledad IMAP Account
 #######################################
 
+
 class IMAPAccount(object):
     """
     An implementation of an imap4 Account
@@ -59,7 +60,7 @@ class IMAPAccount(object):
 
     selected = None
 
-    def __init__(self, user_id, store, d=defer.Deferred()):
+    def __init__(self, store, user_id, d=defer.Deferred()):
         """
         Keeps track of the mailboxes and subscriptions handled by this account.
 
@@ -68,13 +69,14 @@ class IMAPAccount(object):
         You can either pass a deferred to this constructor, or use
         `callWhenReady` method.
 
-        :param user_id: The name of the account (user id, in the form
-                        user@provider).
-        :type user_id: str
-
         :param store: a Soledad instance.
         :type store: Soledad
 
+        :param user_id: The identifier of the user this account belongs to
+                        (user id, in the form user@provider).
+        :type user_id: str
+
+
         :param d: a deferred that will be fired with this IMAPAccount instance
                   when the account is ready to be used.
         :type d: defer.Deferred
@@ -87,7 +89,7 @@ class IMAPAccount(object):
         # about user_id, only the client backend.
 
         self.user_id = user_id
-        self.account = Account(store, ready_cb=lambda: d.callback(self))
+        self.account = Account(store, user_id, ready_cb=lambda: d.callback(self))
 
     def end_session(self):
         """
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index bfc0bfc..d545c00 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -508,7 +508,7 @@ class IMAPMailbox(object):
         def get_range(messages_asked):
             return self._filter_msg_seq(messages_asked)
 
-        d = defer.maybeDeferred(self._bound_seq, messages_asked, uid)
+        d = self._bound_seq(messages_asked, uid)
         if uid:
             d.addCallback(get_range)
         d.addErrback(lambda f: log.err(f))
@@ -520,7 +520,7 @@ class IMAPMailbox(object):
 
         :param messages_asked: IDs of the messages.
         :type messages_asked: MessageSet
-        :rtype: MessageSet
+        :return: a Deferred that will fire with a MessageSet
         """
 
         def set_last_uid(last_uid):
@@ -543,7 +543,7 @@ class IMAPMailbox(object):
                     d = self.collection.all_uid_iter()
                     d.addCallback(set_last_seq)
                 return d
-        return messages_asked
+        return defer.succeed(messages_asked)
 
     def _filter_msg_seq(self, messages_asked):
         """
@@ -713,6 +713,7 @@ class IMAPMailbox(object):
         d_seq.addCallback(get_flags_for_seq)
         return d_seq
 
+    @defer.inlineCallbacks
     def fetch_headers(self, messages_asked, uid):
         """
         A fast method to fetch all headers, tricking just the
@@ -757,14 +758,15 @@ class IMAPMailbox(object):
                     for key, value in
                     self.headers.items())
 
-        messages_asked = self._bound_seq(messages_asked)
-        seq_messg = self._filter_msg_seq(messages_asked)
+        messages_asked = yield self._bound_seq(messages_asked, uid)
+        seq_messg = yield self._filter_msg_seq(messages_asked)
 
-        all_headers = self.messages.all_headers()
-        result = ((msgid, headersPart(
-            msgid, all_headers.get(msgid, {})))
-            for msgid in seq_messg)
-        return result
+        result = []
+        for msgid in seq_messg:
+            msg = yield self.collection.get_message_by_uid(msgid)
+            headers = headersPart(msgid, msg.get_headers())
+            result.append((msgid, headers))
+        defer.returnValue(iter(result))
 
     def store(self, messages_asked, flags, mode, uid):
         """
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 99e7174..5a63af0 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -20,21 +20,17 @@ LEAP IMAP4 Server Implementation.
 import StringIO
 from copy import copy
 
-from twisted import cred
-from twisted.internet import reactor
 from twisted.internet.defer import maybeDeferred
 from twisted.mail import imap4
 from twisted.python import log
 
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.events import emit_async, catalog
-from leap.soledad.client import Soledad
-
 # imports for LITERAL+ patch
 from twisted.internet import defer, interfaces
 from twisted.mail.imap4 import IllegalClientResponse
 from twisted.mail.imap4 import LiteralString, LiteralFile
 
+from leap.common.events import emit_async, catalog
+
 
 def _getContentType(msg):
     """
@@ -72,25 +68,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
     """
     An IMAP4 Server with a LEAP Storage Backend.
     """
-    def __init__(self, *args, **kwargs):
-        # pop extraneous arguments
-        soledad = kwargs.pop('soledad', None)
-        uuid = kwargs.pop('uuid', None)
-        userid = kwargs.pop('userid', None)
-
-        leap_assert(soledad, "need a soledad instance")
-        leap_assert_type(soledad, Soledad)
-        leap_assert(uuid, "need a user in the initialization")
-
-        self._userid = userid
-
-        # initialize imap server!
-        imap4.IMAP4Server.__init__(self, *args, **kwargs)
-
-        # we should initialize the account here,
-        # but we move it to the factory so we can
-        # populate the test account properly (and only once
-        # per session)
 
     #############################################################
     #
@@ -181,10 +158,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
         :param line: the line from the server, without the line delimiter.
         :type line: str
         """
-        if self.theAccount.session_ended is True and self.state != "unauth":
-            log.msg("Closing the session. State: unauth")
-            self.state = "unauth"
-
         if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
@@ -208,25 +181,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
             self.mbox = None
         self.state = 'unauth'
 
-    def authenticateLogin(self, username, password):
-        """
-        Lookup the account with the given parameters, and deny
-        the improper combinations.
-
-        :param username: the username that is attempting authentication.
-        :type username: str
-        :param password: the password to authenticate with.
-        :type password: str
-        """
-        # XXX this should use portal:
-        # return portal.login(cred.credentials.UsernamePassword(user, pass)
-        if username != self._userid:
-            # bad username, reject.
-            raise cred.error.UnauthorizedLogin()
-        # any dummy password is allowed so far. use realm instead!
-        emit_async(catalog.IMAP_CLIENT_LOGIN, "1")
-        return imap4.IAccount, self.theAccount, lambda: None
-
     def do_FETCH(self, tag, messages, query, uid=0):
         """
         Overwritten fetch dispatcher to use the fast fetch_flags
@@ -657,7 +611,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
         d.addCallback(send_response)
         return d
         # XXX patched ---------------------------------
-
     # -----------------------------------------------------------------------
 
     auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist,
@@ -733,3 +686,8 @@ class LEAPIMAPServer(imap4.IMAP4Server):
     #############################################################
     # END of Twisted imap4 patch to support LITERAL+ extension
     #############################################################
+
+    def authenticateLogin(self, user, passwd):
+        result = imap4.IMAP4Server.authenticateLogin(self, user, passwd)
+        emit_async(catalog.IMAP_CLIENT_LOGIN, str(user))
+        return result
diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index 2045757..c4d602d 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # imap-server.tac
 # Copyright (C) 2013,2014 LEAP
@@ -27,6 +28,9 @@ userid = 'user@provider'
 uuid = 'deadbeefdeadabad'
 passwd = 'supersecret' # optional, will get prompted if not found.
 """
+
+# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd
+
 import ConfigParser
 import getpass
 import os
@@ -112,7 +116,7 @@ tempdir = "/tmp/"
 
 print "[~] user:", userid
 soledad = initialize_soledad(uuid, userid, passwd, secrets,
-                             localdb, gnupg_home, tempdir)
+                             localdb, gnupg_home, tempdir, userid=userid)
 km_args = (userid, "https://localhost", soledad)
 km_kwargs = {
     "token": "",
@@ -131,7 +135,8 @@ keymanager = KeyManager(*km_args, **km_kwargs)
 
 
 def getIMAPService():
-    factory = imap.LeapIMAPFactory(uuid, userid, soledad)
+    soledad_sessions = {userid: soledad}
+    factory = imap.LeapIMAPFactory(soledad_sessions)
     return internet.TCPServer(port, factory, interface="localhost")
 
 
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index a50611b..4663854 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -15,21 +15,24 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-IMAP service initialization
+IMAP Service Initialization.
 """
 import logging
 import os
 
 from collections import defaultdict
 
+from twisted.cred.portal import Portal, IRealm
+from twisted.mail.imap4 import IAccount
+from twisted.internet import defer
 from twisted.internet import reactor
 from twisted.internet.error import CannotListenError
 from twisted.internet.protocol import ServerFactory
-from twisted.mail import imap4
 from twisted.python import log
+from zope.interface import implementer
 
 from leap.common.events import emit_async, catalog
-from leap.common.check import leap_check
+from leap.mail.cred import LocalSoledadTokenChecker
 from leap.mail.imap.account import IMAPAccount
 from leap.mail.imap.server import LEAPIMAPServer
 
@@ -41,57 +44,92 @@ DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
 if DO_MANHOLE:
     from leap.mail.imap.service import manhole
 
-DO_PROFILE = os.environ.get("LEAP_PROFILE", None)
-if DO_PROFILE:
-    import cProfile
-    log.msg("Starting PROFILING...")
-
-    PROFILE_DAT = "/tmp/leap_mail_profile.pstats"
-    pr = cProfile.Profile()
-    pr.enable()
-
 # The default port in which imap service will run
+
 IMAP_PORT = 1984
 
+#
+# Credentials Handling
+#
+
+
+@implementer(IRealm)
+class LocalSoledadIMAPRealm(object):
+
+    _encoding = 'utf-8'
+
+    def __init__(self, soledad_sessions):
+        """
+        :param soledad_sessions: a dict-like object, containing instances
+                                 of a Store (soledad instances), indexed by
+                                 userid.
+        """
+        self._soledad_sessions = soledad_sessions
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        if isinstance(avatarId, str):
+            avatarId = avatarId.decode(self._encoding)
+
+        def gotSoledad(soledad):
+            for iface in interfaces:
+                if iface is IAccount:
+                    avatar = IMAPAccount(soledad, avatarId)
+                    return (IAccount, avatar,
+                            getattr(avatar, 'logout', lambda: None))
+            raise NotImplementedError(self, interfaces)
+
+        return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad)
+
+    def lookupSoledadInstance(self, userid):
+        soledad = self._soledad_sessions[userid]
+        # XXX this should return the instance after whenReady callback
+        return defer.succeed(soledad)
+
+
+class IMAPTokenChecker(LocalSoledadTokenChecker):
+    """A credentials checker that will lookup a token for the IMAP service.
+    For now it will be using the same identifier than SMTPTokenChecker"""
+
+    service = 'mail_auth'
+
+
+class LocalSoledadIMAPServer(LEAPIMAPServer):
 
-class IMAPAuthRealm(object):
     """
-    Dummy authentication realm. Do not use in production!
+    An IMAP Server that authenticates against a LocalSoledad store.
     """
-    theAccount = None
 
-    def requestAvatar(self, avatarId, mind, *interfaces):
-        return imap4.IAccount, self.theAccount, lambda: None
+    def __init__(self, soledad_sessions, *args, **kw):
+
+        LEAPIMAPServer.__init__(self, *args, **kw)
+
+        realm = LocalSoledadIMAPRealm(soledad_sessions)
+        portal = Portal(realm)
+        checker = IMAPTokenChecker(soledad_sessions)
+        self.checker = checker
+        self.portal = portal
+        portal.registerChecker(checker)
 
 
 class LeapIMAPFactory(ServerFactory):
+
     """
     Factory for a IMAP4 server with soledad remote sync and gpg-decryption
     capabilities.
     """
-    protocol = LEAPIMAPServer
 
-    def __init__(self, uuid, userid, soledad):
+    protocol = LocalSoledadIMAPServer
+
+    def __init__(self, soledad_sessions):
         """
         Initializes the server factory.
 
-        :param uuid: user uuid
-        :type uuid: str
-
-        :param userid: user id (user@provider.org)
-        :type userid: str
-
-        :param soledad: soledad instance
-        :type soledad: Soledad
+        :param soledad_sessions: a dict-like object, containing instances
+                                 of a Store (soledad instances), indexed by
+                                 userid.
         """
-        self._uuid = uuid
-        self._userid = userid
-        self._soledad = soledad
-
-        theAccount = IMAPAccount(uuid, soledad)
-        self.theAccount = theAccount
+        self._soledad_sessions = soledad_sessions
         self._connections = defaultdict()
-        # XXX how to pass the store along?
 
     def buildProtocol(self, addr):
         """
@@ -103,13 +141,7 @@ class LeapIMAPFactory(ServerFactory):
         # TODO should reject anything from addr != localhost,
         # just in case.
         log.msg("Building protocol for connection %s" % addr)
-        imapProtocol = self.protocol(
-            uuid=self._uuid,
-            userid=self._userid,
-            soledad=self._soledad)
-        imapProtocol.theAccount = self.theAccount
-        imapProtocol.factory = self
-
+        imapProtocol = self.protocol(self._soledad_sessions)
         self._connections[addr] = imapProtocol
         return imapProtocol
 
@@ -123,39 +155,21 @@ class LeapIMAPFactory(ServerFactory):
         """
         Stops imap service (fetcher, factory and port).
         """
-        # mark account as unusable, so any imap command will fail
-        # with unauth state.
-        self.theAccount.end_session()
-
-        # TODO should wait for all the pending deferreds,
-        # the twisted way!
-        if DO_PROFILE:
-            log.msg("Stopping PROFILING")
-            pr.disable()
-            pr.dump_stats(PROFILE_DAT)
-
         return ServerFactory.doStop(self)
 
 
-def run_service(store, **kwargs):
+def run_service(soledad_sessions, port=IMAP_PORT):
     """
     Main entry point to run the service from the client.
 
-    :param store: a soledad instance
+    :param soledad_sessions: a dict-like object, containing instances
+                             of a Store (soledad instances), indexed by userid.
 
     :returns: the port as returned by the reactor when starts listening, and
               the factory for the protocol.
+    :rtype: tuple
     """
-    leap_check(store, "store cannot be None")
-    # XXX this can also be a ProxiedObject, FIXME
-    # leap_assert_type(store, Soledad)
-
-    port = kwargs.get('port', IMAP_PORT)
-    userid = kwargs.get('userid', None)
-    leap_check(userid is not None, "need an user id")
-
-    uuid = store.uuid
-    factory = LeapIMAPFactory(uuid, userid, store)
+    factory = LeapIMAPFactory(soledad_sessions)
 
     try:
         interface = "localhost"
@@ -164,6 +178,7 @@ def run_service(store, **kwargs):
         if os.environ.get("LEAP_DOCKERIZED"):
             interface = ''
 
+        # TODO use Endpoints !!!
         tport = reactor.listenTCP(port, factory,
                                   interface=interface)
     except CannotListenError:
@@ -178,9 +193,9 @@ def run_service(store, **kwargs):
             # TODO get pass from env var.too.
             manhole_factory = manhole.getManholeFactory(
                 {'f': factory,
-                 'a': factory.theAccount,
                  'gm': factory.theAccount.getMailbox},
                 "boss", "leap")
+            # TODO  use Endpoints !!!
             reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory,
                               interface="127.0.0.1")
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 62c3c41..ccce285 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -575,8 +575,8 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin):
         """
         Test login requiring quoting
         """
-        self.server._userid = '{test}user@leap.se'
-        self.server._password = '{test}password'
+        self.server.checker.userid = '{test}user@leap.se'
+        self.server.checker.password = '{test}password'
 
         def login():
             d = self.client.login('{test}user@leap.se', '{test}password')
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index a34538b..64a0326 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -20,10 +20,15 @@ Common utilities for testing Soledad IMAP Server.
 from email import parser
 
 from mock import Mock
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.cred.portal import Portal, IRealm
 from twisted.mail import imap4
 from twisted.internet import defer
 from twisted.protocols import loopback
 from twisted.python import log
+from zope.interface import implementer
 
 from leap.mail.adaptors import soledad as soledad_adaptor
 from leap.mail.imap.account import IMAPAccount
@@ -64,6 +69,57 @@ class SimpleClient(imap4.IMAP4Client):
         self.events.append(['newMessages', exists, recent])
         self.transport.loseConnection()
 
+#
+# Dummy credentials for tests
+#
+
+
+@implementer(IRealm)
+class TestRealm(object):
+
+    def __init__(self, account):
+        self._account = account
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        avatar = self._account
+        return (imap4.IAccount, avatar,
+                getattr(avatar, 'logout', lambda: None))
+
+
+@implementer(ICredentialsChecker)
+class TestCredentialsChecker(object):
+
+    credentialInterfaces = (IUsernamePassword,)
+
+    userid = TEST_USER
+    password = TEST_PASSWD
+
+    def requestAvatarId(self, credentials):
+        username, password = credentials.username, credentials.password
+        d = self.checkTestCredentials(username, password)
+        d.addErrback(lambda f: defer.fail(UnauthorizedLogin()))
+        return d
+
+    def checkTestCredentials(self, username, password):
+        if username == self.userid and password == self.password:
+            return defer.succeed(username)
+        else:
+            return defer.fail(Exception("Wrong credentials"))
+
+
+class TestSoledadIMAPServer(LEAPIMAPServer):
+
+    def __init__(self, account, *args, **kw):
+
+        LEAPIMAPServer.__init__(self, *args, **kw)
+
+        realm = TestRealm(account)
+        portal = Portal(realm)
+        checker = TestCredentialsChecker()
+        self.checker = checker
+        self.portal = portal
+        portal.registerChecker(checker)
+
 
 class IMAP4HelperMixin(SoledadTestMixin):
     """
@@ -77,14 +133,12 @@ class IMAP4HelperMixin(SoledadTestMixin):
 
         soledad_adaptor.cleanup_deferred_locks()
 
-        UUID = 'deadbeef',
         USERID = TEST_USER
 
         def setup_server(account):
-            self.server = LEAPIMAPServer(
-                uuid=UUID, userid=USERID,
-                contextFactory=self.serverCTX,
-                soledad=self._soledad)
+            self.server = TestSoledadIMAPServer(
+                account=account,
+                contextFactory=self.serverCTX)
             self.server.theAccount = account
 
             d_server_ready = defer.Deferred()
@@ -104,7 +158,7 @@ class IMAP4HelperMixin(SoledadTestMixin):
             self._soledad.sync = Mock()
 
             d = defer.Deferred()
-            self.acc = IMAPAccount(USERID, self._soledad, d=d)
+            self.acc = IMAPAccount(self._soledad, USERID, d=d)
             return d
 
         d = super(IMAP4HelperMixin, self).setUp()