Merge branch 'release-0.3.7'0.3.7

4 files changed, 190 insertions, 54 deletions

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index dd65def..3422ed5 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -28,6 +28,7 @@ from email.parser import Parser
 from twisted.python import log
 from twisted.internet.task import LoopingCall
 from twisted.internet.threads import deferToThread
+from zope.proxy import sameProxiedObjects
 
 from leap.common import events as leap_events
 from leap.common.check import leap_assert, leap_assert_type
@@ -39,6 +40,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
+from leap.keymanager.openpgp import OpenPGPKey
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
@@ -66,7 +68,7 @@ class LeapIncomingMail(object):
     fetching_lock = threading.Lock()
 
     def __init__(self, keymanager, soledad, imap_account,
-                 check_period):
+                 check_period, userid):
 
         """
         Initialize LeapIMAP.
@@ -88,14 +90,14 @@ class LeapIncomingMail(object):
         leap_assert_type(soledad, Soledad)
         leap_assert(check_period, "need a period to check incoming mail")
         leap_assert_type(check_period, int)
+        leap_assert(userid, "need a userid to initialize")
 
         self._keymanager = keymanager
         self._soledad = soledad
         self.imapAccount = imap_account
         self._inbox = self.imapAccount.getMailbox('inbox')
+        self._userid = userid
 
-        self._pkey = self._keymanager.get_all_keys_in_local_db(
-            private=True).pop()
         self._loop = None
         self._check_period = check_period
 
@@ -107,6 +109,13 @@ class LeapIncomingMail(object):
         """
         self._soledad.create_index("just-mail", "incoming")
 
+    @property
+    def _pkey(self):
+        if sameProxiedObjects(self._keymanager, None):
+            logger.warning('tried to get key, but null keymanager found')
+            return None
+        return self._keymanager.get_key(self._userid, OpenPGPKey, private=True)
+
     #
     # Public API: fetch, start_loop, stop.
     #
@@ -118,6 +127,8 @@ class LeapIncomingMail(object):
         Calls a deferred that will execute the fetch callback
         in a separate thread
         """
+        logger.debug("fetching mail for: %s %s" % (
+            self._soledad.uuid, self._userid))
         if not self.fetching_lock.locked():
             d = deferToThread(self._sync_soledad)
             d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error)
@@ -130,16 +141,19 @@ class LeapIncomingMail(object):
         """
         Starts a loop to fetch mail.
         """
-        self._loop = LoopingCall(self.fetch)
-        self._loop.start(self._check_period)
+        if self._loop is None:
+            self._loop = LoopingCall(self.fetch)
+            self._loop.start(self._check_period)
+        else:
+            logger.warning("Tried to start an already running fetching loop.")
 
     def stop(self):
         """
         Stops the loop that fetches mail.
         """
-        # XXX should cancel ongoing fetches too.
         if self._loop and self._loop.running is True:
             self._loop.stop()
+            self._loop = None
 
     #
     # Private methods.
@@ -203,7 +217,7 @@ class LeapIncomingMail(object):
         Generic errback
         """
         err = failure.value
-        logger.error("error!: %r" % (err,))
+        logger.exception("error!: %r" % (err,))
 
     def _decryption_error(self, failure):
         """
@@ -334,6 +348,7 @@ class LeapIncomingMail(object):
         :return: data, possibly descrypted.
         :rtype: str
         """
+        # TODO split this method
         leap_assert_type(data, unicode)
 
         parser = Parser()
@@ -375,6 +390,8 @@ class LeapIncomingMail(object):
                 decrdata = decrdata.encode(encoding, 'replace')
 
             decrmsg = parser.parsestr(decrdata)
+            # remove original message's multipart/encrypted content-type
+            del(origmsg['content-type'])
             # replace headers back in original message
             for hkey, hval in decrmsg.items():
                 try:
@@ -417,7 +434,6 @@ class LeapIncomingMail(object):
                          incoming message
         :type msgtuple: (SoledadDocument, str)
         """
-        print "adding message locally....."
         doc, data = msgtuple
         self._inbox.addMessage(data, (self.RECENT_FLAG,))
         leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 7a9f810..bb2830d 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -23,6 +23,7 @@ import StringIO
 import cStringIO
 import time
 
+from collections import defaultdict
 from email.parser import Parser
 
 from zope.interface import implements
@@ -241,6 +242,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 
         :rtype: SoledadDocument
         """
+        # XXX only upper for INBOX ---
         name = name.upper()
         doc = self._soledad.get_from_index(
             self.TYPE_MBOX_IDX, self.MBOX_KEY, name)
@@ -274,6 +276,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :returns: a a SoledadMailbox instance
         :rtype: SoledadMailbox
         """
+        # XXX only upper for INBOX
         name = name.upper()
         if name not in self.mailboxes:
             raise imap4.MailboxException("No such mailbox")
@@ -299,6 +302,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :returns: True if successful
         :rtype: bool
         """
+        # XXX only upper for INBOX
         name = name.upper()
         # XXX should check mailbox name for RFC-compliant form
 
@@ -360,6 +364,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 
         :rtype: bool
         """
+        # XXX only upper for INBOX
         name = name.upper()
 
         if name not in self.mailboxes:
@@ -385,6 +390,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
                       names. use with care.
         :type force: bool
         """
+        # XXX only upper for INBOX
         name = name.upper()
         if not name in self.mailboxes:
             raise imap4.MailboxException("No such mailbox")
@@ -422,6 +428,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :param newname: new name of the mailbox
         :type newname: str
         """
+        # XXX only upper for INBOX
         oldname = oldname.upper()
         newname = newname.upper()
 
@@ -487,7 +494,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         # maybe we should store subscriptions in another
         # document...
         if not name in self.mailboxes:
-            print "not this mbox"
             self.addMailbox(name)
         mbox = self._get_mailbox_by_name(name)
 
@@ -785,6 +791,7 @@ class LeapMessage(WithMsgFields):
         return dict(filter_by_cond)
 
     # --- no multipart for now
+    # XXX Fix MULTIPART SUPPORT!
 
     def isMultipart(self):
         return False
@@ -967,6 +974,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         docs = self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_UID_IDX,
             self.TYPE_MESSAGE_VAL, self.mbox, str(uid))
+
         return docs[0] if docs else None
 
     def get_msg_by_uid(self, uid):
@@ -984,6 +992,47 @@ class MessageCollection(WithMsgFields, IndexedDB):
         if doc:
             return LeapMessage(doc)
 
+    def get_by_index(self, index):
+        """
+        Retrieves a mesage document by mailbox index.
+
+        :param index: the index of the sequence (zero-indexed)
+        :type index: int
+        """
+        try:
+            return self.get_all()[index]
+        except IndexError:
+            return None
+
+    def get_msg_by_index(self, index):
+        """
+        Retrieves a LeapMessage by sequence index.
+
+        :param index: the index of the sequence (zero-indexed)
+        :type index: int
+        """
+        doc = self.get_by_index(index)
+        if doc:
+            return LeapMessage(doc)
+
+    def is_deleted(self, doc):
+        """
+        Returns whether a given doc is deleted or not.
+
+        :param doc: the document to check
+        :rtype: bool
+        """
+        return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
+
+    def get_last(self):
+        """
+        Gets the last LeapMessage
+        """
+        _all = self.get_all()
+        if not _all:
+            return None
+        return LeapMessage(_all[-1])
+
     def get_all(self):
         """
         Get all message documents for the selected mailbox.
@@ -993,9 +1042,13 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :rtype: list of SoledadDocument
         """
         # XXX this should return LeapMessage instances
-        return self._soledad.get_from_index(
+        all_docs = [doc for doc in self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox)
+            self.TYPE_MESSAGE_VAL, self.mbox)]
+            #if not self.is_deleted(doc)]
+        # highly inneficient, but first let's grok it and then
+        # let's worry about efficiency.
+        return sorted(all_docs, key=lambda item: item.content['uid'])
 
     def unseen_iter(self):
         """
@@ -1075,8 +1128,11 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :return: LeapMessage or None if not found.
         :rtype: LeapMessage
         """
+        #try:
+            #return self.get_msg_by_uid(uid)
         try:
-            return self.get_msg_by_uid(uid)
+            return [doc
+                    for doc in self.get_all()][uid - 1]
         except IndexError:
             return None
 
@@ -1116,7 +1172,7 @@ class SoledadMailbox(WithMsgFields):
     CMD_UIDVALIDITY = "UIDVALIDITY"
     CMD_UNSEEN = "UNSEEN"
 
-    listeners = []
+    _listeners = defaultdict(set)
 
     def __init__(self, mbox, soledad=None, rw=1):
         """
@@ -1150,11 +1206,18 @@ class SoledadMailbox(WithMsgFields):
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
 
-        # the server itself is a listener to the mailbox.
-        # so we can notify it (and should!) after chanes in flags
-        # and number of messages.
-        print "emptying the listeners"
-        map(lambda i: self.listeners.remove(i), self.listeners)
+    @property
+    def listeners(self):
+        """
+        Returns listeners for this mbox.
+
+        The server itself is a listener to the mailbox.
+        so we can notify it (and should!) after changes in flags
+        and number of messages.
+
+        :rtype: set
+        """
+        return self._listeners[self.mbox]
 
     def addListener(self, listener):
         """
@@ -1164,7 +1227,7 @@ class SoledadMailbox(WithMsgFields):
         :type listener: an object that implements IMailboxListener
         """
         logger.debug('adding mailbox listener: %s' % listener)
-        self.listeners.append(listener)
+        self.listeners.add(listener)
 
     def removeListener(self, listener):
         """
@@ -1173,25 +1236,24 @@ class SoledadMailbox(WithMsgFields):
         :param listener: listener to remove
         :type listener: an object that implements IMailboxListener
         """
-        logger.debug('removing mailbox listener: %s' % listener)
-        try:
-            self.listeners.remove(listener)
-        except ValueError:
-            logger.error(
-                "removeListener: cannot remove listener %s" % listener)
+        self.listeners.remove(listener)
 
     def _get_mbox(self):
         """
         Returns mailbox document.
 
-        :return: A SoledadDocument containing this mailbox.
-        :rtype: SoledadDocument
+        :return: A SoledadDocument containing this mailbox, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
         """
-        query = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MBOX_VAL, self.mbox)
-        if query:
-            return query.pop()
+        try:
+            query = self._soledad.get_from_index(
+                SoledadBackedAccount.TYPE_MBOX_IDX,
+                self.TYPE_MBOX_VAL, self.mbox)
+            if query:
+                return query.pop()
+        except Exception as exc:
+            logger.error("Unhandled error %r" % exc)
 
     def getFlags(self):
         """
@@ -1288,8 +1350,12 @@ class SoledadMailbox(WithMsgFields):
 
         :rtype: int
         """
-        # XXX reimplement with proper index
-        return self.messages.count() + 1
+        last = self.messages.get_last()
+        if last:
+            nextuid = last.getUID() + 1
+        else:
+            nextuid = 1
+        return nextuid
 
     def getMessageCount(self):
         """
@@ -1376,6 +1442,8 @@ class SoledadMailbox(WithMsgFields):
 
         self.messages.add_msg(message, flags=flags, date=date,
                               uid=uid_next)
+
+        # XXX recent should not include deleted...??
         exists = len(self.messages)
         recent = len(self.messages.get_recent())
         for listener in self.listeners:
@@ -1435,16 +1503,35 @@ class SoledadMailbox(WithMsgFields):
         :rtype: A tuple of two-tuples of message sequence numbers and
                 LeapMessage
         """
-        # XXX implement sequence numbers (uid = 0)
         result = []
+        sequence = True if uid == 0 else False
 
         if not messages.last:
-            messages.last = self.messages.count()
+            try:
+                iter(messages)
+            except TypeError:
+                # looks like we cannot iterate
+                last = self.messages.get_last()
+                uid_last = last.getUID()
+                messages.last = uid_last
+
+        # for sequence numbers (uid = 0)
+        if sequence:
+            for msg_id in messages:
+                msg = self.messages.get_msg_by_index(msg_id - 1)
+                if msg:
+                    result.append((msg.getUID(), msg))
+                else:
+                    print "fetch %s, no msg found!!!" % msg_id
+
+        else:
+            for msg_id in messages:
+                msg = self.messages.get_msg_by_uid(msg_id)
+                if msg:
+                    result.append((msg_id, msg))
+                else:
+                    print "fetch %s, no msg found!!!" % msg_id
 
-        for msg_id in messages:
-            msg = self.messages.get_msg_by_uid(msg_id)
-            if msg:
-                result.append((msg_id, msg))
         return tuple(result)
 
     def _signal_unread_to_ui(self):
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 5f7322a..feb2593 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -25,11 +25,12 @@ from twisted.internet.protocol import ServerFactory
 from twisted.internet.error import CannotListenError
 from twisted.mail import imap4
 from twisted.python import log
+from twisted import cred
 
 logger = logging.getLogger(__name__)
 
 from leap.common import events as leap_events
-from leap.common.check import leap_assert, leap_assert_type
+from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import KeyManager
 from leap.mail.imap.server import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
@@ -54,10 +55,13 @@ class LeapIMAPServer(imap4.IMAP4Server):
     def __init__(self, *args, **kwargs):
         # pop extraneous arguments
         soledad = kwargs.pop('soledad', None)
-        user = kwargs.pop('user', None)
+        uuid = kwargs.pop('uuid', None)
+        userid = kwargs.pop('userid', None)
         leap_assert(soledad, "need a soledad instance")
         leap_assert_type(soledad, Soledad)
-        leap_assert(user, "need a user in the initialization")
+        leap_assert(uuid, "need a user in the initialization")
+
+        self._userid = userid
 
         # initialize imap server!
         imap4.IMAP4Server.__init__(self, *args, **kwargs)
@@ -77,6 +81,12 @@ class LeapIMAPServer(imap4.IMAP4Server):
         #self.theAccount = theAccount
 
     def lineReceived(self, line):
+        """
+        Attempt to parse a single line from the server.
+
+        :param line: the line from the server, without the line delimiter.
+        :type line: str
+        """
         if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
@@ -87,7 +97,21 @@ class LeapIMAPServer(imap4.IMAP4Server):
         imap4.IMAP4Server.lineReceived(self, line)
 
     def authenticateLogin(self, username, password):
-        # all is allowed so far. use realm instead
+        """
+        Lookup the account with the given parameters, and deny
+        the improper combinations.
+
+        :param username: the username that is attempting authentication.
+        :type username: str
+        :param password: the password to authenticate with.
+        :type password: str
+        """
+        # XXX this should use portal:
+        # return portal.login(cred.credentials.UsernamePassword(user, pass)
+        if username != self._userid:
+            # bad username, reject.
+            raise cred.error.UnauthorizedLogin()
+        # any dummy password is allowed so far. use realm instead!
         leap_events.signal(IMAP_CLIENT_LOGIN, "1")
         return imap4.IAccount, self.theAccount, lambda: None
 
@@ -108,28 +132,32 @@ class LeapIMAPFactory(ServerFactory):
     capabilities.
     """
 
-    def __init__(self, user, soledad):
+    def __init__(self, uuid, userid, soledad):
         """
         Initializes the server factory.
 
-        :param user: user ID. **right now it's uuid**
-                     this might change!
-        :type user: str
+        :param uuid: user uuid
+        :type uuid: str
+
+        :param userid: user id (user@provider.org)
+        :type userid: str
 
         :param soledad: soledad instance
         :type soledad: Soledad
         """
-        self._user = user
+        self._uuid = uuid
+        self._userid = userid
         self._soledad = soledad
 
         theAccount = SoledadBackedAccount(
-            user, soledad=soledad)
+            uuid, soledad=soledad)
         self.theAccount = theAccount
 
     def buildProtocol(self, addr):
         "Return a protocol suitable for the job."
         imapProtocol = LeapIMAPServer(
-            user=self._user,
+            uuid=self._uuid,
+            userid=self._userid,
             soledad=self._soledad)
         imapProtocol.theAccount = self.theAccount
         imapProtocol.factory = self
@@ -152,9 +180,11 @@ def run_service(*args, **kwargs):
 
     port = kwargs.get('port', IMAP_PORT)
     check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD)
+    userid = kwargs.get('userid', None)
+    leap_check(userid is not None, "need an user id")
 
     uuid = soledad._get_uuid()
-    factory = LeapIMAPFactory(uuid, soledad)
+    factory = LeapIMAPFactory(uuid, userid, soledad)
 
     from twisted.internet import reactor
 
@@ -165,7 +195,8 @@ def run_service(*args, **kwargs):
             keymanager,
             soledad,
             factory.theAccount,
-            check_period)
+            check_period,
+            userid)
     except CannotListenError:
         logger.error("IMAP Service failed to start: "
                      "cannot listen in port %s" % (port,))
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 3411795..ad11315 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -91,7 +91,7 @@ def initialize_soledad(email, gnupg_home, tempdir):
     """
 
     uuid = "foobar-uuid"
-    passphrase = "verysecretpassphrase"
+    passphrase = u"verysecretpassphrase"
     secret_path = os.path.join(tempdir, "secret.gpg")
     local_db_path = os.path.join(tempdir, "soledad.u1db")
     server_url = "http://provider"
@@ -101,6 +101,8 @@ def initialize_soledad(email, gnupg_home, tempdir):
 
         get_doc = Mock(return_value=None)
         put_doc = Mock()
+        lock = Mock(return_value=('atoken', 300))
+        unlock = Mock(return_value=True)
 
         def __call__(self):
             return self