Merge remote-tracking branch 'refs/remotes/kali/bug/fix-store-iteration' into develop

3 files changed, 110 insertions, 48 deletions

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 94070ac..cf09bc4 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -463,9 +463,40 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
         d = self.messages.remove_all_deleted()
+        d.addCallback(self.messages.reset_last_uid)
         d.addCallback(self._expunge_cb)
         return d
 
+    def _bound_seq(self, messages_asked):
+        """
+        Put an upper bound to a messages sequence if this is open.
+
+        :param messages_asked: IDs of the messages.
+        :type messages_asked: MessageSet
+        :rtype: MessageSet
+        """
+        if not messages_asked.last:
+            try:
+                iter(messages_asked)
+            except TypeError:
+                # looks like we cannot iterate
+                messages_asked.last = self.last_uid
+        return messages_asked
+
+    def _filter_msg_seq(self, messages_asked):
+        """
+        Filter a message sequence returning only the ones that do exist in the
+        collection.
+
+        :param messages_asked: IDs of the messages.
+        :type messages_asked: MessageSet
+        :rtype: set
+        """
+        set_asked = set(messages_asked)
+        set_exist = set(self.messages.all_uid_iter())
+        seq_messg = set_asked.intersection(set_exist)
+        return seq_messg
+
     @deferred
     def fetch(self, messages_asked, uid):
         """
@@ -495,17 +526,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         sequence = False
         #sequence = True if uid == 0 else False
 
-        if not messages_asked.last:
-            try:
-                iter(messages_asked)
-            except TypeError:
-                # looks like we cannot iterate
-                messages_asked.last = self.last_uid
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
 
-        set_asked = set(messages_asked)
-        set_exist = set(self.messages.all_uid_iter())
-        seq_messg = set_asked.intersection(set_exist)
-        getmsg = lambda msgid: self.messages.get_msg_by_uid(msgid)
+        def getmsg(msgid):
+            if self.isWriteable():
+                deferLater(reactor, 2, self._unset_recent_flag, messages_asked)
+            return self.messages.get_msg_by_uid(msgid)
 
         # for sequence numbers (uid = 0)
         if sequence:
@@ -515,12 +542,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         else:
             result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
 
-        if self.isWriteable():
-            deferLater(reactor, 30, self._unset_recent_flag)
-            # XXX I should rewrite the scheduler so it handles a
-            # set of queues with different priority.
-            self._unset_recent_flag()
-
         # this should really be called as a final callback of
         # the do_FETCH method...
         deferLater(reactor, 1, self._signal_unread_to_ui)
@@ -532,6 +553,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         A fast method to fetch all flags, tricking just the
         needed subset of the MIME interface that's needed to satisfy
         a generic FLAGS query.
+
         Given how LEAP Mail is supposed to work without local cache,
         this query is going to be quite common, and also we expect
         it to be in the form 1:* at the beginning of a session, so
@@ -561,23 +583,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             def getFlags(self):
                 return map(str, self.flags)
 
-        if not messages_asked.last:
-            try:
-                iter(messages_asked)
-            except TypeError:
-                # looks like we cannot iterate
-                messages_asked.last = self.last_uid
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
 
-        set_asked = set(messages_asked)
-        set_exist = set(self.messages.all_uid_iter())
-        seq_messg = set_asked.intersection(set_exist)
         all_flags = self.messages.all_flags()
         result = ((msgid, flagsPart(
             msgid, all_flags[msgid])) for msgid in seq_messg)
         return result
 
     @deferred
-    def _unset_recent_flag(self):
+    def _unset_recent_flag(self, message_uid):
         """
         Unsets `Recent` flag from a tuple of messages.
         Called from fetch.
@@ -593,19 +608,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         If it is not possible to determine whether or not this
         session is the first session to be notified about a message,
         then that message SHOULD be considered recent.
-        """
-        # TODO this fucker, for the sake of correctness, is messing with
-        # the whole collection of flag docs.
 
-        # Possible ways of action:
-        # 1. Ignore it, we want fun.
-        # 2. Trigger it with a delay
-        # 3. Route it through a queue with lesser priority than the
-        #    regularar writer.
+        :param message_uids: the sequence of msg ids to update.
+        :type message_uids: sequence
+        """
+        # XXX deprecate this!
+        # move to a mailbox-level call, and do it in batches!
 
-        log.msg('unsetting recent flags...')
-        for msg in self.messages.get_recent():
-            msg.removeFlags((fields.RECENT_FLAG,))
+        log.msg('unsetting recent flag: %s' % message_uid)
+        msg = self.messages.get_msg_by_uid(message_uid)
+        msg.removeFlags((fields.RECENT_FLAG,))
         self._signal_unread_to_ui()
 
     @deferred
@@ -617,7 +629,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
 
     @deferred
-    def store(self, messages, flags, mode, uid):
+    def store(self, messages_asked, flags, mode, uid):
         """
         Sets the flags of one or more messages.
 
@@ -646,25 +658,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :raise ReadOnlyMailbox: Raised if this mailbox is not open for
                                 read-write.
         """
+        from twisted.internet import reactor
         # XXX implement also sequence (uid = 0)
         # XXX we should prevent cclient from setting Recent flag.
         leap_assert(not isinstance(flags, basestring),
                     "flags cannot be a string")
         flags = tuple(flags)
 
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
+
         if not self.isWriteable():
             log.msg('read only mailbox!')
             raise imap4.ReadOnlyMailbox
 
-        if not messages.last:
-            messages.last = self.messages.count()
-
         result = {}
-        for msg_id in messages:
+        for msg_id in seq_messg:
             log.msg("MSG ID = %s" % msg_id)
             msg = self.messages.get_msg_by_uid(msg_id)
             if not msg:
-                return result
+                continue
             if mode == 1:
                 msg.addFlags(flags)
             elif mode == -1:
@@ -673,7 +686,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 msg.setFlags(flags)
             result[msg_id] = msg.getFlags()
 
-        self._signal_unread_to_ui()
+        # this should really be called as a final callback of
+        # the do_FETCH method...
+        deferLater(reactor, 1, self._signal_unread_to_ui)
         return result
 
     # ISearchableMailbox
@@ -740,6 +755,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         new_fdoc[self.MBOX_KEY] = self.mbox
 
         d = self._do_add_doc(new_fdoc)
+        # XXX notify should be done when all the
+        # copies in the batch are finished.
         d.addCallback(self._notify_new)
 
     @deferred
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 22de356..1b996b6 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -467,7 +467,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: C{str}
         :return: An RFC822-formatted date string.
         """
-        return str(self._hdoc.content.get(self.DATE_KEY, ''))
+        date = self._hdoc.content.get(self.DATE_KEY, '')
+        return str(date)
 
     #
     # IMessagePart
@@ -1077,12 +1078,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         hd[self.MSGID_KEY] = msgid
 
         if not subject and self.SUBJECT_FIELD in headers:
-            hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
+            hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
         else:
             hd[self.SUBJECT_KEY] = subject
 
         if not date and self.DATE_FIELD in headers:
-            hd[self.DATE_KEY] = first(headers[self.DATE_FIELD])
+            hd[self.DATE_KEY] = headers[self.DATE_FIELD]
         else:
             hd[self.DATE_KEY] = date
         return hd
@@ -1337,6 +1338,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                         fields.TYPE_FLAGS_VAL, self.mbox))
         return (u for u in sorted(all_uids))
 
+    def reset_last_uid(self, param):
+        """
+        Set the last uid to the highest uid found.
+        Used while expunging, passed as a callback.
+        """
+        try:
+            self.last_uid = max(self.all_uid_iter()) + 1
+        except ValueError:
+            # empty sequence
+            pass
+        return param
+
     def all_flags(self):
         """
         Return a dict with all flags documents for this mailbox.
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index e877869..6e03456 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -124,7 +124,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
         cbFetch = self._IMAP4Server__cbFetch
         ebFetch = self._IMAP4Server__ebFetch
 
-        if str(query[0]) == "flags":
+        print "QUERY: ", query
+
+        if len(query) == 1 and str(query[0]) == "flags":
             self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
             maybeDeferred(
@@ -144,6 +146,36 @@ class LeapIMAPServer(imap4.IMAP4Server):
     select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
                     imap4.IMAP4Server.arg_fetchatt)
 
+    def _cbSelectWork(self, mbox, cmdName, tag):
+        """
+        Callback for selectWork, patched to avoid conformance errors due to
+        incomplete UIDVALIDITY line.
+        """
+        if mbox is None:
+            self.sendNegativeResponse(tag, 'No such mailbox')
+            return
+        if '\\noselect' in [s.lower() for s in mbox.getFlags()]:
+            self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
+            return
+
+        flags = mbox.getFlags()
+        self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS')
+        self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT')
+        self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
+
+        # Patched -------------------------------------------------------
+        # imaptest was complaining about the incomplete line, we're adding
+        # "UIDs valid" here.
+        self.sendPositiveResponse(
+            None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity())
+        # ----------------------------------------------------------------
+
+        s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY'
+        mbox.addListener(self)
+        self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName))
+        self.state = 'select'
+        self.mbox = mbox
+
 
 class IMAPAuthRealm(object):
     """