From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Dec 2013 14:10:14 -0400 Subject: Split the near-2k loc file into more handy modules. ...aaaand not a single fuck was given that day! --- src/leap/mail/imap/mailbox.py | 617 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 617 insertions(+) create mode 100644 src/leap/mail/imap/mailbox.py (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py new file mode 100644 index 0000000..09c06a2 --- /dev/null +++ b/src/leap/mail/imap/mailbox.py @@ -0,0 +1,617 @@ +# *- coding: utf-8 -*- +# mailbox.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Mailbox. +""" +import logging +from collections import defaultdict + +from twisted.internet import defer +from twisted.python import log + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common import events as leap_events +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.decorators import deferred +from leap.mail.imap.fields import WithMsgFields, fields +from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.parser import MBoxParser + +logger = logging.getLogger(__name__) + + +class SoledadMailbox(WithMsgFields, MBoxParser): + """ + A Soledad-backed IMAP mailbox. + + Implements the high-level method needed for the Mailbox interfaces. + The low-level database methods are contained in MessageCollection class, + which we instantiate and make accessible in the `messages` attribute. + """ + implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) + # XXX should finish the implementation of IMailboxListener + # XXX should implement IMessageCopier too + + messages = None + _closed = False + + INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, + WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, + WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, + WithMsgFields.LIST_FLAG) + flags = None + + CMD_MSG = "MESSAGES" + CMD_RECENT = "RECENT" + CMD_UIDNEXT = "UIDNEXT" + CMD_UIDVALIDITY = "UIDVALIDITY" + CMD_UNSEEN = "UNSEEN" + + _listeners = defaultdict(set) + + def __init__(self, mbox, soledad=None, rw=1): + """ + SoledadMailbox constructor. Needs to get passed a name, plus a + Soledad instance. + + :param mbox: the mailbox name + :type mbox: str + + :param soledad: a Soledad instance. + :type soledad: Soledad + + :param rw: read-and-write flags + :type rw: int + """ + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(soledad, "Need a soledad instance to initialize") + + # XXX should move to wrapper + #leap_assert(isinstance(soledad._db, SQLCipherDatabase), + #"soledad._db must be an instance of SQLCipherDatabase") + + self.mbox = self._parse_mailbox_name(mbox) + self.rw = rw + + self._soledad = soledad + + self.messages = MessageCollection( + mbox=mbox, soledad=self._soledad) + + if not self.getFlags(): + self.setFlags(self.INIT_FLAGS) + + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox] + + def addListener(self, listener): + """ + Adds a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + logger.debug('adding mailbox listener: %s' % listener) + self.listeners.add(listener) + + def removeListener(self, listener): + """ + Removes a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + self.listeners.remove(listener) + + def _get_mbox(self): + """ + Returns mailbox document. + + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. + """ + try: + query = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_MBOX_VAL, self.mbox) + if query: + return query.pop() + except Exception as exc: + logger.error("Unhandled error %r" % exc) + + def getFlags(self): + """ + Returns the flags defined for this mailbox. + + :returns: tuple of flags for this mailbox + :rtype: tuple of str + """ + mbox = self._get_mbox() + if not mbox: + return None + flags = mbox.content.get(self.FLAGS_KEY, []) + return map(str, flags) + + def setFlags(self, flags): + """ + Sets flags for this mailbox. + + :param flags: a tuple with the flags + :type flags: tuple of str + """ + leap_assert(isinstance(flags, tuple), + "flags expected to be a tuple") + mbox = self._get_mbox() + if not mbox: + return None + mbox.content[self.FLAGS_KEY] = map(str, flags) + self._soledad.put_doc(mbox) + + # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + + def _get_closed(self): + """ + Return the closed attribute for this mailbox. + + :return: True if the mailbox is closed + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.CLOSED_KEY, False) + + def _set_closed(self, closed): + """ + Set the closed attribute for this mailbox. + + :param closed: the state to be set + :type closed: bool + """ + leap_assert(isinstance(closed, bool), "closed needs to be boolean") + mbox = self._get_mbox() + mbox.content[self.CLOSED_KEY] = closed + self._soledad.put_doc(mbox) + + closed = property( + _get_closed, _set_closed, doc="Closed attribute.") + + def _get_last_uid(self): + """ + Return the last uid for this mailbox. + + :return: the last uid for messages in this mailbox + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.LAST_UID_KEY, 1) + + def _set_last_uid(self, uid): + """ + Sets the last uid for this mailbox. + + :param uid: the uid to be set + :type uid: int + """ + leap_assert(isinstance(uid, int), "uid has to be int") + mbox = self._get_mbox() + key = self.LAST_UID_KEY + + count = self.getMessageCount() + + # XXX safety-catch. If we do get duplicates, + # we want to avoid further duplication. + + if uid >= count: + value = uid + else: + # something is wrong, + # just set the last uid + # beyond the max msg count. + logger.debug("WRONG uid < count. Setting last uid to %s", count) + value = count + + mbox.content[key] = value + self._soledad.put_doc(mbox) + + last_uid = property( + _get_last_uid, _set_last_uid, doc="Last_UID attribute.") + + def getUIDValidity(self): + """ + Return the unique validity identifier for this mailbox. + + :return: unique validity identifier + :rtype: int + """ + mbox = self._get_mbox() + return mbox.content.get(self.CREATED_KEY, 1) + + def getUID(self, message): + """ + Return the UID of a message in the mailbox + + .. note:: this implementation does not make much sense RIGHT NOW, + but in the future will be useful to get absolute UIDs from + message sequence numbers. + + :param message: the message uid + :type message: int + + :rtype: int + """ + msg = self.messages.get_msg_by_uid(message) + return msg.getUID() + + def getUIDNext(self): + """ + Return the likely UID for the next message added to this + mailbox. Currently it returns the higher UID incremented by + one. + + We increment the next uid *each* time this function gets called. + In this way, there will be gaps if the message with the allocated + uid cannot be saved. But that is preferable to having race conditions + if we get to parallel message adding. + + :rtype: int + """ + self.last_uid += 1 + return self.last_uid + + def getMessageCount(self): + """ + Returns the total count of messages in this mailbox. + + :rtype: int + """ + return self.messages.count() + + def getUnseenCount(self): + """ + Returns the number of messages with the 'Unseen' flag. + + :return: count of messages flagged `unseen` + :rtype: int + """ + return self.messages.count_unseen() + + def getRecentCount(self): + """ + Returns the number of messages with the 'Recent' flag. + + :return: count of messages flagged `recent` + :rtype: int + """ + return self.messages.count_recent() + + def isWriteable(self): + """ + Get the read/write status of the mailbox. + + :return: 1 if mailbox is read-writeable, 0 otherwise. + :rtype: int + """ + return self.rw + + def getHierarchicalDelimiter(self): + """ + Returns the character used to delimite hierarchies in mailboxes. + + :rtype: str + """ + return '/' + + def requestStatus(self, names): + """ + Handles a status request by gathering the output of the different + status commands. + + :param names: a list of strings containing the status commands + :type names: iter + """ + r = {} + if self.CMD_MSG in names: + r[self.CMD_MSG] = self.getMessageCount() + if self.CMD_RECENT in names: + r[self.CMD_RECENT] = self.getRecentCount() + if self.CMD_UIDNEXT in names: + r[self.CMD_UIDNEXT] = self.last_uid + 1 + if self.CMD_UIDVALIDITY in names: + r[self.CMD_UIDVALIDITY] = self.getUID() + if self.CMD_UNSEEN in names: + r[self.CMD_UNSEEN] = self.getUnseenCount() + return defer.succeed(r) + + def addMessage(self, message, flags, date=None): + """ + Adds a message to this mailbox. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str + + :return: a deferred that evals to None + """ + # XXX we should treat the message as an IMessage from here + leap_assert_type(message, basestring) + uid_next = self.getUIDNext() + logger.debug('Adding msg with UID :%s' % uid_next) + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + d = self._do_add_messages(message, flags, date, uid_next) + d.addCallback(self._notify_new) + + @deferred + def _do_add_messages(self, message, flags, date, uid_next): + """ + Calls to the messageCollection add_msg method (deferred to thread). + Invoked from addMessage. + """ + self.messages.add_msg(message, flags=flags, date=date, + uid=uid_next) + + def _notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + :param args: ignored. + """ + exists = self.getMessageCount() + recent = self.getRecentCount() + logger.debug("NOTIFY: there are %s messages, %s recent" % ( + exists, + recent)) + + logger.debug("listeners: %s", str(self.listeners)) + for l in self.listeners: + logger.debug('notifying...') + l.newMessages(exists, recent) + + # commands, do not rename methods + + def destroy(self): + """ + Called before this mailbox is permanently deleted. + + Should cleanup resources, and set the \\Noselect flag + on the mailbox. + """ + self.setFlags((self.NOSELECT_FLAG,)) + self.deleteAllDocs() + + # XXX removing the mailbox in situ for now, + # we should postpone the removal + self._soledad.delete_doc(self._get_mbox()) + + def expunge(self): + """ + Remove all messages flagged \\Deleted + """ + if not self.isWriteable(): + raise imap4.ReadOnlyMailbox + delete = [] + deleted = [] + + for m in self.messages.get_all_docs(): + # XXX should operate with LeapMessages instead, + # so we don't expose the implementation. + # (so, iterate for m in self.messages) + if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: + delete.append(m) + for m in delete: + deleted.append(m.content) + self.messages.remove(m) + + # XXX should return the UIDs of the deleted messages + # more generically + return [x for x in range(len(deleted))] + + @deferred + def fetch(self, messages, uid): + """ + Retrieve one or more messages in this mailbox. + + from rfc 3501: The data items to be fetched can be either a single atom + or a parenthesized list. + + :param messages: IDs of the messages to retrieve information about + :type messages: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :rtype: A tuple of two-tuples of message sequence numbers and + LeapMessage + """ + result = [] + sequence = True if uid == 0 else False + + if not messages.last: + try: + iter(messages) + except TypeError: + # looks like we cannot iterate + messages.last = self.last_uid + + # for sequence numbers (uid = 0) + if sequence: + logger.debug("Getting msg by index: INEFFICIENT call!") + raise NotImplementedError + + else: + for msg_id in messages: + msg = self.messages.get_msg_by_uid(msg_id) + if msg: + result.append((msg_id, msg)) + else: + logger.debug("fetch %s, no msg found!!!" % msg_id) + + if self.isWriteable(): + self._unset_recent_flag() + self._signal_unread_to_ui() + + # XXX workaround for hangs in thunderbird + #return tuple(result[:100]) # --- doesn't show all!! + return tuple(result) + + @deferred + def _unset_recent_flag(self): + """ + Unsets `Recent` flag from a tuple of messages. + Called from fetch. + + From RFC, about `Recent`: + + Message is "recently" arrived in this mailbox. This session + is the first session to have been notified about this + message; if the session is read-write, subsequent sessions + will not see \Recent set for this message. This flag can not + be altered by the client. + + If it is not possible to determine whether or not this + session is the first session to be notified about a message, + then that message SHOULD be considered recent. + """ + log.msg('unsetting recent flags...') + for msg in self.messages.get_recent(): + msg.removeFlags((fields.RECENT_FLAG,)) + self._signal_unread_to_ui() + + @deferred + def _signal_unread_to_ui(self): + """ + Sends unread event to ui. + """ + unseen = self.getUnseenCount() + leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + + @deferred + def store(self, messages, flags, mode, uid): + """ + Sets the flags of one or more messages. + + :param messages: The identifiers of the messages to set the flags + :type messages: A MessageSet object with the list of messages requested + + :param flags: The flags to set, unset, or add. + :type flags: sequence of str + + :param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be + added to the specified messages. If mode is 0, all + existing flags should be cleared and these flags should be + added. + :type mode: -1, 0, or 1 + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A dict mapping message sequence numbers to sequences of + str representing the flags set on the message after this + operation has been performed. + :rtype: dict + + :raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + # XXX implement also sequence (uid = 0) + # XXX we should prevent cclient from setting Recent flag. + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) + + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + if not messages.last: + messages.last = self.messages.count() + + result = {} + for msg_id in messages: + log.msg("MSG ID = %s" % msg_id) + msg = self.messages.get_msg_by_uid(msg_id) + if mode == 1: + msg.addFlags(flags) + elif mode == -1: + msg.removeFlags(flags) + elif mode == 0: + msg.setFlags(flags) + result[msg_id] = msg.getFlags() + + self._signal_unread_to_ui() + return result + + @deferred + def close(self): + """ + Expunge and mark as closed + """ + self.expunge() + self.closed = True + + #@deferred + #def copy(self, messageObject): + #""" + #Copy the given message object into this mailbox. + #""" + # XXX should just: + # 1. Get the message._fdoc + # 2. Change the UID to UIDNext for this mailbox + # 3. Add implements IMessageCopier + + # convenience fun + + def deleteAllDocs(self): + """ + Deletes all docs in this mailbox + """ + docs = self.messages.get_all_docs() + for doc in docs: + self.messages._soledad.delete_doc(doc) + + def __repr__(self): + """ + Representation string for this mailbox. + """ + return u"" % ( + self.mbox, self.messages.count()) -- cgit v1.2.3 From 25a0aea875fd0d67238beed1237f7239474673ec Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 27 Dec 2013 02:06:44 -0400 Subject: First stage of the storage schema rewrite. * Separates between flags, docs, body and attachment docs. * Implement IMessageCopier interface: move and have fun! This little change is known to push forward our beloved architect emotional rollercoster. * Message deduplication. * It also fixes a hidden bug that was rendering the multipart mime interface useless (yes, the "True" parameter in the parsestr method). * Does not handle well nested attachs, includes dirty workaround that flattens them. * Includes chiiph's patch for rc2: * return deferred from addMessage * convert StringIO types to string * remove unneeded yields from the chain of deferreds in fetcher --- src/leap/mail/imap/mailbox.py | 103 ++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 30 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 09c06a2..5ea6f55 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -17,7 +17,13 @@ """ Soledad Mailbox. """ +import copy +import threading import logging +import time +import StringIO +import cStringIO + from collections import defaultdict from twisted.internet import defer @@ -45,9 +51,14 @@ class SoledadMailbox(WithMsgFields, MBoxParser): The low-level database methods are contained in MessageCollection class, which we instantiate and make accessible in the `messages` attribute. """ - implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) + implements( + imap4.IMailbox, + imap4.IMailboxInfo, + imap4.ICloseableMailbox, + imap4.IMessageCopier) + # XXX should finish the implementation of IMailboxListener - # XXX should implement IMessageCopier too + # XXX should implement ISearchableMailbox too messages = None _closed = False @@ -65,6 +76,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): CMD_UNSEEN = "UNSEEN" _listeners = defaultdict(set) + next_uid_lock = threading.Lock() def __init__(self, mbox, soledad=None, rw=1): """ @@ -284,8 +296,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: int """ - self.last_uid += 1 - return self.last_uid + with self.next_uid_lock: + self.last_uid += 1 + return self.last_uid def getMessageCount(self): """ @@ -366,6 +379,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: a deferred that evals to None """ + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() # XXX we should treat the message as an IMessage from here leap_assert_type(message, basestring) uid_next = self.getUIDNext() @@ -375,11 +390,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_messages(message, flags, date, uid_next) + d = self._do_add_message(message, flags, date, uid_next) d.addCallback(self._notify_new) + return d @deferred - def _do_add_messages(self, message, flags, date, uid_next): + def _do_add_message(self, message, flags, date, uid_next): """ Calls to the messageCollection add_msg method (deferred to thread). Invoked from addMessage. @@ -420,28 +436,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # we should postpone the removal self._soledad.delete_doc(self._get_mbox()) + @deferred def expunge(self): """ Remove all messages flagged \\Deleted """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - delete = [] deleted = [] - - for m in self.messages.get_all_docs(): - # XXX should operate with LeapMessages instead, - # so we don't expose the implementation. - # (so, iterate for m in self.messages) - if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: - delete.append(m) - for m in delete: - deleted.append(m.content) - self.messages.remove(m) - - # XXX should return the UIDs of the deleted messages - # more generically - return [x for x in range(len(deleted))] + for m in self.messages: + if self.DELETED_FLAG in m.getFlags(): + self.messages.remove(m) + # XXX this would ve more efficient if we can just pass + # a sequence of uids. + deleted.append(m.getUID()) + return deleted @deferred def fetch(self, messages, uid): @@ -510,6 +519,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): session is the first session to be notified about a message, then that message SHOULD be considered recent. """ + # TODO this fucker, for the sake of correctness, is messing with + # the whole collection of flag docs. + + # Possible ways of action: + # 1. Ignore it, we want fun. + # 2. Trigger it with a delay + # 3. Route it through a queue with lesser priority than the + # regularar writer. + + # hmm let's try 2. in a quickndirty way... + time.sleep(1) log.msg('unsetting recent flags...') for msg in self.messages.get_recent(): msg.removeFlags((fields.RECENT_FLAG,)) @@ -570,6 +590,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): for msg_id in messages: log.msg("MSG ID = %s" % msg_id) msg = self.messages.get_msg_by_uid(msg_id) + if not msg: + return result if mode == 1: msg.addFlags(flags) elif mode == -1: @@ -589,15 +611,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.expunge() self.closed = True - #@deferred - #def copy(self, messageObject): - #""" - #Copy the given message object into this mailbox. - #""" - # XXX should just: - # 1. Get the message._fdoc - # 2. Change the UID to UIDNext for this mailbox - # 3. Add implements IMessageCopier + # IMessageCopier + + @deferred + def copy(self, messageObject): + """ + Copy the given message object into this mailbox. + """ + uid_next = self.getUIDNext() + msg = messageObject + + # XXX should use a public api instead + fdoc = msg._fdoc + if not fdoc: + logger.debug("Tried to copy a MSG with no fdoc") + return + + new_fdoc = copy.deepcopy(fdoc.content) + new_fdoc[self.UID_KEY] = uid_next + new_fdoc[self.MBOX_KEY] = self.mbox + + d = self._do_add_doc(new_fdoc) + d.addCallback(self._notify_new) + + @deferred + def _do_add_doc(self, doc): + """ + Defers the adding of a new doc. + :param doc: document to be created in soledad. + """ + self._soledad.create_doc(doc) # convenience fun -- cgit v1.2.3 From 5585ff784940dee267576d097076de66797f9188 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 2 Jan 2014 16:08:09 -0400 Subject: fix tests after rewrite --- src/leap/mail/imap/mailbox.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 5ea6f55..10087f6 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -390,18 +390,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags, date, uid_next) + d = self._do_add_message(message, flags=flags, date=date, uid=uid_next) d.addCallback(self._notify_new) return d @deferred - def _do_add_message(self, message, flags, date, uid_next): + def _do_add_message(self, message, flags, date, uid): """ Calls to the messageCollection add_msg method (deferred to thread). Invoked from addMessage. """ - self.messages.add_msg(message, flags=flags, date=date, - uid=uid_next) + self.messages.add_msg(message, flags=flags, date=date, uid=uid) def _notify_new(self, *args): """ @@ -436,21 +435,29 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # we should postpone the removal self._soledad.delete_doc(self._get_mbox()) - @deferred + def _close_cb(self, result): + self.closed = True + + def close(self): + """ + Expunge and mark as closed + """ + d = self.expunge() + d.addCallback(self._close_cb) + return d + + def _expunge_cb(self, result): + return result + def expunge(self): """ Remove all messages flagged \\Deleted """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - deleted = [] - for m in self.messages: - if self.DELETED_FLAG in m.getFlags(): - self.messages.remove(m) - # XXX this would ve more efficient if we can just pass - # a sequence of uids. - deleted.append(m.getUID()) - return deleted + d = self.messages.remove_all_deleted() + d.addCallback(self._expunge_cb) + return d @deferred def fetch(self, messages, uid): @@ -603,14 +610,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._signal_unread_to_ui() return result - @deferred - def close(self): - """ - Expunge and mark as closed - """ - self.expunge() - self.closed = True - # IMessageCopier @deferred -- cgit v1.2.3 From e1946f1653dbbb6fcf61569bc873ab061965664e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 6 Jan 2014 04:44:05 -0400 Subject: tests infrastructure for multipart --- src/leap/mail/imap/mailbox.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 10087f6..1d76d4d 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -478,6 +478,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): LeapMessage """ result = [] + + # XXX DEBUG ------------- + print "getting uid", uid + print "in mbox", self.mbox + sequence = True if uid == 0 else False if not messages.last: -- cgit v1.2.3 From e4f2914517ea11aeef60aa74be50116e1979f34d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Jan 2014 21:39:27 -0400 Subject: handle all fetches as sequential * this allows quick testing using telnet, and the use of other less sofisticated MUAs. --- src/leap/mail/imap/mailbox.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 1d76d4d..7c01490 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -479,11 +479,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ result = [] - # XXX DEBUG ------------- - print "getting uid", uid - print "in mbox", self.mbox + # For the moment our UID is sequential, so we + # can treat them all the same. + # Change this to the flag that twisted expects when we + # switch to content-hash based index + local UID table. - sequence = True if uid == 0 else False + sequence = False + #sequence = True if uid == 0 else False if not messages.last: try: -- cgit v1.2.3 From 51eaab77deedf0c923fe40cf3d346fa879bf2ae3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Jan 2014 13:20:00 -0400 Subject: Add check for uniqueness when adding mails. Check by mbox + content-hash --- src/leap/mail/imap/mailbox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 7c01490..c9e8684 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -125,7 +125,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def addListener(self, listener): """ - Adds a listener to the listeners queue. + Add a listener to the listeners queue. The server adds itself as a listener when there is a SELECT, so it can send EXIST commands. @@ -137,7 +137,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def removeListener(self, listener): """ - Removes a listener from the listeners queue. + Remove a listener from the listeners queue. :param listener: listener to remove :type listener: an object that implements IMailboxListener @@ -146,7 +146,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def _get_mbox(self): """ - Returns mailbox document. + Return mailbox document. :return: A SoledadDocument containing this mailbox, or None if the query failed. -- cgit v1.2.3 From 4856f32ec75cda000fc794d0ac93990e0d1e42f6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Jan 2014 17:58:02 -0400 Subject: Very limited support for SEARCH Commands. Closes: #4209 limited to HEADER Message-ID. This is a quick workaround for avoiding duplicate saves in Drafts Folder. but we'll get there! --- src/leap/mail/imap/mailbox.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c9e8684..ccbf5c2 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -39,6 +39,7 @@ from leap.mail.decorators import deferred from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.parser import MBoxParser +from leap.mail.utils import first logger = logging.getLogger(__name__) @@ -55,6 +56,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): imap4.IMailbox, imap4.IMailboxInfo, imap4.ICloseableMailbox, + imap4.ISearchableMailbox, imap4.IMessageCopier) # XXX should finish the implementation of IMailboxListener @@ -617,6 +619,49 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._signal_unread_to_ui() return result + # ISearchableMailbox + + def search(self, query, uid): + """ + Search for messages that meet the given query criteria. + + Warning: this is half-baked, and it might give problems since + it offers the SearchableInterface. + We'll be implementing it asap. + + :param query: The search criteria + :type query: list + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A list of message sequence numbers or message UIDs which + match the search criteria or a C{Deferred} whose callback + will be invoked with such a list. + :rtype: C{list} or C{Deferred} + """ + # TODO see if we can raise w/o interrupting flow + #:raise IllegalQueryError: Raised when query is not valid. + # example query: + # ['UNDELETED', 'HEADER', 'Message-ID', + # '52D44F11.9060107@dev.bitmask.net'] + + # TODO hardcoding for now! -- we'll support generic queries later on + # but doing a quickfix for avoiding duplicat saves in the draft folder. + # See issue #4209 + + if query[1] == 'HEADER' and query[2].lower() == "message-id": + msgid = str(query[3]).strip() + d = self.messages._get_uid_from_msgid(str(msgid)) + d1 = defer.gatherResults([d]) + # we want a list, so return it all the same + return d1 + + # nothing implemented for any other query + logger.warning("Cannot process query: %s" % (query,)) + return [] + # IMessageCopier @deferred -- cgit v1.2.3 From 2b53238ce5211bc23da8d1e8903335daa12ca02e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 14 Jan 2014 16:28:07 -0400 Subject: remove locks (moved to soledad client) --- src/leap/mail/imap/mailbox.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index ccbf5c2..cd782b2 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -39,7 +39,6 @@ from leap.mail.decorators import deferred from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.parser import MBoxParser -from leap.mail.utils import first logger = logging.getLogger(__name__) @@ -60,7 +59,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): imap4.IMessageCopier) # XXX should finish the implementation of IMailboxListener - # XXX should implement ISearchableMailbox too + # XXX should complately implement ISearchableMailbox too messages = None _closed = False @@ -78,6 +77,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): CMD_UNSEEN = "UNSEEN" _listeners = defaultdict(set) + next_uid_lock = threading.Lock() def __init__(self, mbox, soledad=None, rw=1): @@ -161,7 +161,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if query: return query.pop() except Exception as exc: - logger.error("Unhandled error %r" % exc) + logger.exception("Unhandled error %r" % exc) def getFlags(self): """ @@ -226,6 +226,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: bool """ mbox = self._get_mbox() + if not mbox: + logger.error("We could not get a mbox!") + # XXX It looks like it has been corrupted. + # We need to be able to survive this. + return None return mbox.content.get(self.LAST_UID_KEY, 1) def _set_last_uid(self, uid): -- cgit v1.2.3 From 90f4338da088394ade1663871a23b8fb0a4c0d66 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jan 2014 17:05:24 -0400 Subject: Performance improvement on FLAGS-only FETCH * Compute the intersection set of the uids on a FETCH, so we avoid iterating through the non-existant UIDs. * Dispatch FLAGS query to our specialized method, that fetches all the flags documents and return objects that only specify one subset of the MessagePart interface, apt to render flags quickly with less queries overhead. * Overwrite the do_FETCH command in the imap Server to use fetch_flags. * Use deferLater for a better dispatch of tasks in the reactor. --- src/leap/mail/imap/mailbox.py | 94 +++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 21 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index cd782b2..94070ac 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -20,13 +20,13 @@ Soledad Mailbox. import copy import threading import logging -import time import StringIO import cStringIO from collections import defaultdict from twisted.internet import defer +from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -59,7 +59,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): imap4.IMessageCopier) # XXX should finish the implementation of IMailboxListener - # XXX should complately implement ISearchableMailbox too + # XXX should completely implement ISearchableMailbox too messages = None _closed = False @@ -467,15 +467,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): return d @deferred - def fetch(self, messages, uid): + def fetch(self, messages_asked, uid): """ Retrieve one or more messages in this mailbox. from rfc 3501: The data items to be fetched can be either a single atom or a parenthesized list. - :param messages: IDs of the messages to retrieve information about - :type messages: MessageSet + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet :param uid: If true, the IDs are UIDs. They are message sequence IDs otherwise. @@ -484,7 +485,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: A tuple of two-tuples of message sequence numbers and LeapMessage """ - result = [] + from twisted.internet import reactor # For the moment our UID is sequential, so we # can treat them all the same. @@ -494,12 +495,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): sequence = False #sequence = True if uid == 0 else False - if not messages.last: + if not messages_asked.last: try: - iter(messages) + iter(messages_asked) except TypeError: # looks like we cannot iterate - messages.last = self.last_uid + messages_asked.last = self.last_uid + + set_asked = set(messages_asked) + set_exist = set(self.messages.all_uid_iter()) + seq_messg = set_asked.intersection(set_exist) + getmsg = lambda msgid: self.messages.get_msg_by_uid(msgid) # for sequence numbers (uid = 0) if sequence: @@ -507,20 +513,68 @@ class SoledadMailbox(WithMsgFields, MBoxParser): raise NotImplementedError else: - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) - else: - logger.debug("fetch %s, no msg found!!!" % msg_id) + result = ((msgid, getmsg(msgid)) for msgid in seq_messg) if self.isWriteable(): + deferLater(reactor, 30, self._unset_recent_flag) + # XXX I should rewrite the scheduler so it handles a + # set of queues with different priority. self._unset_recent_flag() - self._signal_unread_to_ui() - # XXX workaround for hangs in thunderbird - #return tuple(result[:100]) # --- doesn't show all!! - return tuple(result) + # this should really be called as a final callback of + # the do_FETCH method... + deferLater(reactor, 1, self._signal_unread_to_ui) + return result + + @deferred + def fetch_flags(self, messages_asked, uid): + """ + A fast method to fetch all flags, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic FLAGS query. + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + it's not bad to fetch all the flags doc at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :return: A tuple of two-tuples of message sequence numbers and + flagsPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + class flagsPart(object): + def __init__(self, uid, flags): + self.uid = uid + self.flags = flags + + def getUID(self): + return self.uid + + def getFlags(self): + return map(str, self.flags) + + if not messages_asked.last: + try: + iter(messages_asked) + except TypeError: + # looks like we cannot iterate + messages_asked.last = self.last_uid + + set_asked = set(messages_asked) + set_exist = set(self.messages.all_uid_iter()) + seq_messg = set_asked.intersection(set_exist) + all_flags = self.messages.all_flags() + result = ((msgid, flagsPart( + msgid, all_flags[msgid])) for msgid in seq_messg) + return result @deferred def _unset_recent_flag(self): @@ -549,8 +603,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # 3. Route it through a queue with lesser priority than the # regularar writer. - # hmm let's try 2. in a quickndirty way... - time.sleep(1) log.msg('unsetting recent flags...') for msg in self.messages.get_recent(): msg.removeFlags((fields.RECENT_FLAG,)) -- cgit v1.2.3 From ae56191d2d6f2953bd49f43b9dedb322a7f0db8c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 17:18:11 -0400 Subject: reset last uid on expunge --- src/leap/mail/imap/mailbox.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 94070ac..86dac77 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -463,6 +463,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not self.isWriteable(): raise imap4.ReadOnlyMailbox d = self.messages.remove_all_deleted() + d.addCallback(self.messages.reset_last_uid) d.addCallback(self._expunge_cb) return d -- cgit v1.2.3 From 557fac26982aa1360ed51d158869312d6438eb84 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 17:24:27 -0400 Subject: factor out bound and filter for msg seqs --- src/leap/mail/imap/mailbox.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 86dac77..84eb528 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -467,6 +467,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self._expunge_cb) return d + def _bound_seq(self, messages_asked): + """ + Put an upper bound to a messages sequence if this is open. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :rtype: MessageSet + """ + if not messages_asked.last: + try: + iter(messages_asked) + except TypeError: + # looks like we cannot iterate + messages_asked.last = self.last_uid + return messages_asked + + def _filter_msg_seq(self, messages_asked): + """ + Filter a message sequence returning only the ones that do exist in the + collection. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :rtype: set + """ + set_asked = set(messages_asked) + set_exist = set(self.messages.all_uid_iter()) + seq_messg = set_asked.intersection(set_exist) + return seq_messg + @deferred def fetch(self, messages_asked, uid): """ -- cgit v1.2.3 From a660231e918df6698b6dcfad9d1845bd77ee6f8f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 17:26:01 -0400 Subject: Fix grave bug with iteration in STORE This was in the root for problems with Trash behavior. Closes: #4958 Make use of the refactored utilities for bounding and filtering sequences. --- src/leap/mail/imap/mailbox.py | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 84eb528..137f9f5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -526,17 +526,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): sequence = False #sequence = True if uid == 0 else False - if not messages_asked.last: - try: - iter(messages_asked) - except TypeError: - # looks like we cannot iterate - messages_asked.last = self.last_uid + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - getmsg = lambda msgid: self.messages.get_msg_by_uid(msgid) # for sequence numbers (uid = 0) if sequence: @@ -563,6 +555,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): A fast method to fetch all flags, tricking just the needed subset of the MIME interface that's needed to satisfy a generic FLAGS query. + Given how LEAP Mail is supposed to work without local cache, this query is going to be quite common, and also we expect it to be in the form 1:* at the beginning of a session, so @@ -592,16 +585,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def getFlags(self): return map(str, self.flags) - if not messages_asked.last: - try: - iter(messages_asked) - except TypeError: - # looks like we cannot iterate - messages_asked.last = self.last_uid + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) all_flags = self.messages.all_flags() result = ((msgid, flagsPart( msgid, all_flags[msgid])) for msgid in seq_messg) @@ -648,7 +634,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) @deferred - def store(self, messages, flags, mode, uid): + def store(self, messages_asked, flags, mode, uid): """ Sets the flags of one or more messages. @@ -677,25 +663,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ + from twisted.internet import reactor # XXX implement also sequence (uid = 0) # XXX we should prevent cclient from setting Recent flag. leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) + if not self.isWriteable(): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox - if not messages.last: - messages.last = self.messages.count() - result = {} - for msg_id in messages: + for msg_id in seq_messg: log.msg("MSG ID = %s" % msg_id) msg = self.messages.get_msg_by_uid(msg_id) if not msg: - return result + continue if mode == 1: msg.addFlags(flags) elif mode == -1: -- cgit v1.2.3 From 6c7207a5667d8158572b2a900a3506e3c3ecc6e5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 17:33:20 -0400 Subject: Temporal refactor setting of recent flag. This flag is set way too often, and is damaging performance. Will move it to a single doc per mailbox in subsequente commits. --- src/leap/mail/imap/mailbox.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 137f9f5..cf09bc4 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -529,6 +529,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) + def getmsg(msgid): + if self.isWriteable(): + deferLater(reactor, 2, self._unset_recent_flag, messages_asked) + return self.messages.get_msg_by_uid(msgid) # for sequence numbers (uid = 0) if sequence: @@ -538,12 +542,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: result = ((msgid, getmsg(msgid)) for msgid in seq_messg) - if self.isWriteable(): - deferLater(reactor, 30, self._unset_recent_flag) - # XXX I should rewrite the scheduler so it handles a - # set of queues with different priority. - self._unset_recent_flag() - # this should really be called as a final callback of # the do_FETCH method... deferLater(reactor, 1, self._signal_unread_to_ui) @@ -594,7 +592,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): return result @deferred - def _unset_recent_flag(self): + def _unset_recent_flag(self, message_uid): """ Unsets `Recent` flag from a tuple of messages. Called from fetch. @@ -610,19 +608,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): If it is not possible to determine whether or not this session is the first session to be notified about a message, then that message SHOULD be considered recent. - """ - # TODO this fucker, for the sake of correctness, is messing with - # the whole collection of flag docs. - # Possible ways of action: - # 1. Ignore it, we want fun. - # 2. Trigger it with a delay - # 3. Route it through a queue with lesser priority than the - # regularar writer. + :param message_uids: the sequence of msg ids to update. + :type message_uids: sequence + """ + # XXX deprecate this! + # move to a mailbox-level call, and do it in batches! - log.msg('unsetting recent flags...') - for msg in self.messages.get_recent(): - msg.removeFlags((fields.RECENT_FLAG,)) + log.msg('unsetting recent flag: %s' % message_uid) + msg = self.messages.get_msg_by_uid(message_uid) + msg.removeFlags((fields.RECENT_FLAG,)) self._signal_unread_to_ui() @deferred @@ -691,7 +686,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msg.setFlags(flags) result[msg_id] = msg.getFlags() - self._signal_unread_to_ui() + # this should really be called as a final callback of + # the do_FETCH method... + deferLater(reactor, 1, self._signal_unread_to_ui) return result # ISearchableMailbox @@ -758,6 +755,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): new_fdoc[self.MBOX_KEY] = self.mbox d = self._do_add_doc(new_fdoc) + # XXX notify should be done when all the + # copies in the batch are finished. d.addCallback(self._notify_new) @deferred -- cgit v1.2.3 From 9f9701d42be385aa9a6d7e72fd10104b0025971b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 22:01:20 -0400 Subject: Separate RECENT Flag to a mailbox document. this way we avoid a bunch of writes. --- src/leap/mail/imap/mailbox.py | 82 ++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 52 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index cf09bc4..bd69d12 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -398,18 +398,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = tuple(str(flag) for flag in flags) d = self._do_add_message(message, flags=flags, date=date, uid=uid_next) - d.addCallback(self._notify_new) return d - @deferred def _do_add_message(self, message, flags, date, uid): """ Calls to the messageCollection add_msg method (deferred to thread). Invoked from addMessage. """ - self.messages.add_msg(message, flags=flags, date=date, uid=uid) + d = self.messages.add_msg(message, flags=flags, date=date, uid=uid) + # XXX notify after batch APPEND? + d.addCallback(self.notify_new) + return d - def _notify_new(self, *args): + def notify_new(self, *args): """ Notify of new messages to all the listeners. @@ -463,8 +464,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not self.isWriteable(): raise imap4.ReadOnlyMailbox d = self.messages.remove_all_deleted() - d.addCallback(self.messages.reset_last_uid) d.addCallback(self._expunge_cb) + d.addCallback(self.messages.reset_last_uid) return d def _bound_seq(self, messages_asked): @@ -480,7 +481,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): iter(messages_asked) except TypeError: # looks like we cannot iterate - messages_asked.last = self.last_uid + try: + messages_asked.last = self.last_uid + except ValueError: + pass return messages_asked def _filter_msg_seq(self, messages_asked): @@ -529,10 +533,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - def getmsg(msgid): - if self.isWriteable(): - deferLater(reactor, 2, self._unset_recent_flag, messages_asked) - return self.messages.get_msg_by_uid(msgid) + getmsg = lambda uid: self.messages.get_msg_by_uid(uid) # for sequence numbers (uid = 0) if sequence: @@ -544,7 +545,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # this should really be called as a final callback of # the do_FETCH method... - deferLater(reactor, 1, self._signal_unread_to_ui) + return result @deferred @@ -591,37 +592,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msgid, all_flags[msgid])) for msgid in seq_messg) return result - @deferred - def _unset_recent_flag(self, message_uid): - """ - Unsets `Recent` flag from a tuple of messages. - Called from fetch. - - From RFC, about `Recent`: - - Message is "recently" arrived in this mailbox. This session - is the first session to have been notified about this - message; if the session is read-write, subsequent sessions - will not see \Recent set for this message. This flag can not - be altered by the client. - - If it is not possible to determine whether or not this - session is the first session to be notified about a message, - then that message SHOULD be considered recent. - - :param message_uids: the sequence of msg ids to update. - :type message_uids: sequence - """ - # XXX deprecate this! - # move to a mailbox-level call, and do it in batches! - - log.msg('unsetting recent flag: %s' % message_uid) - msg = self.messages.get_msg_by_uid(message_uid) - msg.removeFlags((fields.RECENT_FLAG,)) - self._signal_unread_to_ui() - - @deferred - def _signal_unread_to_ui(self): + def signal_unread_to_ui(self): """ Sends unread event to ui. """ @@ -687,8 +658,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): result[msg_id] = msg.getFlags() # this should really be called as a final callback of - # the do_FETCH method... - deferLater(reactor, 1, self._signal_unread_to_ui) + # the do_STORE method... + # XXX --- + #deferLater(reactor, 1, self._signal_unread_to_ui) return result # ISearchableMailbox @@ -741,6 +713,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Copy the given message object into this mailbox. """ + from twisted.internet import reactor uid_next = self.getUIDNext() msg = messageObject @@ -753,17 +726,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): new_fdoc = copy.deepcopy(fdoc.content) new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = self.mbox + self._do_add_doc(new_fdoc) + deferLater(reactor, 1, self.notify_new) - d = self._do_add_doc(new_fdoc) - # XXX notify should be done when all the - # copies in the batch are finished. - d.addCallback(self._notify_new) - - @deferred def _do_add_doc(self, doc): """ - Defers the adding of a new doc. + Defer the adding of a new doc. + :param doc: document to be created in soledad. + :type doc: dict """ self._soledad.create_doc(doc) @@ -771,12 +742,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def deleteAllDocs(self): """ - Deletes all docs in this mailbox + Delete all docs in this mailbox """ docs = self.messages.get_all_docs() for doc in docs: self.messages._soledad.delete_doc(doc) + def unset_recent_flags(self, uids): + """ + Unset Recent flag for a sequence of UIDs. + """ + seq_messg = self._bound_seq(uids) + self.messages.unset_recent_flags(seq_messg) + def __repr__(self): """ Representation string for this mailbox. -- cgit v1.2.3 From 9ef1cd79397d811575826025b924c615e6ce2aa4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 17 Jan 2014 02:51:31 -0400 Subject: Add a fetch_headers for mass-header fetch queries --- src/leap/mail/imap/mailbox.py | 76 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 12 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index bd69d12..b186e75 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -466,6 +466,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self.messages.remove_all_deleted() d.addCallback(self._expunge_cb) d.addCallback(self.messages.reset_last_uid) + + # XXX DEBUG ------------------- + # FIXME !!! + # XXX should remove the hdocset too!!! return d def _bound_seq(self, messages_asked): @@ -520,8 +524,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: A tuple of two-tuples of message sequence numbers and LeapMessage """ - from twisted.internet import reactor - # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we @@ -532,20 +534,14 @@ class SoledadMailbox(WithMsgFields, MBoxParser): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - getmsg = lambda uid: self.messages.get_msg_by_uid(uid) # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") raise NotImplementedError - else: result = ((msgid, getmsg(msgid)) for msgid in seq_messg) - - # this should really be called as a final callback of - # the do_FETCH method... - return result @deferred @@ -558,7 +554,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Given how LEAP Mail is supposed to work without local cache, this query is going to be quite common, and also we expect it to be in the form 1:* at the beginning of a session, so - it's not bad to fetch all the flags doc at once. + it's not bad to fetch all the FLAGS docs at once. :param messages_asked: IDs of the messages to retrieve information about @@ -592,6 +588,55 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msgid, all_flags[msgid])) for msgid in seq_messg) return result + @deferred + def fetch_headers(self, messages_asked, uid): + """ + A fast method to fetch all headers, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic HEADERS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + **MAYBE** it's not too bad to fetch all the HEADERS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :return: A tuple of two-tuples of message sequence numbers and + headersPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + class headersPart(object): + def __init__(self, uid, headers): + self.uid = uid + self.headers = headers + + def getUID(self): + return self.uid + + def getHeaders(self, _): + return dict( + (str(key), str(value)) + for key, value in + self.headers.items()) + + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) + + all_chash = self.messages.all_flags_chash() + all_headers = self.messages.all_headers() + result = ((msgid, headersPart( + msgid, all_headers.get(all_chash.get(msgid, 'nil'), {}))) + for msgid in seq_messg) + return result + def signal_unread_to_ui(self): """ Sends unread event to ui. @@ -629,7 +674,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ - from twisted.internet import reactor # XXX implement also sequence (uid = 0) # XXX we should prevent cclient from setting Recent flag. leap_assert(not isinstance(flags, basestring), @@ -657,10 +701,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msg.setFlags(flags) result[msg_id] = msg.getFlags() + # After changing flags, we want to signal again to the + # UI because the number of unread might have changed. + # Hoever, we should probably limit this to INBOX only? # this should really be called as a final callback of # the do_STORE method... - # XXX --- - #deferLater(reactor, 1, self._signal_unread_to_ui) + from twisted.internet import reactor + deferLater(reactor, 1, self._signal_unread_to_ui) return result # ISearchableMailbox @@ -727,6 +774,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = self.mbox self._do_add_doc(new_fdoc) + + # XXX should use a public api instead + hdoc = msg._hdoc + self.messages.add_hdocset_docid(hdoc.doc_id) + deferLater(reactor, 1, self.notify_new) def _do_add_doc(self, doc): -- cgit v1.2.3 From a50c880f43d1aef00fd233318d9413e01fb3aa3f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 20 Jan 2014 11:38:19 -0400 Subject: Fix typo in the signal_unread method. Closes: #5001 It had been made public to be called from the overwritten methods in service.imap --- src/leap/mail/imap/mailbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index b186e75..a167531 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -707,7 +707,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # this should really be called as a final callback of # the do_STORE method... from twisted.internet import reactor - deferLater(reactor, 1, self._signal_unread_to_ui) + deferLater(reactor, 1, self.signal_unread_to_ui) return result # ISearchableMailbox -- cgit v1.2.3 From ca9ba607ec09036db387dda6704b5956fc7baae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Mon, 20 Jan 2014 13:29:13 -0300 Subject: Fix search command filter --- src/leap/mail/imap/mailbox.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index a167531..174361f 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -742,12 +742,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # but doing a quickfix for avoiding duplicat saves in the draft folder. # See issue #4209 - if query[1] == 'HEADER' and query[2].lower() == "message-id": - msgid = str(query[3]).strip() - d = self.messages._get_uid_from_msgid(str(msgid)) - d1 = defer.gatherResults([d]) - # we want a list, so return it all the same - return d1 + if len(query) > 2: + if query[1] == 'HEADER' and query[2].lower() == "message-id": + msgid = str(query[3]).strip() + d = self.messages._get_uid_from_msgid(str(msgid)) + d1 = defer.gatherResults([d]) + # we want a list, so return it all the same + return d1 # nothing implemented for any other query logger.warning("Cannot process query: %s" % (query,)) -- cgit v1.2.3 From 22c106a7306446a3fa9689f5942a86a53ec884b4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 21 Jan 2014 01:04:37 -0400 Subject: workaround for recursionlimit due to qtreactor --- src/leap/mail/imap/mailbox.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 174361f..38c58cb 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -765,6 +765,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): uid_next = self.getUIDNext() msg = messageObject + # XXX DEBUG ---------------------------------------- + #print "copying MESSAGE from %s (%s) to %s (%s)" % ( + #msg._mbox, msg._uid, self.mbox, uid_next) + # XXX should use a public api instead fdoc = msg._fdoc if not fdoc: -- cgit v1.2.3 From e9db0eb4802e528142000d7a2f7da0c9135fce44 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 26 Jan 2014 19:49:39 -0400 Subject: temporarily remove notify after adding msg --- src/leap/mail/imap/mailbox.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 38c58cb..0131ce0 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -406,8 +406,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Invoked from addMessage. """ d = self.messages.add_msg(message, flags=flags, date=date, uid=uid) - # XXX notify after batch APPEND? - d.addCallback(self.notify_new) + # XXX Removing notify temporarily. + # This is interfering with imaptest results. I'm not clear if it's + # because we clutter the logging or because the set of listeners is + # ever-growing. We should come up with some smart way of dealing with + # it, or maybe just disabling it using an environmental variable since + # we will only have just a few listeners in the regular desktop case. + #d.addCallback(self.notify_new) return d def notify_new(self, *args): @@ -422,7 +427,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): exists, recent)) - logger.debug("listeners: %s", str(self.listeners)) for l in self.listeners: logger.debug('notifying...') l.newMessages(exists, recent) -- cgit v1.2.3 From 4ae6ad57a0f80143e3ded867c1fdd2264804a775 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 21 Jan 2014 19:22:09 -0400 Subject: memory store for append/fetch/copy --- src/leap/mail/imap/mailbox.py | 62 ++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 21 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 0131ce0..9babe6b 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -37,6 +37,7 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.decorators import deferred from leap.mail.imap.fields import WithMsgFields, fields +from leap.mail.imap.memorystore import MessageDict from leap.mail.imap.messages import MessageCollection from leap.mail.imap.parser import MBoxParser @@ -80,7 +81,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): next_uid_lock = threading.Lock() - def __init__(self, mbox, soledad=None, rw=1): + def __init__(self, mbox, soledad, memstore, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. @@ -91,9 +92,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param soledad: a Soledad instance. :type soledad: Soledad - :param rw: read-and-write flags + :param memstore: a MemoryStore instance + :type memstore: MemoryStore + + :param rw: read-and-write flag for this mailbox :type rw: int """ + print "got memstore: ", memstore leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(soledad, "Need a soledad instance to initialize") @@ -105,9 +110,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.rw = rw self._soledad = soledad + self._memstore = memstore self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad) + mbox=mbox, soledad=self._soledad, memstore=self._memstore) if not self.getFlags(): self.setFlags(self.INIT_FLAGS) @@ -231,7 +237,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # XXX It looks like it has been corrupted. # We need to be able to survive this. return None - return mbox.content.get(self.LAST_UID_KEY, 1) + last = mbox.content.get(self.LAST_UID_KEY, 1) + if self._memstore: + last = max(last, self._memstore.get_last_uid(mbox)) + return last def _set_last_uid(self, uid): """ @@ -259,6 +268,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): value = count mbox.content[key] = value + # XXX this should be set in the memorystore instead!!! self._soledad.put_doc(mbox) last_uid = property( @@ -532,12 +542,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # can treat them all the same. # Change this to the flag that twisted expects when we # switch to content-hash based index + local UID table. + print + print "FETCHING..." sequence = False #sequence = True if uid == 0 else False messages_asked = self._bound_seq(messages_asked) + print "asked: ", messages_asked seq_messg = self._filter_msg_seq(messages_asked) + + print "seq: ", seq_messg getmsg = lambda uid: self.messages.get_msg_by_uid(uid) # for sequence numbers (uid = 0) @@ -769,36 +784,41 @@ class SoledadMailbox(WithMsgFields, MBoxParser): uid_next = self.getUIDNext() msg = messageObject - # XXX DEBUG ---------------------------------------- - #print "copying MESSAGE from %s (%s) to %s (%s)" % ( - #msg._mbox, msg._uid, self.mbox, uid_next) - # XXX should use a public api instead fdoc = msg._fdoc + hdoc = msg._hdoc if not fdoc: logger.debug("Tried to copy a MSG with no fdoc") return + #old_mbox = fdoc.content[self.MBOX_KEY] + #old_uid = fdoc.content[self.UID_KEY] + #old_key = old_mbox, old_uid + #print "copying from OLD MBOX ", old_mbox + + # XXX bit doubt... to duplicate in memory + # or not to...? + # I think it should be ok to duplicate as long as we're + # careful at the hour of writes... + # We could use also proxies, but it will break when + # the original mailbox is flushed. + + # XXX DEBUG ---------------------------------------- + #print "copying MESSAGE from %s (%s) to %s (%s)" % ( + #msg._mbox, msg._uid, self.mbox, uid_next) + new_fdoc = copy.deepcopy(fdoc.content) new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = self.mbox - self._do_add_doc(new_fdoc) + self._memstore.put(self.mbox, uid_next, MessageDict( + new_fdoc, hdoc.content)) - # XXX should use a public api instead - hdoc = msg._hdoc - self.messages.add_hdocset_docid(hdoc.doc_id) + # XXX use memory store + if hasattr(hdoc, 'doc_id'): + self.messages.add_hdocset_docid(hdoc.doc_id) deferLater(reactor, 1, self.notify_new) - def _do_add_doc(self, doc): - """ - Defer the adding of a new doc. - - :param doc: document to be created in soledad. - :type doc: dict - """ - self._soledad.create_doc(doc) - # convenience fun def deleteAllDocs(self): -- cgit v1.2.3 From e2218eec4fd91e4648160a05e3debc05efa0d0d9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 02:36:38 -0400 Subject: add soledadstore class move parts-related bits to messageparts pass soledad in initialization for memory messages --- src/leap/mail/imap/mailbox.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 9babe6b..5e16b4b 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -37,8 +37,8 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.decorators import deferred from leap.mail.imap.fields import WithMsgFields, fields -from leap.mail.imap.memorystore import MessageDict from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.messageparts import MessageWrapper from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) @@ -549,10 +549,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): #sequence = True if uid == 0 else False messages_asked = self._bound_seq(messages_asked) - print "asked: ", messages_asked seq_messg = self._filter_msg_seq(messages_asked) - - print "seq: ", seq_messg getmsg = lambda uid: self.messages.get_msg_by_uid(uid) # for sequence numbers (uid = 0) @@ -791,29 +788,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): logger.debug("Tried to copy a MSG with no fdoc") return - #old_mbox = fdoc.content[self.MBOX_KEY] - #old_uid = fdoc.content[self.UID_KEY] - #old_key = old_mbox, old_uid - #print "copying from OLD MBOX ", old_mbox - - # XXX bit doubt... to duplicate in memory - # or not to...? - # I think it should be ok to duplicate as long as we're - # careful at the hour of writes... - # We could use also proxies, but it will break when - # the original mailbox is flushed. - - # XXX DEBUG ---------------------------------------- - #print "copying MESSAGE from %s (%s) to %s (%s)" % ( - #msg._mbox, msg._uid, self.mbox, uid_next) - new_fdoc = copy.deepcopy(fdoc.content) new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = self.mbox - self._memstore.put(self.mbox, uid_next, MessageDict( - new_fdoc, hdoc.content)) + self._memstore.create_message( + self.mbox, uid_next, + MessageWrapper( + new_fdoc, hdoc.content)) - # XXX use memory store + # XXX use memory store !!! if hasattr(hdoc, 'doc_id'): self.messages.add_hdocset_docid(hdoc.doc_id) -- cgit v1.2.3 From e02db78b1b6d8fe021efd4adb250c64a1dd4bac4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 24 Jan 2014 05:39:13 -0400 Subject: flags use the memstore * add new/dirty deferred dict to notify when written to disk * fix eventual duplication after copy * fix flag flickering on first retrieval. --- src/leap/mail/imap/mailbox.py | 70 +++++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 22 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 5e16b4b..108d0da 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -36,6 +36,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.decorators import deferred +from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.messageparts import MessageWrapper @@ -475,8 +476,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Remove all messages flagged \\Deleted """ + print "EXPUNGE!" if not self.isWriteable(): raise imap4.ReadOnlyMailbox + mstore = self._memstore + if mstore is not None: + deleted = mstore.all_deleted_uid_iter(self.mbox) + print "deleted ", list(deleted) + for uid in deleted: + mstore.remove_message(self.mbox, uid) + + print "now deleting from soledad" d = self.messages.remove_all_deleted() d.addCallback(self._expunge_cb) d.addCallback(self.messages.reset_last_uid) @@ -709,21 +719,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msg = self.messages.get_msg_by_uid(msg_id) if not msg: continue + # We duplicate the set operations here + # to return the result because it's less costly than + # retrieving the flags again. + newflags = set(msg.getFlags()) + if mode == 1: msg.addFlags(flags) + newflags = newflags.union(set(flags)) elif mode == -1: msg.removeFlags(flags) + newflags.difference_update(flags) elif mode == 0: msg.setFlags(flags) - result[msg_id] = msg.getFlags() - - # After changing flags, we want to signal again to the - # UI because the number of unread might have changed. - # Hoever, we should probably limit this to INBOX only? - # this should really be called as a final callback of - # the do_STORE method... - from twisted.internet import reactor - deferLater(reactor, 1, self.signal_unread_to_ui) + newflags = set(flags) + result[msg_id] = newflags return result # ISearchableMailbox @@ -780,6 +790,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): from twisted.internet import reactor uid_next = self.getUIDNext() msg = messageObject + memstore = self._memstore # XXX should use a public api instead fdoc = msg._fdoc @@ -787,20 +798,35 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not fdoc: logger.debug("Tried to copy a MSG with no fdoc") return - new_fdoc = copy.deepcopy(fdoc.content) - new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = self.mbox - self._memstore.create_message( - self.mbox, uid_next, - MessageWrapper( - new_fdoc, hdoc.content)) - - # XXX use memory store !!! - if hasattr(hdoc, 'doc_id'): - self.messages.add_hdocset_docid(hdoc.doc_id) - - deferLater(reactor, 1, self.notify_new) + + fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] + dest_fdoc = memstore.get_fdoc_from_chash( + fdoc_chash, self.mbox) + exist = dest_fdoc and not empty(dest_fdoc.content) + + if exist: + print "Destination message already exists!" + + else: + print "DO COPY MESSAGE!" + new_fdoc[self.UID_KEY] = uid_next + new_fdoc[self.MBOX_KEY] = self.mbox + + # XXX set recent! + + print "****************************" + print "copy message..." + print "new fdoc ", new_fdoc + print "hdoc: ", hdoc + print "****************************" + + self._memstore.create_message( + self.mbox, uid_next, + MessageWrapper( + new_fdoc, hdoc.content)) + + deferLater(reactor, 1, self.notify_new) # convenience fun -- cgit v1.2.3 From f5365ae0c2edb8b3e879f876f2f7e42b25f4616a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jan 2014 16:11:53 -0400 Subject: handle last_uid property in memory store --- src/leap/mail/imap/mailbox.py | 131 +++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 73 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 108d0da..b5c5719 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -26,7 +26,7 @@ import cStringIO from collections import defaultdict from twisted.internet import defer -from twisted.internet.task import deferLater +#from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -119,6 +119,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not self.getFlags(): self.setFlags(self.INIT_FLAGS) + if self._memstore: + self.prime_last_uid_to_memstore() + @property def listeners(self): """ @@ -132,6 +135,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._listeners[self.mbox] + # TODO this grows too crazily when many instances are fired, like + # during imaptest stress testing. Should have a queue of limited size + # instead. def addListener(self, listener): """ Add a listener to the listeners queue. @@ -153,6 +159,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ self.listeners.remove(listener) + # TODO move completely to soledadstore, under memstore reponsibility. def _get_mbox(self): """ Return mailbox document. @@ -228,52 +235,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def _get_last_uid(self): """ Return the last uid for this mailbox. + If we have a memory store, the last UID will be the highest + recorded UID in the message store, or a counter cached from + the mailbox document in soledad if this is higher. :return: the last uid for messages in this mailbox :rtype: bool """ - mbox = self._get_mbox() - if not mbox: - logger.error("We could not get a mbox!") - # XXX It looks like it has been corrupted. - # We need to be able to survive this. - return None - last = mbox.content.get(self.LAST_UID_KEY, 1) - if self._memstore: - last = max(last, self._memstore.get_last_uid(mbox)) + last = self._memstore.get_last_uid(self.mbox) + print "last uid for %s: %s (from memstore)" % (self.mbox, last) return last - def _set_last_uid(self, uid): - """ - Sets the last uid for this mailbox. + last_uid = property( + _get_last_uid, doc="Last_UID attribute.") - :param uid: the uid to be set - :type uid: int + def prime_last_uid_to_memstore(self): """ - leap_assert(isinstance(uid, int), "uid has to be int") - mbox = self._get_mbox() - key = self.LAST_UID_KEY - - count = self.getMessageCount() - - # XXX safety-catch. If we do get duplicates, - # we want to avoid further duplication. - - if uid >= count: - value = uid - else: - # something is wrong, - # just set the last uid - # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to %s", count) - value = count - - mbox.content[key] = value - # XXX this should be set in the memorystore instead!!! - self._soledad.put_doc(mbox) - - last_uid = property( - _get_last_uid, _set_last_uid, doc="Last_UID attribute.") + Prime memstore with last_uid value + """ + set_exist = set(self.messages.all_uid_iter()) + last = max(set_exist) + 1 if set_exist else 1 + logger.info("Priming Soledad last_uid to %s" % (last,)) + self._memstore.set_last_soledad_uid(self.mbox, last) def getUIDValidity(self): """ @@ -315,8 +298,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: int """ with self.next_uid_lock: - self.last_uid += 1 - return self.last_uid + if self._memstore: + return self.last_uid + 1 + else: + # XXX after lock, it should be safe to + # return just the increment here, and + # have a different method that actually increments + # the counter when really adding. + self.last_uid += 1 + return self.last_uid def getMessageCount(self): """ @@ -397,26 +387,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: a deferred that evals to None """ + # TODO have a look at the cases for internal date in the rfc if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): message = message.getvalue() - # XXX we should treat the message as an IMessage from here + + # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) - uid_next = self.getUIDNext() - logger.debug('Adding msg with UID :%s' % uid_next) if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date, uid=uid_next) + d = self._do_add_message(message, flags=flags, date=date) return d - def _do_add_message(self, message, flags, date, uid): + def _do_add_message(self, message, flags, date): """ - Calls to the messageCollection add_msg method (deferred to thread). + Calls to the messageCollection add_msg method. Invoked from addMessage. """ - d = self.messages.add_msg(message, flags=flags, date=date, uid=uid) + d = self.messages.add_msg(message, flags=flags, date=date) # XXX Removing notify temporarily. # This is interfering with imaptest results. I'm not clear if it's # because we clutter the logging or because the set of listeners is @@ -456,6 +446,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # XXX removing the mailbox in situ for now, # we should postpone the removal + + # XXX move to memory store?? self._soledad.delete_doc(self._get_mbox()) def _close_cb(self, result): @@ -466,8 +458,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Expunge and mark as closed """ d = self.expunge() - d.addCallback(self._close_cb) - return d + #d.addCallback(self._close_cb) + #return d def _expunge_cb(self, result): return result @@ -479,22 +471,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): print "EXPUNGE!" if not self.isWriteable(): raise imap4.ReadOnlyMailbox - mstore = self._memstore - if mstore is not None: - deleted = mstore.all_deleted_uid_iter(self.mbox) - print "deleted ", list(deleted) - for uid in deleted: - mstore.remove_message(self.mbox, uid) - - print "now deleting from soledad" - d = self.messages.remove_all_deleted() - d.addCallback(self._expunge_cb) - d.addCallback(self.messages.reset_last_uid) - - # XXX DEBUG ------------------- - # FIXME !!! - # XXX should remove the hdocset too!!! - return d + + return self._memstore.expunge(self.mbox) + + # TODO we can defer this back when it's correct + # but we should make sure the memstore has been synced. + + #d = self._memstore.expunge(self.mbox) + #d.addCallback(self._expunge_cb) + #return d def _bound_seq(self, messages_asked): """ @@ -783,12 +768,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # IMessageCopier @deferred + #@profile def copy(self, messageObject): """ Copy the given message object into this mailbox. """ from twisted.internet import reactor - uid_next = self.getUIDNext() msg = messageObject memstore = self._memstore @@ -796,7 +781,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): fdoc = msg._fdoc hdoc = msg._hdoc if not fdoc: - logger.debug("Tried to copy a MSG with no fdoc") + logger.warning("Tried to copy a MSG with no fdoc") return new_fdoc = copy.deepcopy(fdoc.content) @@ -807,11 +792,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if exist: print "Destination message already exists!" - else: print "DO COPY MESSAGE!" + mbox = self.mbox + uid_next = memstore.increment_last_soledad_uid(mbox) new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = self.mbox + new_fdoc[self.MBOX_KEY] = mbox # XXX set recent! @@ -824,9 +810,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._memstore.create_message( self.mbox, uid_next, MessageWrapper( - new_fdoc, hdoc.content)) - - deferLater(reactor, 1, self.notify_new) + new_fdoc, hdoc.content), + notify_on_disk=False) # convenience fun -- cgit v1.2.3 From a7e0054b595822325f749b0b1df7d25cab4e6486 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 28 Jan 2014 18:39:59 -0400 Subject: docstring fixes Also some fixes for None comparisons. --- src/leap/mail/imap/mailbox.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index b5c5719..a0eb0a9 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -26,7 +26,6 @@ import cStringIO from collections import defaultdict from twisted.internet import defer -#from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -99,7 +98,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param rw: read-and-write flag for this mailbox :type rw: int """ - print "got memstore: ", memstore leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(soledad, "Need a soledad instance to initialize") @@ -240,10 +238,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): the mailbox document in soledad if this is higher. :return: the last uid for messages in this mailbox - :rtype: bool + :rtype: int """ last = self._memstore.get_last_uid(self.mbox) - print "last uid for %s: %s (from memstore)" % (self.mbox, last) + logger.debug("last uid for %s: %s (from memstore)" % ( + repr(self.mbox), last)) return last last_uid = property( @@ -468,7 +467,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Remove all messages flagged \\Deleted """ - print "EXPUNGE!" if not self.isWriteable(): raise imap4.ReadOnlyMailbox @@ -537,8 +535,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # can treat them all the same. # Change this to the flag that twisted expects when we # switch to content-hash based index + local UID table. - print - print "FETCHING..." sequence = False #sequence = True if uid == 0 else False @@ -648,9 +644,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): for msgid in seq_messg) return result - def signal_unread_to_ui(self): + def signal_unread_to_ui(self, *args, **kwargs): """ Sends unread event to ui. + + :param args: ignored + :param kwargs: ignored """ unseen = self.getUnseenCount() leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) @@ -767,13 +766,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # IMessageCopier - @deferred + #@deferred #@profile def copy(self, messageObject): """ Copy the given message object into this mailbox. """ - from twisted.internet import reactor msg = messageObject memstore = self._memstore @@ -791,23 +789,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): exist = dest_fdoc and not empty(dest_fdoc.content) if exist: - print "Destination message already exists!" + logger.warning("Destination message already exists!") else: - print "DO COPY MESSAGE!" mbox = self.mbox uid_next = memstore.increment_last_soledad_uid(mbox) new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = mbox - # XXX set recent! - - print "****************************" - print "copy message..." - print "new fdoc ", new_fdoc - print "hdoc: ", hdoc - print "****************************" + # FIXME set recent! - self._memstore.create_message( + return self._memstore.create_message( self.mbox, uid_next, MessageWrapper( new_fdoc, hdoc.content), -- cgit v1.2.3 From 1b71ba510a2e6680f1ecc84eacfc492b0bbe24fc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 29 Jan 2014 00:54:20 -0400 Subject: Fix copy and deletion problems * reorganize and simplify STORE command processing * add the notification after the processing of the whole sequence --- src/leap/mail/imap/mailbox.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index a0eb0a9..3a6937f 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -654,7 +654,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): unseen = self.getUnseenCount() leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) - @deferred def store(self, messages_asked, flags, mode, uid): """ Sets the flags of one or more messages. @@ -697,28 +696,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox - result = {} - for msg_id in seq_messg: - log.msg("MSG ID = %s" % msg_id) - msg = self.messages.get_msg_by_uid(msg_id) - if not msg: - continue - # We duplicate the set operations here - # to return the result because it's less costly than - # retrieving the flags again. - newflags = set(msg.getFlags()) - - if mode == 1: - msg.addFlags(flags) - newflags = newflags.union(set(flags)) - elif mode == -1: - msg.removeFlags(flags) - newflags.difference_update(flags) - elif mode == 0: - msg.setFlags(flags) - newflags = set(flags) - result[msg_id] = newflags - return result + return self.messages.set_flags(self.mbox, seq_messg, flags, mode) # ISearchableMailbox -- cgit v1.2.3 From bd06be63eac85a29e32768e55ab52a46043f3493 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 29 Jan 2014 16:19:37 -0400 Subject: Fix UIDVALIDITY command. thanks to evolution for complaining about this. --- src/leap/mail/imap/mailbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3a6937f..2d1ab88 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -366,7 +366,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if self.CMD_UIDNEXT in names: r[self.CMD_UIDNEXT] = self.last_uid + 1 if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUID() + r[self.CMD_UIDVALIDITY] = self.getUIDValidity() if self.CMD_UNSEEN in names: r[self.CMD_UNSEEN] = self.getUnseenCount() return defer.succeed(r) -- cgit v1.2.3 From 75da338c765ffb935290f5ca16ea2df406dc89d8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 30 Jan 2014 17:23:27 -0400 Subject: skip notifications --- src/leap/mail/imap/mailbox.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 2d1ab88..6c8d78d 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -22,6 +22,7 @@ import threading import logging import StringIO import cStringIO +import os from collections import defaultdict @@ -43,6 +44,12 @@ from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) +""" +If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid +notifying clients of new messages. Use during stress tests. +""" +NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) + class SoledadMailbox(WithMsgFields, MBoxParser): """ @@ -77,6 +84,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" + # FIXME we should turn this into a datastructure with limited capacity _listeners = defaultdict(set) next_uid_lock = threading.Lock() @@ -145,6 +153,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param listener: listener to add :type listener: an object that implements IMailboxListener """ + if not NOTIFY_NEW: + return logger.debug('adding mailbox listener: %s' % listener) self.listeners.add(listener) @@ -421,6 +431,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param args: ignored. """ + if not NOTIFY_NEW: + return exists = self.getMessageCount() recent = self.getRecentCount() logger.debug("NOTIFY: there are %s messages, %s recent" % ( -- cgit v1.2.3 From ff7de0c9bc760e097c0286d2d62a19095be3f35e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 30 Jan 2014 18:35:03 -0400 Subject: prime-uids We pre-fetch the uids from soledad on mailbox initialization --- src/leap/mail/imap/mailbox.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 6c8d78d..802ebf3 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -126,6 +126,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.setFlags(self.INIT_FLAGS) if self._memstore: + self.prime_known_uids_to_memstore() self.prime_last_uid_to_memstore() @property @@ -263,10 +264,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Prime memstore with last_uid value """ set_exist = set(self.messages.all_uid_iter()) - last = max(set_exist) + 1 if set_exist else 1 + last = max(set_exist) if set_exist else 0 logger.info("Priming Soledad last_uid to %s" % (last,)) self._memstore.set_last_soledad_uid(self.mbox, last) + def prime_known_uids_to_memstore(self): + """ + Prime memstore with the set of all known uids. + + We do this to be able to filter the requests efficiently. + """ + known_uids = self.messages.all_soledad_uid_iter() + self._memstore.set_known_uids(self.mbox, known_uids) + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -525,6 +535,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): return seq_messg @deferred + #@profile def fetch(self, messages_asked, uid): """ Retrieve one or more messages in this mailbox. -- cgit v1.2.3 From 0f6a8e1c83995cffec51e81f626d4bb29d4f7345 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 31 Jan 2014 03:34:03 -0400 Subject: properly implement deferreds in several commands Passing along a deferred as an observer whose callback will be called with the proper result. Returning to thread in the appropiate points. just let's remember that twisted APIs are not thread safe! SoledadStore process_item also properly returned to thread. Changed @deferred to @deferred_to_thread so it results less confusing to read. "know the territory". aha! --- src/leap/mail/imap/mailbox.py | 112 +++++++++++++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 23 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 802ebf3..79fb476 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -27,6 +27,7 @@ import os from collections import defaultdict from twisted.internet import defer +from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -35,7 +36,7 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type -from leap.mail.decorators import deferred +from leap.mail.decorators import deferred_to_thread from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection @@ -51,6 +52,11 @@ notifying clients of new messages. Use during stress tests. NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) +class MessageCopyError(Exception): + """ + """ + + class SoledadMailbox(WithMsgFields, MBoxParser): """ A Soledad-backed IMAP mailbox. @@ -534,7 +540,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): seq_messg = set_asked.intersection(set_exist) return seq_messg - @deferred + @deferred_to_thread #@profile def fetch(self, messages_asked, uid): """ @@ -574,7 +580,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): result = ((msgid, getmsg(msgid)) for msgid in seq_messg) return result - @deferred + @deferred_to_thread def fetch_flags(self, messages_asked, uid): """ A fast method to fetch all flags, tricking just the @@ -615,10 +621,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): all_flags = self.messages.all_flags() result = ((msgid, flagsPart( - msgid, all_flags[msgid])) for msgid in seq_messg) + msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) return result - @deferred + @deferred_to_thread def fetch_headers(self, messages_asked, uid): """ A fast method to fetch all headers, tricking just the @@ -698,28 +704,43 @@ class SoledadMailbox(WithMsgFields, MBoxParser): otherwise they are message sequence IDs. :type uid: bool - :return: A dict mapping message sequence numbers to sequences of - str representing the flags set on the message after this - operation has been performed. - :rtype: dict + :return: A deferred, that will be called with a dict mapping message + sequence numbers to sequences of str representing the flags + set on the message after this operation has been performed. + :rtype: deferred :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ + from twisted.internet import reactor + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + d = defer.Deferred() + deferLater(reactor, 0, self._do_store, messages_asked, flags, + mode, uid, d) + return d + + def _do_store(self, messages_asked, flags, mode, uid, observer): + """ + Helper method, invoke set_flags method in the MessageCollection. + + See the documentation for the `store` method for the parameters. + + :param observer: a deferred that will be called with the dictionary + mapping UIDs to flags after the operation has been + done. + :type observer: deferred + """ # XXX implement also sequence (uid = 0) - # XXX we should prevent cclient from setting Recent flag. + # XXX we should prevent cclient from setting Recent flag? leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) - messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - - if not self.isWriteable(): - log.msg('read only mailbox!') - raise imap4.ReadOnlyMailbox - - return self.messages.set_flags(self.mbox, seq_messg, flags, mode) + self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer) # ISearchableMailbox @@ -767,13 +788,46 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # IMessageCopier - #@deferred - #@profile - def copy(self, messageObject): + def copy(self, message): """ Copy the given message object into this mailbox. - """ - msg = messageObject + + :param message: an IMessage implementor + :type message: LeapMessage + :return: a deferred that will be fired with the message + uid when the copy succeed. + :rtype: Deferred + """ + from twisted.internet import reactor + print "COPY :", message + d = defer.Deferred() + + # XXX this should not happen ... track it down, + # probably to FETCH... + if message is None: + log.msg("BUG: COPY found a None in passed message") + d.calback(None) + deferLater(reactor, 0, self._do_copy, message, d) + return d + + #@profile + def _do_copy(self, message, observer): + """ + Call invoked from the deferLater in `copy`. This will + copy the flags and header documents, and pass them to the + `create_message` method in the MemoryStore, together with + the observer deferred that we've been passed along. + + :param message: an IMessage implementor + :type message: LeapMessage + :param observer: the deferred that will fire with the + UID of the message + :type observer: Deferred + """ + # XXX for clarity, this could be delegated to a + # MessageCollection mixin that implements copy too, and + # moved out of here. + msg = message memstore = self._memstore # XXX should use a public api instead @@ -785,12 +839,23 @@ class SoledadMailbox(WithMsgFields, MBoxParser): new_fdoc = copy.deepcopy(fdoc.content) fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] + + # XXX is this hitting the db??? --- probably. + # We should profile after the pre-fetch. dest_fdoc = memstore.get_fdoc_from_chash( fdoc_chash, self.mbox) exist = dest_fdoc and not empty(dest_fdoc.content) if exist: + # Should we signal error on the callback? logger.warning("Destination message already exists!") + + # XXX I'm still not clear if we should raise the + # callback. This actually rases an ugly warning + # in some muas like thunderbird. I guess the user does + # not deserve that. + #observer.errback(MessageCopyError("Already exists!")) + observer.callback(True) else: mbox = self.mbox uid_next = memstore.increment_last_soledad_uid(mbox) @@ -799,10 +864,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # FIXME set recent! - return self._memstore.create_message( + self._memstore.create_message( self.mbox, uid_next, MessageWrapper( new_fdoc, hdoc.content), + observer=observer, notify_on_disk=False) # convenience fun -- cgit v1.2.3 From 18fed49c4143eb764ae9e806882d24f8f4e95744 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 2 Feb 2014 09:26:37 -0400 Subject: fix missing content after in-memory add because THE KEYS WILL BE STRINGS AFTER ADDED TO SOLEDAD Can I remember that? * Fix copy from local folders * Fix copy when we already have a copy of the message in the inbox, marked as deleted. * Fix also bad deferred.succeed in add_msg when it already exist. --- src/leap/mail/imap/mailbox.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 79fb476..688f941 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -162,6 +162,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ if not NOTIFY_NEW: return + logger.debug('adding mailbox listener: %s' % listener) self.listeners.add(listener) @@ -801,7 +802,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): from twisted.internet import reactor print "COPY :", message d = defer.Deferred() - # XXX this should not happen ... track it down, # probably to FETCH... if message is None: @@ -810,7 +810,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): deferLater(reactor, 0, self._do_copy, message, d) return d - #@profile def _do_copy(self, message, observer): """ Call invoked from the deferLater in `copy`. This will @@ -851,7 +850,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): logger.warning("Destination message already exists!") # XXX I'm still not clear if we should raise the - # callback. This actually rases an ugly warning + # errback. This actually rases an ugly warning # in some muas like thunderbird. I guess the user does # not deserve that. #observer.errback(MessageCopyError("Already exists!")) -- cgit v1.2.3 From 8201146254a204fec92395bf497a2a6f76274b85 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 2 Feb 2014 16:26:58 -0400 Subject: re-add expunge deferred --- src/leap/mail/imap/mailbox.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 688f941..40d3420 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -486,8 +486,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Expunge and mark as closed """ d = self.expunge() - #d.addCallback(self._close_cb) - #return d + d.addCallback(self._close_cb) + return d def _expunge_cb(self, result): return result @@ -498,15 +498,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - - return self._memstore.expunge(self.mbox) - - # TODO we can defer this back when it's correct - # but we should make sure the memstore has been synced. - - #d = self._memstore.expunge(self.mbox) - #d.addCallback(self._expunge_cb) - #return d + d = defer.Deferred() + return self._memstore.expunge(self.mbox, d) + self._memstore.expunge(self.mbox) + d.addCallback(self._expunge_cb, d) + return d def _bound_seq(self, messages_asked): """ @@ -800,7 +796,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: Deferred """ from twisted.internet import reactor - print "COPY :", message + d = defer.Deferred() # XXX this should not happen ... track it down, # probably to FETCH... -- cgit v1.2.3 From 23e28bae2c3cb74e00e29ee8add0b73adeb65c2b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 4 Feb 2014 10:57:49 -0400 Subject: fixes after review * Some more docstring completion/fixes. * Removed unneeded str coertion. * Handle mailbox name in logs. * Separate manhole boilerplate into its own file. --- src/leap/mail/imap/mailbox.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 40d3420..c682578 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -52,11 +52,6 @@ notifying clients of new messages. Use during stress tests. NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) -class MessageCopyError(Exception): - """ - """ - - class SoledadMailbox(WithMsgFields, MBoxParser): """ A Soledad-backed IMAP mailbox. @@ -802,7 +797,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # probably to FETCH... if message is None: log.msg("BUG: COPY found a None in passed message") - d.calback(None) + d.callback(None) deferLater(reactor, 0, self._do_copy, message, d) return d @@ -849,7 +844,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # errback. This actually rases an ugly warning # in some muas like thunderbird. I guess the user does # not deserve that. - #observer.errback(MessageCopyError("Already exists!")) observer.callback(True) else: mbox = self.mbox -- cgit v1.2.3 From 423624e5f2c4d3f8cfe8f15f4d6649ed3eea11dc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 5 Feb 2014 11:48:20 -0400 Subject: fix expunge deferreds so they wait --- src/leap/mail/imap/mailbox.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c682578..d8af0a5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -484,9 +484,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self._close_cb) return d - def _expunge_cb(self, result): - return result - def expunge(self): """ Remove all messages flagged \\Deleted @@ -494,9 +491,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not self.isWriteable(): raise imap4.ReadOnlyMailbox d = defer.Deferred() - return self._memstore.expunge(self.mbox, d) - self._memstore.expunge(self.mbox) - d.addCallback(self._expunge_cb, d) + self._memstore.expunge(self.mbox, d) return d def _bound_seq(self, messages_asked): -- cgit v1.2.3 From 860e407ba0a86be30865a77ec29c6ecacf7899a4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 01:39:47 -0400 Subject: defer copy and soledad writes --- src/leap/mail/imap/mailbox.py | 68 ++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 27 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index d8af0a5..84bfa54 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -447,7 +447,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): return exists = self.getMessageCount() recent = self.getRecentCount() - logger.debug("NOTIFY: there are %s messages, %s recent" % ( + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox, exists, recent)) @@ -528,7 +529,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): return seq_messg @deferred_to_thread - #@profile def fetch(self, messages_asked, uid): """ Retrieve one or more messages in this mailbox. @@ -809,6 +809,44 @@ class SoledadMailbox(WithMsgFields, MBoxParser): UID of the message :type observer: Deferred """ + memstore = self._memstore + + def createCopy(result): + exist, new_fdoc, hdoc = result + if exist: + # Should we signal error on the callback? + logger.warning("Destination message already exists!") + + # XXX I'm still not clear if we should raise the + # errback. This actually rases an ugly warning + # in some muas like thunderbird. I guess the user does + # not deserve that. + observer.callback(True) + else: + mbox = self.mbox + uid_next = memstore.increment_last_soledad_uid(mbox) + new_fdoc[self.UID_KEY] = uid_next + new_fdoc[self.MBOX_KEY] = mbox + + # FIXME set recent! + + self._memstore.create_message( + self.mbox, uid_next, + MessageWrapper( + new_fdoc, hdoc.content), + observer=observer, + notify_on_disk=False) + + d = self._get_msg_copy(message) + d.addCallback(createCopy) + d.addErrback(lambda f: log.msg(f.getTraceback())) + + @deferred_to_thread + def _get_msg_copy(self, message): + """ + Get a copy of the fdoc for this message, and check whether + it already exists. + """ # XXX for clarity, this could be delegated to a # MessageCollection mixin that implements copy too, and # moved out of here. @@ -822,7 +860,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): logger.warning("Tried to copy a MSG with no fdoc") return new_fdoc = copy.deepcopy(fdoc.content) - fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] # XXX is this hitting the db??? --- probably. @@ -830,30 +867,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): dest_fdoc = memstore.get_fdoc_from_chash( fdoc_chash, self.mbox) exist = dest_fdoc and not empty(dest_fdoc.content) - - if exist: - # Should we signal error on the callback? - logger.warning("Destination message already exists!") - - # XXX I'm still not clear if we should raise the - # errback. This actually rases an ugly warning - # in some muas like thunderbird. I guess the user does - # not deserve that. - observer.callback(True) - else: - mbox = self.mbox - uid_next = memstore.increment_last_soledad_uid(mbox) - new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = mbox - - # FIXME set recent! - - self._memstore.create_message( - self.mbox, uid_next, - MessageWrapper( - new_fdoc, hdoc.content), - observer=observer, - notify_on_disk=False) + return exist, new_fdoc, hdoc # convenience fun -- cgit v1.2.3 From bd83f834920709db3350c58dedd3cd2181c1b2cc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 02:28:54 -0400 Subject: prefetch flag docs --- src/leap/mail/imap/mailbox.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 84bfa54..f319bf0 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -90,6 +90,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): next_uid_lock = threading.Lock() + _fdoc_primed = {} + def __init__(self, mbox, soledad, memstore, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a @@ -129,6 +131,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if self._memstore: self.prime_known_uids_to_memstore() self.prime_last_uid_to_memstore() + self.prime_flag_docs_to_memstore() @property def listeners(self): @@ -279,6 +282,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): known_uids = self.messages.all_soledad_uid_iter() self._memstore.set_known_uids(self.mbox, known_uids) + def prime_flag_docs_to_memstore(self): + """ + Prime memstore with all the flags documents. + """ + primed = self._fdoc_primed.get(self.mbox, False) + if not primed: + all_flag_docs = self.messages.get_all_soledad_flag_docs() + self._memstore.load_flag_docs(self.mbox, all_flag_docs) + self._fdoc_primed[self.mbox] = True + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -606,7 +619,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - all_flags = self.messages.all_flags() + all_flags = self._memstore.all_flags(self.mbox) result = ((msgid, flagsPart( msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) return result @@ -833,7 +846,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._memstore.create_message( self.mbox, uid_next, MessageWrapper( - new_fdoc, hdoc.content), + new_fdoc, hdoc), observer=observer, notify_on_disk=False) @@ -860,6 +873,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): logger.warning("Tried to copy a MSG with no fdoc") return new_fdoc = copy.deepcopy(fdoc.content) + copy_hdoc = copy.deepcopy(hdoc.content) fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] # XXX is this hitting the db??? --- probably. @@ -867,7 +881,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): dest_fdoc = memstore.get_fdoc_from_chash( fdoc_chash, self.mbox) exist = dest_fdoc and not empty(dest_fdoc.content) - return exist, new_fdoc, hdoc + return exist, new_fdoc, copy_hdoc # convenience fun -- cgit v1.2.3 From 3b6ff2133e477441eb8f6956a17be2412fa1ac7c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 10:27:55 -0400 Subject: do not defer fetches to thread I think this is not a good idea now that all is done in the memstore, overhead from passing the data to thread and gathering the result seems to be much higher than just retreiving the data we need from the memstore. --- src/leap/mail/imap/mailbox.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index f319bf0..1fa0554 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -541,7 +541,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): seq_messg = set_asked.intersection(set_exist) return seq_messg - @deferred_to_thread def fetch(self, messages_asked, uid): """ Retrieve one or more messages in this mailbox. @@ -580,7 +579,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): result = ((msgid, getmsg(msgid)) for msgid in seq_messg) return result - @deferred_to_thread def fetch_flags(self, messages_asked, uid): """ A fast method to fetch all flags, tricking just the @@ -624,7 +622,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) return result - @deferred_to_thread def fetch_headers(self, messages_asked, uid): """ A fast method to fetch all headers, tricking just the -- cgit v1.2.3 From ff3a6a640fdb345449a5f9cd3379bbaefa36111e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 15:46:17 -0400 Subject: take recent count from memstore --- src/leap/mail/imap/mailbox.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 1fa0554..c188f91 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -559,6 +559,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: A tuple of two-tuples of message sequence numbers and LeapMessage """ + from twisted.internet import reactor # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we @@ -577,6 +578,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): raise NotImplementedError else: result = ((msgid, getmsg(msgid)) for msgid in seq_messg) + reactor.callLater(0, self.unset_recent_flags, seq_messg) return result def fetch_flags(self, messages_asked, uid): @@ -838,6 +840,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = mbox + flags = list(new_fdoc[self.FLAGS_KEY]) + flags.append(fields.RECENT_FLAG) + new_fdoc[self.FLAGS_KEY] = flags + # FIXME set recent! self._memstore.create_message( @@ -890,12 +896,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): for doc in docs: self.messages._soledad.delete_doc(doc) - def unset_recent_flags(self, uids): + def unset_recent_flags(self, uid_seq): """ Unset Recent flag for a sequence of UIDs. """ - seq_messg = self._bound_seq(uids) - self.messages.unset_recent_flags(seq_messg) + self.messages.unset_recent_flags(uid_seq) def __repr__(self): """ -- cgit v1.2.3 From 813db4a356141592337f39f9c801203367c63193 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 7 Feb 2014 02:54:52 -0400 Subject: remove hdoc copy since it's in its own structure now --- src/leap/mail/imap/mailbox.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c188f91..6e472ee 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -824,12 +824,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): memstore = self._memstore def createCopy(result): - exist, new_fdoc, hdoc = result + exist, new_fdoc = result if exist: # Should we signal error on the callback? logger.warning("Destination message already exists!") - # XXX I'm still not clear if we should raise the + # XXX I'm not sure if we should raise the # errback. This actually rases an ugly warning # in some muas like thunderbird. I guess the user does # not deserve that. @@ -848,8 +848,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._memstore.create_message( self.mbox, uid_next, - MessageWrapper( - new_fdoc, hdoc), + MessageWrapper(new_fdoc), observer=observer, notify_on_disk=False) @@ -862,6 +861,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Get a copy of the fdoc for this message, and check whether it already exists. + + :return: exist, new_fdoc + :rtype: tuple """ # XXX for clarity, this could be delegated to a # MessageCollection mixin that implements copy too, and @@ -869,22 +871,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): msg = message memstore = self._memstore - # XXX should use a public api instead - fdoc = msg._fdoc - hdoc = msg._hdoc - if not fdoc: + if empty(msg.fdoc): logger.warning("Tried to copy a MSG with no fdoc") return - new_fdoc = copy.deepcopy(fdoc.content) - copy_hdoc = copy.deepcopy(hdoc.content) + new_fdoc = copy.deepcopy(msg.fdoc.content) fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] - # XXX is this hitting the db??? --- probably. - # We should profile after the pre-fetch. dest_fdoc = memstore.get_fdoc_from_chash( fdoc_chash, self.mbox) exist = dest_fdoc and not empty(dest_fdoc.content) - return exist, new_fdoc, copy_hdoc + return exist, new_fdoc # convenience fun -- cgit v1.2.3 From 0d61ed62ed1a2b3bead50b16324d50acfe71727d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 01:35:35 -0400 Subject: add profile-command utility --- src/leap/mail/imap/mailbox.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 6e472ee..122875b 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -50,6 +50,25 @@ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. """ NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + import time + + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + def do_profile_cmd(d, name): + """ + Add the profiling debug to the passed callback. + :param d: deferred + :param name: name of the command + :type name: str + """ + d.addCallback(_debugProfiling, name, time.time()) + d.addErrback(lambda f: log.msg(f.getTraceback())) class SoledadMailbox(WithMsgFields, MBoxParser): @@ -133,6 +152,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.prime_last_uid_to_memstore() self.prime_flag_docs_to_memstore() + from twisted.internet import reactor + self.reactor = reactor + @property def listeners(self): """ @@ -711,14 +733,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ - from twisted.internet import reactor if not self.isWriteable(): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox d = defer.Deferred() - deferLater(reactor, 0, self._do_store, messages_asked, flags, - mode, uid, d) + self.reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "STORE") return d def _do_store(self, messages_asked, flags, mode, uid, observer): @@ -797,15 +820,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): uid when the copy succeed. :rtype: Deferred """ - from twisted.internet import reactor - d = defer.Deferred() - # XXX this should not happen ... track it down, - # probably to FETCH... - if message is None: - log.msg("BUG: COPY found a None in passed message") - d.callback(None) - deferLater(reactor, 0, self._do_copy, message, d) + if PROFILE_CMD: + do_profile_cmd(d, "COPY") + d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + deferLater(self.reactor, 0, self._do_copy, message, d) return d def _do_copy(self, message, observer): -- cgit v1.2.3 From 912873a939214bc805fb398bc5a2fe1949fe34d6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 01:37:23 -0400 Subject: do not get last_uid from the set of soledad messages but always from the counter instead. once assigned, the uid must never be reused, unless the uidvalidity mailbox value changes. doing otherwise will cause messages not to be shown until next session. Also, renamed get_mbox method for clarity. --- src/leap/mail/imap/mailbox.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 122875b..018f88e 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -108,6 +108,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): _listeners = defaultdict(set) next_uid_lock = threading.Lock() + last_uid_lock = threading.Lock() _fdoc_primed = {} @@ -196,7 +197,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.listeners.remove(listener) # TODO move completely to soledadstore, under memstore reponsibility. - def _get_mbox(self): + def _get_mbox_doc(self): """ Return mailbox document. @@ -220,7 +221,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - mbox = self._get_mbox() + mbox = self._get_mbox_doc() if not mbox: return None flags = mbox.content.get(self.FLAGS_KEY, []) @@ -235,7 +236,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") - mbox = self._get_mbox() + mbox = self._get_mbox_doc() if not mbox: return None mbox.content[self.FLAGS_KEY] = map(str, flags) @@ -250,7 +251,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: True if the mailbox is closed :rtype: bool """ - mbox = self._get_mbox() + mbox = self._get_mbox_doc() return mbox.content.get(self.CLOSED_KEY, False) def _set_closed(self, closed): @@ -261,7 +262,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type closed: bool """ leap_assert(isinstance(closed, bool), "closed needs to be boolean") - mbox = self._get_mbox() + mbox = self._get_mbox_doc() mbox.content[self.CLOSED_KEY] = closed self._soledad.put_doc(mbox) @@ -290,8 +291,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Prime memstore with last_uid value """ - set_exist = set(self.messages.all_uid_iter()) - last = max(set_exist) if set_exist else 0 + mbox = self._get_mbox_doc() + last = mbox.content.get('lastuid', 0) logger.info("Priming Soledad last_uid to %s" % (last,)) self._memstore.set_last_soledad_uid(self.mbox, last) @@ -321,7 +322,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: unique validity identifier :rtype: int """ - mbox = self._get_mbox() + mbox = self._get_mbox_doc() return mbox.content.get(self.CREATED_KEY, 1) def getUID(self, message): @@ -483,12 +484,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): exists = self.getMessageCount() recent = self.getRecentCount() logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( - self.mbox, - exists, - recent)) + self.mbox, exists, recent)) for l in self.listeners: - logger.debug('notifying...') l.newMessages(exists, recent) # commands, do not rename methods @@ -507,7 +505,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # we should postpone the removal # XXX move to memory store?? - self._soledad.delete_doc(self._get_mbox()) + self._soledad.delete_doc(self._get_mbox_doc()) def _close_cb(self, result): self.closed = True @@ -756,7 +754,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type observer: deferred """ # XXX implement also sequence (uid = 0) - # XXX we should prevent cclient from setting Recent flag? + # XXX we should prevent client from setting Recent flag? leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) -- cgit v1.2.3 From 5e96a249ab541cedcc79e4e60f46cd4a187e47fb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 01:39:43 -0400 Subject: fix repeated recent flag --- src/leap/mail/imap/mailbox.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 018f88e..fa97512 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -854,12 +854,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: mbox = self.mbox uid_next = memstore.increment_last_soledad_uid(mbox) + new_fdoc[self.UID_KEY] = uid_next new_fdoc[self.MBOX_KEY] = mbox flags = list(new_fdoc[self.FLAGS_KEY]) flags.append(fields.RECENT_FLAG) - new_fdoc[self.FLAGS_KEY] = flags + new_fdoc[self.FLAGS_KEY] = tuple(set(flags)) # FIXME set recent! @@ -896,7 +897,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): dest_fdoc = memstore.get_fdoc_from_chash( fdoc_chash, self.mbox) - exist = dest_fdoc and not empty(dest_fdoc.content) + + exist = not empty(dest_fdoc) return exist, new_fdoc # convenience fun -- cgit v1.2.3 From fd9c8c2e3c88476b90805b689f6914fe5eac16df Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 02:53:28 -0400 Subject: defer fetch to thread also, dispatch query for all headers to its own method. --- src/leap/mail/imap/mailbox.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index fa97512..21f0554 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -211,6 +211,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): fields.TYPE_MBOX_VAL, self.mbox) if query: return query.pop() + else: + logger.error("Could not find mbox document for %r" % + (self.mbox,)) except Exception as exc: logger.exception("Unhandled error %r" % exc) @@ -576,10 +579,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser): otherwise. :type uid: bool + :rtype: deferred + """ + d = defer.Deferred() + self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "FETCH") + return d + + # called in thread + def _do_fetch(self, messages_asked, uid, d): + """ + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + :param d: deferred whose callback will be called with result. + :type d: Deferred + :rtype: A tuple of two-tuples of message sequence numbers and LeapMessage """ - from twisted.internet import reactor # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we @@ -597,9 +620,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): logger.debug("Getting msg by index: INEFFICIENT call!") raise NotImplementedError else: - result = ((msgid, getmsg(msgid)) for msgid in seq_messg) - reactor.callLater(0, self.unset_recent_flags, seq_messg) - return result + got_msg = [(msgid, getmsg(msgid)) for msgid in seq_messg] + result = ((msgid, msg) for msgid, msg in got_msg + if msg is not None) + self.reactor.callLater(0, self.unset_recent_flags, seq_messg) + self.reactor.callFromThread(d.callback, result) def fetch_flags(self, messages_asked, uid): """ @@ -668,6 +693,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): MessagePart. :rtype: tuple """ + # TODO how often is thunderbird doing this? + class headersPart(object): def __init__(self, uid, headers): self.uid = uid @@ -685,10 +712,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - all_chash = self.messages.all_flags_chash() all_headers = self.messages.all_headers() result = ((msgid, headersPart( - msgid, all_headers.get(all_chash.get(msgid, 'nil'), {}))) + msgid, all_headers.get(msgid, {}))) for msgid in seq_messg) return result -- cgit v1.2.3 From f6566fe83c93625b918664526e8858f7be667354 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 16:20:26 -0400 Subject: defer appends too and cut some more time by firing the callback as soon as we've got an UID. --- src/leap/mail/imap/mailbox.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 21f0554..7083316 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -111,6 +111,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): last_uid_lock = threading.Lock() _fdoc_primed = {} + _last_uid_primed = {} def __init__(self, mbox, soledad, memstore, rw=1): """ @@ -294,10 +295,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Prime memstore with last_uid value """ - mbox = self._get_mbox_doc() - last = mbox.content.get('lastuid', 0) - logger.info("Priming Soledad last_uid to %s" % (last,)) - self._memstore.set_last_soledad_uid(self.mbox, last) + primed = self._last_uid_primed.get(self.mbox, False) + if not primed: + mbox = self._get_mbox_doc() + last = mbox.content.get('lastuid', 0) + logger.info("Priming Soledad last_uid to %s" % (last,)) + self._memstore.set_last_soledad_uid(self.mbox, last) + self._last_uid_primed[self.mbox] = True def prime_known_uids_to_memstore(self): """ @@ -459,6 +463,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = tuple(str(flag) for flag in flags) d = self._do_add_message(message, flags=flags, date=date) + if PROFILE_CMD: + do_profile_cmd(d, "APPEND") + # XXX should notify here probably return d def _do_add_message(self, message, flags, date): @@ -467,13 +474,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Invoked from addMessage. """ d = self.messages.add_msg(message, flags=flags, date=date) - # XXX Removing notify temporarily. - # This is interfering with imaptest results. I'm not clear if it's - # because we clutter the logging or because the set of listeners is - # ever-growing. We should come up with some smart way of dealing with - # it, or maybe just disabling it using an environmental variable since - # we will only have just a few listeners in the regular desktop case. - #d.addCallback(self.notify_new) return d def notify_new(self, *args): -- cgit v1.2.3 From 54114126d0b8e16784b67ee972e549e5c152c9d0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Feb 2014 12:37:31 -0400 Subject: purge empty fdocs on select --- src/leap/mail/imap/mailbox.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 7083316..087780f 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -157,6 +157,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): from twisted.internet import reactor self.reactor = reactor + # purge memstore from empty fdocs. + self._memstore.purge_fdoc_store(mbox) + @property def listeners(self): """ -- cgit v1.2.3 From b520a60d0e48f36dcebe03d19b65839afc460fe9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Feb 2014 12:39:33 -0400 Subject: move mbox-doc handling to soledadstore, and lock it --- src/leap/mail/imap/mailbox.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 087780f..d18bc9a 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -200,7 +200,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ self.listeners.remove(listener) - # TODO move completely to soledadstore, under memstore reponsibility. def _get_mbox_doc(self): """ Return mailbox document. @@ -209,17 +208,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ - try: - query = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() - else: - logger.error("Could not find mbox document for %r" % - (self.mbox,)) - except Exception as exc: - logger.exception("Unhandled error %r" % exc) + return self._memstore.get_mbox_doc(self.mbox) def getFlags(self): """ @@ -234,6 +223,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = mbox.content.get(self.FLAGS_KEY, []) return map(str, flags) + # XXX move to memstore->soledadstore def setFlags(self, flags): """ Sets flags for this mailbox. @@ -258,8 +248,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: True if the mailbox is closed :rtype: bool """ - mbox = self._get_mbox_doc() - return mbox.content.get(self.CLOSED_KEY, False) + return self._memstore.get_mbox_closed(self.mbox) def _set_closed(self, closed): """ @@ -268,10 +257,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param closed: the state to be set :type closed: bool """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - mbox = self._get_mbox_doc() - mbox.content[self.CLOSED_KEY] = closed - self._soledad.put_doc(mbox) + self._memstore.set_mbox_closed(self.mbox, closed) closed = property( _get_closed, _set_closed, doc="Closed attribute.") -- cgit v1.2.3 From ac4c70f0be36c985e16e3f4ec0a38ef6f8d48166 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Feb 2014 12:42:02 -0400 Subject: remove all refs during removal, and protect from empty docs --- src/leap/mail/imap/mailbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index d18bc9a..045de82 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -609,7 +609,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): logger.debug("Getting msg by index: INEFFICIENT call!") raise NotImplementedError else: - got_msg = [(msgid, getmsg(msgid)) for msgid in seq_messg] + got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) result = ((msgid, msg) for msgid, msg in got_msg if msg is not None) self.reactor.callLater(0, self.unset_recent_flags, seq_messg) -- cgit v1.2.3 From 45733a231128cc06e123f352b4eb9886d6820878 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 14 Feb 2014 12:41:58 -0400 Subject: docstring fixes --- src/leap/mail/imap/mailbox.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 045de82..d55cae6 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -895,6 +895,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Get a copy of the fdoc for this message, and check whether it already exists. + :param message: an IMessage implementor + :type message: LeapMessage :return: exist, new_fdoc :rtype: tuple """ -- cgit v1.2.3 From f72fd69e637aa252f64c0e0ec8f7e3ebeb2290bb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 17 Feb 2014 10:50:31 -0400 Subject: speedup mailbox select --- src/leap/mail/imap/mailbox.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index d55cae6..57505f0 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -110,8 +110,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): next_uid_lock = threading.Lock() last_uid_lock = threading.Lock() + # TODO unify all the `primed` dicts _fdoc_primed = {} _last_uid_primed = {} + _known_uids_primed = {} def __init__(self, mbox, soledad, memstore, rw=1): """ @@ -130,6 +132,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param rw: read-and-write flag for this mailbox :type rw: int """ + logger.debug("Initializing mailbox %r" % (mbox,)) leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(soledad, "Need a soledad instance to initialize") @@ -146,6 +149,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.messages = MessageCollection( mbox=mbox, soledad=self._soledad, memstore=self._memstore) + # XXX careful with this get/set (it would be + # hitting db unconditionally, move to memstore too) + # Now it's returning a fixed amount of flags from mem + # as a workaround. if not self.getFlags(): self.setFlags(self.INIT_FLAGS) @@ -159,6 +166,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # purge memstore from empty fdocs. self._memstore.purge_fdoc_store(mbox) + logger.debug("DONE initializing mailbox %r" % (mbox,)) @property def listeners(self): @@ -217,10 +225,18 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - mbox = self._get_mbox_doc() - if not mbox: - return None - flags = mbox.content.get(self.FLAGS_KEY, []) + flags = self.INIT_FLAGS + + # XXX returning fixed flags always + # Since I have not found a case where the client + # wants to modify this, as a way of speeding up + # selects. To do it right, we probably should keep + # track of the set of all flags used by msgs + # in this mailbox. Does it matter? + #mbox = self._get_mbox_doc() + #if not mbox: + #return None + #flags = mbox.content.get(self.FLAGS_KEY, []) return map(str, flags) # XXX move to memstore->soledadstore @@ -237,6 +253,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not mbox: return None mbox.content[self.FLAGS_KEY] = map(str, flags) + logger.debug("Writing mbox document for %r to Soledad" + % (self.mbox,)) self._soledad.put_doc(mbox) # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. @@ -298,8 +316,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): We do this to be able to filter the requests efficiently. """ - known_uids = self.messages.all_soledad_uid_iter() - self._memstore.set_known_uids(self.mbox, known_uids) + primed = self._known_uids_primed.get(self.mbox, False) + if not primed: + known_uids = self.messages.all_soledad_uid_iter() + self._memstore.set_known_uids(self.mbox, known_uids) + self._known_uids_primed[self.mbox] = True def prime_flag_docs_to_memstore(self): """ @@ -465,6 +486,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self.messages.add_msg(message, flags=flags, date=date) return d + @deferred_to_thread def notify_new(self, *args): """ Notify of new messages to all the listeners. @@ -836,7 +858,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = defer.Deferred() if PROFILE_CMD: do_profile_cmd(d, "COPY") - d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) deferLater(self.reactor, 0, self._do_copy, message, d) return d @@ -863,9 +884,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # XXX I'm not sure if we should raise the # errback. This actually rases an ugly warning - # in some muas like thunderbird. I guess the user does - # not deserve that. - observer.callback(True) + # in some muas like thunderbird. + # UID 0 seems a good convention for no uid. + observer.callback(0) else: mbox = self.mbox uid_next = memstore.increment_last_soledad_uid(mbox) -- cgit v1.2.3 From daa0c9fd588c61f2e440453be83af137779ce207 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 17 Feb 2014 12:18:41 -0400 Subject: cache uidvalidity --- src/leap/mail/imap/mailbox.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 57505f0..6513db9 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -149,6 +149,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.messages = MessageCollection( mbox=mbox, soledad=self._soledad, memstore=self._memstore) + self._uidvalidity = None + # XXX careful with this get/set (it would be # hitting db unconditionally, move to memstore too) # Now it's returning a fixed amount of flags from mem @@ -339,8 +341,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: unique validity identifier :rtype: int """ - mbox = self._get_mbox_doc() - return mbox.content.get(self.CREATED_KEY, 1) + if self._uidvalidity is None: + mbox = self._get_mbox_doc() + self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) + return self._uidvalidity def getUID(self, message): """ -- cgit v1.2.3 From 0f2f53c8819133e36780e521fecbfadda331255a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 17 Feb 2014 13:00:41 -0400 Subject: defer fetch-all-flags too --- src/leap/mail/imap/mailbox.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 6513db9..be8b429 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -656,15 +656,37 @@ class SoledadMailbox(WithMsgFields, MBoxParser): about :type messages_asked: MessageSet - :param uid: If true, the IDs are UIDs. They are message sequence IDs + :param uid: If 1, the IDs are UIDs. They are message sequence IDs otherwise. - :type uid: bool + :type uid: int :return: A tuple of two-tuples of message sequence numbers and flagsPart, which is a only a partial implementation of MessagePart. :rtype: tuple """ + d = defer.Deferred() + self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "FETCH-ALL-FLAGS") + return d + + # called in thread + def _do_fetch_flags(self, messages_asked, uid, d): + """ + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + :param d: deferred whose callback will be called with result. + :type d: Deferred + + :rtype: A tuple of two-tuples of message sequence numbers and + flagsPart + """ class flagsPart(object): def __init__(self, uid, flags): self.uid = uid @@ -682,7 +704,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): all_flags = self._memstore.all_flags(self.mbox) result = ((msgid, flagsPart( msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - return result + self.reactor.callFromThread(d.callback, result) def fetch_headers(self, messages_asked, uid): """ -- cgit v1.2.3 From 99ec94f08fb2d062eb2c350b64971ea9ad8d87dd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 17 Feb 2014 13:59:06 -0400 Subject: avoid unneeded db index updates and rdoc creation --- src/leap/mail/imap/mailbox.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index be8b429..d7be662 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -132,14 +132,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param rw: read-and-write flag for this mailbox :type rw: int """ - logger.debug("Initializing mailbox %r" % (mbox,)) leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(soledad, "Need a soledad instance to initialize") - # XXX should move to wrapper - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - self.mbox = self._parse_mailbox_name(mbox) self.rw = rw @@ -168,7 +163,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # purge memstore from empty fdocs. self._memstore.purge_fdoc_store(mbox) - logger.debug("DONE initializing mailbox %r" % (mbox,)) @property def listeners(self): -- cgit v1.2.3 From 4bcb32639bff9a5aab076dba2bdc7667cea60c7f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 01:11:26 -0400 Subject: fix rdoc duplication --- src/leap/mail/imap/mailbox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index d7be662..59b2b40 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -135,6 +135,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(soledad, "Need a soledad instance to initialize") + from twisted.internet import reactor + self.reactor = reactor + self.mbox = self._parse_mailbox_name(mbox) self.rw = rw @@ -158,9 +161,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.prime_last_uid_to_memstore() self.prime_flag_docs_to_memstore() - from twisted.internet import reactor - self.reactor = reactor - # purge memstore from empty fdocs. self._memstore.purge_fdoc_store(mbox) -- cgit v1.2.3 From bd476d7ba97a479db14a9b72b8b52ef5997d98f6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 17:07:58 -0400 Subject: Fix regression on "duplicate drafts" issue. Not a permanent solution, but it looks for fdoc matching a given msgid to avoid duplication of drafts in thunderbird folders. --- src/leap/mail/imap/mailbox.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 59b2b40..947cf1b 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -354,7 +354,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: int """ msg = self.messages.get_msg_by_uid(message) - return msg.getUID() + if msg is not None: + return msg.getUID() def getUIDNext(self): """ @@ -854,6 +855,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if len(query) > 2: if query[1] == 'HEADER' and query[2].lower() == "message-id": msgid = str(query[3]).strip() + logger.debug("Searching for %s" % (msgid,)) d = self.messages._get_uid_from_msgid(str(msgid)) d1 = defer.gatherResults([d]) # we want a list, so return it all the same -- cgit v1.2.3 From cffce1a7dfbca91278862e7a173e661e6644e6ec Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Feb 2014 12:19:21 -0400 Subject: Workaround for broken notify-after-copy --- src/leap/mail/imap/mailbox.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 947cf1b..9b1f4e5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -474,7 +474,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date) if PROFILE_CMD: do_profile_cmd(d, "APPEND") - # XXX should notify here probably + + # A better place for this would be the COPY/APPEND dispatcher + # in server.py, but qtreactor hangs when I do that, so this seems + # to work fine for now. + d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) return d def _do_add_message(self, message, flags, date): @@ -485,7 +489,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self.messages.add_msg(message, flags=flags, date=date) return d - @deferred_to_thread def notify_new(self, *args): """ Notify of new messages to all the listeners. @@ -494,13 +497,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ if not NOTIFY_NEW: return + + def cbNotifyNew(result): + exists, recent = result + for l in self.listeners: + l.newMessages(exists, recent) + d = self._get_notify_count() + d.addCallback(cbNotifyNew) + + @deferred_to_thread + def _get_notify_count(self): + """ + Get message count and recent count for this mailbox + Executed in a separate thread. Called from notify_new. + + :return: number of messages and number of recent messages. + :rtype: tuple + """ exists = self.getMessageCount() recent = self.getRecentCount() logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( self.mbox, exists, recent)) - - for l in self.listeners: - l.newMessages(exists, recent) + return exists, recent # commands, do not rename methods @@ -880,6 +898,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = defer.Deferred() if PROFILE_CMD: do_profile_cmd(d, "COPY") + + # A better place for this would be the COPY/APPEND dispatcher + # in server.py, but qtreactor hangs when I do that, so this seems + # to work fine for now. + d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) deferLater(self.reactor, 0, self._do_copy, message, d) return d -- cgit v1.2.3 From 6480fe087e764ace849f552bef3339e1fcd85eff Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Feb 2014 21:41:53 -0400 Subject: fix unread notification to UI --- src/leap/mail/imap/mailbox.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 9b1f4e5..d8e6cb1 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -371,15 +371,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: int """ with self.next_uid_lock: - if self._memstore: - return self.last_uid + 1 - else: - # XXX after lock, it should be safe to - # return just the increment here, and - # have a different method that actually increments - # the counter when really adding. - self.last_uid += 1 - return self.last_uid + return self.last_uid + 1 def getMessageCount(self): """ @@ -474,11 +466,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date) if PROFILE_CMD: do_profile_cmd(d, "APPEND") - # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + d.addCallback(self.cb_signal_unread_to_ui) + d.addErrback(lambda f: log.msg(f.getTraceback())) return d def _do_add_message(self, message, flags, date): @@ -613,6 +606,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) if PROFILE_CMD: do_profile_cmd(d, "FETCH") + d.addCallback(self.cb_signal_unread_to_ui) return d # called in thread @@ -768,14 +762,27 @@ class SoledadMailbox(WithMsgFields, MBoxParser): for msgid in seq_messg) return result - def signal_unread_to_ui(self, *args, **kwargs): + def cb_signal_unread_to_ui(self, result): """ Sends unread event to ui. + Used as a callback in several commands. + + :param result: ignored + """ + d = self._get_unseen_deferred() + d.addCallback(self.__cb_signal_unread_to_ui) + return result + + @deferred_to_thread + def _get_unseen_deferred(self): + return self.getUnseenCount() - :param args: ignored - :param kwargs: ignored + def __cb_signal_unread_to_ui(self, unseen): + """ + Send the unread signal to UI. + :param unseen: number of unseen messages. + :type unseen: int """ - unseen = self.getUnseenCount() leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) def store(self, messages_asked, flags, mode, uid): @@ -816,6 +823,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") + d.addCallback(self.cb_signal_unread_to_ui) + d.addErrback(lambda f: log.msg(f.getTraceback())) return d def _do_store(self, messages_asked, flags, mode, uid, observer): -- cgit v1.2.3 From 733994d68b9f3ce528b552f67e9cbec005e57e9f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Feb 2014 22:38:29 -0400 Subject: rename all fdocs when folder is renamed --- src/leap/mail/imap/mailbox.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index d8e6cb1..503e38b 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -337,6 +337,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ if self._uidvalidity is None: mbox = self._get_mbox_doc() + if mbox is None: + return 0 self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) return self._uidvalidity -- cgit v1.2.3 From eef78aae4164a740f5673c38202f0a32b3615c1e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 26 Mar 2014 12:06:26 -0400 Subject: fix wrong object being passed in the messageSaved callback this was the result of a bad merge during the last fetch refactor. --- src/leap/mail/imap/mailbox.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 503e38b..47c7ff1 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -439,7 +439,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): r[self.CMD_UNSEEN] = self.getUnseenCount() return defer.succeed(r) - def addMessage(self, message, flags, date=None): + def addMessage(self, message, flags, date=None, notify_on_disk=False): """ Adds a message to this mailbox. @@ -465,23 +465,29 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date) + d = self._do_add_message(message, flags=flags, date=date, + notify_on_disk=notify_on_disk) if PROFILE_CMD: do_profile_cmd(d, "APPEND") # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. - d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - d.addCallback(self.cb_signal_unread_to_ui) + + def notifyCallback(x): + self.reactor.callLater(0, self.notify_new) + return x + + d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d - def _do_add_message(self, message, flags, date): + def _do_add_message(self, message, flags, date, notify_on_disk=False): """ Calls to the messageCollection add_msg method. Invoked from addMessage. """ - d = self.messages.add_msg(message, flags=flags, date=date) + d = self.messages.add_msg(message, flags=flags, date=date, + notify_on_disk=notify_on_disk) return d def notify_new(self, *args): @@ -499,6 +505,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): l.newMessages(exists, recent) d = self._get_notify_count() d.addCallback(cbNotifyNew) + d.addCallback(self.cb_signal_unread_to_ui) @deferred_to_thread def _get_notify_count(self): -- cgit v1.2.3