From 1f6687d1375ff97f1ad0746e45f91f922866f32d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Oct 2014 14:51:53 +0200 Subject: adapt to soledad 0.7 async API --- src/leap/mail/imap/mailbox.py | 57 +++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 15 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 34cf535..3c1769a 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -303,21 +303,32 @@ class SoledadMailbox(WithMsgFields, MBoxParser): We do this to be able to filter the requests efficiently. """ primed = self._known_uids_primed.get(self.mbox, False) - if not primed: - known_uids = self.messages.all_soledad_uid_iter() + # XXX handle the maybeDeferred + + def set_primed(known_uids): self._memstore.set_known_uids(self.mbox, known_uids) self._known_uids_primed[self.mbox] = True + if not primed: + d = self.messages.all_soledad_uid_iter() + d.addCallback(set_primed) + return d + def prime_flag_docs_to_memstore(self): """ Prime memstore with all the flags documents. """ primed = self._fdoc_primed.get(self.mbox, False) - if not primed: - all_flag_docs = self.messages.get_all_soledad_flag_docs() - self._memstore.load_flag_docs(self.mbox, all_flag_docs) + + def set_flag_docs(flag_docs): + self._memstore.load_flag_docs(self.mbox, flag_docs) self._fdoc_primed[self.mbox] = True + if not primed: + d = self.messages.get_all_soledad_flag_docs() + d.addCallback(set_flag_docs) + return d + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -522,21 +533,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Should cleanup resources, and set the \\Noselect flag on the mailbox. + """ # XXX this will overwrite all the existing flags! # should better simply addFlag self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() # XXX removing the mailbox in situ for now, # we should postpone the removal - # XXX move to memory store?? - mbox_doc = self._get_mbox_doc() - if mbox_doc is None: - # memory-only store! - return - self._soledad.delete_doc(self._get_mbox_doc()) + def remove_mbox_doc(ignored): + # XXX move to memory store?? + + def _remove_mbox_doc(doc): + if doc is None: + # memory-only store! + return defer.succeed(True) + return self._soledad.delete_doc(doc) + + doc = self._get_mbox_doc() + return _remove_mbox_doc(doc) + + d = self.deleteAllDocs() + d.addCallback(remove_mbox_doc) + return d def _close_cb(self, result): self.closed = True @@ -1006,9 +1026,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Delete all docs in this mailbox """ - docs = self.messages.get_all_docs() - for doc in docs: - self.messages._soledad.delete_doc(doc) + def del_all_docs(docs): + todelete = [] + for doc in docs: + d = self.messages._soledad.delete_doc(doc) + todelete.append(d) + return defer.gatherResults(todelete) + + d = self.messages.get_all_docs() + d.addCallback(del_all_docs) + return d def unset_recent_flags(self, uid_seq): """ -- cgit v1.2.3 From ea4373132458f906b6270744bcfd3e76b64dbd0a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Nov 2014 15:04:26 +0100 Subject: Serializable Models + Soledad Adaptor --- src/leap/mail/imap/mailbox.py | 76 ++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 30 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3c1769a..ea54d33 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ Soledad Mailbox. """ import copy +import re import threading import logging import StringIO @@ -27,6 +28,7 @@ import os from collections import defaultdict from twisted.internet import defer +from twisted.internet import reactor from twisted.internet.task import deferLater from twisted.python import log @@ -36,15 +38,18 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import INBOX_NAME from leap.mail.decorators import deferred_to_thread from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) +# TODO +# [ ] Restore profile_cmd instrumentation + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. @@ -71,7 +76,9 @@ if PROFILE_CMD: d.addErrback(lambda f: log.msg(f.getTraceback())) -class SoledadMailbox(WithMsgFields, MBoxParser): +# TODO Rename to Mailbox +# TODO Remove WithMsgFields +class SoledadMailbox(WithMsgFields): """ A Soledad-backed IMAP mailbox. @@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): _last_uid_primed = {} _known_uids_primed = {} - def __init__(self, mbox, soledad, memstore, rw=1): + # TODO pass the collection to the constructor + # TODO pass the mbox_doc too + def __init__(self, mbox, store, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. @@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param mbox: the mailbox name :type mbox: str - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + :param store: + :type store: Soledad :param rw: read-and-write flag for this mailbox :type rw: int """ leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert(store, "Need a store instance to initialize") - from twisted.internet import reactor - self.reactor = reactor - - self.mbox = self._parse_mailbox_name(mbox) + self.mbox = normalize_mailbox(mbox) self.rw = rw - self._soledad = soledad - self._memstore = memstore - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad, memstore=self._memstore) + self.store = store + self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None # XXX careful with this get/set (it would be @@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._memstore.get_mbox_doc(self.mbox) - # XXX the memstore->soledadstore method in memstore is not complete def getFlags(self): """ Returns the flags defined for this mailbox. @@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = self.INIT_FLAGS return map(str, flags) - # XXX the memstore->soledadstore method in memstore is not complete def setFlags(self, flags): """ Sets flags for this mailbox. @@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date, notify_on_disk=notify_on_disk) - if PROFILE_CMD: - do_profile_cmd(d, "APPEND") + #if PROFILE_CMD: + #do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # to work fine for now. def notifyCallback(x): - self.reactor.callLater(0, self.notify_new) + reactor.callLater(0, self.notify_new) return x d.addCallback(notifyCallback) @@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: deferred """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH") + + # XXX do not need no thread... + reactor.callInThread(self._do_fetch, messages_asked, uid, d) d.addCallback(self.cb_signal_unread_to_ui) return d @@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self.__cb_signal_unread_to_ui) return result - @deferred_to_thread def _get_unseen_deferred(self): return self.getUnseenCount() @@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: C{list} or C{Deferred} """ # TODO see if we can raise w/o interrupting flow - #:raise IllegalQueryError: Raised when query is not valid. + # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', # '52D44F11.9060107@dev.bitmask.net'] @@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(createCopy) d.addErrback(lambda f: log.msg(f.getTraceback())) - @deferred_to_thread + #@deferred_to_thread def _get_msg_copy(self, message): """ Get a copy of the fdoc for this message, and check whether @@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return u"" % ( self.mbox, self.messages.count()) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name -- cgit v1.2.3 From 30387b83756ba078d04f4af701e3f595e30d9c50 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 1 Jan 2015 18:21:44 -0400 Subject: cleanup imap implementation --- src/leap/mail/imap/mailbox.py | 472 +++++++++++------------------------------- 1 file changed, 125 insertions(+), 347 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index ea54d33..faeba9d 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013, 2014 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,11 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Soledad Mailbox. +IMAP Mailbox. """ -import copy import re -import threading import logging import StringIO import cStringIO @@ -29,7 +27,6 @@ from collections import defaultdict from twisted.internet import defer from twisted.internet import reactor -from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -38,17 +35,15 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type -from leap.mail.constants import INBOX_NAME -from leap.mail.decorators import deferred_to_thread -from leap.mail.utils import empty -from leap.mail.imap.fields import WithMsgFields, fields -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.messageparts import MessageWrapper +from leap.mail.constants import INBOX_NAME, MessageFlags logger = logging.getLogger(__name__) -# TODO +# TODO LIST # [ ] Restore profile_cmd instrumentation +# [ ] finish the implementation of IMailboxListener +# [ ] implement the rest of ISearchableMailbox + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid @@ -75,16 +70,20 @@ if PROFILE_CMD: d.addCallback(_debugProfiling, name, time.time()) d.addErrback(lambda f: log.msg(f.getTraceback())) +INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, + MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, + MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, + MessageFlags.LIST_FLAG) + -# TODO Rename to Mailbox -# TODO Remove WithMsgFields -class SoledadMailbox(WithMsgFields): +class IMAPMailbox(object): """ A Soledad-backed IMAP mailbox. Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. + The low-level database methods are contained in IMAPMessageCollection + class, which we instantiate and make accessible in the `messages` + attribute. """ implements( imap4.IMailbox, @@ -93,17 +92,7 @@ class SoledadMailbox(WithMsgFields): imap4.ISearchableMailbox, imap4.IMessageCopier) - # XXX should finish the implementation of IMailboxListener - # XXX should completely implement ISearchableMailbox too - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None + init_flags = INIT_FLAGS CMD_MSG = "MESSAGES" CMD_RECENT = "RECENT" @@ -111,58 +100,31 @@ class SoledadMailbox(WithMsgFields): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" - # FIXME we should turn this into a datastructure with limited capacity + # TODO we should turn this into a datastructure with limited capacity _listeners = defaultdict(set) - next_uid_lock = threading.Lock() - last_uid_lock = threading.Lock() - - # TODO unify all the `primed` dicts - _fdoc_primed = {} - _last_uid_primed = {} - _known_uids_primed = {} - - # TODO pass the collection to the constructor - # TODO pass the mbox_doc too - def __init__(self, mbox, store, rw=1): + def __init__(self, collection, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. - :param mbox: the mailbox name - :type mbox: str - - :param store: - :type store: Soledad + :param collection: instance of IMAPMessageCollection + :type collection: IMAPMessageCollection :param rw: read-and-write flag for this mailbox :type rw: int """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(store, "Need a store instance to initialize") - - self.mbox = normalize_mailbox(mbox) self.rw = rw - self.store = store - - self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None + self.collection = collection - # XXX careful with this get/set (it would be - # hitting db unconditionally, move to memstore too) - # Now it's returning a fixed amount of flags from mem - # as a workaround. if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - if self._memstore: - self.prime_known_uids_to_memstore() - self.prime_last_uid_to_memstore() - self.prime_flag_docs_to_memstore() + self.setFlags(self.init_flags) - # purge memstore from empty fdocs. - self._memstore.purge_fdoc_store(mbox) + @property + def mbox_name(self): + return self.collection.mbox_name @property def listeners(self): @@ -175,11 +137,12 @@ class SoledadMailbox(WithMsgFields): :rtype: set """ - return self._listeners[self.mbox] + return self._listeners[self.mbox_name] - # TODO this grows too crazily when many instances are fired, like + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. + def addListener(self, listener): """ Add a listener to the listeners queue. @@ -204,16 +167,6 @@ class SoledadMailbox(WithMsgFields): """ self.listeners.remove(listener) - def _get_mbox_doc(self): - """ - Return mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - return self._memstore.get_mbox_doc(self.mbox) - def getFlags(self): """ Returns the flags defined for this mailbox. @@ -221,10 +174,11 @@ class SoledadMailbox(WithMsgFields): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - flags = self._memstore.get_mbox_flags(self.mbox) + flags = self.collection.mbox_wrapper.flags if not flags: - flags = self.INIT_FLAGS - return map(str, flags) + flags = self.init_flags + flags_str = map(str, flags) + return flags_str def setFlags(self, flags): """ @@ -234,98 +188,31 @@ class SoledadMailbox(WithMsgFields): :type flags: tuple of str """ # XXX this is setting (overriding) old flags. + # Better pass a mode flag leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") - self._memstore.set_mbox_flags(self.mbox, flags) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + return self.collection.set_mbox_attr("flags", flags) - def _get_closed(self): + @property + def is_closed(self): """ Return the closed attribute for this mailbox. :return: True if the mailbox is closed :rtype: bool """ - return self._memstore.get_mbox_closed(self.mbox) + return self.collection.get_mbox_attr("closed") - def _set_closed(self, closed): + def set_closed(self, closed): """ Set the closed attribute for this mailbox. :param closed: the state to be set :type closed: bool - """ - self._memstore.set_mbox_closed(self.mbox, closed) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - If we have a memory store, the last UID will be the highest - recorded UID in the message store, or a counter cached from - the mailbox document in soledad if this is higher. - - :return: the last uid for messages in this mailbox - :rtype: int - """ - last = self._memstore.get_last_uid(self.mbox) - logger.debug("last uid for %s: %s (from memstore)" % ( - repr(self.mbox), last)) - return last - - last_uid = property( - _get_last_uid, doc="Last_UID attribute.") - def prime_last_uid_to_memstore(self): - """ - Prime memstore with last_uid value - """ - primed = self._last_uid_primed.get(self.mbox, False) - if not primed: - mbox = self._get_mbox_doc() - if mbox is None: - # memory-only store - return - last = mbox.content.get('lastuid', 0) - logger.info("Priming Soledad last_uid to %s" % (last,)) - self._memstore.set_last_soledad_uid(self.mbox, last) - self._last_uid_primed[self.mbox] = True - - def prime_known_uids_to_memstore(self): - """ - Prime memstore with the set of all known uids. - - We do this to be able to filter the requests efficiently. - """ - primed = self._known_uids_primed.get(self.mbox, False) - # XXX handle the maybeDeferred - - def set_primed(known_uids): - self._memstore.set_known_uids(self.mbox, known_uids) - self._known_uids_primed[self.mbox] = True - - if not primed: - d = self.messages.all_soledad_uid_iter() - d.addCallback(set_primed) - return d - - def prime_flag_docs_to_memstore(self): - """ - Prime memstore with all the flags documents. + :rtype: Deferred """ - primed = self._fdoc_primed.get(self.mbox, False) - - def set_flag_docs(flag_docs): - self._memstore.load_flag_docs(self.mbox, flag_docs) - self._fdoc_primed[self.mbox] = True - - if not primed: - d = self.messages.get_all_soledad_flag_docs() - d.addCallback(set_flag_docs) - return d + return self.collection.set_mbox_attr("closed", closed) def getUIDValidity(self): """ @@ -334,12 +221,7 @@ class SoledadMailbox(WithMsgFields): :return: unique validity identifier :rtype: int """ - if self._uidvalidity is None: - mbox = self._get_mbox_doc() - if mbox is None: - return 0 - self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) - return self._uidvalidity + return self.collection.get_mbox_attr("created") def getUID(self, message): """ @@ -354,9 +236,9 @@ class SoledadMailbox(WithMsgFields): :rtype: int """ - msg = self.messages.get_msg_by_uid(message) - if msg is not None: - return msg.getUID() + d = self.collection.get_msg_by_uid(message) + d.addCallback(lambda m: m.getUID()) + return d def getUIDNext(self): """ @@ -364,23 +246,20 @@ class SoledadMailbox(WithMsgFields): mailbox. Currently it returns the higher UID incremented by one. - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int + :return: deferred with int + :rtype: Deferred """ - with self.next_uid_lock: - return self.last_uid + 1 + d = self.collection.get_uid_next() + return d def getMessageCount(self): """ Returns the total count of messages in this mailbox. - :rtype: int + :return: deferred with int + :rtype: Deferred """ - return self.messages.count() + return self.collection.count() def getUnseenCount(self): """ @@ -389,7 +268,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `unseen` :rtype: int """ - return self.messages.count_unseen() + return self.collection.count_unseen() def getRecentCount(self): """ @@ -398,7 +277,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `recent` :rtype: int """ - return self.messages.count_recent() + return self.collection.count_recent() def isWriteable(self): """ @@ -407,6 +286,8 @@ class SoledadMailbox(WithMsgFields): :return: 1 if mailbox is read-writeable, 0 otherwise. :rtype: int """ + # XXX We don't need to store it in the mbox doc, do we? + # return int(self.collection.get_mbox_attr('rw')) return self.rw def getHierarchicalDelimiter(self): @@ -431,14 +312,14 @@ class SoledadMailbox(WithMsgFields): if self.CMD_RECENT in names: r[self.CMD_RECENT] = self.getRecentCount() if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 + r[self.CMD_UIDNEXT] = self.getUIDNext() if self.CMD_UIDVALIDITY in names: r[self.CMD_UIDVALIDITY] = self.getUIDValidity() if self.CMD_UNSEEN in names: r[self.CMD_UNSEEN] = self.getUnseenCount() return defer.succeed(r) - def addMessage(self, message, flags, date=None, notify_on_disk=False): + def addMessage(self, message, flags, date=None): """ Adds a message to this mailbox. @@ -464,10 +345,8 @@ class SoledadMailbox(WithMsgFields): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - #if PROFILE_CMD: - #do_profile_cmd(d, "APPEND") + # if PROFILE_CMD: + # do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -478,19 +357,11 @@ class SoledadMailbox(WithMsgFields): reactor.callLater(0, self.notify_new) return x + d = self.collection.add_message(flags=flags, date=date) d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d - def _do_add_message(self, message, flags, date, notify_on_disk=False): - """ - Calls to the messageCollection add_msg method. - Invoked from addMessage. - """ - d = self.messages.add_msg(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - return d - def notify_new(self, *args): """ Notify of new messages to all the listeners. @@ -502,26 +373,34 @@ class SoledadMailbox(WithMsgFields): def cbNotifyNew(result): exists, recent = result - for l in self.listeners: - l.newMessages(exists, recent) + for listener in self.listeners: + listener.newMessages(exists, recent) + d = self._get_notify_count() d.addCallback(cbNotifyNew) d.addCallback(self.cb_signal_unread_to_ui) - @deferred_to_thread def _get_notify_count(self): """ Get message count and recent count for this mailbox Executed in a separate thread. Called from notify_new. - :return: number of messages and number of recent messages. - :rtype: tuple + :return: a deferred that will fire with a tuple, with number of + messages and number of recent messages. + :rtype: Deferred """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( - self.mbox, exists, recent)) - return exists, recent + d_exists = self.getMessageCount() + d_recent = self.getRecentCount() + d_list = [d_exists, d_recent] + + def log_num_msg(result): + exists, recent = result + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox_name, exists, recent)) + + d = defer.gatherResults(d_list) + d.addCallback(log_num_msg) + return d # commands, do not rename methods @@ -533,27 +412,18 @@ class SoledadMailbox(WithMsgFields): on the mailbox. """ - # XXX this will overwrite all the existing flags! + # XXX this will overwrite all the existing flags # should better simply addFlag - self.setFlags((self.NOSELECT_FLAG,)) - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - - def remove_mbox_doc(ignored): - # XXX move to memory store?? + self.setFlags((MessageFlags.NOSELECT_FLAG,)) - def _remove_mbox_doc(doc): - if doc is None: - # memory-only store! - return defer.succeed(True) - return self._soledad.delete_doc(doc) - - doc = self._get_mbox_doc() - return _remove_mbox_doc(doc) + def remove_mbox(_): + # FIXME collection does not have a delete_mbox method, + # it's in account. + # XXX should take care of deleting the uid table too. + return self.collection.delete_mbox(self.mbox_name) d = self.deleteAllDocs() - d.addCallback(remove_mbox_doc) + d.addCallback(remove_mbox) return d def _close_cb(self, result): @@ -574,9 +444,11 @@ class SoledadMailbox(WithMsgFields): if not self.isWriteable(): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self._memstore.expunge(self.mbox, d) + # FIXME actually broken. + # Iterate through index, and do a expunge. return d + # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): """ Put an upper bound to a messages sequence if this is open. @@ -596,6 +468,7 @@ class SoledadMailbox(WithMsgFields): pass return messages_asked + # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -627,29 +500,6 @@ class SoledadMailbox(WithMsgFields): :rtype: deferred """ - d = defer.Deferred() - - # XXX do not need no thread... - reactor.callInThread(self._do_fetch, messages_asked, uid, d) - d.addCallback(self.cb_signal_unread_to_ui) - return d - - # called in thread - def _do_fetch(self, messages_asked, uid, d): - """ - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - :param d: deferred whose callback will be called with result. - :type d: Deferred - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we @@ -660,18 +510,23 @@ class SoledadMailbox(WithMsgFields): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - getmsg = lambda uid: self.messages.get_msg_by_uid(uid) + getmsg = self.collection.get_msg_by_uid # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") + # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) result = ((msgid, msg) for msgid, msg in got_msg if msg is not None) - self.reactor.callLater(0, self.unset_recent_flags, seq_messg) - self.reactor.callFromThread(d.callback, result) + reactor.callLater(0, self.unset_recent_flags, seq_messg) + + # TODO -- call signal_to_ui + # d.addCallback(self.cb_signal_unread_to_ui) + + return result def fetch_flags(self, messages_asked, uid): """ @@ -698,12 +553,11 @@ class SoledadMailbox(WithMsgFields): :rtype: tuple """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d) + reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: do_profile_cmd(d, "FETCH-ALL-FLAGS") return d - # called in thread def _do_fetch_flags(self, messages_asked, uid, d): """ :param messages_asked: IDs of the messages to retrieve information @@ -733,10 +587,11 @@ class SoledadMailbox(WithMsgFields): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - all_flags = self._memstore.all_flags(self.mbox) + # FIXME use deferreds here + all_flags = self.collection.get_all_flags(self.mbox_name) result = ((msgid, flagsPart( msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - self.reactor.callFromThread(d.callback, result) + d.callback(result) def fetch_headers(self, messages_asked, uid): """ @@ -843,8 +698,8 @@ class SoledadMailbox(WithMsgFields): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self.reactor.callLater(0, self._do_store, messages_asked, flags, - mode, uid, d) + reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") d.addCallback(self.cb_signal_unread_to_ui) @@ -853,7 +708,7 @@ class SoledadMailbox(WithMsgFields): def _do_store(self, messages_asked, flags, mode, uid, observer): """ - Helper method, invoke set_flags method in the MessageCollection. + Helper method, invoke set_flags method in the IMAPMessageCollection. See the documentation for the `store` method for the parameters. @@ -869,7 +724,8 @@ class SoledadMailbox(WithMsgFields): flags = tuple(flags) messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer) + self.collection.set_flags( + self.mbox_name, seq_messg, flags, mode, observer) # ISearchableMailbox @@ -908,6 +764,7 @@ class SoledadMailbox(WithMsgFields): msgid = str(query[3]).strip() logger.debug("Searching for %s" % (msgid,)) d = self.messages._get_uid_from_msgid(str(msgid)) + # XXX remove gatherResults d1 = defer.gatherResults([d]) # we want a list, so return it all the same return d1 @@ -928,94 +785,18 @@ class SoledadMailbox(WithMsgFields): uid when the copy succeed. :rtype: Deferred """ - d = defer.Deferred() if PROFILE_CMD: do_profile_cmd(d, "COPY") # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. - d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - deferLater(self.reactor, 0, self._do_copy, message, d) - return d + #d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + #deferLater(self.reactor, 0, self._do_copy, message, d) + #return d - def _do_copy(self, message, observer): - """ - Call invoked from the deferLater in `copy`. This will - copy the flags and header documents, and pass them to the - `create_message` method in the MemoryStore, together with - the observer deferred that we've been passed along. - - :param message: an IMessage implementor - :type message: LeapMessage - :param observer: the deferred that will fire with the - UID of the message - :type observer: Deferred - """ - memstore = self._memstore - - def createCopy(result): - exist, new_fdoc = result - if exist: - # Should we signal error on the callback? - logger.warning("Destination message already exists!") - - # XXX I'm not sure if we should raise the - # errback. This actually rases an ugly warning - # in some muas like thunderbird. - # UID 0 seems a good convention for no uid. - observer.callback(0) - else: - mbox = self.mbox - uid_next = memstore.increment_last_soledad_uid(mbox) - - new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = mbox - - flags = list(new_fdoc[self.FLAGS_KEY]) - flags.append(fields.RECENT_FLAG) - new_fdoc[self.FLAGS_KEY] = tuple(set(flags)) - - # FIXME set recent! - - self._memstore.create_message( - self.mbox, uid_next, - MessageWrapper(new_fdoc), - observer=observer, - notify_on_disk=False) - - d = self._get_msg_copy(message) - d.addCallback(createCopy) - d.addErrback(lambda f: log.msg(f.getTraceback())) - - #@deferred_to_thread - def _get_msg_copy(self, message): - """ - Get a copy of the fdoc for this message, and check whether - it already exists. - - :param message: an IMessage implementor - :type message: LeapMessage - :return: exist, new_fdoc - :rtype: tuple - """ - # XXX for clarity, this could be delegated to a - # MessageCollection mixin that implements copy too, and - # moved out of here. - msg = message - memstore = self._memstore - - if empty(msg.fdoc): - logger.warning("Tried to copy a MSG with no fdoc") - return - new_fdoc = copy.deepcopy(msg.fdoc.content) - fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] - - dest_fdoc = memstore.get_fdoc_from_chash( - fdoc_chash, self.mbox) - - exist = not empty(dest_fdoc) - return exist, new_fdoc + # FIXME not implemented !!! --- + return self.collection.copy_msg(message, self.mbox_name) # convenience fun @@ -1023,29 +804,25 @@ class SoledadMailbox(WithMsgFields): """ Delete all docs in this mailbox """ - def del_all_docs(docs): - todelete = [] - for doc in docs: - d = self.messages._soledad.delete_doc(doc) - todelete.append(d) - return defer.gatherResults(todelete) - - d = self.messages.get_all_docs() - d.addCallback(del_all_docs) - return d + # FIXME not implemented + return self.collection.delete_all_docs() def unset_recent_flags(self, uid_seq): """ Unset Recent flag for a sequence of UIDs. """ - self.messages.unset_recent_flags(uid_seq) + # FIXME not implemented + return self.collection.unset_recent_flags(uid_seq) def __repr__(self): """ Representation string for this mailbox. """ - return u"" % ( - self.mbox, self.messages.count()) + return u"" % ( + self.mbox_name, self.messages.count()) + + +_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) def normalize_mailbox(name): @@ -1060,7 +837,8 @@ def normalize_mailbox(name): :rtype: unicode """ - _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + # XXX maybe it would make sense to normalize common folders too: + # trash, sent, drafts, etc... if _INBOX_RE.match(name): # ensure inital INBOX is uppercase return INBOX_NAME + name[len(INBOX_NAME):] -- cgit v1.2.3 From c1fc9b52d8b577814e921d128357afdbd9278662 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 12 Jan 2015 20:47:29 -0400 Subject: Use mailbox uuids The previous implementation is naive, since it imposes a burden when renaming mailboxes. We're using uuids in the local uid tables instead, which is more cryptic but way more efficient. * receive mbox uuid instead of name * use mailbox uuid in identifiers --- src/leap/mail/imap/mailbox.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index faeba9d..f2cbf75 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -236,6 +236,7 @@ class IMAPMailbox(object): :rtype: int """ + # TODO --- return the uid if it has it!!! d = self.collection.get_msg_by_uid(message) d.addCallback(lambda m: m.getUID()) return d @@ -357,7 +358,7 @@ class IMAPMailbox(object): reactor.callLater(0, self.notify_new) return x - d = self.collection.add_message(flags=flags, date=date) + d = self.collection.add_msg(message, flags=flags, date=date) d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d @@ -389,14 +390,15 @@ class IMAPMailbox(object): messages and number of recent messages. :rtype: Deferred """ - d_exists = self.getMessageCount() - d_recent = self.getRecentCount() + d_exists = defer.maybeDeferred(self.getMessageCount) + d_recent = defer.maybeDeferred(self.getRecentCount) d_list = [d_exists, d_recent] def log_num_msg(result): - exists, recent = result + exists, recent = tuple(result) logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( self.mbox_name, exists, recent)) + return result d = defer.gatherResults(d_list) d.addCallback(log_num_msg) @@ -654,7 +656,7 @@ class IMAPMailbox(object): return result def _get_unseen_deferred(self): - return self.getUnseenCount() + return defer.maybeDeferred(self.getUnseenCount) def __cb_signal_unread_to_ui(self, unseen): """ -- cgit v1.2.3 From 68500fb15dbb7531eeb397ccee2c160d71284d97 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 7 Jan 2015 12:12:24 -0400 Subject: Complete IMAP implementation, update tests --- src/leap/mail/imap/mailbox.py | 116 ++++++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 38 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index f2cbf75..e1eb6bf 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -21,9 +21,11 @@ import re import logging import StringIO import cStringIO +import time import os from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -36,6 +38,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage logger = logging.getLogger(__name__) @@ -88,7 +91,7 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - imap4.ICloseableMailbox, + #imap4.ICloseableMailbox, imap4.ISearchableMailbox, imap4.IMessageCopier) @@ -105,8 +108,7 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. + SoledadMailbox constructor. :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -115,6 +117,7 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw + self.closed = False self._uidvalidity = None self.collection = collection @@ -139,6 +142,11 @@ class IMAPMailbox(object): """ return self._listeners[self.mbox_name] + def get_imap_message(self, message): + msg = IMAPMessage(message) + msg.store = self.collection.store + return msg + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. @@ -308,17 +316,24 @@ class IMAPMailbox(object): :type names: iter """ r = {} + maybe = defer.maybeDeferred if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() + r[self.CMD_MSG] = maybe(self.getMessageCount) if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() + r[self.CMD_RECENT] = maybe(self.getRecentCount) if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getUIDNext() + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d def addMessage(self, message, flags, date=None): """ @@ -341,11 +356,15 @@ class IMAPMailbox(object): # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) + if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) + if date is None: + date = formatdate(time.time()) + # if PROFILE_CMD: # do_profile_cmd(d, "APPEND") @@ -419,10 +438,11 @@ class IMAPMailbox(object): self.setFlags((MessageFlags.NOSELECT_FLAG,)) def remove_mbox(_): - # FIXME collection does not have a delete_mbox method, - # it's in account. - # XXX should take care of deleting the uid table too. - return self.collection.delete_mbox(self.mbox_name) + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d d = self.deleteAllDocs() d.addCallback(remove_mbox) @@ -431,13 +451,14 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - def close(self): - """ - Expunge and mark as closed - """ - d = self.expunge() - d.addCallback(self._close_cb) - return d + # TODO server already calls expunge for closing + #def close(self): + #""" + #Expunge and mark as closed + #""" + #d = self.expunge() + #d.addCallback(self._close_cb) + #return d def expunge(self): """ @@ -445,10 +466,7 @@ class IMAPMailbox(object): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - d = defer.Deferred() - # FIXME actually broken. - # Iterate through index, and do a expunge. - return d + return self.collection.delete_all_flagged() # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): @@ -465,12 +483,12 @@ class IMAPMailbox(object): except TypeError: # looks like we cannot iterate try: + # XXX fixme, does not exist messages_asked.last = self.last_uid except ValueError: pass return messages_asked - # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -480,10 +498,16 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: set """ - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - return seq_messg + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(sequence): + set_asked = set(messages_asked) + set_exist = set(sequence) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d def fetch(self, messages_asked, uid): """ @@ -509,26 +533,41 @@ class IMAPMailbox(object): sequence = False # sequence = True if uid == 0 else False + getmsg = self.collection.get_message_by_uid messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - getmsg = self.collection.get_msg_by_uid + d_sequence = self._filter_msg_seq(messages_asked) + + def get_imap_messages_for_sequence(sequence): + def _zip_msgid(messages): + return zip( + list(sequence), + map(self.get_imap_message, messages)) + + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in sequence: + d_msg.append(getmsg(msgid)) + + d = defer.gatherResults(d_msg) + d.addCallback(_zip_msgid) + return d # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError + else: - got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) - result = ((msgid, msg) for msgid, msg in got_msg - if msg is not None) - reactor.callLater(0, self.unset_recent_flags, seq_messg) + d_sequence.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - - return result + return d_sequence def fetch_flags(self, messages_asked, uid): """ @@ -755,6 +794,7 @@ class IMAPMailbox(object): # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on @@ -821,7 +861,7 @@ class IMAPMailbox(object): Representation string for this mailbox. """ return u"" % ( - self.mbox_name, self.messages.count()) + self.mbox_name, self.collection.count()) _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) -- cgit v1.2.3 From 8c65f09a16e4e00452dffa7d72771d9fac21c9c0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 13:48:21 -0400 Subject: imap: complete FETCH implementation --- src/leap/mail/imap/mailbox.py | 95 ++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 32 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index e1eb6bf..a000133 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -143,9 +143,9 @@ class IMAPMailbox(object): return self._listeners[self.mbox_name] def get_imap_message(self, message): - msg = IMAPMessage(message) - msg.store = self.collection.store - return msg + d = defer.Deferred() + IMAPMessage(message, store=self.collection.store, d=d) + return d # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size @@ -468,7 +468,6 @@ class IMAPMailbox(object): raise imap4.ReadOnlyMailbox return self.collection.delete_all_flagged() - # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): """ Put an upper bound to a messages sequence if this is open. @@ -477,16 +476,18 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: MessageSet """ + def set_last(last_uid): + messages_asked.last = last_uid + return messages_asked + if not messages_asked.last: try: iter(messages_asked) except TypeError: # looks like we cannot iterate - try: - # XXX fixme, does not exist - messages_asked.last = self.last_uid - except ValueError: - pass + d = self.collection.get_last_uid() + d.addCallback(set_last) + return d return messages_asked def _filter_msg_seq(self, messages_asked): @@ -524,50 +525,64 @@ class IMAPMailbox(object): otherwise. :type uid: bool - :rtype: deferred + :rtype: deferred with a generator that yields... """ # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we # switch to content-hash based index + local UID table. - sequence = False - # sequence = True if uid == 0 else False + is_sequence = True if uid == 0 else False getmsg = self.collection.get_message_by_uid + getimapmsg = self.get_imap_message - messages_asked = self._bound_seq(messages_asked) - d_sequence = self._filter_msg_seq(messages_asked) + def get_imap_messages_for_sequence(msg_sequence): + + def _get_imap_msg(messages): + d_imapmsg = [] + for msg in messages: + d_imapmsg.append(getimapmsg(msg)) + return defer.gatherResults(d_imapmsg) - def get_imap_messages_for_sequence(sequence): - def _zip_msgid(messages): - return zip( - list(sequence), - map(self.get_imap_message, messages)) + def _zip_msgid(imap_messages): + zipped = zip( + list(msg_sequence), imap_messages) + return (item for item in zipped) def _unset_recent(sequence): reactor.callLater(0, self.unset_recent_flags, sequence) return sequence d_msg = [] - for msgid in sequence: + for msgid in msg_sequence: d_msg.append(getmsg(msgid)) d = defer.gatherResults(d_msg) + d.addCallback(_get_imap_msg) d.addCallback(_zip_msgid) return d # for sequence numbers (uid = 0) - if sequence: + if is_sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: - d_sequence.addCallback(get_imap_messages_for_sequence) + d = self._get_sequence_of_messages(messages_asked) + d.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - return d_sequence + return d + + def _get_sequence_of_messages(self, messages_asked): + def get_sequence(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = defer.maybeDeferred(self._bound_seq, messages_asked) + d.addCallback(get_sequence) + return d def fetch_flags(self, messages_asked, uid): """ @@ -611,8 +626,8 @@ class IMAPMailbox(object): :param d: deferred whose callback will be called with result. :type d: Deferred - :rtype: A tuple of two-tuples of message sequence numbers and - flagsPart + :rtype: A generator that yields two-tuples of message sequence numbers + and flagsPart """ class flagsPart(object): def __init__(self, uid, flags): @@ -625,14 +640,30 @@ class IMAPMailbox(object): def getFlags(self): return map(str, self.flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) + def pack_flags(result): + #if result is None: + #print "No result" + #return + _uid, _flags = result + return _uid, flagsPart(_uid, _flags) - # FIXME use deferreds here - all_flags = self.collection.get_all_flags(self.mbox_name) - result = ((msgid, flagsPart( - msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - d.callback(result) + def get_flags_for_seq(sequence): + d_all_flags = [] + for msgid in sequence: + d_flags_per_uid = self.collection.get_flags_by_uid(msgid) + d_flags_per_uid.addCallback(pack_flags) + d_all_flags.append(d_flags_per_uid) + gotflags = defer.gatherResults(d_all_flags) + gotflags.addCallback(get_uid_flag_generator) + return gotflags + + def get_uid_flag_generator(result): + generator = (item for item in result) + d.callback(generator) + + d_seq = self._get_sequence_of_messages(messages_asked) + d_seq.addCallback(get_flags_for_seq) + return d_seq def fetch_headers(self, messages_asked, uid): """ -- cgit v1.2.3 From d327d8416475493864b1e0922b51812be566e028 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 21 Jan 2015 00:50:27 -0400 Subject: imap: implement setting of message flags --- src/leap/mail/imap/mailbox.py | 51 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 23 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index a000133..58fc514 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -231,7 +231,7 @@ class IMAPMailbox(object): """ return self.collection.get_mbox_attr("created") - def getUID(self, message): + def getUID(self, message_number): """ Return the UID of a message in the mailbox @@ -239,15 +239,15 @@ class IMAPMailbox(object): but in the future will be useful to get absolute UIDs from message sequence numbers. - :param message: the message uid + :param message: the message sequence number. :type message: int :rtype: int + :return: the UID of the message. """ - # TODO --- return the uid if it has it!!! - d = self.collection.get_msg_by_uid(message) - d.addCallback(lambda m: m.getUID()) - return d + # TODO support relative sequences. The (imap) message should + # receive a sequence number attribute: a deferred is not expected + return message_number def getUIDNext(self): """ @@ -451,15 +451,6 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - # TODO server already calls expunge for closing - #def close(self): - #""" - #Expunge and mark as closed - #""" - #d = self.expunge() - #d.addCallback(self._close_cb) - #return d - def expunge(self): """ Remove all messages flagged \\Deleted @@ -641,9 +632,6 @@ class IMAPMailbox(object): return map(str, self.flags) def pack_flags(result): - #if result is None: - #print "No result" - #return _uid, _flags = result return _uid, flagsPart(_uid, _flags) @@ -790,14 +778,31 @@ class IMAPMailbox(object): :type observer: deferred """ # XXX implement also sequence (uid = 0) - # XXX we should prevent client from setting Recent flag? + # TODO we should prevent client from setting Recent flag leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - self.collection.set_flags( - self.mbox_name, seq_messg, flags, mode, observer) + + def set_flags_for_seq(sequence): + + def return_result_dict(list_of_flags): + result = dict(zip(list(sequence), list_of_flags)) + observer.callback(result) + return result + + d_all_set = [] + for msgid in sequence: + d = self.collection.get_message_by_uid(msgid) + d.addCallback(lambda msg: self.collection.update_flags( + msg, flags, mode)) + d_all_set.append(d) + got_flags_setted = defer.gatherResults(d_all_set) + got_flags_setted.addCallback(return_result_dict) + return got_flags_setted + + d_seq = self._get_sequence_of_messages(messages_asked) + d_seq.addCallback(set_flags_for_seq) + return d_seq # ISearchableMailbox -- cgit v1.2.3 From aab87b503184619b5637d6b326ec7717e34dfb99 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:20:37 -0400 Subject: lots of little fixes after meskio's review mostly having to do with poor, missing or outdated documentation, naming of confusing things and reordering of code blocks for improved readability. --- src/leap/mail/imap/mailbox.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 58fc514..52f4dd5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -91,8 +91,9 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - #imap4.ICloseableMailbox, imap4.ISearchableMailbox, + # XXX I think we do not need to implement CloseableMailbox, do we? + # imap4.ICloseableMailbox imap4.IMessageCopier) init_flags = INIT_FLAGS @@ -108,8 +109,6 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. - :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -117,14 +116,10 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw - self.closed = False self._uidvalidity = None self.collection = collection - if not self.getFlags(): - self.setFlags(self.init_flags) - @property def mbox_name(self): return self.collection.mbox_name @@ -201,6 +196,7 @@ class IMAPMailbox(object): "flags expected to be a tuple") return self.collection.set_mbox_attr("flags", flags) + # TODO - not used? @property def is_closed(self): """ @@ -211,6 +207,7 @@ class IMAPMailbox(object): """ return self.collection.get_mbox_attr("closed") + # TODO - not used? def set_closed(self, closed): """ Set the closed attribute for this mailbox. @@ -448,9 +445,6 @@ class IMAPMailbox(object): d.addCallback(remove_mbox) return d - def _close_cb(self, result): - self.closed = True - def expunge(self): """ Remove all messages flagged \\Deleted @@ -555,24 +549,23 @@ class IMAPMailbox(object): # for sequence numbers (uid = 0) if is_sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: - d = self._get_sequence_of_messages(messages_asked) + d = self._get_messages_range(messages_asked) d.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) return d - def _get_sequence_of_messages(self, messages_asked): - def get_sequence(messages_asked): + def _get_messages_range(self, messages_asked): + def get_range(messages_asked): return self._filter_msg_seq(messages_asked) d = defer.maybeDeferred(self._bound_seq, messages_asked) - d.addCallback(get_sequence) + d.addCallback(get_range) return d def fetch_flags(self, messages_asked, uid): @@ -599,6 +592,10 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError + d = defer.Deferred() reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: @@ -649,7 +646,7 @@ class IMAPMailbox(object): generator = (item for item in result) d.callback(generator) - d_seq = self._get_sequence_of_messages(messages_asked) + d_seq = self._get_messages_range(messages_asked) d_seq.addCallback(get_flags_for_seq) return d_seq @@ -677,7 +674,11 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ + # TODO implement sequences # TODO how often is thunderbird doing this? + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError class headersPart(object): def __init__(self, uid, headers): @@ -753,6 +754,12 @@ class IMAPMailbox(object): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ + # TODO implement sequences + # TODO how often is thunderbird doing this? + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError + if not self.isWriteable(): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox @@ -777,7 +784,7 @@ class IMAPMailbox(object): done. :type observer: deferred """ - # XXX implement also sequence (uid = 0) + # TODO implement also sequence (uid = 0) # TODO we should prevent client from setting Recent flag leap_assert(not isinstance(flags, basestring), "flags cannot be a string") @@ -800,7 +807,7 @@ class IMAPMailbox(object): got_flags_setted.addCallback(return_result_dict) return got_flags_setted - d_seq = self._get_sequence_of_messages(messages_asked) + d_seq = self._get_messages_range(messages_asked) d_seq.addCallback(set_flags_for_seq) return d_seq -- cgit v1.2.3 From 98def315e5f48df6eec713dbe175df8bdfe406dd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 03:16:32 -0400 Subject: re-add support for basic multipart messages --- src/leap/mail/imap/mailbox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/mailbox.py') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 52f4dd5..045636e 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -540,7 +540,11 @@ class IMAPMailbox(object): d_msg = [] for msgid in msg_sequence: - d_msg.append(getmsg(msgid)) + # XXX We want cdocs because we "probably" are asked for the + # body. We should be smarted at do_FETCH and pass a parameter + # to this method in order not to prefetch cdocs if they're not + # going to be used. + d_msg.append(getmsg(msgid, get_cdocs=True)) d = defer.gatherResults(d_msg) d.addCallback(_get_imap_msg) -- cgit v1.2.3