diff options
| -rw-r--r-- | mail/src/leap/mail/imap/account.py | 253 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/index.py | 51 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/mailbox.py | 57 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/memorystore.py | 39 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/messages.py | 136 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/server.py | 204 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/soledadstore.py | 11 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/tests/test_imap.py | 185 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/tests/utils.py | 25 |
9 files changed, 698 insertions, 263 deletions
diff --git a/mail/src/leap/mail/imap/account.py b/mail/src/leap/mail/imap/account.py index 70ed13bd..fe466cb4 100644 --- a/mail/src/leap/mail/imap/account.py +++ b/mail/src/leap/mail/imap/account.py @@ -22,6 +22,7 @@ import logging import os import time +from twisted.internet import defer from twisted.mail import imap4 from twisted.python import log from zope.interface import implements @@ -65,6 +66,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): _soledad = None selected = None closed = False + _initialized = False def __init__(self, account_name, soledad, memstore=None): """ @@ -93,14 +95,39 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): self.__mailboxes = set([]) - self.initialize_db() + self._deferred_initialization = defer.Deferred() + self._initialize_storage() - # every user should have the right to an inbox folder - # at least, so let's make one! - self._load_mailboxes() + def _initialize_storage(self): - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.addMailbox(self.INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.initialize_db() + + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization def _get_empty_mailbox(self): """ @@ -120,10 +147,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ # XXX use soledadstore instead ...; - doc = self._soledad.get_from_index( + def get_first_if_any(docs): + return docs[0] if docs else None + + d = self._soledad.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, self._parse_mailbox_name(name)) - return doc[0] if doc else None + d.addCallback(get_first_if_any) + return d @property def mailboxes(self): @@ -134,19 +165,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): return sorted(self.__mailboxes) def _load_mailboxes(self): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)]) - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + def update_mailboxes(db_indexes): + self.__mailboxes.update( + [doc.content[self.MBOX_KEY] for doc in db_indexes]) + d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) + d.addCallback(update_mailboxes) + return d def getMailbox(self, name): """ @@ -182,7 +206,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): one is provided. :type creation_ts: int - :returns: True if successful + :returns: a Deferred that will contain the document if successful. :rtype: bool """ name = self._parse_mailbox_name(name) @@ -203,21 +227,29 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): mbox[self.MBOX_KEY] = name mbox[self.CREATED_KEY] = creation_ts - doc = self._soledad.create_doc(mbox) - self._load_mailboxes() - return bool(doc) + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self._soledad.create_doc(mbox) + d.addCallback(load_mbox_cache) + return d def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. + :param pathspec: + The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. :type pathspec: str - :return: A true value if the creation succeeds. - :rtype: bool + :return: + A deferred that will fire with a true value if the creation + succeeds. + :rtype: Deferred :raise MailboxException: Raised if this mailbox cannot be added. """ @@ -225,18 +257,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): paths = filter( None, self._parse_mailbox_name(pathspec).split('/')) + + subs = [] + sep = '/' + for accum in range(1, len(paths)): try: - self.addMailbox('/'.join(paths[:accum])) + partial = sep.join(paths[:accum]) + d = self.addMailbox(partial) + subs.append(d) except imap4.MailboxCollision: pass try: - self.addMailbox('/'.join(paths)) + df = self.addMailbox(sep.join(paths)) except imap4.MailboxCollision: if not pathspec.endswith('/'): - return False - self._load_mailboxes() - return True + df = defer.succeed(False) + else: + df = defer.succeed(True) + finally: + subs.append(df) + + def all_good(result): + return all(result) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + if subs: + d1 = defer.gatherResults(subs, consumeErrors=True) + d1.addCallback(load_mbox_cache) + d1.addCallback(all_good) + else: + d1 = defer.succeed(False) + d1.addCallback(load_mbox_cache) + return d1 def select(self, name, readwrite=1): """ @@ -275,17 +332,20 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: the mailbox to be deleted :type name: str - :param force: if True, it will not check for noselect flag or inferior - names. use with care. + :param force: + if True, it will not check for noselect flag or inferior + names. use with care. :type force: bool + :rtype: Deferred """ name = self._parse_mailbox_name(name) if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) + err = imap4.MailboxException("No such mailbox: %r" % name) + return defer.fail(err) mbox = self.getMailbox(name) - if force is False: + if not force: # See if this box is flagged \Noselect # XXX use mbox.flags instead? mbox_flags = mbox.getFlags() @@ -294,11 +354,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # as part of their root. for others in self.mailboxes: if others != name and others.startswith(name): - raise imap4.MailboxException, ( + err = imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") + return defer.fail(err) self.__mailboxes.discard(name) - mbox.destroy() + return mbox.destroy() # XXX FIXME --- not honoring the inferior names... @@ -331,14 +392,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): if new in self.mailboxes: raise imap4.MailboxCollision(repr(new)) + rename_deferreds = [] + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + def update_mbox_doc_name(mbox, oldname, newname, update_deferred): + mbox.content[self.MBOX_KEY] = newname + d = self._soledad.put_doc(mbox) + d.addCallback(lambda r: update_deferred.callback(True)) + for (old, new) in inferiors: - self._memstore.rename_fdocs_mailbox(old, new) - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new self.__mailboxes.discard(old) - self._soledad.put_doc(mbox) + self._memstore.rename_fdocs_mailbox(old, new) + + d0 = defer.Deferred() + d = self._get_mailbox_by_name(old) + d.addCallback(update_mbox_doc_name, old, new, d0) + rename_deferreds.append(d0) - self._load_mailboxes() + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + d1.addCallback(load_mbox_cache) + return d1 def _inferiorNames(self, name): """ @@ -354,6 +431,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO ------------------ can we preserve the attr? + # maybe add to memory store. def isSubscribed(self, name): """ Returns True if user is subscribed to this mailbox. @@ -361,10 +440,35 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: the mailbox to be checked. :type name: str - :rtype: bool + :rtype: Deferred (will fire with bool) + """ + subscribed = self.SUBSCRIBED_KEY + + def is_subscribed(mbox): + subs_bool = bool(mbox.content.get(subscribed, False)) + return subs_bool + + d = self._get_mailbox_by_name(name) + d.addCallback(is_subscribed) + return d + + # TODO ------------------ can we preserve the property? + # maybe add to memory store. + + def _get_subscriptions(self): """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) + Return a list of the current subscriptions for this account. + + :returns: A deferred that will fire with the subscriptions. + :rtype: Deferred + """ + def get_docs_content(docs): + return [doc.content[self.MBOX_KEY] for doc in docs] + + d = self._soledad.get_from_index( + self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') + d.addCallback(get_docs_content) + return d def _set_subscription(self, name, value): """ @@ -376,26 +480,42 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param value: the boolean value :type value: bool """ + # XXX Note that this kind of operation has + # no guarantees of atomicity. We should not be accessing mbox + # documents concurrently. + + subscribed = self.SUBSCRIBED_KEY + + def update_subscribed_value(mbox): + mbox.content[subscribed] = value + return self._soledad.put_doc(mbox) + # maybe we should store subscriptions in another # document... if name not in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) + d = self.addMailbox(name) + d.addCallback(lambda v: self._get_mailbox_by_name(name)) + else: + d = self._get_mailbox_by_name(name) + d.addCallback(update_subscribed_value) + return d def subscribe(self, name): """ - Subscribe to this mailbox + Subscribe to this mailbox if not already subscribed. :param name: name of the mailbox :type name: str + :rtype: Deferred """ name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) + + def check_and_subscribe(subscriptions): + if name not in subscriptions: + return self._set_subscription(name, True) + d = self._get_subscriptions() + d.addCallback(check_and_subscribe) + return d def unsubscribe(self, name): """ @@ -403,12 +523,21 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: name of the mailbox :type name: str + :rtype: Deferred """ name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException( - "Not currently subscribed to %r" % name) - self._set_subscription(name, False) + + def check_and_unsubscribe(subscriptions): + if name not in subscriptions: + raise imap4.MailboxException( + "Not currently subscribed to %r" % name) + return self._set_subscription(name, False) + d = self._get_subscriptions() + d.addCallback(check_and_unsubscribe) + return d + + def getSubscriptions(self): + return self._get_subscriptions() def listMailboxes(self, ref, wildcard): """ diff --git a/mail/src/leap/mail/imap/index.py b/mail/src/leap/mail/imap/index.py index 5f0919a4..ea35fff7 100644 --- a/mail/src/leap/mail/imap/index.py +++ b/mail/src/leap/mail/imap/index.py @@ -19,6 +19,8 @@ Index for SoledadBackedAccount, Mailbox and Messages. """ import logging +from twisted.internet import defer + from leap.common.check import leap_assert, leap_assert_type from leap.mail.imap.fields import fields @@ -39,6 +41,9 @@ class IndexedDB(object): """ # TODO we might want to move this to soledad itself, check + _index_creation_deferreds = [] + index_ready = False + def initialize_db(self): """ Initialize the database. @@ -46,24 +51,40 @@ class IndexedDB(object): leap_assert(self._soledad, "Need a soledad attribute accesible in the instance") leap_assert_type(self.INDEXES, dict) + self._index_creation_deferreds = [] + + def _on_indexes_created(ignored): + self.index_ready = True + + def _create_index(name, expression): + d = self._soledad.create_index(name, *expression) + self._index_creation_deferreds.append(d) + + def _create_indexes(db_indexes): + db_indexes = dict(db_indexes) + for name, expression in fields.INDEXES.items(): + if name not in db_indexes: + # The index does not yet exist. + _create_index(name, expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so + # we delete it and add the proper index expression. + d1 = self._soledad.delete_index(name) + d1.addCallback(lambda _: _create_index(name, expression)) + + all_created = defer.gatherResults(self._index_creation_deferreds) + all_created.addCallback(_on_indexes_created) + return all_created # Ask the database for currently existing indexes. if not self._soledad: logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") return - db_indexes = dict() if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) + d = self._soledad.list_indexes() + d.addCallback(_create_indexes) + return d diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py index 34cf5355..3c1769a3 100644 --- a/mail/src/leap/mail/imap/mailbox.py +++ b/mail/src/leap/mail/imap/mailbox.py @@ -303,21 +303,32 @@ class SoledadMailbox(WithMsgFields, MBoxParser): We do this to be able to filter the requests efficiently. """ primed = self._known_uids_primed.get(self.mbox, False) - if not primed: - known_uids = self.messages.all_soledad_uid_iter() + # XXX handle the maybeDeferred + + def set_primed(known_uids): self._memstore.set_known_uids(self.mbox, known_uids) self._known_uids_primed[self.mbox] = True + if not primed: + d = self.messages.all_soledad_uid_iter() + d.addCallback(set_primed) + return d + def prime_flag_docs_to_memstore(self): """ Prime memstore with all the flags documents. """ primed = self._fdoc_primed.get(self.mbox, False) - if not primed: - all_flag_docs = self.messages.get_all_soledad_flag_docs() - self._memstore.load_flag_docs(self.mbox, all_flag_docs) + + def set_flag_docs(flag_docs): + self._memstore.load_flag_docs(self.mbox, flag_docs) self._fdoc_primed[self.mbox] = True + if not primed: + d = self.messages.get_all_soledad_flag_docs() + d.addCallback(set_flag_docs) + return d + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -522,21 +533,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Should cleanup resources, and set the \\Noselect flag on the mailbox. + """ # XXX this will overwrite all the existing flags! # should better simply addFlag self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() # XXX removing the mailbox in situ for now, # we should postpone the removal - # XXX move to memory store?? - mbox_doc = self._get_mbox_doc() - if mbox_doc is None: - # memory-only store! - return - self._soledad.delete_doc(self._get_mbox_doc()) + def remove_mbox_doc(ignored): + # XXX move to memory store?? + + def _remove_mbox_doc(doc): + if doc is None: + # memory-only store! + return defer.succeed(True) + return self._soledad.delete_doc(doc) + + doc = self._get_mbox_doc() + return _remove_mbox_doc(doc) + + d = self.deleteAllDocs() + d.addCallback(remove_mbox_doc) + return d def _close_cb(self, result): self.closed = True @@ -1006,9 +1026,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Delete all docs in this mailbox """ - docs = self.messages.get_all_docs() - for doc in docs: - self.messages._soledad.delete_doc(doc) + def del_all_docs(docs): + todelete = [] + for doc in docs: + d = self.messages._soledad.delete_doc(doc) + todelete.append(d) + return defer.gatherResults(todelete) + + d = self.messages.get_all_docs() + d.addCallback(del_all_docs) + return d def unset_recent_flags(self, uid_seq): """ diff --git a/mail/src/leap/mail/imap/memorystore.py b/mail/src/leap/mail/imap/memorystore.py index e0753944..eda5b96e 100644 --- a/mail/src/leap/mail/imap/memorystore.py +++ b/mail/src/leap/mail/imap/memorystore.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # memorystore.py # Copyright (C) 2014 LEAP # @@ -112,8 +112,6 @@ class MemoryStore(object): :param write_period: the interval to dump messages to disk, in seconds. :type write_period: int """ - self.reactor = reactor - self._permanent_store = permanent_store self._write_period = write_period @@ -241,6 +239,7 @@ class MemoryStore(object): self.producer = None self._write_loop = None + # TODO -- remove def _start_write_loop(self): """ Start loop for writing to disk database. @@ -250,6 +249,7 @@ class MemoryStore(object): if not self._write_loop.running: self._write_loop.start(self._write_period, now=True) + # TODO -- remove def _stop_write_loop(self): """ Stop loop for writing to disk database. @@ -278,17 +278,18 @@ class MemoryStore(object): :type uid: int :param message: a message to be added :type message: MessageWrapper - :param observer: the deferred that will fire with the - UID of the message. If notify_on_disk is True, - this will happen when the message is written to - Soledad. Otherwise it will fire as soon as we've - added the message to the memory store. + :param observer: + the deferred that will fire with the UID of the message. If + notify_on_disk is True, this will happen when the message is + written to Soledad. Otherwise it will fire as soon as we've added + the message to the memory store. :type observer: Deferred - :param notify_on_disk: whether the `observer` deferred should - wait until the message is written to disk to - be fired. + :param notify_on_disk: + whether the `observer` deferred should wait until the message is + written to disk to be fired. :type notify_on_disk: bool """ + # TODO -- return a deferred log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) key = mbox, uid @@ -306,7 +307,7 @@ class MemoryStore(object): else: # Caller does not care, just fired and forgot, so we pass # a defer that will inmediately have its callback triggered. - self.reactor.callFromThread(observer.callback, uid) + reactor.callFromThread(observer.callback, uid) def put_message(self, mbox, uid, message, notify_on_disk=True): """ @@ -442,6 +443,7 @@ class MemoryStore(object): :return: MessageWrapper or None """ + # TODO -- return deferred if dirtystate == DirtyState.dirty: flags_only = True @@ -467,6 +469,7 @@ class MemoryStore(object): chash = fdoc.get(fields.CONTENT_HASH_KEY) hdoc = self._hdoc_store[chash] if empty(hdoc): + # XXX this will be a deferred hdoc = self._permanent_store.get_headers_doc(chash) if empty(hdoc): return None @@ -531,7 +534,8 @@ class MemoryStore(object): # IMessageStoreWriter - @deferred_to_thread + # TODO -- I think we don't need this anymore. + # instead, we can have def write_messages(self, store): """ Write the message documents in this MemoryStore to a different store. @@ -657,7 +661,7 @@ class MemoryStore(object): with self._last_uid_lock: self._last_uid[mbox] += 1 value = self._last_uid[mbox] - self.reactor.callInThread(self.write_last_uid, mbox, value) + reactor.callInThread(self.write_last_uid, mbox, value) return value def write_last_uid(self, mbox, value): @@ -1077,6 +1081,7 @@ class MemoryStore(object): return None return self._rflags_store[mbox]['set'] + # XXX -- remove def all_rdocs_iter(self): """ Return an iterator through all in-memory recent flag dicts, wrapped @@ -1125,6 +1130,7 @@ class MemoryStore(object): self.remove_message(mbox, uid) return mem_deleted + # TODO -- remove def stop_and_flush(self): """ Stop the write loop and trigger a write to the producer. @@ -1180,6 +1186,7 @@ class MemoryStore(object): :type observer: Deferred """ mem_deleted = self.remove_all_deleted(mbox) + # TODO return a DeferredList observer.callback(mem_deleted) def _delete_from_soledad_and_memory(self, result, mbox, observer): @@ -1313,8 +1320,8 @@ class MemoryStore(object): :rtype: bool """ # FIXME this should return a deferred !!! - # XXX ----- can fire when all new + dirty deferreds - # are done (gatherResults) + # TODO this should be moved to soledadStore instead + # (all pending deferreds) return getattr(self, self.WRITING_FLAG) @property diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py index e8d64d1c..c7610910 100644 --- a/mail/src/leap/mail/imap/messages.py +++ b/mail/src/leap/mail/imap/messages.py @@ -71,6 +71,7 @@ def try_unique_query(curried): :param curried: a curried function :type curried: callable """ + # XXX FIXME ---------- convert to deferreds leap_assert(callable(curried), "A callable is expected") try: query = curried() @@ -134,10 +135,11 @@ class LeapMessage(fields, MBoxParser): self.__chash = None self.__bdoc = None - self.reactor = reactor - # XXX make these properties public + # XXX FIXME ------ the documents can be + # deferreds too.... niice. + @property def fdoc(self): """ @@ -506,18 +508,15 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the flags for this message. """ - result = {} - try: - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % self._uid) - logger.exception(exc) - finally: - return result + def get_first_if_any(docs): + result = first(docs) + return result if result else {} + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + d.addCallback(get_first_if_any) + return d # TODO move to soledadstore instead of accessing soledad directly def _get_headers_doc(self): @@ -525,10 +524,11 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the headers for this message. """ - head_docs = self._soledad.get_from_index( + d = self._soledad.get_from_index( fields.TYPE_C_HASH_IDX, fields.TYPE_HEADERS_VAL, str(self.chash)) - return first(head_docs) + d.addCallback(lambda docs: first(docs)) + return d # TODO move to soledadstore instead of accessing soledad directly def _get_body_doc(self): @@ -536,6 +536,8 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the body for this message. """ + # XXX FIXME --- this might need a maybedeferred + # on the receiving side... hdoc_content = self.hdoc.content body_phash = hdoc_content.get( fields.BODY_KEY, None) @@ -554,13 +556,11 @@ class LeapMessage(fields, MBoxParser): return bdoc # no memstore, or no body doc found there - if self._soledad: - body_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(body_phash)) - return first(body_docs) - else: - logger.error("No phash in container, and no soledad found!") + d = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(body_phash)) + d.addCallback(lambda docs: first(docs)) + return d def __getitem__(self, key): """ @@ -739,8 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): else: self._initialized[mbox] = True - self.reactor = reactor - def _get_empty_doc(self, _type=FLAGS_DOC): """ Returns an empty doc for storing different message parts. @@ -887,9 +885,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) + # TODO return soledad deferred instead observer = defer.Deferred() d = self._do_parse(raw) - d.addCallback(lambda result: self.reactor.callInThread( + d.addCallback(lambda result: reactor.callInThread( self._do_add_msg, result, flags, subject, date, notify_on_disk, observer)) return observer @@ -924,17 +923,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): msg = self.get_msg_by_uid(existing_uid) # We can say the observer that we're done - self.reactor.callFromThread(observer.callback, existing_uid) + # TODO return soledad deferred instead + reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return - # XXX get FUCKING UID from autoincremental table + # TODO S2 -- get FUCKING UID from autoincremental table uid = self.memstore.increment_last_soledad_uid(self.mbox) # We can say the observer that we're done at this point, but # before that we should make sure it has no serious consequences # if we're issued, for instance, a fetch command right after... - # self.reactor.callFromThread(observer.callback, uid) + # reactor.callFromThread(observer.callback, uid) # if we did the notify, we need to invalidate the deferred # so not to try to fire it twice. # observer = None @@ -960,6 +960,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): self.set_recent_flag(uid) msg_container = MessageWrapper(fd, hd, cdocs) + + # TODO S1 -- just pass this to memstore and return that deferred. self.memstore.create_message( self.mbox, uid, msg_container, observer=observer, notify_on_disk=notify_on_disk) @@ -1011,6 +1013,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): Get recent-flags document from Soledad for this mailbox. :rtype: SoledadDocument or None """ + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_IDX, @@ -1029,6 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uids: the uids to unset :type uid: sequence """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags.difference_update( set(uids)) @@ -1042,11 +1046,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uid: the uid to unset :type uid: int """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags.difference_update( set([uid])) - @deferred_to_thread def set_recent_flag(self, uid): """ Set Recent flag for a given uid. @@ -1054,6 +1058,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uid: the uid to set :type uid: int """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags = self.recent_flags.union( set([uid])) @@ -1068,6 +1073,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_C_HASH_IDX, @@ -1125,7 +1131,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): return None return fdoc.content.get(fields.UID_KEY, None) - @deferred_to_thread def _get_uid_from_msgid(self, msgid): """ Return a UID for a given message-id. @@ -1144,7 +1149,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # XXX is this working? return self._get_uid_from_msgidCb(msgid) - @deferred_to_thread def set_flags(self, mbox, messages, flags, mode, observer): """ Set flags for a sequence of messages. @@ -1162,7 +1166,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): done. :type observer: deferred """ - reactor = self.reactor getmsg = self.get_msg_by_uid def set_flags(uid, flags, mode): @@ -1173,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): setted_flags = [set_flags(uid, flags, mode) for uid in messages] result = dict(filter(None, setted_flags)) + # TODO -- remove reactor.callFromThread(observer.callback, result) # getters: generic for a mailbox @@ -1223,37 +1227,45 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): If you want acess to the content, use __iter__ instead - :return: a list of u1db documents - :rtype: list of SoledadDocument + :return: a Deferred, that will fire with a list of u1db documents + :rtype: Deferred (promise of list of SoledadDocument) """ if _type not in fields.__dict__.values(): raise TypeError("Wrong type passed to get_all_docs") + # FIXME ----- either raise or return a deferred wrapper. if sameProxiedObjects(self._soledad, None): logger.warning('Tried to get messages but soledad is None!') return [] - all_docs = [doc for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - _type, self.mbox)] + def get_sorted_docs(docs): + all_docs = [doc for doc in docs] + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad + # FIXME ---------------------------------------------- + return sorted(all_docs, key=lambda item: item.content['uid']) - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, _type, self.mbox) + d.addCallback(get_sorted_docs) + return d def all_soledad_uid_iter(self): """ Return an iterator through the UIDs of all messages, sorted in ascending order. """ - db_uids = set([doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc)]) - return db_uids + # XXX FIXME ------ sorted??? + + def get_uids(docs): + return set([ + doc.content[self.UID_KEY] for doc in docs if not empty(doc)]) + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) + d.addCallback(get_uids) + return d def all_uid_iter(self): """ @@ -1277,16 +1289,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # XXX we really could return a reduced version with # just {'uid': (flags-tuple,) since the prefetch is # only oriented to get the flag tuples. - all_docs = [( - doc.content[self.UID_KEY], - dict(doc.content)) - for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc.content)] - all_flags = dict(all_docs) - return all_flags + + def get_content(docs): + all_docs = [( + doc.content[self.UID_KEY], + dict(doc.content)) + for doc in docs + if not empty(doc.content)] + all_flags = dict(all_docs) + return all_flags + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + d.addCallback(get_content) + return d def all_headers(self): """ @@ -1339,6 +1356,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # recent messages # XXX take it from memstore + # XXX Used somewhere? def count_recent(self): """ Count all messages with the `Recent` flag. diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py index fe56ea67..cf0ba74b 100644 --- a/mail/src/leap/mail/imap/server.py +++ b/mail/src/leap/mail/imap/server.py @@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation. from copy import copy from twisted import cred +from twisted.internet import reactor from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log @@ -50,6 +51,7 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid + self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -59,9 +61,6 @@ class LeapIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) - from twisted.internet import reactor - self.reactor = reactor - def lineReceived(self, line): """ Attempt to parse a single line from the server. @@ -311,21 +310,203 @@ class LeapIMAPServer(imap4.IMAP4Server): return self._fileLiteral(size, literalPlus) ############################# + # --------------------------------- isSubscribed patch + # TODO -- send patch upstream. + # There is a bug in twisted implementation: + # in cbListWork, it's assumed that account.isSubscribed IS a callable, + # although in the interface documentation it's stated that it can be + # a deferred. + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = self._parseMbox(mbox) + mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) + mailboxes.addCallback(self._cbSubscribed) + mailboxes.addCallback( + self._cbListWork, tag, sub, cmdName, + ).addErrback(self._ebListWork, tag) + + def _cbSubscribed(self, mailboxes): + subscribed = [ + maybeDeferred(self.account.isSubscribed, name) + for (name, box) in mailboxes] + + def get_mailboxes_and_subs(result): + subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] + return mailboxes, subscribed + + d = defer.gatherResults(subscribed) + d.addCallback(get_mailboxes_and_subs) + return d + + def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): + mailboxes, subscribed = mailboxes_subscribed + + for (name, box) in mailboxes: + if not sub or name in subscribed: + flags = box.getFlags() + delim = box.getHierarchicalDelimiter() + resp = (imap4.DontQuoteMe(cmdName), + map(imap4.DontQuoteMe, flags), + delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse( + imap4.collapseNestedLists(resp)) + self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) + # -------------------- end isSubscribed patch ----------- + + # TODO ---- + # subscribe method had also to be changed to accomodate + # deferred + # Revert to regular methods as soon as we implement non-deferred memory + # cache. + def do_SUBSCRIBE(self, tag, name): + name = self._parseMbox(name) + + def _subscribeCb(_): + self.sendPositiveResponse(tag, 'Subscribed') + + def _subscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while subscribing to mailbox") + + d = self.account.subscribe(name) + d.addCallbacks(_subscribeCb, _subscribeEb) + return d + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + def do_UNSUBSCRIBE(self, tag, name): + # unsubscribe method had also to be changed to accomodate + # deferred + name = self._parseMbox(name) + + def _unsubscribeCb(_): + self.sendPositiveResponse(tag, 'Unsubscribed') + + def _unsubscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while unsubscribing " + "from mailbox") + + d = self.account.unsubscribe(name) + d.addCallbacks(_unsubscribeCb, _unsubscribeEb) + return d + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [self._parseMbox(n) for n in oldname, newname] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse( + tag, + 'You cannot rename the inbox, or ' + 'rename another mailbox to inbox.') + return + + def _renameCb(_): + self.sendPositiveResponse(tag, 'Mailbox renamed') + + def _renameEb(failure): + m = failure.value + print "rename failure!" + if failure.check(TypeError): + self.sendBadResponse(tag, 'Invalid command syntax') + elif failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + log.err() + self.sendBadResponse( + tag, + "Server error encountered while " + "renaming mailbox") + + d = self.account.rename(oldname, newname) + d.addCallbacks(_renameCb, _renameEb) + return d + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + def do_CREATE(self, tag, name): + name = self._parseMbox(name) + + def _createCb(result): + if result: + self.sendPositiveResponse(tag, 'Mailbox created') + else: + self.sendNegativeResponse(tag, 'Mailbox not created') + + def _createEb(failure): + c = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(c)) + else: + log.err() + self.sendBadResponse( + tag, "Server error encountered while creating mailbox") + + d = self.account.create(name) + d.addCallbacks(_createCb, _createEb) + return d + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + def do_DELETE(self, tag, name): + name = self._parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, 'You cannot delete the inbox') + return + + def _deleteCb(result): + self.sendPositiveResponse(tag, 'Mailbox deleted') + + def _deleteEb(failure): + m = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + print "other error" + log.err() + self.sendBadResponse( + tag, + "Server error encountered while deleting mailbox") + + d = self.account.delete(name) + d.addCallbacks(_deleteCb, _deleteEb) + return d + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + # Need to override the command table after patching # arg_astring and arg_literal + # do_DELETE = imap4.IMAP4Server.do_DELETE + # do_CREATE = imap4.IMAP4Server.do_CREATE + # do_RENAME = imap4.IMAP4Server.do_RENAME + # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_LOGIN = imap4.IMAP4Server.do_LOGIN - do_CREATE = imap4.IMAP4Server.do_CREATE - do_DELETE = imap4.IMAP4Server.do_DELETE - do_RENAME = imap4.IMAP4Server.do_RENAME - do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE - do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_STATUS = imap4.IMAP4Server.do_STATUS do_APPEND = imap4.IMAP4Server.do_APPEND do_COPY = imap4.IMAP4Server.do_COPY _selectWork = imap4.IMAP4Server._selectWork - _listWork = imap4.IMAP4Server._listWork + arg_plist = imap4.IMAP4Server.arg_plist arg_seqset = imap4.IMAP4Server.arg_seqset opt_plist = imap4.IMAP4Server.opt_plist @@ -342,8 +523,8 @@ class LeapIMAPServer(imap4.IMAP4Server): auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') select_EXAMINE = auth_EXAMINE - auth_DELETE = (do_DELETE, arg_astring) - select_DELETE = auth_DELETE + # auth_DELETE = (do_DELETE, arg_astring) + # select_DELETE = auth_DELETE auth_RENAME = (do_RENAME, arg_astring, arg_astring) select_RENAME = auth_RENAME @@ -369,7 +550,6 @@ class LeapIMAPServer(imap4.IMAP4Server): select_COPY = (do_COPY, arg_seqset, arg_astring) - ############################################################# # END of Twisted imap4 patch to support LITERAL+ extension ############################################################# diff --git a/mail/src/leap/mail/imap/soledadstore.py b/mail/src/leap/mail/imap/soledadstore.py index f3de8ebe..fc8ea558 100644 --- a/mail/src/leap/mail/imap/soledadstore.py +++ b/mail/src/leap/mail/imap/soledadstore.py @@ -40,11 +40,6 @@ from leap.mail.utils import first, empty, accumulator_queue logger = logging.getLogger(__name__) -# TODO -# [ ] Implement a retry queue? -# [ ] Consider journaling of operations. - - class ContentDedup(object): """ Message deduplication. @@ -132,6 +127,7 @@ A lock per document. # http://stackoverflow.com/a/2437645/1157664 # Setting this to twice the number of threads in the threadpool # should be safe. + put_locks = defaultdict(lambda: threading.Lock()) mbox_doc_locks = defaultdict(lambda: threading.Lock()) @@ -429,7 +425,6 @@ class SoledadStore(ContentDedup): continue if item.part == MessagePartType.fdoc: - #logger.debug("PUT dirty fdoc") yield item, call # XXX also for linkage-doc !!! @@ -479,7 +474,7 @@ class SoledadStore(ContentDedup): return query.pop() else: logger.error("Could not find mbox document for %r" % - (mbox,)) + (mbox,)) except Exception as exc: logger.exception("Unhandled error %r" % exc) @@ -552,8 +547,10 @@ class SoledadStore(ContentDedup): :type uid: int :rtype: SoledadDocument or None """ + # TODO -- inlineCallbacks result = None try: + # TODO -- yield flag_docs = self._soledad.get_from_index( fields.TYPE_MBOX_UID_IDX, fields.TYPE_FLAGS_VAL, mbox, str(uid)) diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py index 7837aaa3..dd4294c2 100644 --- a/mail/src/leap/mail/imap/tests/test_imap.py +++ b/mail/src/leap/mail/imap/tests/test_imap.py @@ -68,7 +68,6 @@ def sortNest(l): class TestRealm: - """ A minimal auth realm for testing purposes only """ @@ -83,7 +82,6 @@ class TestRealm: # class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): - """ Tests for the MessageCollection class """ @@ -254,14 +252,18 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.login(TEST_USER, TEST_PASSWD) def create(): + create_deferreds = [] for name in succeed + fail: d = self.client.create(name) d.addCallback(strip(cb)).addErrback(eb) - d.addCallbacks(self._cbStopClient, self._ebGeneral) + create_deferreds.append(d) + dd = defer.gatherResults(create_deferreds) + dd.addCallbacks(self._cbStopClient, self._ebGeneral) + return dd self.result = [] - d1 = self.connected.addCallback(strip(login)).addCallback( - strip(create)) + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create)) d2 = self.loopback() d = defer.gatherResults([d1, d2]) return d.addCallback(self._cbTestCreate, succeed, fail) @@ -269,24 +271,27 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestCreate(self, ignored, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = list(LeapIMAPServer.theAccount.mailboxes) - answers = ([u'INBOX', u'foobox', 'test', u'test/box', - u'test/box/box', 'testbox']) - self.assertEqual(mboxes, [a for a in answers]) + mboxes = LeapIMAPServer.theAccount.mailboxes + + answers = ([u'INBOX', u'testbox', u'test/box', u'test', + u'test/box/box', 'foobox']) + self.assertEqual(sorted(mboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = LeapIMAPServer.theAccount + d0 = lambda: acc.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) def delete(): - return self.client.delete('delete/me') + return self.client.delete('test-delete/me') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -352,11 +357,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - LeapIMAPServer.theAccount.addMailbox('delete') - to_delete = LeapIMAPServer.theAccount.getMailbox('delete') - to_delete.setFlags((r'\Noselect',)) - to_delete.getFlags() - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = LeapIMAPServer.theAccount + d_del0 = lambda: acc.addMailbox('delete') + d_del1 = lambda: acc.addMailbox('delete/me') + + def set_noselect_flag(): + mbox = acc.getMailbox('delete') + mbox.setFlags((r'\Noselect',)) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -369,6 +376,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.failure = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d_del0)) + d1.addCallback(strip(d_del1)) + d1.addCallback(strip(set_noselect_flag)) d1.addCallback(strip(delete)).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -385,7 +395,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can rename a mailbox """ - LeapIMAPServer.theAccount.addMailbox('oldmbox') + d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -394,6 +404,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.rename('oldmbox', 'newname') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -435,8 +446,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - LeapIMAPServer.theAccount.create('oldmbox/m1') - LeapIMAPServer.theAccount.create('oldmbox/m2') + acc = LeapIMAPServer.theAccount + dc1 = lambda: acc.create('oldmbox/m1') + dc2 = lambda: acc.create('oldmbox/m2') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -445,6 +457,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.rename('oldmbox', 'newname') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -454,7 +468,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestHierarchicalRename(self, ignored): mboxes = LeapIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] - self.assertEqual(mboxes, [s for s in expected]) + self.assertEqual(sorted(mboxes), sorted([s for s in expected])) def testSubscribe(self): """ @@ -466,23 +480,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def subscribe(): return self.client.subscribe('this/mbox') + def get_subscriptions(ignored): + return LeapIMAPServer.theAccount.getSubscriptions() + d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['this/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['this/mbox'])) return d def testUnsubscribe(self): """ Test whether we can unsubscribe from a set of mailboxes """ - LeapIMAPServer.theAccount.subscribe('this/mbox') - LeapIMAPServer.theAccount.subscribe('that/mbox') + acc = LeapIMAPServer.theAccount + + dc1 = lambda: acc.subscribe('this/mbox') + dc2 = lambda: acc.subscribe('that/mbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -490,22 +509,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def unsubscribe(): return self.client.unsubscribe('this/mbox') + def get_subscriptions(ignored): + return LeapIMAPServer.theAccount.getSubscriptions() + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['that/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['that/mbox'])) return d def testSelect(self): """ Try to select a mailbox """ - self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + acc = self.server.theAccount + d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) self.selectedArgs = None def login(): @@ -520,6 +545,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallback(strip(select)) d1.addErrback(self._ebGeneral) @@ -754,13 +780,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) - def _listSetup(self, f): - LeapIMAPServer.theAccount.addMailbox('root/subthingl', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('root/another-thing', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('non-root/subthing', - creation_ts=42) + def _listSetup(self, f, f2=None): + acc = LeapIMAPServer.theAccount + + dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -770,6 +795,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.listed = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallback(strip(dc3)) + + if f2 is not None: + d1.addCallback(f2) + d1.addCallbacks(strip(f), self._ebGeneral) d1.addCallbacks(listed, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -786,7 +818,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"), + (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") ]) )) @@ -796,20 +828,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - LeapIMAPServer.theAccount.subscribe('root/subthingl2') + acc = LeapIMAPServer.theAccount + + def subs_mailbox(): + # why not client.subscribe instead? + return acc.subscribe('root/subthing') def lsub(): return self.client.lsub('root', '%') - d = self._listSetup(lsub) + + d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")]) + [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) return d def testStatus(self): """ Test Status command """ - LeapIMAPServer.theAccount.addMailbox('root/subthings') + acc = LeapIMAPServer.theAccount + + def add_mailbox(): + return acc.addMailbox('root/subthings') + # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -824,7 +865,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.statused = result self.statused = None - d1 = self.connected.addCallback(strip(login)) + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(status), self._ebGeneral) d1.addCallbacks(statused, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -930,7 +973,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -946,6 +989,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ) ) d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -995,10 +1039,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test closing the mailbox. We expect to get deleted all messages flagged as such. """ + acc = self.server.theAccount name = 'mailbox-close' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + d0 = lambda: acc.addMailbox(name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1006,14 +1050,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def select(): return self.client.select(name) + def get_mailbox(): + self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + def add_messages(): - d1 = m.messages.add_msg( + d1 = self.mailbox.messages.add_msg( 'test 1', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( + d2 = self.mailbox.messages.add_msg( 'test 2', subject="Message 2", flags=('AnotherFlag',)) - d3 = m.messages.add_msg( + d3 = self.mailbox.messages.add_msg( 'test 3', subject="Message 3", flags=('\\Deleted',)) d = defer.gatherResults([d1, d2, d3]) @@ -1023,30 +1070,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.close() d = self.connected.addCallback(strip(login)) + d.addCallback(strip(d0)) d.addCallbacks(strip(select), self._ebGeneral) + d.addCallback(strip(get_mailbox)) d.addCallbacks(strip(add_messages), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m) + return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - def _cbTestClose(self, ignored, m): - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) + def _cbTestClose(self, ignored): + self.assertEqual(len(self.mailbox.messages), 1) + msg = self.mailbox.messages.get_msg_by_uid(2) self.assertTrue(msg is not None) self.assertEqual( dict(msg.hdoc.content)['subject'], 'Message 2') - self.failUnless(m.closed) + self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ + acc = self.server.theAccount name = 'mailbox-expunge' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + + d0 = lambda: acc.addMailbox(name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1054,14 +1104,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def select(): return self.client.select('mailbox-expunge') + def get_mailbox(): + self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + def add_messages(): - d1 = m.messages.add_msg( + d1 = self.mailbox.messages.add_msg( 'test 1', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( + d2 = self.mailbox.messages.add_msg( 'test 2', subject="Message 2", flags=('AnotherFlag',)) - d3 = m.messages.add_msg( + d3 = self.mailbox.messages.add_msg( 'test 3', subject="Message 3", flags=('\\Deleted',)) d = defer.gatherResults([d1, d2, d3]) @@ -1076,21 +1129,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestExpunge, m) + return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored, m): + def _cbTestExpunge(self, ignored): # we only left 1 mssage with no deleted flag - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) + self.assertEqual(len(self.mailbox.messages), 1) + msg = self.mailbox.messages.get_msg_by_uid(2) - msg = list(m.messages)[0] + msg = list(self.mailbox.messages)[0] self.assertTrue(msg is not None) self.assertEqual( diff --git a/mail/src/leap/mail/imap/tests/utils.py b/mail/src/leap/mail/imap/tests/utils.py index 5339acf0..9a3868c5 100644 --- a/mail/src/leap/mail/imap/tests/utils.py +++ b/mail/src/leap/mail/imap/tests/utils.py @@ -139,31 +139,32 @@ class IMAP4HelperMixin(BaseLeapTest): ########### - d = defer.Deferred() + d_server_ready = defer.Deferred() + self.server = LeapIMAPServer( uuid=UUID, userid=USERID, contextFactory=self.serverCTX, - # XXX do we really need this?? soledad=self._soledad) - self.client = SimpleClient(d, contextFactory=self.clientCTX) - self.connected = d - - # XXX REVIEW-ME. - # We're adding theAccount here to server - # but it was also passed to initialization - # as it was passed to realm. - # I THINK we ONLY need to do it at one place now. + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) theAccount = SoledadBackedAccount( USERID, soledad=self._soledad, memstore=memstore) + d_account_ready = theAccount.callWhenReady(lambda r: None) LeapIMAPServer.theAccount = theAccount + self.connected = defer.gatherResults( + [d_server_ready, d_account_ready]) + + # XXX FIXME -------------------------------------------- + # XXX this needs to be done differently, + # have to be hooked on initialization callback instead. # in case we get something from previous tests... - for mb in self.server.theAccount.mailboxes: - self.server.theAccount.delete(mb) + #for mb in self.server.theAccount.mailboxes: + #self.server.theAccount.delete(mb) # email parser self.parser = parser.Parser() |