adapt to soledad 0.7 async API

9 files changed, 698 insertions, 263 deletions

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 70ed13b..fe466cb 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -22,6 +22,7 @@ import logging
 import os
 import time
 
+from twisted.internet import defer
 from twisted.mail import imap4
 from twisted.python import log
 from zope.interface import implements
@@ -65,6 +66,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     _soledad = None
     selected = None
     closed = False
+    _initialized = False
 
     def __init__(self, account_name, soledad, memstore=None):
         """
@@ -93,14 +95,39 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         self.__mailboxes = set([])
 
-        self.initialize_db()
+        self._deferred_initialization = defer.Deferred()
+        self._initialize_storage()
 
-        # every user should have the right to an inbox folder
-        # at least, so let's make one!
-        self._load_mailboxes()
+    def _initialize_storage(self):
 
-        if not self.mailboxes:
-            self.addMailbox(self.INBOX_NAME)
+        def add_mailbox_if_none(result):
+            # every user should have the right to an inbox folder
+            # at least, so let's make one!
+            if not self.mailboxes:
+                self.addMailbox(self.INBOX_NAME)
+
+        def finish_initialization(result):
+            self._initialized = True
+            self._deferred_initialization.callback(None)
+
+        def load_mbox_cache(result):
+            d = self._load_mailboxes()
+            d.addCallback(lambda _: result)
+            return d
+
+        d = self.initialize_db()
+
+        d.addCallback(load_mbox_cache)
+        d.addCallback(add_mailbox_if_none)
+        d.addCallback(finish_initialization)
+
+    def callWhenReady(self, cb):
+        if self._initialized:
+            cb(self)
+            return defer.succeed(None)
+        else:
+            self._deferred_initialization.addCallback(cb)
+            return self._deferred_initialization
 
     def _get_empty_mailbox(self):
         """
@@ -120,10 +147,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :rtype: SoledadDocument
         """
         # XXX use soledadstore instead ...;
-        doc = self._soledad.get_from_index(
+        def get_first_if_any(docs):
+            return docs[0] if docs else None
+
+        d = self._soledad.get_from_index(
             self.TYPE_MBOX_IDX, self.MBOX_KEY,
             self._parse_mailbox_name(name))
-        return doc[0] if doc else None
+        d.addCallback(get_first_if_any)
+        return d
 
     @property
     def mailboxes(self):
@@ -134,19 +165,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         return sorted(self.__mailboxes)
 
     def _load_mailboxes(self):
-        self.__mailboxes.update(
-            [doc.content[self.MBOX_KEY]
-             for doc in self._soledad.get_from_index(
-                 self.TYPE_IDX, self.MBOX_KEY)])
-
-    @property
-    def subscriptions(self):
-        """
-        A list of the current subscriptions for this account.
-        """
-        return [doc.content[self.MBOX_KEY]
-                for doc in self._soledad.get_from_index(
-                    self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
+        def update_mailboxes(db_indexes):
+            self.__mailboxes.update(
+                [doc.content[self.MBOX_KEY] for doc in db_indexes])
+        d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY)
+        d.addCallback(update_mailboxes)
+        return d
 
     def getMailbox(self, name):
         """
@@ -182,7 +206,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
                             one is provided.
         :type creation_ts: int
 
-        :returns: True if successful
+        :returns: a Deferred that will contain the document if successful.
         :rtype: bool
         """
         name = self._parse_mailbox_name(name)
@@ -203,21 +227,29 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         mbox[self.MBOX_KEY] = name
         mbox[self.CREATED_KEY] = creation_ts
 
-        doc = self._soledad.create_doc(mbox)
-        self._load_mailboxes()
-        return bool(doc)
+        def load_mbox_cache(result):
+            d = self._load_mailboxes()
+            d.addCallback(lambda _: result)
+            return d
+
+        d = self._soledad.create_doc(mbox)
+        d.addCallback(load_mbox_cache)
+        return d
 
     def create(self, pathspec):
         """
         Create a new mailbox from the given hierarchical name.
 
-        :param pathspec: The full hierarchical name of a new mailbox to create.
-                         If any of the inferior hierarchical names to this one
-                         do not exist, they are created as well.
+        :param pathspec:
+            The full hierarchical name of a new mailbox to create.
+            If any of the inferior hierarchical names to this one
+            do not exist, they are created as well.
         :type pathspec: str
 
-        :return: A true value if the creation succeeds.
-        :rtype: bool
+        :return:
+            A deferred that will fire with a true value if the creation
+            succeeds.
+        :rtype: Deferred
 
         :raise MailboxException: Raised if this mailbox cannot be added.
         """
@@ -225,18 +257,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         paths = filter(
             None,
             self._parse_mailbox_name(pathspec).split('/'))
+
+        subs = []
+        sep = '/'
+
         for accum in range(1, len(paths)):
             try:
-                self.addMailbox('/'.join(paths[:accum]))
+                partial = sep.join(paths[:accum])
+                d = self.addMailbox(partial)
+                subs.append(d)
             except imap4.MailboxCollision:
                 pass
         try:
-            self.addMailbox('/'.join(paths))
+            df = self.addMailbox(sep.join(paths))
         except imap4.MailboxCollision:
             if not pathspec.endswith('/'):
-                return False
-        self._load_mailboxes()
-        return True
+                df = defer.succeed(False)
+            else:
+                df = defer.succeed(True)
+        finally:
+            subs.append(df)
+
+        def all_good(result):
+            return all(result)
+
+        def load_mbox_cache(result):
+            d = self._load_mailboxes()
+            d.addCallback(lambda _: result)
+            return d
+
+        if subs:
+            d1 = defer.gatherResults(subs, consumeErrors=True)
+            d1.addCallback(load_mbox_cache)
+            d1.addCallback(all_good)
+        else:
+            d1 = defer.succeed(False)
+            d1.addCallback(load_mbox_cache)
+        return d1
 
     def select(self, name, readwrite=1):
         """
@@ -275,17 +332,20 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :param name: the mailbox to be deleted
         :type name: str
 
-        :param force: if True, it will not check for noselect flag or inferior
-                      names. use with care.
+        :param force:
+            if True, it will not check for noselect flag or inferior
+            names. use with care.
         :type force: bool
+        :rtype: Deferred
         """
         name = self._parse_mailbox_name(name)
 
         if name not in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox: %r" % name)
+            err = imap4.MailboxException("No such mailbox: %r" % name)
+            return defer.fail(err)
         mbox = self.getMailbox(name)
 
-        if force is False:
+        if not force:
             # See if this box is flagged \Noselect
             # XXX use mbox.flags instead?
             mbox_flags = mbox.getFlags()
@@ -294,11 +354,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
                 # as part of their root.
                 for others in self.mailboxes:
                     if others != name and others.startswith(name):
-                        raise imap4.MailboxException, (
+                        err = imap4.MailboxException(
                             "Hierarchically inferior mailboxes "
                             "exist and \\Noselect is set")
+                        return defer.fail(err)
         self.__mailboxes.discard(name)
-        mbox.destroy()
+        return mbox.destroy()
 
         # XXX FIXME --- not honoring the inferior names...
 
@@ -331,14 +392,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
             if new in self.mailboxes:
                 raise imap4.MailboxCollision(repr(new))
 
+        rename_deferreds = []
+
+        def load_mbox_cache(result):
+            d = self._load_mailboxes()
+            d.addCallback(lambda _: result)
+            return d
+
+        def update_mbox_doc_name(mbox, oldname, newname, update_deferred):
+            mbox.content[self.MBOX_KEY] = newname
+            d = self._soledad.put_doc(mbox)
+            d.addCallback(lambda r: update_deferred.callback(True))
+
         for (old, new) in inferiors:
-            self._memstore.rename_fdocs_mailbox(old, new)
-            mbox = self._get_mailbox_by_name(old)
-            mbox.content[self.MBOX_KEY] = new
             self.__mailboxes.discard(old)
-            self._soledad.put_doc(mbox)
+            self._memstore.rename_fdocs_mailbox(old, new)
+
+            d0 = defer.Deferred()
+            d = self._get_mailbox_by_name(old)
+            d.addCallback(update_mbox_doc_name, old, new, d0)
+            rename_deferreds.append(d0)
 
-        self._load_mailboxes()
+        d1 = defer.gatherResults(rename_deferreds, consumeErrors=True)
+        d1.addCallback(load_mbox_cache)
+        return d1
 
     def _inferiorNames(self, name):
         """
@@ -354,6 +431,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
                 inferiors.append(infname)
         return inferiors
 
+    # TODO ------------------ can we preserve the attr?
+    # maybe add to memory store.
     def isSubscribed(self, name):
         """
         Returns True if user is subscribed to this mailbox.
@@ -361,10 +440,35 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :param name: the mailbox to be checked.
         :type name: str
 
-        :rtype: bool
+        :rtype: Deferred (will fire with bool)
+        """
+        subscribed = self.SUBSCRIBED_KEY
+
+        def is_subscribed(mbox):
+            subs_bool = bool(mbox.content.get(subscribed, False))
+            return subs_bool
+
+        d = self._get_mailbox_by_name(name)
+        d.addCallback(is_subscribed)
+        return d
+
+    # TODO ------------------ can we preserve the property?
+    # maybe add to memory store.
+
+    def _get_subscriptions(self):
         """
-        mbox = self._get_mailbox_by_name(name)
-        return mbox.content.get('subscribed', False)
+        Return a list of the current subscriptions for this account.
+
+        :returns: A deferred that will fire with the subscriptions.
+        :rtype: Deferred
+        """
+        def get_docs_content(docs):
+            return [doc.content[self.MBOX_KEY] for doc in docs]
+
+        d = self._soledad.get_from_index(
+            self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')
+        d.addCallback(get_docs_content)
+        return d
 
     def _set_subscription(self, name, value):
         """
@@ -376,26 +480,42 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :param value: the boolean value
         :type value: bool
         """
+        # XXX Note that this kind of operation has
+        # no guarantees of atomicity. We should not be accessing mbox
+        # documents concurrently.
+
+        subscribed = self.SUBSCRIBED_KEY
+
+        def update_subscribed_value(mbox):
+            mbox.content[subscribed] = value
+            return self._soledad.put_doc(mbox)
+
         # maybe we should store subscriptions in another
         # document...
         if name not in self.mailboxes:
-            self.addMailbox(name)
-        mbox = self._get_mailbox_by_name(name)
-
-        if mbox:
-            mbox.content[self.SUBSCRIBED_KEY] = value
-            self._soledad.put_doc(mbox)
+            d = self.addMailbox(name)
+            d.addCallback(lambda v: self._get_mailbox_by_name(name))
+        else:
+            d = self._get_mailbox_by_name(name)
+        d.addCallback(update_subscribed_value)
+        return d
 
     def subscribe(self, name):
         """
-        Subscribe to this mailbox
+        Subscribe to this mailbox if not already subscribed.
 
         :param name: name of the mailbox
         :type name: str
+        :rtype: Deferred
         """
         name = self._parse_mailbox_name(name)
-        if name not in self.subscriptions:
-            self._set_subscription(name, True)
+
+        def check_and_subscribe(subscriptions):
+            if name not in subscriptions:
+                return self._set_subscription(name, True)
+        d = self._get_subscriptions()
+        d.addCallback(check_and_subscribe)
+        return d
 
     def unsubscribe(self, name):
         """
@@ -403,12 +523,21 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         :param name: name of the mailbox
         :type name: str
+        :rtype: Deferred
         """
         name = self._parse_mailbox_name(name)
-        if name not in self.subscriptions:
-            raise imap4.MailboxException(
-                "Not currently subscribed to %r" % name)
-        self._set_subscription(name, False)
+
+        def check_and_unsubscribe(subscriptions):
+            if name not in subscriptions:
+                raise imap4.MailboxException(
+                    "Not currently subscribed to %r" % name)
+            return self._set_subscription(name, False)
+        d = self._get_subscriptions()
+        d.addCallback(check_and_unsubscribe)
+        return d
+
+    def getSubscriptions(self):
+        return self._get_subscriptions()
 
     def listMailboxes(self, ref, wildcard):
         """
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
index 5f0919a..ea35fff 100644
--- a/src/leap/mail/imap/index.py
+++ b/src/leap/mail/imap/index.py
@@ -19,6 +19,8 @@ Index for SoledadBackedAccount, Mailbox and Messages.
 """
 import logging
 
+from twisted.internet import defer
+
 from leap.common.check import leap_assert, leap_assert_type
 
 from leap.mail.imap.fields import fields
@@ -39,6 +41,9 @@ class IndexedDB(object):
     """
     # TODO we might want to move this to soledad itself, check
 
+    _index_creation_deferreds = []
+    index_ready = False
+
     def initialize_db(self):
         """
         Initialize the database.
@@ -46,24 +51,40 @@ class IndexedDB(object):
         leap_assert(self._soledad,
                     "Need a soledad attribute accesible in the instance")
         leap_assert_type(self.INDEXES, dict)
+        self._index_creation_deferreds = []
+
+        def _on_indexes_created(ignored):
+            self.index_ready = True
+
+        def _create_index(name, expression):
+            d = self._soledad.create_index(name, *expression)
+            self._index_creation_deferreds.append(d)
+
+        def _create_indexes(db_indexes):
+            db_indexes = dict(db_indexes)
+            for name, expression in fields.INDEXES.items():
+                if name not in db_indexes:
+                    # The index does not yet exist.
+                    _create_index(name, expression)
+                    continue
+
+                if expression == db_indexes[name]:
+                    # The index exists and is up to date.
+                    continue
+                # The index exists but the definition is not what expected, so
+                # we delete it and add the proper index expression.
+                d1 = self._soledad.delete_index(name)
+                d1.addCallback(lambda _: _create_index(name, expression))
+
+            all_created = defer.gatherResults(self._index_creation_deferreds)
+            all_created.addCallback(_on_indexes_created)
+            return all_created
 
         # Ask the database for currently existing indexes.
         if not self._soledad:
             logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
             return
-        db_indexes = dict()
         if self._soledad is not None:
-            db_indexes = dict(self._soledad.list_indexes())
-        for name, expression in fields.INDEXES.items():
-            if name not in db_indexes:
-                # The index does not yet exist.
-                self._soledad.create_index(name, *expression)
-                continue
-
-            if expression == db_indexes[name]:
-                # The index exists and is up to date.
-                continue
-            # The index exists but the definition is not what expected, so we
-            # delete it and add the proper index expression.
-            self._soledad.delete_index(name)
-            self._soledad.create_index(name, *expression)
+            d = self._soledad.list_indexes()
+            d.addCallback(_create_indexes)
+            return d
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 34cf535..3c1769a 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -303,21 +303,32 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         We do this to be able to filter the requests efficiently.
         """
         primed = self._known_uids_primed.get(self.mbox, False)
-        if not primed:
-            known_uids = self.messages.all_soledad_uid_iter()
+        # XXX handle the maybeDeferred
+
+        def set_primed(known_uids):
             self._memstore.set_known_uids(self.mbox, known_uids)
             self._known_uids_primed[self.mbox] = True
 
+        if not primed:
+            d = self.messages.all_soledad_uid_iter()
+            d.addCallback(set_primed)
+            return d
+
     def prime_flag_docs_to_memstore(self):
         """
         Prime memstore with all the flags documents.
         """
         primed = self._fdoc_primed.get(self.mbox, False)
-        if not primed:
-            all_flag_docs = self.messages.get_all_soledad_flag_docs()
-            self._memstore.load_flag_docs(self.mbox, all_flag_docs)
+
+        def set_flag_docs(flag_docs):
+            self._memstore.load_flag_docs(self.mbox, flag_docs)
             self._fdoc_primed[self.mbox] = True
 
+        if not primed:
+            d = self.messages.get_all_soledad_flag_docs()
+            d.addCallback(set_flag_docs)
+            return d
+
     def getUIDValidity(self):
         """
         Return the unique validity identifier for this mailbox.
@@ -522,21 +533,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         Should cleanup resources, and set the \\Noselect flag
         on the mailbox.
+
         """
         # XXX this will overwrite all the existing flags!
         # should better simply addFlag
         self.setFlags((self.NOSELECT_FLAG,))
-        self.deleteAllDocs()
 
         # XXX removing the mailbox in situ for now,
         # we should postpone the removal
 
-        # XXX move to memory store??
-        mbox_doc = self._get_mbox_doc()
-        if mbox_doc is None:
-            # memory-only store!
-            return
-        self._soledad.delete_doc(self._get_mbox_doc())
+        def remove_mbox_doc(ignored):
+            # XXX move to memory store??
+
+            def _remove_mbox_doc(doc):
+                if doc is None:
+                    # memory-only store!
+                    return defer.succeed(True)
+                return self._soledad.delete_doc(doc)
+
+            doc = self._get_mbox_doc()
+            return _remove_mbox_doc(doc)
+
+        d = self.deleteAllDocs()
+        d.addCallback(remove_mbox_doc)
+        return d
 
     def _close_cb(self, result):
         self.closed = True
@@ -1006,9 +1026,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Delete all docs in this mailbox
         """
-        docs = self.messages.get_all_docs()
-        for doc in docs:
-            self.messages._soledad.delete_doc(doc)
+        def del_all_docs(docs):
+            todelete = []
+            for doc in docs:
+                d = self.messages._soledad.delete_doc(doc)
+                todelete.append(d)
+            return defer.gatherResults(todelete)
+
+        d = self.messages.get_all_docs()
+        d.addCallback(del_all_docs)
+        return d
 
     def unset_recent_flags(self, uid_seq):
         """
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index e075394..eda5b96 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+
 # memorystore.py
 # Copyright (C) 2014 LEAP
 #
@@ -112,8 +112,6 @@ class MemoryStore(object):
         :param write_period: the interval to dump messages to disk, in seconds.
         :type write_period: int
         """
-        self.reactor = reactor
-
         self._permanent_store = permanent_store
         self._write_period = write_period
 
@@ -241,6 +239,7 @@ class MemoryStore(object):
             self.producer = None
             self._write_loop = None
 
+    # TODO -- remove
     def _start_write_loop(self):
         """
         Start loop for writing to disk database.
@@ -250,6 +249,7 @@ class MemoryStore(object):
         if not self._write_loop.running:
             self._write_loop.start(self._write_period, now=True)
 
+    # TODO -- remove
     def _stop_write_loop(self):
         """
         Stop loop for writing to disk database.
@@ -278,17 +278,18 @@ class MemoryStore(object):
         :type uid: int
         :param message: a message to be added
         :type message: MessageWrapper
-        :param observer: the deferred that will fire with the
-                         UID of the message. If notify_on_disk is True,
-                         this will happen when the message is written to
-                         Soledad. Otherwise it will fire as soon as we've
-                         added the message to the memory store.
+        :param observer:
+            the deferred that will fire with the UID of the message. If
+            notify_on_disk is True, this will happen when the message is
+            written to Soledad. Otherwise it will fire as soon as we've added
+            the message to the memory store.
         :type observer: Deferred
-        :param notify_on_disk: whether the `observer` deferred should
-                               wait until the message is written to disk to
-                               be fired.
+        :param notify_on_disk:
+            whether the `observer` deferred should wait until the message is
+            written to disk to be fired.
         :type notify_on_disk: bool
         """
+        # TODO -- return a deferred
         log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid))
         key = mbox, uid
 
@@ -306,7 +307,7 @@ class MemoryStore(object):
             else:
                 # Caller does not care, just fired and forgot, so we pass
                 # a defer that will inmediately have its callback triggered.
-                self.reactor.callFromThread(observer.callback, uid)
+                reactor.callFromThread(observer.callback, uid)
 
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
@@ -442,6 +443,7 @@ class MemoryStore(object):
 
         :return: MessageWrapper or None
         """
+        # TODO -- return deferred
         if dirtystate == DirtyState.dirty:
             flags_only = True
 
@@ -467,6 +469,7 @@ class MemoryStore(object):
             chash = fdoc.get(fields.CONTENT_HASH_KEY)
             hdoc = self._hdoc_store[chash]
             if empty(hdoc):
+                # XXX this will be a deferred
                 hdoc = self._permanent_store.get_headers_doc(chash)
                 if empty(hdoc):
                     return None
@@ -531,7 +534,8 @@ class MemoryStore(object):
 
     # IMessageStoreWriter
 
-    @deferred_to_thread
+    # TODO -- I think we don't need this anymore.
+    # instead, we can have
     def write_messages(self, store):
         """
         Write the message documents in this MemoryStore to a different store.
@@ -657,7 +661,7 @@ class MemoryStore(object):
         with self._last_uid_lock:
             self._last_uid[mbox] += 1
             value = self._last_uid[mbox]
-            self.reactor.callInThread(self.write_last_uid, mbox, value)
+            reactor.callInThread(self.write_last_uid, mbox, value)
             return value
 
     def write_last_uid(self, mbox, value):
@@ -1077,6 +1081,7 @@ class MemoryStore(object):
             return None
         return self._rflags_store[mbox]['set']
 
+    # XXX -- remove
     def all_rdocs_iter(self):
         """
         Return an iterator through all in-memory recent flag dicts, wrapped
@@ -1125,6 +1130,7 @@ class MemoryStore(object):
             self.remove_message(mbox, uid)
         return mem_deleted
 
+    # TODO -- remove
     def stop_and_flush(self):
         """
         Stop the write loop and trigger a write to the producer.
@@ -1180,6 +1186,7 @@ class MemoryStore(object):
         :type observer: Deferred
         """
         mem_deleted = self.remove_all_deleted(mbox)
+        # TODO return a DeferredList
         observer.callback(mem_deleted)
 
     def _delete_from_soledad_and_memory(self, result, mbox, observer):
@@ -1313,8 +1320,8 @@ class MemoryStore(object):
         :rtype: bool
         """
         # FIXME this should return a deferred !!!
-        # XXX ----- can fire when all new + dirty deferreds
-        # are done (gatherResults)
+        # TODO this should be moved to soledadStore instead
+        # (all pending deferreds)
         return getattr(self, self.WRITING_FLAG)
 
     @property
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index e8d64d1..c761091 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -71,6 +71,7 @@ def try_unique_query(curried):
     :param curried: a curried function
     :type curried: callable
     """
+    # XXX FIXME ---------- convert to deferreds
     leap_assert(callable(curried), "A callable is expected")
     try:
         query = curried()
@@ -134,10 +135,11 @@ class LeapMessage(fields, MBoxParser):
         self.__chash = None
         self.__bdoc = None
 
-        self.reactor = reactor
-
     # XXX make these properties public
 
+    # XXX FIXME ------ the documents can be
+    # deferreds too.... niice.
+
     @property
     def fdoc(self):
         """
@@ -506,18 +508,15 @@ class LeapMessage(fields, MBoxParser):
         Return the document that keeps the flags for this
         message.
         """
-        result = {}
-        try:
-            flag_docs = self._soledad.get_from_index(
-                fields.TYPE_MBOX_UID_IDX,
-                fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
-            result = first(flag_docs)
-        except Exception as exc:
-            # ugh! Something's broken down there!
-            logger.warning("ERROR while getting flags for UID: %s" % self._uid)
-            logger.exception(exc)
-        finally:
-            return result
+        def get_first_if_any(docs):
+            result = first(docs)
+            return result if result else {}
+
+        d = self._soledad.get_from_index(
+            fields.TYPE_MBOX_UID_IDX,
+            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+        d.addCallback(get_first_if_any)
+        return d
 
     # TODO move to soledadstore instead of accessing soledad directly
     def _get_headers_doc(self):
@@ -525,10 +524,11 @@ class LeapMessage(fields, MBoxParser):
         Return the document that keeps the headers for this
         message.
         """
-        head_docs = self._soledad.get_from_index(
+        d = self._soledad.get_from_index(
             fields.TYPE_C_HASH_IDX,
             fields.TYPE_HEADERS_VAL, str(self.chash))
-        return first(head_docs)
+        d.addCallback(lambda docs: first(docs))
+        return d
 
     # TODO move to soledadstore instead of accessing soledad directly
     def _get_body_doc(self):
@@ -536,6 +536,8 @@ class LeapMessage(fields, MBoxParser):
         Return the document that keeps the body for this
         message.
         """
+        # XXX FIXME --- this might need a maybedeferred
+        # on the receiving side...
         hdoc_content = self.hdoc.content
         body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
@@ -554,13 +556,11 @@ class LeapMessage(fields, MBoxParser):
                 return bdoc
 
         # no memstore, or no body doc found there
-        if self._soledad:
-            body_docs = self._soledad.get_from_index(
-                fields.TYPE_P_HASH_IDX,
-                fields.TYPE_CONTENT_VAL, str(body_phash))
-            return first(body_docs)
-        else:
-            logger.error("No phash in container, and no soledad found!")
+        d = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(body_phash))
+        d.addCallback(lambda docs: first(docs))
+        return d
 
     def __getitem__(self, key):
         """
@@ -739,8 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             else:
                 self._initialized[mbox] = True
 
-        self.reactor = reactor
-
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
         Returns an empty doc for storing different message parts.
@@ -887,9 +885,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             flags = tuple()
         leap_assert_type(flags, tuple)
 
+        # TODO return soledad deferred instead
         observer = defer.Deferred()
         d = self._do_parse(raw)
-        d.addCallback(lambda result: self.reactor.callInThread(
+        d.addCallback(lambda result: reactor.callInThread(
             self._do_add_msg, result, flags, subject, date,
             notify_on_disk, observer))
         return observer
@@ -924,17 +923,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             msg = self.get_msg_by_uid(existing_uid)
 
             # We can say the observer that we're done
-            self.reactor.callFromThread(observer.callback, existing_uid)
+            # TODO return soledad deferred instead
+            reactor.callFromThread(observer.callback, existing_uid)
             msg.setFlags((fields.DELETED_FLAG,), -1)
             return
 
-        # XXX get FUCKING UID from autoincremental table
+        # TODO S2 -- get FUCKING UID from autoincremental table
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
 
         # We can say the observer that we're done at this point, but
         # before that we should make sure it has no serious consequences
         # if we're issued, for instance, a fetch command right after...
-        # self.reactor.callFromThread(observer.callback, uid)
+        # reactor.callFromThread(observer.callback, uid)
         # if we did the notify, we need to invalidate the deferred
         # so not to try to fire it twice.
         # observer = None
@@ -960,6 +960,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
 
         self.set_recent_flag(uid)
         msg_container = MessageWrapper(fd, hd, cdocs)
+
+        # TODO S1 -- just pass this to memstore and return that deferred.
         self.memstore.create_message(
             self.mbox, uid, msg_container,
             observer=observer, notify_on_disk=notify_on_disk)
@@ -1011,6 +1013,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         Get recent-flags document from Soledad for this mailbox.
         :rtype: SoledadDocument or None
         """
+        # FIXME ----- use deferreds.
         curried = partial(
             self._soledad.get_from_index,
             fields.TYPE_MBOX_IDX,
@@ -1029,6 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :param uids: the uids to unset
         :type uid: sequence
         """
+        # FIXME ----- use deferreds.
         with self._rdoc_property_lock[self.mbox]:
             self.recent_flags.difference_update(
                 set(uids))
@@ -1042,11 +1046,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :param uid: the uid to unset
         :type uid: int
         """
+        # FIXME ----- use deferreds.
         with self._rdoc_property_lock[self.mbox]:
             self.recent_flags.difference_update(
                 set([uid]))
 
-    @deferred_to_thread
     def set_recent_flag(self, uid):
         """
         Set Recent flag for a given uid.
@@ -1054,6 +1058,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :param uid: the uid to set
         :type uid: int
         """
+        # FIXME ----- use deferreds.
         with self._rdoc_property_lock[self.mbox]:
             self.recent_flags = self.recent_flags.union(
                 set([uid]))
@@ -1068,6 +1073,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
                  the query failed.
         :rtype: SoledadDocument or None.
         """
+        # FIXME ----- use deferreds.
         curried = partial(
             self._soledad.get_from_index,
             fields.TYPE_MBOX_C_HASH_IDX,
@@ -1125,7 +1131,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             return None
         return fdoc.content.get(fields.UID_KEY, None)
 
-    @deferred_to_thread
     def _get_uid_from_msgid(self, msgid):
         """
         Return a UID for a given message-id.
@@ -1144,7 +1149,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
-    @deferred_to_thread
     def set_flags(self, mbox, messages, flags, mode, observer):
         """
         Set flags for a sequence of messages.
@@ -1162,7 +1166,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
                          done.
         :type observer: deferred
         """
-        reactor = self.reactor
         getmsg = self.get_msg_by_uid
 
         def set_flags(uid, flags, mode):
@@ -1173,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         setted_flags = [set_flags(uid, flags, mode) for uid in messages]
         result = dict(filter(None, setted_flags))
 
+        # TODO -- remove
         reactor.callFromThread(observer.callback, result)
 
     # getters: generic for a mailbox
@@ -1223,37 +1227,45 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
 
         If you want acess to the content, use __iter__ instead
 
-        :return: a list of u1db documents
-        :rtype: list of SoledadDocument
+        :return: a Deferred, that will fire with a list of u1db documents
+        :rtype: Deferred (promise of list of SoledadDocument)
         """
         if _type not in fields.__dict__.values():
             raise TypeError("Wrong type passed to get_all_docs")
 
+        # FIXME ----- either raise or return a deferred wrapper.
         if sameProxiedObjects(self._soledad, None):
             logger.warning('Tried to get messages but soledad is None!')
             return []
 
-        all_docs = [doc for doc in self._soledad.get_from_index(
-            fields.TYPE_MBOX_IDX,
-            _type, self.mbox)]
+        def get_sorted_docs(docs):
+            all_docs = [doc for doc in docs]
+            # inneficient, but first let's grok it and then
+            # let's worry about efficiency.
+            # XXX FIXINDEX -- should implement order by in soledad
+            # FIXME ----------------------------------------------
+            return sorted(all_docs, key=lambda item: item.content['uid'])
 
-        # inneficient, but first let's grok it and then
-        # let's worry about efficiency.
-        # XXX FIXINDEX -- should implement order by in soledad
-        # FIXME ----------------------------------------------
-        return sorted(all_docs, key=lambda item: item.content['uid'])
+        d = self._soledad.get_from_index(
+            fields.TYPE_MBOX_IDX, _type, self.mbox)
+        d.addCallback(get_sorted_docs)
+        return d
 
     def all_soledad_uid_iter(self):
         """
         Return an iterator through the UIDs of all messages, sorted in
         ascending order.
         """
-        db_uids = set([doc.content[self.UID_KEY] for doc in
-                       self._soledad.get_from_index(
-                           fields.TYPE_MBOX_IDX,
-                           fields.TYPE_FLAGS_VAL, self.mbox)
-                       if not empty(doc)])
-        return db_uids
+        # XXX FIXME ------ sorted???
+
+        def get_uids(docs):
+            return set([
+                doc.content[self.UID_KEY] for doc in docs if not empty(doc)])
+
+        d = self._soledad.get_from_index(
+            fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox)
+        d.addCallback(get_uids)
+        return d
 
     def all_uid_iter(self):
         """
@@ -1277,16 +1289,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         # XXX we really could return a reduced version with
         # just {'uid': (flags-tuple,) since the prefetch is
         # only oriented to get the flag tuples.
-        all_docs = [(
-            doc.content[self.UID_KEY],
-            dict(doc.content))
-            for doc in
-            self._soledad.get_from_index(
-                fields.TYPE_MBOX_IDX,
-                fields.TYPE_FLAGS_VAL, self.mbox)
-            if not empty(doc.content)]
-        all_flags = dict(all_docs)
-        return all_flags
+
+        def get_content(docs):
+            all_docs = [(
+                doc.content[self.UID_KEY],
+                dict(doc.content))
+                for doc in docs
+                if not empty(doc.content)]
+            all_flags = dict(all_docs)
+            return all_flags
+
+        d = self._soledad.get_from_index(
+            fields.TYPE_MBOX_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox)
+        d.addCallback(get_content)
+        return d
 
     def all_headers(self):
         """
@@ -1339,6 +1356,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
     # recent messages
 
     # XXX take it from memstore
+    # XXX Used somewhere?
     def count_recent(self):
         """
         Count all messages with the `Recent` flag.
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index fe56ea6..cf0ba74 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation.
 from copy import copy
 
 from twisted import cred
+from twisted.internet import reactor
 from twisted.internet.defer import maybeDeferred
 from twisted.mail import imap4
 from twisted.python import log
@@ -50,6 +51,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         leap_assert(uuid, "need a user in the initialization")
 
         self._userid = userid
+        self.reactor = reactor
 
         # initialize imap server!
         imap4.IMAP4Server.__init__(self, *args, **kwargs)
@@ -59,9 +61,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
         # populate the test account properly (and only once
         # per session)
 
-        from twisted.internet import reactor
-        self.reactor = reactor
-
     def lineReceived(self, line):
         """
         Attempt to parse a single line from the server.
@@ -311,21 +310,203 @@ class LeapIMAPServer(imap4.IMAP4Server):
         return self._fileLiteral(size, literalPlus)
         #############################
 
+    # --------------------------------- isSubscribed patch
+    # TODO -- send patch upstream.
+    # There is a bug in twisted implementation:
+    # in cbListWork, it's assumed that account.isSubscribed IS a callable,
+    # although in the interface documentation it's stated that it can be
+    # a deferred.
+
+    def _listWork(self, tag, ref, mbox, sub, cmdName):
+        mbox = self._parseMbox(mbox)
+        mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox)
+        mailboxes.addCallback(self._cbSubscribed)
+        mailboxes.addCallback(
+            self._cbListWork, tag, sub, cmdName,
+            ).addErrback(self._ebListWork, tag)
+
+    def _cbSubscribed(self, mailboxes):
+        subscribed = [
+            maybeDeferred(self.account.isSubscribed, name)
+            for (name, box) in mailboxes]
+
+        def get_mailboxes_and_subs(result):
+            subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes]
+            return mailboxes, subscribed
+
+        d = defer.gatherResults(subscribed)
+        d.addCallback(get_mailboxes_and_subs)
+        return d
+
+    def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName):
+        mailboxes, subscribed = mailboxes_subscribed
+
+        for (name, box) in mailboxes:
+            if not sub or name in subscribed:
+                flags = box.getFlags()
+                delim = box.getHierarchicalDelimiter()
+                resp = (imap4.DontQuoteMe(cmdName),
+                        map(imap4.DontQuoteMe, flags),
+                        delim, name.encode('imap4-utf-7'))
+                self.sendUntaggedResponse(
+                    imap4.collapseNestedLists(resp))
+        self.sendPositiveResponse(tag, '%s completed' % (cmdName,))
+    # -------------------- end isSubscribed patch -----------
+
+    # TODO ----
+    # subscribe method had also to be changed to accomodate
+    # deferred
+    # Revert to regular methods as soon as we implement non-deferred memory
+    # cache.
+    def do_SUBSCRIBE(self, tag, name):
+        name = self._parseMbox(name)
+
+        def _subscribeCb(_):
+            self.sendPositiveResponse(tag, 'Subscribed')
+
+        def _subscribeEb(failure):
+            m = failure.value
+            log.err()
+            if failure.check(imap4.MailboxException):
+                self.sendNegativeResponse(tag, str(m))
+            else:
+                self.sendBadResponse(
+                    tag,
+                    "Server error encountered while subscribing to mailbox")
+
+        d = self.account.subscribe(name)
+        d.addCallbacks(_subscribeCb, _subscribeEb)
+        return d
+
+    auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
+    select_SUBSCRIBE = auth_SUBSCRIBE
+
+    def do_UNSUBSCRIBE(self, tag, name):
+        # unsubscribe method had also to be changed to accomodate
+        # deferred
+        name = self._parseMbox(name)
+
+        def _unsubscribeCb(_):
+            self.sendPositiveResponse(tag, 'Unsubscribed')
+
+        def _unsubscribeEb(failure):
+            m = failure.value
+            log.err()
+            if failure.check(imap4.MailboxException):
+                self.sendNegativeResponse(tag, str(m))
+            else:
+                self.sendBadResponse(
+                    tag,
+                    "Server error encountered while unsubscribing "
+                    "from mailbox")
+
+        d = self.account.unsubscribe(name)
+        d.addCallbacks(_unsubscribeCb, _unsubscribeEb)
+        return d
+
+    auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
+    select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+    def do_RENAME(self, tag, oldname, newname):
+        oldname, newname = [self._parseMbox(n) for n in oldname, newname]
+        if oldname.lower() == 'inbox' or newname.lower() == 'inbox':
+            self.sendNegativeResponse(
+                tag,
+                'You cannot rename the inbox, or '
+                'rename another mailbox to inbox.')
+            return
+
+        def _renameCb(_):
+            self.sendPositiveResponse(tag, 'Mailbox renamed')
+
+        def _renameEb(failure):
+            m = failure.value
+            print "rename failure!"
+            if failure.check(TypeError):
+                self.sendBadResponse(tag, 'Invalid command syntax')
+            elif failure.check(imap4.MailboxException):
+                self.sendNegativeResponse(tag, str(m))
+            else:
+                log.err()
+                self.sendBadResponse(
+                    tag,
+                    "Server error encountered while "
+                    "renaming mailbox")
+
+        d = self.account.rename(oldname, newname)
+        d.addCallbacks(_renameCb, _renameEb)
+        return d
+
+    auth_RENAME = (do_RENAME, arg_astring, arg_astring)
+    select_RENAME = auth_RENAME
+
+    def do_CREATE(self, tag, name):
+        name = self._parseMbox(name)
+
+        def _createCb(result):
+            if result:
+                self.sendPositiveResponse(tag, 'Mailbox created')
+            else:
+                self.sendNegativeResponse(tag, 'Mailbox not created')
+
+        def _createEb(failure):
+            c = failure.value
+            if failure.check(imap4.MailboxException):
+                self.sendNegativeResponse(tag, str(c))
+            else:
+                log.err()
+                self.sendBadResponse(
+                    tag, "Server error encountered while creating mailbox")
+
+        d = self.account.create(name)
+        d.addCallbacks(_createCb, _createEb)
+        return d
+
+    auth_CREATE = (do_CREATE, arg_astring)
+    select_CREATE = auth_CREATE
+
+    def do_DELETE(self, tag, name):
+        name = self._parseMbox(name)
+        if name.lower() == 'inbox':
+            self.sendNegativeResponse(tag, 'You cannot delete the inbox')
+            return
+
+        def _deleteCb(result):
+            self.sendPositiveResponse(tag, 'Mailbox deleted')
+
+        def _deleteEb(failure):
+            m = failure.value
+            if failure.check(imap4.MailboxException):
+                self.sendNegativeResponse(tag, str(m))
+            else:
+                print "other error"
+                log.err()
+                self.sendBadResponse(
+                    tag,
+                    "Server error encountered while deleting mailbox")
+
+        d = self.account.delete(name)
+        d.addCallbacks(_deleteCb, _deleteEb)
+        return d
+
+    auth_DELETE = (do_DELETE, arg_astring)
+    select_DELETE = auth_DELETE
+
     # Need to override the command table after patching
     # arg_astring and arg_literal
 
+    # do_DELETE = imap4.IMAP4Server.do_DELETE
+    # do_CREATE = imap4.IMAP4Server.do_CREATE
+    # do_RENAME = imap4.IMAP4Server.do_RENAME
+    # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
+    # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
     do_LOGIN = imap4.IMAP4Server.do_LOGIN
-    do_CREATE = imap4.IMAP4Server.do_CREATE
-    do_DELETE = imap4.IMAP4Server.do_DELETE
-    do_RENAME = imap4.IMAP4Server.do_RENAME
-    do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
-    do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
     do_STATUS = imap4.IMAP4Server.do_STATUS
     do_APPEND = imap4.IMAP4Server.do_APPEND
     do_COPY = imap4.IMAP4Server.do_COPY
 
     _selectWork = imap4.IMAP4Server._selectWork
-    _listWork = imap4.IMAP4Server._listWork
+
     arg_plist = imap4.IMAP4Server.arg_plist
     arg_seqset = imap4.IMAP4Server.arg_seqset
     opt_plist = imap4.IMAP4Server.opt_plist
@@ -342,8 +523,8 @@ class LeapIMAPServer(imap4.IMAP4Server):
     auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE')
     select_EXAMINE = auth_EXAMINE
 
-    auth_DELETE = (do_DELETE, arg_astring)
-    select_DELETE = auth_DELETE
+    # auth_DELETE = (do_DELETE, arg_astring)
+    # select_DELETE = auth_DELETE
 
     auth_RENAME = (do_RENAME, arg_astring, arg_astring)
     select_RENAME = auth_RENAME
@@ -369,7 +550,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
 
     select_COPY = (do_COPY, arg_seqset, arg_astring)
 
-
     #############################################################
     # END of Twisted imap4 patch to support LITERAL+ extension
     #############################################################
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index f3de8eb..fc8ea55 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -40,11 +40,6 @@ from leap.mail.utils import first, empty, accumulator_queue
 logger = logging.getLogger(__name__)
 
 
-# TODO
-# [ ] Implement a retry queue?
-# [ ] Consider journaling of operations.
-
-
 class ContentDedup(object):
     """
     Message deduplication.
@@ -132,6 +127,7 @@ A lock per document.
 # http://stackoverflow.com/a/2437645/1157664
 # Setting this to twice the number of threads in the threadpool
 # should be safe.
+
 put_locks = defaultdict(lambda: threading.Lock())
 mbox_doc_locks = defaultdict(lambda: threading.Lock())
 
@@ -429,7 +425,6 @@ class SoledadStore(ContentDedup):
                     continue
 
                 if item.part == MessagePartType.fdoc:
-                    #logger.debug("PUT dirty fdoc")
                     yield item, call
 
                 # XXX also for linkage-doc !!!
@@ -479,7 +474,7 @@ class SoledadStore(ContentDedup):
                 return query.pop()
             else:
                 logger.error("Could not find mbox document for %r" %
-                            (mbox,))
+                             (mbox,))
         except Exception as exc:
             logger.exception("Unhandled error %r" % exc)
 
@@ -552,8 +547,10 @@ class SoledadStore(ContentDedup):
         :type uid: int
         :rtype: SoledadDocument or None
         """
+        # TODO -- inlineCallbacks
         result = None
         try:
+            # TODO -- yield
             flag_docs = self._soledad.get_from_index(
                 fields.TYPE_MBOX_UID_IDX,
                 fields.TYPE_FLAGS_VAL, mbox, str(uid))
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 7837aaa..dd4294c 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -68,7 +68,6 @@ def sortNest(l):
 
 
 class TestRealm:
-
     """
     A minimal auth realm for testing purposes only
     """
@@ -83,7 +82,6 @@ class TestRealm:
 #
 
 class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
-
     """
     Tests for the MessageCollection class
     """
@@ -254,14 +252,18 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return self.client.login(TEST_USER, TEST_PASSWD)
 
         def create():
+            create_deferreds = []
             for name in succeed + fail:
                 d = self.client.create(name)
                 d.addCallback(strip(cb)).addErrback(eb)
-            d.addCallbacks(self._cbStopClient, self._ebGeneral)
+                create_deferreds.append(d)
+            dd = defer.gatherResults(create_deferreds)
+            dd.addCallbacks(self._cbStopClient, self._ebGeneral)
+            return dd
 
         self.result = []
-        d1 = self.connected.addCallback(strip(login)).addCallback(
-            strip(create))
+        d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(create))
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
         return d.addCallback(self._cbTestCreate, succeed, fail)
@@ -269,24 +271,27 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def _cbTestCreate(self, ignored, succeed, fail):
         self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail))
 
-        mboxes = list(LeapIMAPServer.theAccount.mailboxes)
-        answers = ([u'INBOX', u'foobox', 'test', u'test/box',
-                    u'test/box/box', 'testbox'])
-        self.assertEqual(mboxes, [a for a in answers])
+        mboxes = LeapIMAPServer.theAccount.mailboxes
+
+        answers = ([u'INBOX', u'testbox', u'test/box', u'test',
+                    u'test/box/box', 'foobox'])
+        self.assertEqual(sorted(mboxes), sorted([a for a in answers]))
 
     def testDelete(self):
         """
         Test whether we can delete mailboxes
         """
-        LeapIMAPServer.theAccount.addMailbox('delete/me')
+        acc = LeapIMAPServer.theAccount
+        d0 = lambda: acc.addMailbox('test-delete/me')
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
 
         def delete():
-            return self.client.delete('delete/me')
+            return self.client.delete('test-delete/me')
 
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(d0))
         d1.addCallbacks(strip(delete), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -352,11 +357,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Try deleting a mailbox with sub-folders, and \NoSelect flag set.
         An exception is expected.
         """
-        LeapIMAPServer.theAccount.addMailbox('delete')
-        to_delete = LeapIMAPServer.theAccount.getMailbox('delete')
-        to_delete.setFlags((r'\Noselect',))
-        to_delete.getFlags()
-        LeapIMAPServer.theAccount.addMailbox('delete/me')
+        acc = LeapIMAPServer.theAccount
+        d_del0 = lambda: acc.addMailbox('delete')
+        d_del1 = lambda: acc.addMailbox('delete/me')
+
+        def set_noselect_flag():
+            mbox = acc.getMailbox('delete')
+            mbox.setFlags((r'\Noselect',))
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -369,6 +376,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.failure = None
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(d_del0))
+        d1.addCallback(strip(d_del1))
+        d1.addCallback(strip(set_noselect_flag))
         d1.addCallback(strip(delete)).addErrback(deleteFailed)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -385,7 +395,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test whether we can rename a mailbox
         """
-        LeapIMAPServer.theAccount.addMailbox('oldmbox')
+        d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox')
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -394,6 +404,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return self.client.rename('oldmbox', 'newname')
 
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(d0))
         d1.addCallbacks(strip(rename), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -435,8 +446,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Try to rename hierarchical mailboxes
         """
-        LeapIMAPServer.theAccount.create('oldmbox/m1')
-        LeapIMAPServer.theAccount.create('oldmbox/m2')
+        acc = LeapIMAPServer.theAccount
+        dc1 = lambda: acc.create('oldmbox/m1')
+        dc2 = lambda: acc.create('oldmbox/m2')
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -445,6 +457,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return self.client.rename('oldmbox', 'newname')
 
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(dc1))
+        d1.addCallback(strip(dc2))
         d1.addCallbacks(strip(rename), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -454,7 +468,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def _cbTestHierarchicalRename(self, ignored):
         mboxes = LeapIMAPServer.theAccount.mailboxes
         expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2']
-        self.assertEqual(mboxes, [s for s in expected])
+        self.assertEqual(sorted(mboxes), sorted([s for s in expected]))
 
     def testSubscribe(self):
         """
@@ -466,23 +480,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         def subscribe():
             return self.client.subscribe('this/mbox')
 
+        def get_subscriptions(ignored):
+            return LeapIMAPServer.theAccount.getSubscriptions()
+
         d1 = self.connected.addCallback(strip(login))
         d1.addCallbacks(strip(subscribe), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
-        d.addCallback(lambda _:
-                      self.assertEqual(
-                          LeapIMAPServer.theAccount.subscriptions,
-                          ['this/mbox']))
+        d.addCallback(get_subscriptions)
+        d.addCallback(lambda subscriptions:
+                      self.assertEqual(subscriptions,
+                                       ['this/mbox']))
         return d
 
     def testUnsubscribe(self):
         """
         Test whether we can unsubscribe from a set of mailboxes
         """
-        LeapIMAPServer.theAccount.subscribe('this/mbox')
-        LeapIMAPServer.theAccount.subscribe('that/mbox')
+        acc = LeapIMAPServer.theAccount
+
+        dc1 = lambda: acc.subscribe('this/mbox')
+        dc2 = lambda: acc.subscribe('that/mbox')
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -490,22 +509,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         def unsubscribe():
             return self.client.unsubscribe('this/mbox')
 
+        def get_subscriptions(ignored):
+            return LeapIMAPServer.theAccount.getSubscriptions()
+
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(dc1))
+        d1.addCallback(strip(dc2))
         d1.addCallbacks(strip(unsubscribe), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
-        d.addCallback(lambda _:
-                      self.assertEqual(
-                          LeapIMAPServer.theAccount.subscriptions,
-                          ['that/mbox']))
+        d.addCallback(get_subscriptions)
+        d.addCallback(lambda subscriptions:
+                      self.assertEqual(subscriptions,
+                                       ['that/mbox']))
         return d
 
     def testSelect(self):
         """
         Try to select a mailbox
         """
-        self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42)
+        acc = self.server.theAccount
+        d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42)
         self.selectedArgs = None
 
         def login():
@@ -520,6 +545,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return d
 
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(d0))
         d1.addCallback(strip(select))
         d1.addErrback(self._ebGeneral)
 
@@ -754,13 +780,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                       '\\Deleted', '\\Draft', '\\Recent', 'List'),
             'READ-WRITE': False})
 
-    def _listSetup(self, f):
-        LeapIMAPServer.theAccount.addMailbox('root/subthingl',
-                                             creation_ts=42)
-        LeapIMAPServer.theAccount.addMailbox('root/another-thing',
-                                             creation_ts=42)
-        LeapIMAPServer.theAccount.addMailbox('non-root/subthing',
-                                             creation_ts=42)
+    def _listSetup(self, f, f2=None):
+        acc = LeapIMAPServer.theAccount
+
+        dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42)
+        dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42)
+        dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42)
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -770,6 +795,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.listed = None
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(dc1))
+        d1.addCallback(strip(dc2))
+        d1.addCallback(strip(dc3))
+
+        if f2 is not None:
+            d1.addCallback(f2)
+
         d1.addCallbacks(strip(f), self._ebGeneral)
         d1.addCallbacks(listed, self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -786,7 +818,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d.addCallback(lambda listed: self.assertEqual(
             sortNest(listed),
             sortNest([
-                (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"),
+                (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"),
                 (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing")
             ])
         ))
@@ -796,20 +828,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test LSub command
         """
-        LeapIMAPServer.theAccount.subscribe('root/subthingl2')
+        acc = LeapIMAPServer.theAccount
+
+        def subs_mailbox():
+            # why not client.subscribe instead?
+            return acc.subscribe('root/subthing')
 
         def lsub():
             return self.client.lsub('root', '%')
-        d = self._listSetup(lsub)
+
+        d = self._listSetup(lsub, strip(subs_mailbox))
         d.addCallback(self.assertEqual,
-                      [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")])
+                      [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")])
         return d
 
     def testStatus(self):
         """
         Test Status command
         """
-        LeapIMAPServer.theAccount.addMailbox('root/subthings')
+        acc = LeapIMAPServer.theAccount
+
+        def add_mailbox():
+            return acc.addMailbox('root/subthings')
+
         # XXX FIXME ---- should populate this a little bit,
         # with unseen etc...
 
@@ -824,7 +865,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             self.statused = result
 
         self.statused = None
-        d1 = self.connected.addCallback(strip(login))
+
+        d1 = self.connected.addCallback(strip(add_mailbox))
+        d1.addCallback(strip(login))
         d1.addCallbacks(strip(status), self._ebGeneral)
         d1.addCallbacks(statused, self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -930,7 +973,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test partially appending a message to the mailbox
         """
         infile = util.sibpath(__file__, 'rfc822.message')
-        LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
+        d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -946,6 +989,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                 )
             )
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(d0))
         d1.addCallbacks(strip(append), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -995,10 +1039,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test closing the mailbox. We expect to get deleted all messages flagged
         as such.
         """
+        acc = self.server.theAccount
         name = 'mailbox-close'
-        self.server.theAccount.addMailbox(name)
 
-        m = LeapIMAPServer.theAccount.getMailbox(name)
+        d0 = lambda: acc.addMailbox(name)
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -1006,14 +1050,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         def select():
             return self.client.select(name)
 
+        def get_mailbox():
+            self.mailbox = LeapIMAPServer.theAccount.getMailbox(name)
+
         def add_messages():
-            d1 = m.messages.add_msg(
+            d1 = self.mailbox.messages.add_msg(
                 'test 1', subject="Message 1",
                 flags=('\\Deleted', 'AnotherFlag'))
-            d2 = m.messages.add_msg(
+            d2 = self.mailbox.messages.add_msg(
                 'test 2', subject="Message 2",
                 flags=('AnotherFlag',))
-            d3 = m.messages.add_msg(
+            d3 = self.mailbox.messages.add_msg(
                 'test 3', subject="Message 3",
                 flags=('\\Deleted',))
             d = defer.gatherResults([d1, d2, d3])
@@ -1023,30 +1070,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return self.client.close()
 
         d = self.connected.addCallback(strip(login))
+        d.addCallback(strip(d0))
         d.addCallbacks(strip(select), self._ebGeneral)
+        d.addCallback(strip(get_mailbox))
         d.addCallbacks(strip(add_messages), self._ebGeneral)
         d.addCallbacks(strip(close), self._ebGeneral)
         d.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
-        return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m)
+        return defer.gatherResults([d, d2]).addCallback(self._cbTestClose)
 
-    def _cbTestClose(self, ignored, m):
-        self.assertEqual(len(m.messages), 1)
-        msg = m.messages.get_msg_by_uid(2)
+    def _cbTestClose(self, ignored):
+        self.assertEqual(len(self.mailbox.messages), 1)
+        msg = self.mailbox.messages.get_msg_by_uid(2)
         self.assertTrue(msg is not None)
 
         self.assertEqual(
             dict(msg.hdoc.content)['subject'],
             'Message 2')
-        self.failUnless(m.closed)
+        self.failUnless(self.mailbox.closed)
 
     def testExpunge(self):
         """
         Test expunge command
         """
+        acc = self.server.theAccount
         name = 'mailbox-expunge'
-        self.server.theAccount.addMailbox(name)
-        m = LeapIMAPServer.theAccount.getMailbox(name)
+
+        d0 = lambda: acc.addMailbox(name)
 
         def login():
             return self.client.login(TEST_USER, TEST_PASSWD)
@@ -1054,14 +1104,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         def select():
             return self.client.select('mailbox-expunge')
 
+        def get_mailbox():
+            self.mailbox = LeapIMAPServer.theAccount.getMailbox(name)
+
         def add_messages():
-            d1 = m.messages.add_msg(
+            d1 = self.mailbox.messages.add_msg(
                 'test 1', subject="Message 1",
                 flags=('\\Deleted', 'AnotherFlag'))
-            d2 = m.messages.add_msg(
+            d2 = self.mailbox.messages.add_msg(
                 'test 2', subject="Message 2",
                 flags=('AnotherFlag',))
-            d3 = m.messages.add_msg(
+            d3 = self.mailbox.messages.add_msg(
                 'test 3', subject="Message 3",
                 flags=('\\Deleted',))
             d = defer.gatherResults([d1, d2, d3])
@@ -1076,21 +1129,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.results = None
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallback(strip(d0))
         d1.addCallbacks(strip(select), self._ebGeneral)
+        d1.addCallback(strip(get_mailbox))
         d1.addCallbacks(strip(add_messages), self._ebGeneral)
         d1.addCallbacks(strip(expunge), self._ebGeneral)
         d1.addCallbacks(expunged, self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
-        return d.addCallback(self._cbTestExpunge, m)
+        return d.addCallback(self._cbTestExpunge)
 
-    def _cbTestExpunge(self, ignored, m):
+    def _cbTestExpunge(self, ignored):
         # we only left 1 mssage with no deleted flag
-        self.assertEqual(len(m.messages), 1)
-        msg = m.messages.get_msg_by_uid(2)
+        self.assertEqual(len(self.mailbox.messages), 1)
+        msg = self.mailbox.messages.get_msg_by_uid(2)
 
-        msg = list(m.messages)[0]
+        msg = list(self.mailbox.messages)[0]
         self.assertTrue(msg is not None)
 
         self.assertEqual(
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index 5339acf..9a3868c 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -139,31 +139,32 @@ class IMAP4HelperMixin(BaseLeapTest):
 
         ###########
 
-        d = defer.Deferred()
+        d_server_ready = defer.Deferred()
+
         self.server = LeapIMAPServer(
             uuid=UUID, userid=USERID,
             contextFactory=self.serverCTX,
-            # XXX do we really need this??
             soledad=self._soledad)
 
-        self.client = SimpleClient(d, contextFactory=self.clientCTX)
-        self.connected = d
-
-        # XXX REVIEW-ME.
-        # We're adding theAccount here to server
-        # but it was also passed to initialization
-        # as it was passed to realm.
-        # I THINK we ONLY need to do it at one place now.
+        self.client = SimpleClient(
+            d_server_ready, contextFactory=self.clientCTX)
 
         theAccount = SoledadBackedAccount(
             USERID,
             soledad=self._soledad,
             memstore=memstore)
+        d_account_ready = theAccount.callWhenReady(lambda r: None)
         LeapIMAPServer.theAccount = theAccount
 
+        self.connected = defer.gatherResults(
+            [d_server_ready, d_account_ready])
+
+        # XXX FIXME --------------------------------------------
+        # XXX this needs to be done differently,
+        # have to be hooked on initialization callback instead.
         # in case we get something from previous tests...
-        for mb in self.server.theAccount.mailboxes:
-            self.server.theAccount.delete(mb)
+        #for mb in self.server.theAccount.mailboxes:
+            #self.server.theAccount.delete(mb)
 
         # email parser
         self.parser = parser.Parser()