Merge remote-tracking branch 'refs/remotes/kali/feature/split_message_docs' into develop

20 files changed, 4250 insertions, 1999 deletions

diff --git a/mail/changes/feature_split_message_docs b/mail/changes/feature_split_message_docs
new file mode 100644
index 0000000..0109501
--- /dev/null
+++ b/mail/changes/feature_split_message_docs
@@ -0,0 +1,7 @@
+  o Defer costly operations to a pool of threads.
+  o Split the internal representation of messages into three distinct documents:
+    1) Flags 2) Headers 3) Content.
+  o Make use of the Twisted MIME interface.
+  o Add deduplication ability to the save operation, for body and attachments.
+  o Add IMessageCopier interface to mailbox implementation, so bulk moves
+    are costless. Closes: #4654
diff --git a/mail/src/leap/mail/decorators.py b/mail/src/leap/mail/decorators.py
new file mode 100644
index 0000000..d5eac97
--- /dev/null
+++ b/mail/src/leap/mail/decorators.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# decorators.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Useful decorators for mail package.
+"""
+import logging
+import os
+
+from functools import wraps
+
+from twisted.internet.threads import deferToThread
+
+logger = logging.getLogger(__name__)
+
+
+# TODO
+# Should write a helper to be able to pass a timeout argument.
+# See this answer: http://stackoverflow.com/a/19019648/1157664
+# And the notes by glyph and jpcalderone
+
+def deferred(f):
+    """
+    Decorator, for deferring methods to Threads.
+
+    It will do a deferToThread of the decorated method
+    unless the environment variable LEAPMAIL_DEBUG is set.
+
+    It uses a descriptor to delay the definition of the
+    method wrapper.
+    """
+    class descript(object):
+        """
+        The class to be used as decorator.
+
+        It takes any method as the passed object.
+        """
+
+        def __init__(self, f):
+            """
+            Initializes the decorator object.
+
+            :param f: the decorated function
+            :type f: callable
+            """
+            self.f = f
+
+        def __get__(self, instance, klass):
+            """
+            Descriptor implementation.
+
+            At creation time, the decorated `method` is unbound.
+
+            It will dispatch the make_unbound method if we still do not
+            have an instance available, and the make_bound method when the
+            method has already been bound to the instance.
+
+            :param instance: the instance of the class, or None if not exist.
+            :type instance: instantiated class or None.
+            """
+            if instance is None:
+                # Class method was requested
+                return self.make_unbound(klass)
+            return self.make_bound(instance)
+
+        def _errback(self, failure):
+            """
+            Errorback that logs the exception catched.
+
+            :param failure: a twisted failure
+            :type failure: Failure
+            """
+            logger.warning('Error in method: %s' % (self.f.__name__))
+            logger.exception(failure.getTraceback())
+
+        def make_unbound(self, klass):
+            """
+            Return a wrapped function with the unbound call, during the
+            early access to the decortad method. This gets passed
+            only the class (not the instance since it does not yet exist).
+
+            :param klass: the class to which the still unbound method belongs
+            :type klass: type
+            """
+
+            @wraps(self.f)
+            def wrapper(*args, **kwargs):
+                """
+                We're temporarily wrapping the decorated method, but this
+                should not be called, since our application should use
+                the bound-wrapped method after this decorator class has been
+                used.
+
+                This documentation will vanish at runtime.
+                """
+                raise TypeError(
+                    'unbound method {}() must be called with {} instance '
+                    'as first argument (got nothing instead)'.format(
+                        self.f.__name__,
+                        klass.__name__)
+                )
+            return wrapper
+
+        def make_bound(self, instance):
+            """
+            Return a function that wraps the bound method call,
+            after we are able to access the instance object.
+
+            :param instance: an instance of the class the decorated method,
+                             now bound, belongs to.
+            :type instance: object
+            """
+
+            @wraps(self.f)
+            def wrapper(*args, **kwargs):
+                """
+                Do a proper function wrapper that defers the decorated method
+                call to a separated thread if the LEAPMAIL_DEBUG
+                environment variable is set.
+
+                This documentation will vanish at runtime.
+                """
+                if not os.environ.get('LEAPMAIL_DEBUG'):
+                    d = deferToThread(self.f, instance, *args, **kwargs)
+                    d.addErrback(self._errback)
+                    return d
+                else:
+                    return self.f(instance, *args, **kwargs)
+
+            # This instance does not need the descriptor anymore,
+            # let it find the wrapper directly next time:
+            setattr(instance, self.f.__name__, wrapper)
+            return wrapper
+
+    return descript(f)
diff --git a/mail/src/leap/mail/imap/account.py b/mail/src/leap/mail/imap/account.py
new file mode 100644
index 0000000..fd861e7
--- /dev/null
+++ b/mail/src/leap/mail/imap/account.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+# account.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Backed Account.
+"""
+import copy
+import time
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.imap.index import IndexedDB
+from leap.mail.imap.fields import WithMsgFields
+from leap.mail.imap.parser import MBoxParser
+from leap.mail.imap.mailbox import SoledadMailbox
+from leap.soledad.client import Soledad
+
+
+#######################################
+# Soledad Account
+#######################################
+
+
+class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+    """
+    An implementation of IAccount and INamespacePresenteer
+    that is backed by Soledad Encrypted Documents.
+    """
+
+    implements(imap4.IAccount, imap4.INamespacePresenter)
+
+    _soledad = None
+    selected = None
+
+    def __init__(self, account_name, soledad=None):
+        """
+        Creates a SoledadAccountIndex that keeps track of the mailboxes
+        and subscriptions handled by this account.
+
+        :param acct_name: The name of the account (user id).
+        :type acct_name: str
+
+        :param soledad: a Soledad instance.
+        :param soledad: Soledad
+        """
+        leap_assert(soledad, "Need a soledad instance to initialize")
+        leap_assert_type(soledad, Soledad)
+
+        # XXX SHOULD assert too that the name matches the user/uuid with which
+        # soledad has been initialized.
+
+        self._account_name = self._parse_mailbox_name(account_name)
+        self._soledad = soledad
+
+        self.initialize_db()
+
+        # every user should have the right to an inbox folder
+        # at least, so let's make one!
+
+        if not self.mailboxes:
+            self.addMailbox(self.INBOX_NAME)
+
+    def _get_empty_mailbox(self):
+        """
+        Returns an empty mailbox.
+
+        :rtype: dict
+        """
+        return copy.deepcopy(self.EMPTY_MBOX)
+
+    def _get_mailbox_by_name(self, name):
+        """
+        Return an mbox document by name.
+
+        :param name: the name of the mailbox
+        :type name: str
+
+        :rtype: SoledadDocument
+        """
+        doc = self._soledad.get_from_index(
+            self.TYPE_MBOX_IDX, self.MBOX_KEY,
+            self._parse_mailbox_name(name))
+        return doc[0] if doc else None
+
+    @property
+    def mailboxes(self):
+        """
+        A list of the current mailboxes for this account.
+        """
+        return [doc.content[self.MBOX_KEY]
+                for doc in self._soledad.get_from_index(
+                    self.TYPE_IDX, self.MBOX_KEY)]
+
+    @property
+    def subscriptions(self):
+        """
+        A list of the current subscriptions for this account.
+        """
+        return [doc.content[self.MBOX_KEY]
+                for doc in self._soledad.get_from_index(
+                    self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
+
+    def getMailbox(self, name):
+        """
+        Returns a Mailbox with that name, without selecting it.
+
+        :param name: name of the mailbox
+        :type name: str
+
+        :returns: a a SoledadMailbox instance
+        :rtype: SoledadMailbox
+        """
+        name = self._parse_mailbox_name(name)
+
+        if name not in self.mailboxes:
+            raise imap4.MailboxException("No such mailbox")
+
+        return SoledadMailbox(name, soledad=self._soledad)
+
+    ##
+    ## IAccount
+    ##
+
+    def addMailbox(self, name, creation_ts=None):
+        """
+        Add a mailbox to the account.
+
+        :param name: the name of the mailbox
+        :type name: str
+
+        :param creation_ts: an optional creation timestamp to be used as
+                            mailbox id. A timestamp will be used if no
+                            one is provided.
+        :type creation_ts: int
+
+        :returns: True if successful
+        :rtype: bool
+        """
+        name = self._parse_mailbox_name(name)
+
+        if name in self.mailboxes:
+            raise imap4.MailboxCollision, name
+
+        if not creation_ts:
+            # by default, we pass an int value
+            # taken from the current time
+            # we make sure to take enough decimals to get a unique
+            # mailbox-uidvalidity.
+            creation_ts = int(time.time() * 10E2)
+
+        mbox = self._get_empty_mailbox()
+        mbox[self.MBOX_KEY] = name
+        mbox[self.CREATED_KEY] = creation_ts
+
+        doc = self._soledad.create_doc(mbox)
+        return bool(doc)
+
+    def create(self, pathspec):
+        """
+        Create a new mailbox from the given hierarchical name.
+
+        :param pathspec: The full hierarchical name of a new mailbox to create.
+                         If any of the inferior hierarchical names to this one
+                         do not exist, they are created as well.
+        :type pathspec: str
+
+        :return: A true value if the creation succeeds.
+        :rtype: bool
+
+        :raise MailboxException: Raised if this mailbox cannot be added.
+        """
+        # TODO raise MailboxException
+        paths = filter(
+            None,
+            self._parse_mailbox_name(pathspec).split('/'))
+        for accum in range(1, len(paths)):
+            try:
+                self.addMailbox('/'.join(paths[:accum]))
+            except imap4.MailboxCollision:
+                pass
+        try:
+            self.addMailbox('/'.join(paths))
+        except imap4.MailboxCollision:
+            if not pathspec.endswith('/'):
+                return False
+        return True
+
+    def select(self, name, readwrite=1):
+        """
+        Selects a mailbox.
+
+        :param name: the mailbox to select
+        :type name: str
+
+        :param readwrite: 1 for readwrite permissions.
+        :type readwrite: int
+
+        :rtype: bool
+        """
+        name = self._parse_mailbox_name(name)
+
+        if name not in self.mailboxes:
+            return None
+
+        self.selected = name
+
+        return SoledadMailbox(
+            name, rw=readwrite,
+            soledad=self._soledad)
+
+    def delete(self, name, force=False):
+        """
+        Deletes a mailbox.
+
+        Right now it does not purge the messages, but just removes the mailbox
+        name from the mailboxes list!!!
+
+        :param name: the mailbox to be deleted
+        :type name: str
+
+        :param force: if True, it will not check for noselect flag or inferior
+                      names. use with care.
+        :type force: bool
+        """
+        name = self._parse_mailbox_name(name)
+
+        if not name in self.mailboxes:
+            raise imap4.MailboxException("No such mailbox")
+
+        mbox = self.getMailbox(name)
+
+        if force is False:
+            # See if this box is flagged \Noselect
+            # XXX use mbox.flags instead?
+            if self.NOSELECT_FLAG in mbox.getFlags():
+                # Check for hierarchically inferior mailboxes with this one
+                # as part of their root.
+                for others in self.mailboxes:
+                    if others != name and others.startswith(name):
+                        raise imap4.MailboxException, (
+                            "Hierarchically inferior mailboxes "
+                            "exist and \\Noselect is set")
+        mbox.destroy()
+
+        # XXX FIXME --- not honoring the inferior names...
+
+        # if there are no hierarchically inferior names, we will
+        # delete it from our ken.
+        #if self._inferiorNames(name) > 1:
+            # ??! -- can this be rite?
+            #self._index.removeMailbox(name)
+
+    def rename(self, oldname, newname):
+        """
+        Renames a mailbox.
+
+        :param oldname: old name of the mailbox
+        :type oldname: str
+
+        :param newname: new name of the mailbox
+        :type newname: str
+        """
+        oldname = self._parse_mailbox_name(oldname)
+        newname = self._parse_mailbox_name(newname)
+
+        if oldname not in self.mailboxes:
+            raise imap4.NoSuchMailbox, oldname
+
+        inferiors = self._inferiorNames(oldname)
+        inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
+
+        for (old, new) in inferiors:
+            if new in self.mailboxes:
+                raise imap4.MailboxCollision, new
+
+        for (old, new) in inferiors:
+            mbox = self._get_mailbox_by_name(old)
+            mbox.content[self.MBOX_KEY] = new
+            self._soledad.put_doc(mbox)
+
+        # XXX ---- FIXME!!!! ------------------------------------
+        # until here we just renamed the index...
+        # We have to rename also the occurrence of this
+        # mailbox on ALL the messages that are contained in it!!!
+        # ... we maybe could use a reference to the doc_id
+        # in each msg, instead of the "mbox" field in msgs
+        # -------------------------------------------------------
+
+    def _inferiorNames(self, name):
+        """
+        Return hierarchically inferior mailboxes.
+
+        :param name: name of the mailbox
+        :rtype: list
+        """
+        # XXX use wildcard query instead
+        inferiors = []
+        for infname in self.mailboxes:
+            if infname.startswith(name):
+                inferiors.append(infname)
+        return inferiors
+
+    def isSubscribed(self, name):
+        """
+        Returns True if user is subscribed to this mailbox.
+
+        :param name: the mailbox to be checked.
+        :type name: str
+
+        :rtype: bool
+        """
+        mbox = self._get_mailbox_by_name(name)
+        return mbox.content.get('subscribed', False)
+
+    def _set_subscription(self, name, value):
+        """
+        Sets the subscription value for a given mailbox
+
+        :param name: the mailbox
+        :type name: str
+
+        :param value: the boolean value
+        :type value: bool
+        """
+        # maybe we should store subscriptions in another
+        # document...
+        if not name in self.mailboxes:
+            self.addMailbox(name)
+        mbox = self._get_mailbox_by_name(name)
+
+        if mbox:
+            mbox.content[self.SUBSCRIBED_KEY] = value
+            self._soledad.put_doc(mbox)
+
+    def subscribe(self, name):
+        """
+        Subscribe to this mailbox
+
+        :param name: name of the mailbox
+        :type name: str
+        """
+        name = self._parse_mailbox_name(name)
+        if name not in self.subscriptions:
+            self._set_subscription(name, True)
+
+    def unsubscribe(self, name):
+        """
+        Unsubscribe from this mailbox
+
+        :param name: name of the mailbox
+        :type name: str
+        """
+        name = self._parse_mailbox_name(name)
+        if name not in self.subscriptions:
+            raise imap4.MailboxException, "Not currently subscribed to " + name
+        self._set_subscription(name, False)
+
+    def listMailboxes(self, ref, wildcard):
+        """
+        List the mailboxes.
+
+        from rfc 3501:
+        returns a subset of names from the complete set
+        of all names available to the client.  Zero or more untagged LIST
+        replies are returned, containing the name attributes, hierarchy
+        delimiter, and name.
+
+        :param ref: reference name
+        :type ref: str
+
+        :param wildcard: mailbox name with possible wildcards
+        :type wildcard: str
+        """
+        # XXX use wildcard in index query
+        ref = self._inferiorNames(
+            self._parse_mailbox_name(ref))
+        wildcard = imap4.wildcardToRegexp(wildcard, '/')
+        return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+
+    ##
+    ## INamespacePresenter
+    ##
+
+    def getPersonalNamespaces(self):
+        return [["", "/"]]
+
+    def getSharedNamespaces(self):
+        return None
+
+    def getOtherNamespaces(self):
+        return None
+
+    # extra, for convenience
+
+    def deleteAllMessages(self, iknowhatiamdoing=False):
+        """
+        Deletes all messages from all mailboxes.
+        Danger! high voltage!
+
+        :param iknowhatiamdoing: confirmation parameter, needs to be True
+                                 to proceed.
+        """
+        if iknowhatiamdoing is True:
+            for mbox in self.mailboxes:
+                self.delete(mbox, force=True)
+
+    def __repr__(self):
+        """
+        Representation string for this object.
+        """
+        return "<SoledadBackedAccount (%s)>" % self._account_name
diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index b1c34ba..604a2ea 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -17,21 +17,24 @@
 """
 Incoming mail fetcher.
 """
-import logging
+import copy
 import json
-import ssl
+import logging
+#import ssl
 import threading
 import time
-import copy
-from StringIO import StringIO
+import sys
+import traceback
 
 from email.parser import Parser
 from email.generator import Generator
 from email.utils import parseaddr
+from StringIO import StringIO
 
 from twisted.python import log
+from twisted.internet import defer
 from twisted.internet.task import LoopingCall
-from twisted.internet.threads import deferToThread
+#from twisted.internet.threads import deferToThread
 from zope.proxy import sameProxiedObjects
 
 from leap.common import events as leap_events
@@ -45,12 +48,18 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
 from leap.keymanager.openpgp import OpenPGPKey
+from leap.mail.decorators import deferred
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
 
 logger = logging.getLogger(__name__)
 
+MULTIPART_ENCRYPTED = "multipart/encrypted"
+MULTIPART_SIGNED = "multipart/signed"
+PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
+PGP_END = "-----END PGP MESSAGE-----"
+
 
 class MalformedMessage(Exception):
     """
@@ -125,6 +134,9 @@ class LeapIncomingMail(object):
 
         self._create_soledad_indexes()
 
+        # initialize a mail parser only once
+        self._parser = Parser()
+
     def _create_soledad_indexes(self):
         """
         Create needed indexes on soledad.
@@ -152,9 +164,10 @@ class LeapIncomingMail(object):
         logger.debug("fetching mail for: %s %s" % (
             self._soledad.uuid, self._userid))
         if not self.fetching_lock.locked():
-            d = deferToThread(self._sync_soledad)
-            d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error)
-            d.addCallbacks(self._process_doclist, self._sync_soledad_error)
+            d1 = self._sync_soledad()
+            d = defer.gatherResults([d1], consumeErrors=True)
+            d.addCallbacks(self._signal_fetch_to_ui, self._errback)
+            d.addCallbacks(self._signal_unread_to_ui, self._errback)
             return d
         else:
             logger.debug("Already fetching mail.")
@@ -184,6 +197,11 @@ class LeapIncomingMail(object):
 
     # synchronize incoming mail
 
+    def _errback(self, failure):
+        logger.exception(failure.value)
+        traceback.print_tb(*sys.exc_info())
+
+    @deferred
     def _sync_soledad(self):
         """
         Synchronizes with remote soledad.
@@ -196,10 +214,9 @@ class LeapIncomingMail(object):
             self._soledad.sync()
             log.msg('soledad synced.')
             doclist = self._soledad.get_from_index("just-mail", "*")
+        self._process_doclist(doclist)
 
-        return doclist
-
-    def _signal_unread_to_ui(self):
+    def _signal_unread_to_ui(self, *args):
         """
         Sends unread event to ui.
         """
@@ -215,53 +232,18 @@ class LeapIncomingMail(object):
         :returns: doclist
         :rtype: iterable
         """
+        doclist = doclist[0]  # gatherResults pass us a list
         fetched_ts = time.mktime(time.gmtime())
-        num_mails = len(doclist)
-        log.msg("there are %s mails" % (num_mails,))
+        num_mails = len(doclist) if doclist is not None else 0
+        if num_mails != 0:
+            log.msg("there are %s mails" % (num_mails,))
         leap_events.signal(
             IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts))
-        self._signal_unread_to_ui()
         return doclist
 
-    def _sync_soledad_error(self, failure):
-        """
-        Errback for sync errors.
-        """
-        # XXX should signal unrecoverable maybe.
-        err = failure.value
-        logger.error("error syncing soledad: %s" % (err,))
-        if failure.check(ssl.SSLError):
-            logger.warning('SSL Error while '
-                           'syncing soledad: %r' % (err,))
-        elif failure.check(Exception):
-            logger.warning('Unknown error while '
-                           'syncing soledad: %r' % (err,))
-
-    def _log_err(self, failure):
-        """
-        Generic errback
-        """
-        err = failure.value
-        logger.exception("error!: %r" % (err,))
-
-    def _decryption_error(self, failure):
-        """
-        Errback for decryption errors.
-        """
-        # XXX should signal unrecoverable maybe.
-        err = failure.value
-        logger.error("error decrypting msg: %s" % (err,))
-
-    def _saving_error(self, failure):
-        """
-        Errback for local save errors.
-        """
-        # XXX should signal unrecoverable maybe.
-        err = failure.value
-        logger.error("error saving msg locally: %s" % (err,))
-
     # process incoming mail.
 
+    @defer.inlineCallbacks
     def _process_doclist(self, doclist):
         """
         Iterates through the doclist, checks if each doc
@@ -278,7 +260,6 @@ class LeapIncomingMail(object):
             return
         num_mails = len(doclist)
 
-        docs_cb = []
         for index, doc in enumerate(doclist):
             logger.debug("processing doc %d of %d" % (index + 1, num_mails))
             leap_events.signal(
@@ -287,35 +268,18 @@ class LeapIncomingMail(object):
             if self._is_msg(keys):
                 # Ok, this looks like a legit msg.
                 # Let's process it!
-                # Deferred chain for individual messages
-
-                # XXX use an IConsumer instead... ?
-                d = deferToThread(self._decrypt_doc, doc)
-                d.addCallback(self._process_decrypted_doc)
-                d.addErrback(self._log_err)
-                d.addCallback(self._add_message_locally)
-                d.addErrback(self._log_err)
-                docs_cb.append(d)
+                decrypted = list(self._decrypt_doc(doc))[0]
+                res = self._add_message_locally(decrypted)
+                yield res
+
             else:
                 # Ooops, this does not.
                 logger.debug('This does not look like a proper msg.')
-        return docs_cb
 
     #
     # operations on individual messages
     #
 
-    def _is_msg(self, keys):
-        """
-        Checks if the keys of a dictionary match the signature
-        of the document type we use for messages.
-
-        :param keys: iterable containing the strings to match.
-        :type keys: iterable of strings.
-        :rtype: bool
-        """
-        return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
-
     def _decrypt_doc(self, doc):
         """
         Decrypt the contents of a document.
@@ -339,7 +303,9 @@ class LeapIncomingMail(object):
             logger.error("Error while decrypting msg: %r" % (exc,))
             decrdata = ""
         leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0")
-        return doc, decrdata
+
+        data = list(self._process_decrypted_doc((doc, decrdata)))
+        yield (doc, data)
 
     def _process_decrypted_doc(self, msgtuple):
         """
@@ -357,16 +323,15 @@ class LeapIncomingMail(object):
         doc, data = msgtuple
         msg = json.loads(data)
         if not isinstance(msg, dict):
-            return False
+            defer.returnValue(False)
         if not msg.get(self.INCOMING_KEY, False):
-            return False
+            defer.returnValue(False)
 
         # ok, this is an incoming message
         rawmsg = msg.get(self.CONTENT_KEY, None)
         if not rawmsg:
             return False
-        data = self._maybe_decrypt_msg(rawmsg)
-        return doc, data
+        return self._maybe_decrypt_msg(rawmsg)
 
     def _maybe_decrypt_msg(self, data):
         """
@@ -381,17 +346,16 @@ class LeapIncomingMail(object):
         leap_assert_type(data, unicode)
 
         # parse the original message
-        parser = Parser()
         encoding = get_email_charset(data)
         data = data.encode(encoding)
-        msg = parser.parsestr(data)
+        msg = self._parser.parsestr(data)
 
         # try to obtain sender public key
         senderPubkey = None
         fromHeader = msg.get('from', None)
-        if fromHeader is not None \
-                and (msg.get_content_type() == 'multipart/encrypted' \
-                     or msg.get_content_type() == 'multipart/signed'):
+        if (fromHeader is not None
+            and (msg.get_content_type() == MULTIPART_ENCRYPTED
+                 or msg.get_content_type() == MULTIPART_SIGNED)):
             _, senderAddress = parseaddr(fromHeader)
             try:
                 senderPubkey = self._keymanager.get_key_from_cache(
@@ -400,11 +364,14 @@ class LeapIncomingMail(object):
                 pass
 
         valid_sig = False  # we will add a header saying if sig is valid
-        if msg.get_content_type() == 'multipart/encrypted':
-            decrmsg, valid_sig = self._decrypt_multipart_encrypted_msg(
+        decrypt_multi = self._decrypt_multipart_encrypted_msg
+        decrypt_inline = self._maybe_decrypt_inline_encrypted_msg
+
+        if msg.get_content_type() == MULTIPART_ENCRYPTED:
+            decrmsg, valid_sig = decrypt_multi(
                 msg, encoding, senderPubkey)
         else:
-            decrmsg, valid_sig = self._maybe_decrypt_inline_encrypted_msg(
+            decrmsg, valid_sig = decrypt_inline(
                 msg, encoding, senderPubkey)
 
         # add x-leap-signature header
@@ -419,7 +386,7 @@ class LeapIncomingMail(object):
                 self.LEAP_SIGNATURE_INVALID,
                 pubkey=senderPubkey.key_id)
 
-        return decrmsg.as_string()
+        yield decrmsg.as_string()
 
     def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey):
         """
@@ -437,25 +404,12 @@ class LeapIncomingMail(object):
         """
         log.msg('decrypting multipart encrypted msg')
         msg = copy.deepcopy(msg)
-        # sanity check
-        payload = msg.get_payload()
-        if len(payload) != 2:
-            raise MalformedMessage(
-                'Multipart/encrypted messages should have exactly 2 body '
-                'parts (instead of %d).' % len(payload))
-        if payload[0].get_content_type() != 'application/pgp-encrypted':
-            raise MalformedMessage(
-                "Multipart/encrypted messages' first body part should "
-                "have content type equal to 'application/pgp-encrypted' "
-                "(instead of %s)." % payload[0].get_content_type())
-        if payload[1].get_content_type() != 'application/octet-stream':
-            raise MalformedMessage(
-                "Multipart/encrypted messages' second body part should "
-                "have content type equal to 'octet-stream' (instead of "
-                "%s)." % payload[1].get_content_type())
+        self._msg_multipart_sanity_check(msg)
+
         # parse message and get encrypted content
         pgpencmsg = msg.get_payload()[1]
         encdata = pgpencmsg.get_payload()
+
         # decrypt or fail gracefully
         try:
             decrdata, valid_sig = self._decrypt_and_verify_data(
@@ -463,17 +417,20 @@ class LeapIncomingMail(object):
         except keymanager_errors.DecryptError as e:
             logger.warning('Failed to decrypt encrypted message (%s). '
                            'Storing message without modifications.' % str(e))
-            return msg, False  # return original message
+            # Bailing out!
+            return (msg, False)
+
         # decrypted successully, now fix encoding and parse
         try:
             decrdata = decrdata.encode(encoding)
         except (UnicodeEncodeError, UnicodeDecodeError) as e:
             logger.error("Unicode error {0}".format(e))
             decrdata = decrdata.encode(encoding, 'replace')
-        parser = Parser()
-        decrmsg = parser.parsestr(decrdata)
+
+        decrmsg = self._parser.parsestr(decrdata)
         # remove original message's multipart/encrypted content-type
         del(msg['content-type'])
+
         # replace headers back in original message
         for hkey, hval in decrmsg.items():
             try:
@@ -481,9 +438,10 @@ class LeapIncomingMail(object):
                 msg.replace_header(hkey, hval)
             except KeyError:
                 msg[hkey] = hval
-        # replace payload by unencrypted payload
+
+        # all ok, replace payload by unencrypted payload
         msg.set_payload(decrmsg.get_payload())
-        return msg, valid_sig
+        return (msg, valid_sig)
 
     def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
                                             senderPubkey):
@@ -497,8 +455,9 @@ class LeapIncomingMail(object):
         :param senderPubkey: The key of the sender of the message.
         :type senderPubkey: OpenPGPKey
 
-        :return: A unitary tuple containing a decrypted message.
-        :rtype: (Message)
+        :return: A tuple containing a decrypted message and
+                 a bool indicating whether the signature is valid.
+        :rtype: (Message, bool)
         """
         log.msg('maybe decrypting inline encrypted msg')
         # serialize the original message
@@ -507,8 +466,6 @@ class LeapIncomingMail(object):
         g.flatten(origmsg)
         data = buf.getvalue()
         # handle exactly one inline PGP message
-        PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
-        PGP_END = "-----END PGP MESSAGE-----"
         valid_sig = False
         if PGP_BEGIN in data:
             begin = data.find(PGP_BEGIN)
@@ -522,11 +479,11 @@ class LeapIncomingMail(object):
             except keymanager_errors.DecryptError:
                 logger.warning('Failed to decrypt potential inline encrypted '
                                'message. Storing message as is...')
+
         # if message is not encrypted, return raw data
         if isinstance(data, unicode):
             data = data.encode(encoding, 'replace')
-        parser = Parser()
-        return parser.parsestr(data), valid_sig
+        return (self._parser.parsestr(data), valid_sig)
 
     def _decrypt_and_verify_data(self, data, senderPubkey):
         """
@@ -555,7 +512,7 @@ class LeapIncomingMail(object):
         except keymanager_errors.InvalidSignature:
             decrdata = self._keymanager.decrypt(
                 data, self._pkey)
-        return decrdata, valid_sig
+        return (decrdata, valid_sig)
 
     def _add_message_locally(self, msgtuple):
         """
@@ -570,10 +527,55 @@ class LeapIncomingMail(object):
         """
         log.msg('adding message to local db')
         doc, data = msgtuple
-        self._inbox.addMessage(data, (self.RECENT_FLAG,))
+
+        if isinstance(data, list):
+            data = data[0]
+
+        self._inbox.addMessage(data, flags=(self.RECENT_FLAG,))
+
         leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
         doc_id = doc.doc_id
         self._soledad.delete_doc(doc)
         log.msg("deleted doc %s from incoming" % doc_id)
         leap_events.signal(IMAP_MSG_DELETED_INCOMING)
         self._signal_unread_to_ui()
+        return True
+
+    #
+    # helpers
+    #
+
+    def _msg_multipart_sanity_check(self, msg):
+        """
+        Performs a sanity check against a multipart encrypted msg
+
+        :param msg: The original encrypted message.
+        :type msg: Message
+        """
+        # sanity check
+        payload = msg.get_payload()
+        if len(payload) != 2:
+            raise MalformedMessage(
+                'Multipart/encrypted messages should have exactly 2 body '
+                'parts (instead of %d).' % len(payload))
+        if payload[0].get_content_type() != 'application/pgp-encrypted':
+            raise MalformedMessage(
+                "Multipart/encrypted messages' first body part should "
+                "have content type equal to 'application/pgp-encrypted' "
+                "(instead of %s)." % payload[0].get_content_type())
+        if payload[1].get_content_type() != 'application/octet-stream':
+            raise MalformedMessage(
+                "Multipart/encrypted messages' second body part should "
+                "have content type equal to 'octet-stream' (instead of "
+                "%s)." % payload[1].get_content_type())
+
+    def _is_msg(self, keys):
+        """
+        Checks if the keys of a dictionary match the signature
+        of the document type we use for messages.
+
+        :param keys: iterable containing the strings to match.
+        :type keys: iterable of strings.
+        :rtype: bool
+        """
+        return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
diff --git a/mail/src/leap/mail/imap/fields.py b/mail/src/leap/mail/imap/fields.py
new file mode 100644
index 0000000..2545adf
--- /dev/null
+++ b/mail/src/leap/mail/imap/fields.py
@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+# fields.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Fields for Mailbox and Message.
+"""
+from leap.mail.imap.parser import MBoxParser
+
+
+class WithMsgFields(object):
+    """
+    Container class for class-attributes to be shared by
+    several message-related classes.
+    """
+    # indexing
+    CONTENT_HASH_KEY = "chash"
+    PAYLOAD_HASH_KEY = "phash"
+
+    # Internal representation of Message
+
+    # flags doc
+    UID_KEY = "uid"
+    MBOX_KEY = "mbox"
+    SEEN_KEY = "seen"
+    DEL_KEY = "deleted"
+    RECENT_KEY = "recent"
+    FLAGS_KEY = "flags"
+    MULTIPART_KEY = "multi"
+    SIZE_KEY = "size"
+
+    # headers
+    HEADERS_KEY = "headers"
+    DATE_KEY = "date"
+    SUBJECT_KEY = "subject"
+    # XXX DELETE-ME
+    #NUM_PARTS_KEY = "numparts"  # not needed?!
+    PARTS_MAP_KEY = "part_map"
+    BODY_KEY = "body"  # link to phash of body
+
+    # content
+    LINKED_FROM_KEY = "lkf"
+    RAW_KEY = "raw"
+    CTYPE_KEY = "ctype"
+
+    # Mailbox specific keys
+    CLOSED_KEY = "closed"
+    CREATED_KEY = "created"
+    SUBSCRIBED_KEY = "subscribed"
+    RW_KEY = "rw"
+    LAST_UID_KEY = "lastuid"
+
+    # Document Type, for indexing
+    TYPE_KEY = "type"
+    TYPE_MBOX_VAL = "mbox"
+    TYPE_FLAGS_VAL = "flags"
+    TYPE_HEADERS_VAL = "head"
+    TYPE_CONTENT_VAL = "cnt"
+
+    # XXX DEPRECATE
+    #TYPE_MESSAGE_VAL = "msg"
+    #TYPE_ATTACHMENT_VAL = "attach"
+
+    INBOX_VAL = "inbox"
+
+    # Flags in Mailbox and Message
+    SEEN_FLAG = "\\Seen"
+    RECENT_FLAG = "\\Recent"
+    ANSWERED_FLAG = "\\Answered"
+    FLAGGED_FLAG = "\\Flagged"  # yo dawg
+    DELETED_FLAG = "\\Deleted"
+    DRAFT_FLAG = "\\Draft"
+    NOSELECT_FLAG = "\\Noselect"
+    LIST_FLAG = "List"  # is this OK? (no \. ie, no system flag)
+
+    # Fields in mail object
+    SUBJECT_FIELD = "Subject"
+    DATE_FIELD = "Date"
+
+    # Index  types
+    # --------------
+
+    TYPE_IDX = 'by-type'
+    TYPE_MBOX_IDX = 'by-type-and-mbox'
+    TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
+    TYPE_SUBS_IDX = 'by-type-and-subscribed'
+    TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+    TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+    TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+    TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+    TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+    TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
+
+    # Tomas created the `recent and seen index`, but the semantic is not too
+    # correct since the recent flag is volatile.
+    TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
+
+    KTYPE = TYPE_KEY
+    MBOX_VAL = TYPE_MBOX_VAL
+    CHASH_VAL = CONTENT_HASH_KEY
+    PHASH_VAL = PAYLOAD_HASH_KEY
+
+    INDEXES = {
+        # generic
+        TYPE_IDX: [KTYPE],
+        TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
+        TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
+
+        # mailboxes
+        TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
+
+        # content, headers doc
+        TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
+
+        # attachment payload dedup
+        TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
+
+        # messages
+        TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
+        TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
+        TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
+        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
+                                  'bool(recent)', 'bool(seen)'],
+    }
+
+    MBOX_KEY = MBOX_VAL
+
+    EMPTY_MBOX = {
+        TYPE_KEY: MBOX_KEY,
+        TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
+        SUBJECT_KEY: "",
+        FLAGS_KEY: [],
+        CLOSED_KEY: False,
+        SUBSCRIBED_KEY: False,
+        RW_KEY: 1,
+        LAST_UID_KEY: 0
+    }
+
+fields = WithMsgFields  # alias for convenience
diff --git a/mail/src/leap/mail/imap/index.py b/mail/src/leap/mail/imap/index.py
new file mode 100644
index 0000000..5f0919a
--- /dev/null
+++ b/mail/src/leap/mail/imap/index.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# index.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Index for SoledadBackedAccount, Mailbox and Messages.
+"""
+import logging
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail.imap.fields import fields
+
+
+logger = logging.getLogger(__name__)
+
+
+class IndexedDB(object):
+    """
+    Methods dealing with the index.
+
+    This is a MixIn that needs access to the soledad instance,
+    and also assumes that a INDEXES attribute is accessible to the instance.
+
+    INDEXES must be a dictionary of type:
+    {'index-name': ['field1', 'field2']}
+    """
+    # TODO we might want to move this to soledad itself, check
+
+    def initialize_db(self):
+        """
+        Initialize the database.
+        """
+        leap_assert(self._soledad,
+                    "Need a soledad attribute accesible in the instance")
+        leap_assert_type(self.INDEXES, dict)
+
+        # Ask the database for currently existing indexes.
+        if not self._soledad:
+            logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
+            return
+        db_indexes = dict()
+        if self._soledad is not None:
+            db_indexes = dict(self._soledad.list_indexes())
+        for name, expression in fields.INDEXES.items():
+            if name not in db_indexes:
+                # The index does not yet exist.
+                self._soledad.create_index(name, *expression)
+                continue
+
+            if expression == db_indexes[name]:
+                # The index exists and is up to date.
+                continue
+            # The index exists but the definition is not what expected, so we
+            # delete it and add the proper index expression.
+            self._soledad.delete_index(name)
+            self._soledad.create_index(name, *expression)
diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py
new file mode 100644
index 0000000..7c01490
--- /dev/null
+++ b/mail/src/leap/mail/imap/mailbox.py
@@ -0,0 +1,666 @@
+# *- coding: utf-8 -*-
+# mailbox.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Mailbox.
+"""
+import copy
+import threading
+import logging
+import time
+import StringIO
+import cStringIO
+
+from collections import defaultdict
+
+from twisted.internet import defer
+from twisted.python import log
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common import events as leap_events
+from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
+from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.decorators import deferred
+from leap.mail.imap.fields import WithMsgFields, fields
+from leap.mail.imap.messages import MessageCollection
+from leap.mail.imap.parser import MBoxParser
+
+logger = logging.getLogger(__name__)
+
+
+class SoledadMailbox(WithMsgFields, MBoxParser):
+    """
+    A Soledad-backed IMAP mailbox.
+
+    Implements the high-level method needed for the Mailbox interfaces.
+    The low-level database methods are contained in MessageCollection class,
+    which we instantiate and make accessible in the `messages` attribute.
+    """
+    implements(
+        imap4.IMailbox,
+        imap4.IMailboxInfo,
+        imap4.ICloseableMailbox,
+        imap4.IMessageCopier)
+
+    # XXX should finish the implementation of IMailboxListener
+    # XXX should implement ISearchableMailbox too
+
+    messages = None
+    _closed = False
+
+    INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
+                  WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
+                  WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
+                  WithMsgFields.LIST_FLAG)
+    flags = None
+
+    CMD_MSG = "MESSAGES"
+    CMD_RECENT = "RECENT"
+    CMD_UIDNEXT = "UIDNEXT"
+    CMD_UIDVALIDITY = "UIDVALIDITY"
+    CMD_UNSEEN = "UNSEEN"
+
+    _listeners = defaultdict(set)
+    next_uid_lock = threading.Lock()
+
+    def __init__(self, mbox, soledad=None, rw=1):
+        """
+        SoledadMailbox constructor. Needs to get passed a name, plus a
+        Soledad instance.
+
+        :param mbox: the mailbox name
+        :type mbox: str
+
+        :param soledad: a Soledad instance.
+        :type soledad: Soledad
+
+        :param rw: read-and-write flags
+        :type rw: int
+        """
+        leap_assert(mbox, "Need a mailbox name to initialize")
+        leap_assert(soledad, "Need a soledad instance to initialize")
+
+        # XXX should move to wrapper
+        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
+                    #"soledad._db must be an instance of SQLCipherDatabase")
+
+        self.mbox = self._parse_mailbox_name(mbox)
+        self.rw = rw
+
+        self._soledad = soledad
+
+        self.messages = MessageCollection(
+            mbox=mbox, soledad=self._soledad)
+
+        if not self.getFlags():
+            self.setFlags(self.INIT_FLAGS)
+
+    @property
+    def listeners(self):
+        """
+        Returns listeners for this mbox.
+
+        The server itself is a listener to the mailbox.
+        so we can notify it (and should!) after changes in flags
+        and number of messages.
+
+        :rtype: set
+        """
+        return self._listeners[self.mbox]
+
+    def addListener(self, listener):
+        """
+        Adds a listener to the listeners queue.
+        The server adds itself as a listener when there is a SELECT,
+        so it can send EXIST commands.
+
+        :param listener: listener to add
+        :type listener: an object that implements IMailboxListener
+        """
+        logger.debug('adding mailbox listener: %s' % listener)
+        self.listeners.add(listener)
+
+    def removeListener(self, listener):
+        """
+        Removes a listener from the listeners queue.
+
+        :param listener: listener to remove
+        :type listener: an object that implements IMailboxListener
+        """
+        self.listeners.remove(listener)
+
+    def _get_mbox(self):
+        """
+        Returns mailbox document.
+
+        :return: A SoledadDocument containing this mailbox, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
+        """
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MBOX_IDX,
+                fields.TYPE_MBOX_VAL, self.mbox)
+            if query:
+                return query.pop()
+        except Exception as exc:
+            logger.error("Unhandled error %r" % exc)
+
+    def getFlags(self):
+        """
+        Returns the flags defined for this mailbox.
+
+        :returns: tuple of flags for this mailbox
+        :rtype: tuple of str
+        """
+        mbox = self._get_mbox()
+        if not mbox:
+            return None
+        flags = mbox.content.get(self.FLAGS_KEY, [])
+        return map(str, flags)
+
+    def setFlags(self, flags):
+        """
+        Sets flags for this mailbox.
+
+        :param flags: a tuple with the flags
+        :type flags: tuple of str
+        """
+        leap_assert(isinstance(flags, tuple),
+                    "flags expected to be a tuple")
+        mbox = self._get_mbox()
+        if not mbox:
+            return None
+        mbox.content[self.FLAGS_KEY] = map(str, flags)
+        self._soledad.put_doc(mbox)
+
+    # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
+
+    def _get_closed(self):
+        """
+        Return the closed attribute for this mailbox.
+
+        :return: True if the mailbox is closed
+        :rtype: bool
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.CLOSED_KEY, False)
+
+    def _set_closed(self, closed):
+        """
+        Set the closed attribute for this mailbox.
+
+        :param closed: the state to be set
+        :type closed: bool
+        """
+        leap_assert(isinstance(closed, bool), "closed needs to be boolean")
+        mbox = self._get_mbox()
+        mbox.content[self.CLOSED_KEY] = closed
+        self._soledad.put_doc(mbox)
+
+    closed = property(
+        _get_closed, _set_closed, doc="Closed attribute.")
+
+    def _get_last_uid(self):
+        """
+        Return the last uid for this mailbox.
+
+        :return: the last uid for messages in this mailbox
+        :rtype: bool
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.LAST_UID_KEY, 1)
+
+    def _set_last_uid(self, uid):
+        """
+        Sets the last uid for this mailbox.
+
+        :param uid: the uid to be set
+        :type uid: int
+        """
+        leap_assert(isinstance(uid, int), "uid has to be int")
+        mbox = self._get_mbox()
+        key = self.LAST_UID_KEY
+
+        count = self.getMessageCount()
+
+        # XXX safety-catch. If we do get duplicates,
+        # we want to avoid further duplication.
+
+        if uid >= count:
+            value = uid
+        else:
+            # something is wrong,
+            # just set the last uid
+            # beyond the max msg count.
+            logger.debug("WRONG uid < count. Setting last uid to %s", count)
+            value = count
+
+        mbox.content[key] = value
+        self._soledad.put_doc(mbox)
+
+    last_uid = property(
+        _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
+
+    def getUIDValidity(self):
+        """
+        Return the unique validity identifier for this mailbox.
+
+        :return: unique validity identifier
+        :rtype: int
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.CREATED_KEY, 1)
+
+    def getUID(self, message):
+        """
+        Return the UID of a message in the mailbox
+
+        .. note:: this implementation does not make much sense RIGHT NOW,
+        but in the future will be useful to get absolute UIDs from
+        message sequence numbers.
+
+        :param message: the message uid
+        :type message: int
+
+        :rtype: int
+        """
+        msg = self.messages.get_msg_by_uid(message)
+        return msg.getUID()
+
+    def getUIDNext(self):
+        """
+        Return the likely UID for the next message added to this
+        mailbox. Currently it returns the higher UID incremented by
+        one.
+
+        We increment the next uid *each* time this function gets called.
+        In this way, there will be gaps if the message with the allocated
+        uid cannot be saved. But that is preferable to having race conditions
+        if we get to parallel message adding.
+
+        :rtype: int
+        """
+        with self.next_uid_lock:
+            self.last_uid += 1
+            return self.last_uid
+
+    def getMessageCount(self):
+        """
+        Returns the total count of messages in this mailbox.
+
+        :rtype: int
+        """
+        return self.messages.count()
+
+    def getUnseenCount(self):
+        """
+        Returns the number of messages with the 'Unseen' flag.
+
+        :return: count of messages flagged `unseen`
+        :rtype: int
+        """
+        return self.messages.count_unseen()
+
+    def getRecentCount(self):
+        """
+        Returns the number of messages with the 'Recent' flag.
+
+        :return: count of messages flagged `recent`
+        :rtype: int
+        """
+        return self.messages.count_recent()
+
+    def isWriteable(self):
+        """
+        Get the read/write status of the mailbox.
+
+        :return: 1 if mailbox is read-writeable, 0 otherwise.
+        :rtype: int
+        """
+        return self.rw
+
+    def getHierarchicalDelimiter(self):
+        """
+        Returns the character used to delimite hierarchies in mailboxes.
+
+        :rtype: str
+        """
+        return '/'
+
+    def requestStatus(self, names):
+        """
+        Handles a status request by gathering the output of the different
+        status commands.
+
+        :param names: a list of strings containing the status commands
+        :type names: iter
+        """
+        r = {}
+        if self.CMD_MSG in names:
+            r[self.CMD_MSG] = self.getMessageCount()
+        if self.CMD_RECENT in names:
+            r[self.CMD_RECENT] = self.getRecentCount()
+        if self.CMD_UIDNEXT in names:
+            r[self.CMD_UIDNEXT] = self.last_uid + 1
+        if self.CMD_UIDVALIDITY in names:
+            r[self.CMD_UIDVALIDITY] = self.getUID()
+        if self.CMD_UNSEEN in names:
+            r[self.CMD_UNSEEN] = self.getUnseenCount()
+        return defer.succeed(r)
+
+    def addMessage(self, message, flags, date=None):
+        """
+        Adds a message to this mailbox.
+
+        :param message: the raw message
+        :type message: str
+
+        :param flags: flag list
+        :type flags: list of str
+
+        :param date: timestamp
+        :type date: str
+
+        :return: a deferred that evals to None
+        """
+        if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)):
+            message = message.getvalue()
+        # XXX we should treat the message as an IMessage from here
+        leap_assert_type(message, basestring)
+        uid_next = self.getUIDNext()
+        logger.debug('Adding msg with UID :%s' % uid_next)
+        if flags is None:
+            flags = tuple()
+        else:
+            flags = tuple(str(flag) for flag in flags)
+
+        d = self._do_add_message(message, flags=flags, date=date, uid=uid_next)
+        d.addCallback(self._notify_new)
+        return d
+
+    @deferred
+    def _do_add_message(self, message, flags, date, uid):
+        """
+        Calls to the messageCollection add_msg method (deferred to thread).
+        Invoked from addMessage.
+        """
+        self.messages.add_msg(message, flags=flags, date=date, uid=uid)
+
+    def _notify_new(self, *args):
+        """
+        Notify of new messages to all the listeners.
+
+        :param args: ignored.
+        """
+        exists = self.getMessageCount()
+        recent = self.getRecentCount()
+        logger.debug("NOTIFY: there are %s messages, %s recent" % (
+            exists,
+            recent))
+
+        logger.debug("listeners: %s", str(self.listeners))
+        for l in self.listeners:
+            logger.debug('notifying...')
+            l.newMessages(exists, recent)
+
+    # commands, do not rename methods
+
+    def destroy(self):
+        """
+        Called before this mailbox is permanently deleted.
+
+        Should cleanup resources, and set the \\Noselect flag
+        on the mailbox.
+        """
+        self.setFlags((self.NOSELECT_FLAG,))
+        self.deleteAllDocs()
+
+        # XXX removing the mailbox in situ for now,
+        # we should postpone the removal
+        self._soledad.delete_doc(self._get_mbox())
+
+    def _close_cb(self, result):
+        self.closed = True
+
+    def close(self):
+        """
+        Expunge and mark as closed
+        """
+        d = self.expunge()
+        d.addCallback(self._close_cb)
+        return d
+
+    def _expunge_cb(self, result):
+        return result
+
+    def expunge(self):
+        """
+        Remove all messages flagged \\Deleted
+        """
+        if not self.isWriteable():
+            raise imap4.ReadOnlyMailbox
+        d = self.messages.remove_all_deleted()
+        d.addCallback(self._expunge_cb)
+        return d
+
+    @deferred
+    def fetch(self, messages, uid):
+        """
+        Retrieve one or more messages in this mailbox.
+
+        from rfc 3501: The data items to be fetched can be either a single atom
+        or a parenthesized list.
+
+        :param messages: IDs of the messages to retrieve information about
+        :type messages: MessageSet
+
+        :param uid: If true, the IDs are UIDs. They are message sequence IDs
+                    otherwise.
+        :type uid: bool
+
+        :rtype: A tuple of two-tuples of message sequence numbers and
+                LeapMessage
+        """
+        result = []
+
+        # For the moment our UID is sequential, so we
+        # can treat them all the same.
+        # Change this to the flag that twisted expects when we
+        # switch to content-hash based index + local UID table.
+
+        sequence = False
+        #sequence = True if uid == 0 else False
+
+        if not messages.last:
+            try:
+                iter(messages)
+            except TypeError:
+                # looks like we cannot iterate
+                messages.last = self.last_uid
+
+        # for sequence numbers (uid = 0)
+        if sequence:
+            logger.debug("Getting msg by index: INEFFICIENT call!")
+            raise NotImplementedError
+
+        else:
+            for msg_id in messages:
+                msg = self.messages.get_msg_by_uid(msg_id)
+                if msg:
+                    result.append((msg_id, msg))
+                else:
+                    logger.debug("fetch %s, no msg found!!!" % msg_id)
+
+        if self.isWriteable():
+            self._unset_recent_flag()
+        self._signal_unread_to_ui()
+
+        # XXX workaround for hangs in thunderbird
+        #return tuple(result[:100])  # --- doesn't show all!!
+        return tuple(result)
+
+    @deferred
+    def _unset_recent_flag(self):
+        """
+        Unsets `Recent` flag from a tuple of messages.
+        Called from fetch.
+
+        From RFC, about `Recent`:
+
+        Message is "recently" arrived in this mailbox.  This session
+        is the first session to have been notified about this
+        message; if the session is read-write, subsequent sessions
+        will not see \Recent set for this message.  This flag can not
+        be altered by the client.
+
+        If it is not possible to determine whether or not this
+        session is the first session to be notified about a message,
+        then that message SHOULD be considered recent.
+        """
+        # TODO this fucker, for the sake of correctness, is messing with
+        # the whole collection of flag docs.
+
+        # Possible ways of action:
+        # 1. Ignore it, we want fun.
+        # 2. Trigger it with a delay
+        # 3. Route it through a queue with lesser priority than the
+        #    regularar writer.
+
+        # hmm let's try 2. in a quickndirty way...
+        time.sleep(1)
+        log.msg('unsetting recent flags...')
+        for msg in self.messages.get_recent():
+            msg.removeFlags((fields.RECENT_FLAG,))
+        self._signal_unread_to_ui()
+
+    @deferred
+    def _signal_unread_to_ui(self):
+        """
+        Sends unread event to ui.
+        """
+        unseen = self.getUnseenCount()
+        leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
+
+    @deferred
+    def store(self, messages, flags, mode, uid):
+        """
+        Sets the flags of one or more messages.
+
+        :param messages: The identifiers of the messages to set the flags
+        :type messages: A MessageSet object with the list of messages requested
+
+        :param flags: The flags to set, unset, or add.
+        :type flags: sequence of str
+
+        :param mode: If mode is -1, these flags should be removed from the
+                     specified messages.  If mode is 1, these flags should be
+                     added to the specified messages.  If mode is 0, all
+                     existing flags should be cleared and these flags should be
+                     added.
+        :type mode: -1, 0, or 1
+
+        :param uid: If true, the IDs specified in the query are UIDs;
+                    otherwise they are message sequence IDs.
+        :type uid: bool
+
+        :return: A dict mapping message sequence numbers to sequences of
+                 str representing the flags set on the message after this
+                 operation has been performed.
+        :rtype: dict
+
+        :raise ReadOnlyMailbox: Raised if this mailbox is not open for
+                                read-write.
+        """
+        # XXX implement also sequence (uid = 0)
+        # XXX we should prevent cclient from setting Recent flag.
+        leap_assert(not isinstance(flags, basestring),
+                    "flags cannot be a string")
+        flags = tuple(flags)
+
+        if not self.isWriteable():
+            log.msg('read only mailbox!')
+            raise imap4.ReadOnlyMailbox
+
+        if not messages.last:
+            messages.last = self.messages.count()
+
+        result = {}
+        for msg_id in messages:
+            log.msg("MSG ID = %s" % msg_id)
+            msg = self.messages.get_msg_by_uid(msg_id)
+            if not msg:
+                return result
+            if mode == 1:
+                msg.addFlags(flags)
+            elif mode == -1:
+                msg.removeFlags(flags)
+            elif mode == 0:
+                msg.setFlags(flags)
+            result[msg_id] = msg.getFlags()
+
+        self._signal_unread_to_ui()
+        return result
+
+    # IMessageCopier
+
+    @deferred
+    def copy(self, messageObject):
+        """
+        Copy the given message object into this mailbox.
+        """
+        uid_next = self.getUIDNext()
+        msg = messageObject
+
+        # XXX should use a public api instead
+        fdoc = msg._fdoc
+        if not fdoc:
+            logger.debug("Tried to copy a MSG with no fdoc")
+            return
+
+        new_fdoc = copy.deepcopy(fdoc.content)
+        new_fdoc[self.UID_KEY] = uid_next
+        new_fdoc[self.MBOX_KEY] = self.mbox
+
+        d = self._do_add_doc(new_fdoc)
+        d.addCallback(self._notify_new)
+
+    @deferred
+    def _do_add_doc(self, doc):
+        """
+        Defers the adding of a new doc.
+        :param doc: document to be created in soledad.
+        """
+        self._soledad.create_doc(doc)
+
+    # convenience fun
+
+    def deleteAllDocs(self):
+        """
+        Deletes all docs in this mailbox
+        """
+        docs = self.messages.get_all_docs()
+        for doc in docs:
+            self.messages._soledad.delete_doc(doc)
+
+    def __repr__(self):
+        """
+        Representation string for this mailbox.
+        """
+        return u"<SoledadMailbox: mbox '%s' (%s)>" % (
+            self.mbox, self.messages.count())
diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py
new file mode 100644
index 0000000..37e4311
--- /dev/null
+++ b/mail/src/leap/mail/imap/messages.py
@@ -0,0 +1,1346 @@
+# -*- coding: utf-8 -*-
+# messages.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+LeapMessage and MessageCollection.
+"""
+import copy
+import logging
+import StringIO
+
+from collections import defaultdict, namedtuple
+
+from twisted.mail import imap4
+from twisted.internet import defer
+from twisted.python import log
+from u1db import errors as u1db_errors
+from zope.interface import implements
+from zope.proxy import sameProxiedObjects
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.decorators import memoized_method
+from leap.common.mail import get_email_charset
+from leap.mail import walk
+from leap.mail.utils import first
+from leap.mail.decorators import deferred
+from leap.mail.imap.index import IndexedDB
+from leap.mail.imap.fields import fields, WithMsgFields
+from leap.mail.imap.parser import MailParser, MBoxParser
+from leap.mail.messageflow import IMessageConsumer, MessageProducer
+
+logger = logging.getLogger(__name__)
+
+
+# TODO ------------------------------------------------------------
+
+# [ ] Add linked-from info.
+# [ ] Delete incoming mail only after successful write!
+# [ ] Remove UID from syncable db. Store only those indexes locally.
+# [ ] Send patch to twisted for bug in imap4.py:5717  (content-type can be
+#     none? lower-case?)
+
+def lowerdict(_dict):
+    """
+    Return a dict with the keys in lowercase.
+
+    :param _dict: the dict to convert
+    :rtype: dict
+    """
+    return dict((key.lower(), value)
+                for key, value in _dict.items())
+
+
+class MessagePart(object):
+    """
+    IMessagePart implementor.
+    It takes a subpart message and is able to find
+    the inner parts.
+
+    Excusatio non petita: see the interface documentation.
+    """
+
+    implements(imap4.IMessagePart)
+
+    def __init__(self, soledad, part_map):
+        """
+        Initializes the MessagePart.
+
+        :param part_map: a dictionary containing the parts map for this
+                         message
+        :type part_map: dict
+        """
+        # TODO
+        # It would be good to pass the uid/mailbox also
+        # for references while debugging.
+
+        # We have a problem on bulk moves, and is
+        # that when the fetch on the new mailbox is done
+        # the parts maybe are not complete.
+        # So we should be able to fail with empty
+        # docs until we solve that. The ideal would be
+        # to gather the results of the deferred operations
+        # to signal the operation is complete.
+        #leap_assert(part_map, "part map dict cannot be null")
+        self._soledad = soledad
+        self._pmap = part_map
+
+    def getSize(self):
+        """
+        Return the total size, in octets, of this message part.
+
+        :return: size of the message, in octets
+        :rtype: int
+        """
+        if not self._pmap:
+            return 0
+        size = self._pmap.get('size', None)
+        if not size:
+            logger.error("Message part cannot find size in the partmap")
+        return size
+
+    def getBodyFile(self):
+        """
+        Retrieve a file object containing only the body of this message.
+
+        :return: file-like object opened for reading
+        :rtype: StringIO
+        """
+        fd = StringIO.StringIO()
+        if self._pmap:
+            multi = self._pmap.get('multi')
+            if not multi:
+                phash = self._pmap.get("phash", None)
+            else:
+                pmap = self._pmap.get('part_map')
+                first_part = pmap.get('1', None)
+                if first_part:
+                    phash = first_part['phash']
+
+            if not phash:
+                logger.warning("Could not find phash for this subpart!")
+                payload = str("")
+            else:
+                payload = self._get_payload_from_document(phash)
+
+        else:
+            logger.warning("Message with no part_map!")
+            payload = str("")
+
+        if payload:
+            #headers = self.getHeaders(True)
+            #headers = lowerdict(headers)
+            #content_type = headers.get('content-type', "")
+            content_type = self._get_ctype_from_document(phash)
+            charset_split = content_type.split('charset=')
+            # XXX fuck all this, use a regex!
+            if len(charset_split) > 1:
+                charset = charset_split[1]
+                if charset:
+                    charset = charset.strip()
+            else:
+                charset = None
+            if not charset:
+                charset = self._get_charset(payload)
+            try:
+                payload = payload.encode(charset)
+            except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                logger.error("Unicode error {0}".format(e))
+                payload = payload.encode(charset, 'replace')
+
+        fd.write(payload)
+        fd.seek(0)
+        return fd
+
+    # TODO cache the phash retrieval
+    def _get_payload_from_document(self, phash):
+        """
+        Gets the message payload from the content document.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: basestring
+        """
+        cdocs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+
+        cdoc = first(cdocs)
+        if not cdoc:
+            logger.warning(
+                "Could not find the content doc "
+                "for phash %s" % (phash,))
+        payload = cdoc.content.get(fields.RAW_KEY, "")
+        return payload
+
+    # TODO cache the pahash retrieval
+    def _get_ctype_from_document(self, phash):
+        """
+        Gets the content-type from the content document.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: basestring
+        """
+        cdocs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+
+        cdoc = first(cdocs)
+        if not cdoc:
+            logger.warning(
+                "Could not find the content doc "
+                "for phash %s" % (phash,))
+        ctype = cdoc.content.get('ctype', "")
+        return ctype
+
+    @memoized_method
+    def _get_charset(self, stuff):
+        # TODO put in a common class with LeapMessage
+        """
+        Gets (guesses?) the charset of a payload.
+
+        :param stuff: the stuff to guess about.
+        :type stuff: basestring
+        :returns: charset
+        """
+        # XXX existential doubt 2. shouldn't we make the scope
+        # of the decorator somewhat more persistent?
+        # ah! yes! and put memory bounds.
+        return get_email_charset(unicode(stuff))
+
+    def getHeaders(self, negate, *names):
+        """
+        Retrieve a group of message headers.
+
+        :param names: The names of the headers to retrieve or omit.
+        :type names: tuple of str
+
+        :param negate: If True, indicates that the headers listed in names
+                       should be omitted from the return value, rather
+                       than included.
+        :type negate: bool
+
+        :return: A mapping of header field names to header field values
+        :rtype: dict
+        """
+        if not self._pmap:
+            logger.warning("No pmap in Subpart!")
+            return {}
+        headers = dict(self._pmap.get("headers", []))
+
+        # twisted imap server expects *some* headers to be lowercase
+        # We could use a CaseInsensitiveDict here...
+        headers = dict(
+            (str(key), str(value)) if key.lower() != "content-type"
+            else (str(key.lower()), str(value))
+            for (key, value) in headers.items())
+
+        names = map(lambda s: s.upper(), names)
+        if negate:
+            cond = lambda key: key.upper() not in names
+        else:
+            cond = lambda key: key.upper() in names
+
+        # unpack and filter original dict by negate-condition
+        filter_by_cond = [
+            map(str, (key, val)) for
+            key, val in headers.items()
+            if cond(key)]
+        filtered = dict(filter_by_cond)
+        return filtered
+
+    def isMultipart(self):
+        """
+        Return True if this message is multipart.
+        """
+        if not self._pmap:
+            logger.warning("Could not get part map!")
+            return False
+        multi = self._pmap.get("multi", False)
+        return multi
+
+    def getSubPart(self, part):
+        """
+        Retrieve a MIME submessage
+
+        :type part: C{int}
+        :param part: The number of the part to retrieve, indexed from 0.
+        :raise IndexError: Raised if the specified part does not exist.
+        :raise TypeError: Raised if this message is not multipart.
+        :rtype: Any object implementing C{IMessagePart}.
+        :return: The specified sub-part.
+        """
+        if not self.isMultipart():
+            raise TypeError
+        sub_pmap = self._pmap.get("part_map", {})
+        try:
+            part_map = sub_pmap[str(part + 1)]
+        except KeyError:
+            logger.debug("getSubpart for %s: KeyError" % (part,))
+            raise IndexError
+
+        # XXX check for validity
+        return MessagePart(self._soledad, part_map)
+
+
+class LeapMessage(fields, MailParser, MBoxParser):
+    """
+    The main representation of a message.
+
+    It indexes the messages in one mailbox by a combination
+    of uid+mailbox name.
+    """
+
+    # TODO this has to change.
+    # Should index primarily by chash, and keep a local-lonly
+    # UID table.
+
+    implements(imap4.IMessage)
+
+    def __init__(self, soledad, uid, mbox):
+        """
+        Initializes a LeapMessage.
+
+        :param soledad: a Soledad instance
+        :type soledad: Soledad
+        :param uid: the UID for the message.
+        :type uid: int or basestring
+        :param mbox: the mbox this message belongs to
+        :type mbox: basestring
+        """
+        MailParser.__init__(self)
+        self._soledad = soledad
+        self._uid = int(uid)
+        self._mbox = self._parse_mailbox_name(mbox)
+
+        self.__chash = None
+        self.__bdoc = None
+
+    @property
+    def _fdoc(self):
+        """
+        An accessor to the flags document.
+        """
+        if all(map(bool, (self._uid, self._mbox))):
+            fdoc = self._get_flags_doc()
+            if fdoc:
+                self.__chash = fdoc.content.get(
+                    fields.CONTENT_HASH_KEY, None)
+            return fdoc
+
+    @property
+    def _chash(self):
+        """
+        An accessor to the content hash for this message.
+        """
+        if not self._fdoc:
+            return None
+        if not self.__chash and self._fdoc:
+            self.__chash = self._fdoc.content.get(
+                fields.CONTENT_HASH_KEY, None)
+        return self.__chash
+
+    @property
+    def _hdoc(self):
+        """
+        An accessor to the headers document.
+        """
+        return self._get_headers_doc()
+
+    @property
+    def _bdoc(self):
+        """
+        An accessor to the body document.
+        """
+        if not self._hdoc:
+            return None
+        if not self.__bdoc:
+            self.__bdoc = self._get_body_doc()
+        return self.__bdoc
+
+    # IMessage implementation
+
+    def getUID(self):
+        """
+        Retrieve the unique identifier associated with this message
+
+        :return: uid for this message
+        :rtype: int
+        """
+        return self._uid
+
+    def getFlags(self):
+        """
+        Retrieve the flags associated with this message
+
+        :return: The flags, represented as strings
+        :rtype: tuple
+        """
+        if self._uid is None:
+            return []
+
+        flags = []
+        fdoc = self._fdoc
+        if fdoc:
+            flags = fdoc.content.get(self.FLAGS_KEY, None)
+        if flags:
+            flags = map(str, flags)
+        return tuple(flags)
+
+    # setFlags, addFlags, removeFlags are not in the interface spec
+    # but we use them with store command.
+
+    def setFlags(self, flags):
+        """
+        Sets the flags for this message
+
+        Returns a SoledadDocument that needs to be updated by the caller.
+
+        :param flags: the flags to update in the message.
+        :type flags: tuple of str
+
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
+        """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+        log.msg('setting flags: %s' % (self._uid))
+
+        doc = self._fdoc
+        if not doc:
+            logger.warning(
+                "Could not find FDOC for %s:%s while setting flags!" %
+                (self._mbox, self._uid))
+            return
+        doc.content[self.FLAGS_KEY] = flags
+        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+        doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
+        doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
+        self._soledad.put_doc(doc)
+
+    def addFlags(self, flags):
+        """
+        Adds flags to this message.
+
+        Returns a SoledadDocument that needs to be updated by the caller.
+
+        :param flags: the flags to add to the message.
+        :type flags: tuple of str
+
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
+        """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+        oldflags = self.getFlags()
+        self.setFlags(tuple(set(flags + oldflags)))
+
+    def removeFlags(self, flags):
+        """
+        Remove flags from this message.
+
+        Returns a SoledadDocument that needs to be updated by the caller.
+
+        :param flags: the flags to be removed from the message.
+        :type flags: tuple of str
+
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
+        """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+        oldflags = self.getFlags()
+        self.setFlags(tuple(set(oldflags) - set(flags)))
+
+    def getInternalDate(self):
+        """
+        Retrieve the date internally associated with this message
+
+        :rtype: C{str}
+        :return: An RFC822-formatted date string.
+        """
+        return str(self._hdoc.content.get(self.DATE_KEY, ''))
+
+    #
+    # IMessagePart
+    #
+
+    # XXX we should implement this interface too for the subparts
+    # so we allow nested parts...
+
+    def getBodyFile(self):
+        """
+        Retrieve a file object containing only the body of this message.
+
+        :return: file-like object opened for reading
+        :rtype: StringIO
+        """
+        fd = StringIO.StringIO()
+        bdoc = self._bdoc
+        if bdoc:
+            body = str(self._bdoc.content.get(self.RAW_KEY, ""))
+        else:
+            logger.warning("No BDOC found for message.")
+            body = str("")
+
+        # XXX not needed, isn't it? ---- ivan?
+        #if bdoc:
+            #content_type = bdoc.content.get('content-type', "")
+            #charset = content_type.split('charset=')[1]
+            #if charset:
+                #charset = charset.strip()
+            #if not charset:
+                #charset = self._get_charset(body)
+            #try:
+                #body = str(body.encode(charset))
+            #except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                #logger.error("Unicode error {0}".format(e))
+                #body = str(body.encode(charset, 'replace'))
+
+        fd.write(body)
+        fd.seek(0)
+        return fd
+
+    @memoized_method
+    def _get_charset(self, stuff):
+        """
+        Gets (guesses?) the charset of a payload.
+
+        :param stuff: the stuff to guess about.
+        :type stuff: basestring
+        :returns: charset
+        """
+        # TODO get from subpart headers
+        # XXX existential doubt 2. shouldn't we make the scope
+        # of the decorator somewhat more persistent?
+        # ah! yes! and put memory bounds.
+        return get_email_charset(unicode(stuff))
+
+    def getSize(self):
+        """
+        Return the total size, in octets, of this message.
+
+        :return: size of the message, in octets
+        :rtype: int
+        """
+        size = None
+        if self._fdoc:
+            size = self._fdoc.content.get(self.SIZE_KEY, False)
+        else:
+            logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
+                                                       self._uid))
+        if not size:
+            # XXX fallback, should remove when all migrated.
+            size = self.getBodyFile().len
+        return size
+
+    def getHeaders(self, negate, *names):
+        """
+        Retrieve a group of message headers.
+
+        :param names: The names of the headers to retrieve or omit.
+        :type names: tuple of str
+
+        :param negate: If True, indicates that the headers listed in names
+                       should be omitted from the return value, rather
+                       than included.
+        :type negate: bool
+
+        :return: A mapping of header field names to header field values
+        :rtype: dict
+        """
+        # TODO split in smaller methods
+        headers = self._get_headers()
+        if not headers:
+            logger.warning("No headers found")
+            return {str('content-type'): str('')}
+
+        names = map(lambda s: s.upper(), names)
+        if negate:
+            cond = lambda key: key.upper() not in names
+        else:
+            cond = lambda key: key.upper() in names
+
+        if isinstance(headers, list):
+            headers = dict(headers)
+
+        # twisted imap server expects *some* headers to be lowercase
+        # XXX refactor together with MessagePart method
+        headers = dict(
+            (str(key), str(value)) if key.lower() != "content-type"
+            else (str(key.lower()), str(value))
+            for (key, value) in headers.items())
+
+        # unpack and filter original dict by negate-condition
+        filter_by_cond = [(key, val) for key, val
+                          in headers.items() if cond(key)]
+
+        return dict(filter_by_cond)
+
+    def _get_headers(self):
+        """
+        Return the headers dict for this message.
+        """
+        if self._hdoc is not None:
+            headers = self._hdoc.content.get(self.HEADERS_KEY, {})
+            return headers
+
+        else:
+            logger.warning(
+                "No HEADERS doc for msg %s:%s" % (
+                    self._mbox,
+                    self._uid))
+
+    def isMultipart(self):
+        """
+        Return True if this message is multipart.
+        """
+        if self._fdoc:
+            is_multipart = self._fdoc.content.get(self.MULTIPART_KEY, False)
+            return is_multipart
+        else:
+            logger.warning(
+                "No FLAGS doc for msg %s:%s" % (
+                    self._mbox,
+                    self._uid))
+
+    def getSubPart(self, part):
+        """
+        Retrieve a MIME submessage
+
+        :type part: C{int}
+        :param part: The number of the part to retrieve, indexed from 0.
+        :raise IndexError: Raised if the specified part does not exist.
+        :raise TypeError: Raised if this message is not multipart.
+        :rtype: Any object implementing C{IMessagePart}.
+        :return: The specified sub-part.
+        """
+        if not self.isMultipart():
+            raise TypeError
+        try:
+            pmap_dict = self._get_part_from_parts_map(part + 1)
+        except KeyError:
+            logger.debug("getSubpart for %s: KeyError" % (part,))
+            raise IndexError
+        return MessagePart(self._soledad, pmap_dict)
+
+    #
+    # accessors
+    #
+
+    def _get_part_from_parts_map(self, part):
+        """
+        Get a part map from the headers doc
+
+        :raises: KeyError if key does not exist
+        :rtype: dict
+        """
+        if not self._hdoc:
+            logger.warning("Tried to get part but no HDOC found!")
+            return None
+
+        pmap = self._hdoc.content.get(fields.PARTS_MAP_KEY, {})
+        return pmap[str(part)]
+
+    def _get_flags_doc(self):
+        """
+        Return the document that keeps the flags for this
+        message.
+        """
+        flag_docs = self._soledad.get_from_index(
+            fields.TYPE_MBOX_UID_IDX,
+            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+        return first(flag_docs)
+
+    def _get_headers_doc(self):
+        """
+        Return the document that keeps the headers for this
+        message.
+        """
+        head_docs = self._soledad.get_from_index(
+            fields.TYPE_C_HASH_IDX,
+            fields.TYPE_HEADERS_VAL, str(self._chash))
+        return first(head_docs)
+
+    def _get_body_doc(self):
+        """
+        Return the document that keeps the body for this
+        message.
+        """
+        body_phash = self._hdoc.content.get(
+            fields.BODY_KEY, None)
+        if not body_phash:
+            logger.warning("No body phash for this document!")
+            return None
+        body_docs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(body_phash))
+
+        return first(body_docs)
+
+    def __getitem__(self, key):
+        """
+        Return an item from the content of the flags document,
+        for convenience.
+
+        :param key: The key
+        :type key: str
+
+        :return: The content value indexed by C{key} or None
+        :rtype: str
+        """
+        return self._fdoc.content.get(key, None)
+
+    # setters
+
+    # XXX to be used in the messagecopier interface?!
+
+    def set_uid(self, uid):
+        """
+        Set new uid for this message.
+
+        :param uid: the new uid
+        :type uid: basestring
+        """
+        # XXX dangerous! lock?
+        self._uid = uid
+        d = self._fdoc
+        d.content[self.UID_KEY] = uid
+        self._soledad.put_doc(d)
+
+    def set_mbox(self, mbox):
+        """
+        Set new mbox for this message.
+
+        :param mbox: the new mbox
+        :type mbox: basestring
+        """
+        # XXX dangerous! lock?
+        self._mbox = mbox
+        d = self._fdoc
+        d.content[self.MBOX_KEY] = mbox
+        self._soledad.put_doc(d)
+
+    # destructor
+
+    @deferred
+    def remove(self):
+        """
+        Remove all docs associated with this message.
+        """
+        # XXX For the moment we are only removing the flags and headers
+        # docs. The rest we leave there polluting your hard disk,
+        # until we think about a good way of deorphaning.
+        # Maybe a crawler of unreferenced docs.
+
+        # XXX implement elijah's idea of using a PUT document as a
+        # token to ensure consistency in the removal.
+
+        uid = self._uid
+
+        fd = self._get_flags_doc()
+        #hd = self._get_headers_doc()
+        #bd = self._get_body_doc()
+        #docs = [fd, hd, bd]
+
+        docs = [fd]
+
+        for d in filter(None, docs):
+            try:
+                self._soledad.delete_doc(d)
+            except Exception as exc:
+                logger.error(exc)
+        return uid
+
+    def does_exist(self):
+        """
+        Return True if there is actually a flags message for this
+        UID and mbox.
+        """
+        return self._fdoc is not None
+
+
+SoledadWriterPayload = namedtuple(
+    'SoledadWriterPayload', ['mode', 'payload'])
+
+# TODO we could consider using enum here:
+# https://pypi.python.org/pypi/enum
+
+SoledadWriterPayload.CREATE = 1
+SoledadWriterPayload.PUT = 2
+SoledadWriterPayload.CONTENT_CREATE = 3
+
+
+class SoledadDocWriter(object):
+    """
+    This writer will create docs serially in the local soledad database.
+    """
+
+    implements(IMessageConsumer)
+
+    def __init__(self, soledad):
+        """
+        Initialize the writer.
+
+        :param soledad: the soledad instance
+        :type soledad: Soledad
+        """
+        self._soledad = soledad
+
+    def _get_call_for_item(self, item):
+        """
+        Return the proper call type for a given item.
+
+        :param item: one of the types defined under the
+                     attributes of SoledadWriterPayload
+        :type item: int
+        """
+        call = None
+        payload = item.payload
+
+        if item.mode == SoledadWriterPayload.CREATE:
+            call = self._soledad.create_doc
+        elif (item.mode == SoledadWriterPayload.CONTENT_CREATE
+              and not self._content_does_exist(payload)):
+                call = self._soledad.create_doc
+        elif item.mode == SoledadWriterPayload.PUT:
+            call = self._soledad.put_doc
+        return call
+
+    def _process(self, queue):
+        """
+        Return the item and the proper call type for the next
+        item in the queue if any.
+
+        :param queue: the queue from where we'll pick item.
+        :type queue: Queue
+        """
+        item = queue.get()
+        call = self._get_call_for_item(item)
+        return item, call
+
+    def consume(self, queue):
+        """
+        Creates a new document in soledad db.
+
+        :param queue: queue to get item from, with content of the document
+                      to be inserted.
+        :type queue: Queue
+        """
+        empty = queue.empty()
+        while not empty:
+            item, call = self._process(queue)
+
+            if call:
+                # XXX should handle the delete case
+                # should handle errors
+                try:
+                    call(item.payload)
+                except u1db_errors.RevisionConflict as exc:
+                    logger.error("Error: %r" % (exc,))
+                    raise exc
+
+            empty = queue.empty()
+
+    """
+    Message deduplication.
+
+    We do a query for the content hashes before writing to our beloved
+    sqlcipher backend of Soledad. This means, by now, that:
+
+    1. We will not store the same attachment twice, only the hash of it.
+    2. We will not store the same message body twice, only the hash of it.
+
+    The first case is useful if you are always receiving the same old memes
+    from unwary friends that still have not discovered that 4chan is the
+    generator of the internet. The second will save your day if you have
+    initiated session with the same account in two different machines. I also
+    wonder why would you do that, but let's respect each other choices, like
+    with the religious celebrations, and assume that one day we'll be able
+    to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
+    Stack.
+    """
+
+    def _content_does_exist(self, doc):
+        """
+        Check whether we already have a content document for a payload
+        with this hash in our database.
+
+        :param doc: tentative body document
+        :type doc: dict
+        :returns: True if that happens, False otherwise.
+        """
+        if not doc:
+            return False
+        phash = doc[fields.PAYLOAD_HASH_KEY]
+        attach_docs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+        if not attach_docs:
+            return False
+
+        if len(attach_docs) != 1:
+            logger.warning("Found more than one copy of phash %s!"
+                           % (phash,))
+        logger.debug("Found attachment doc with that hash! Skipping save!")
+        return True
+
+
+class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
+    """
+    A collection of messages, surprisingly.
+
+    It is tied to a selected mailbox name that is passed to constructor.
+    Implements a filter query over the messages contained in a soledad
+    database.
+    """
+    # XXX this should be able to produce a MessageSet methinks
+    # could validate these kinds of objects turning them
+    # into a template for the class.
+    FLAGS_DOC = "FLAGS"
+    HEADERS_DOC = "HEADERS"
+    CONTENT_DOC = "CONTENT"
+
+    templates = {
+
+        FLAGS_DOC: {
+            fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
+            fields.UID_KEY: 1,  # XXX moe to a local table
+            fields.MBOX_KEY: fields.INBOX_VAL,
+            fields.CONTENT_HASH_KEY: "",
+
+            fields.SEEN_KEY: False,
+            fields.RECENT_KEY: True,
+            fields.DEL_KEY: False,
+            fields.FLAGS_KEY: [],
+            fields.MULTIPART_KEY: False,
+            fields.SIZE_KEY: 0
+        },
+
+        HEADERS_DOC: {
+            fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
+            fields.CONTENT_HASH_KEY: "",
+
+            fields.DATE_KEY: "",
+            fields.SUBJECT_KEY: "",
+
+            fields.HEADERS_KEY: {},
+            fields.PARTS_MAP_KEY: {},
+        },
+
+        CONTENT_DOC: {
+            fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
+            fields.PAYLOAD_HASH_KEY: "",
+            fields.LINKED_FROM_KEY: [],
+            fields.CTYPE_KEY: "",  # should index by this too
+
+            # should only get inmutable headers parts
+            # (for indexing)
+            fields.HEADERS_KEY: {},
+            fields.RAW_KEY: "",
+            fields.PARTS_MAP_KEY: {},
+            fields.HEADERS_KEY: {},
+            fields.MULTIPART_KEY: False,
+        },
+
+    }
+
+    def __init__(self, mbox=None, soledad=None):
+        """
+        Constructor for MessageCollection.
+
+        :param mbox: the name of the mailbox. It is the name
+                     with which we filter the query over the
+                     messages database
+        :type mbox: str
+
+        :param soledad: Soledad database
+        :type soledad: Soledad instance
+        """
+        MailParser.__init__(self)
+        leap_assert(mbox, "Need a mailbox name to initialize")
+        leap_assert(mbox.strip() != "", "mbox cannot be blank space")
+        leap_assert(isinstance(mbox, (str, unicode)),
+                    "mbox needs to be a string")
+        leap_assert(soledad, "Need a soledad instance to initialize")
+
+        # okay, all in order, keep going...
+        self.mbox = self._parse_mailbox_name(mbox)
+        self._soledad = soledad
+        self.initialize_db()
+
+        # I think of someone like nietzsche when reading this
+
+        # this will be the producer that will enqueue the content
+        # to be processed serially by the consumer (the writer). We just
+        # need to `put` the new material on its plate.
+
+        self.soledad_writer = MessageProducer(
+            SoledadDocWriter(soledad),
+            period=0.02)
+
+    def _get_empty_doc(self, _type=FLAGS_DOC):
+        """
+        Returns an empty doc for storing different message parts.
+        Defaults to returning a template for a flags document.
+        :return: a dict with the template
+        :rtype: dict
+        """
+        if not _type in self.templates.keys():
+            raise TypeError("Improper type passed to _get_empty_doc")
+        return copy.deepcopy(self.templates[_type])
+
+    def _do_parse(self, raw):
+        """
+        Parse raw message and return it along with
+        relevant information about its outer level.
+
+        :param raw: the raw message
+        :type raw: StringIO or basestring
+        :return: msg, chash, size, multi
+        :rtype: tuple
+        """
+        msg = self._get_parsed_msg(raw)
+        chash = self._get_hash(msg)
+        size = len(msg.as_string())
+        multi = msg.is_multipart()
+        return msg, chash, size, multi
+
+    def _populate_flags(self, flags, uid, chash, size, multi):
+        """
+        Return a flags doc.
+
+        XXX Missing DOC -----------
+        """
+        fd = self._get_empty_doc(self.FLAGS_DOC)
+
+        fd[self.MBOX_KEY] = self.mbox
+        fd[self.UID_KEY] = uid
+        fd[self.CONTENT_HASH_KEY] = chash
+        fd[self.SIZE_KEY] = size
+        fd[self.MULTIPART_KEY] = multi
+        if flags:
+            fd[self.FLAGS_KEY] = map(self._stringify, flags)
+            fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
+            fd[self.DEL_KEY] = self.DELETED_FLAG in flags
+            fd[self.RECENT_KEY] = True  # set always by default
+        return fd
+
+    def _populate_headr(self, msg, chash, subject, date):
+        """
+        Return a headers doc.
+
+        XXX Missing DOC -----------
+        """
+        headers = defaultdict(list)
+        for k, v in msg.items():
+            headers[k].append(v)
+
+        # "fix" for repeated headers.
+        for k, v in headers.items():
+            newline = "\n%s: " % (k,)
+            headers[k] = newline.join(v)
+
+        hd = self._get_empty_doc(self.HEADERS_DOC)
+        hd[self.CONTENT_HASH_KEY] = chash
+        hd[self.HEADERS_KEY] = headers
+
+        if not subject and self.SUBJECT_FIELD in headers:
+            hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
+        else:
+            hd[self.SUBJECT_KEY] = subject
+
+        if not date and self.DATE_FIELD in headers:
+            hd[self.DATE_KEY] = first(headers[self.DATE_FIELD])
+        else:
+            hd[self.DATE_KEY] = date
+        return hd
+
+    @deferred
+    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
+        """
+        Creates a new message document.
+
+        :param raw: the raw message
+        :type raw: str
+
+        :param subject: subject of the message.
+        :type subject: str
+
+        :param flags: flags
+        :type flags: list
+
+        :param date: the received date for the message
+        :type date: str
+
+        :param uid: the message uid for this mailbox
+        :type uid: int
+        """
+        # TODO signal that we can delete the original message!-----
+        # when all the processing is done.
+
+        # TODO add the linked-from info !
+
+        logger.debug('adding message')
+        if flags is None:
+            flags = tuple()
+        leap_assert_type(flags, tuple)
+
+        # parse
+        msg, chash, size, multi = self._do_parse(raw)
+
+        fd = self._populate_flags(flags, uid, chash, size, multi)
+        hd = self._populate_headr(msg, chash, subject, date)
+
+        parts = walk.get_parts(msg)
+        body_phash_fun = [walk.get_body_phash_simple,
+                          walk.get_body_phash_multi][int(multi)]
+        body_phash = body_phash_fun(walk.get_payloads(msg))
+        parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+
+        # add parts map to header doc
+        # (body, multi, part_map)
+        for key in parts_map:
+            hd[key] = parts_map[key]
+        del parts_map
+
+        docs = [fd, hd]
+        cdocs = walk.get_raw_docs(msg, parts)
+
+        # Saving
+        logger.debug('enqueuing message docs for write')
+        ptuple = SoledadWriterPayload
+
+        # first, regular docs: flags and headers
+        for doc in docs:
+            self.soledad_writer.put(ptuple(
+                mode=ptuple.CREATE, payload=doc))
+
+        # and last, but not least, try to create
+        # content docs if not already there.
+        for cd in cdocs:
+            self.soledad_writer.put(ptuple(
+                mode=ptuple.CONTENT_CREATE, payload=cd))
+
+    def _remove_cb(self, result):
+        return result
+
+    def remove_all_deleted(self):
+        """
+        Removes all messages flagged as deleted.
+        """
+        delete_deferl = []
+        for msg in self.get_deleted():
+            delete_deferl.append(msg.remove())
+        d1 = defer.gatherResults(delete_deferl, consumeErrors=True)
+        d1.addCallback(self._remove_cb)
+        return d1
+
+    def remove(self, msg):
+        """
+        Remove a given msg.
+        :param msg: the message to be removed
+        :type msg: LeapMessage
+        """
+        d = msg.remove()
+        d.addCallback(self._remove_cb)
+        return d
+
+    # getters
+
+    def get_msg_by_uid(self, uid):
+        """
+        Retrieves a LeapMessage by UID.
+
+        :param uid: the message uid to query by
+        :type uid: int
+
+        :return: A LeapMessage instance matching the query,
+                 or None if not found.
+        :rtype: LeapMessage
+        """
+        msg = LeapMessage(self._soledad, uid, self.mbox)
+        if not msg.does_exist():
+            return None
+        return msg
+
+    def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
+        """
+        Get all documents for the selected mailbox of the
+        passed type. By default, it returns the flag docs.
+
+        If you want acess to the content, use __iter__ instead
+
+        :return: a list of u1db documents
+        :rtype: list of SoledadDocument
+        """
+        if _type not in fields.__dict__.values():
+            raise TypeError("Wrong type passed to get_all_docs")
+
+        if sameProxiedObjects(self._soledad, None):
+            logger.warning('Tried to get messages but soledad is None!')
+            return []
+
+        all_docs = [doc for doc in self._soledad.get_from_index(
+            fields.TYPE_MBOX_IDX,
+            _type, self.mbox)]
+
+        # inneficient, but first let's grok it and then
+        # let's worry about efficiency.
+        # XXX FIXINDEX -- should implement order by in soledad
+        return sorted(all_docs, key=lambda item: item.content['uid'])
+
+    def all_msg_iter(self):
+        """
+        Return an iterator trhough the UIDs of all messages, sorted in
+        ascending order.
+        """
+        all_uids = (doc.content[self.UID_KEY] for doc in
+                    self._soledad.get_from_index(
+                        fields.TYPE_MBOX_IDX,
+                        fields.TYPE_FLAGS_VAL, self.mbox))
+        return (u for u in sorted(all_uids))
+
+    def count(self):
+        """
+        Return the count of messages for this mailbox.
+
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            fields.TYPE_MBOX_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox)
+        return count
+
+    # unseen messages
+
+    def unseen_iter(self):
+        """
+        Get an iterator for the message UIDs with no `seen` flag
+        for this mailbox.
+
+        :return: iterator through unseen message doc UIDs
+        :rtype: iterable
+        """
+        return (doc.content[self.UID_KEY] for doc in
+                self._soledad.get_from_index(
+                    fields.TYPE_MBOX_SEEN_IDX,
+                    fields.TYPE_FLAGS_VAL, self.mbox, '0'))
+
+    def count_unseen(self):
+        """
+        Count all messages with the `Unseen` flag.
+
+        :returns: count
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            fields.TYPE_MBOX_SEEN_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox, '0')
+        return count
+
+    def get_unseen(self):
+        """
+        Get all messages with the `Unseen` flag
+
+        :returns: a list of LeapMessages
+        :rtype: list
+        """
+        return [LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.unseen_iter()]
+
+    # recent messages
+
+    def recent_iter(self):
+        """
+        Get an iterator for the message UIDs with `recent` flag.
+
+        :return: iterator through recent message docs
+        :rtype: iterable
+        """
+        return (doc.content[self.UID_KEY] for doc in
+                self._soledad.get_from_index(
+                    fields.TYPE_MBOX_RECT_IDX,
+                    fields.TYPE_FLAGS_VAL, self.mbox, '1'))
+
+    def get_recent(self):
+        """
+        Get all messages with the `Recent` flag.
+
+        :returns: a list of LeapMessages
+        :rtype: list
+        """
+        return [LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.recent_iter()]
+
+    def count_recent(self):
+        """
+        Count all messages with the `Recent` flag.
+
+        :returns: count
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            fields.TYPE_MBOX_RECT_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox, '1')
+        return count
+
+    # deleted messages
+
+    def deleted_iter(self):
+        """
+        Get an iterator for the message UIDs with `deleted` flag.
+
+        :return: iterator through deleted message docs
+        :rtype: iterable
+        """
+        return (doc.content[self.UID_KEY] for doc in
+                self._soledad.get_from_index(
+                    fields.TYPE_MBOX_DEL_IDX,
+                    fields.TYPE_FLAGS_VAL, self.mbox, '1'))
+
+    def get_deleted(self):
+        """
+        Get all messages with the `Deleted` flag.
+
+        :returns: a generator of LeapMessages
+        :rtype: generator
+        """
+        return (LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.deleted_iter())
+
+    def __len__(self):
+        """
+        Returns the number of messages on this mailbox.
+
+        :rtype: int
+        """
+        return self.count()
+
+    def __iter__(self):
+        """
+        Returns an iterator over all messages.
+
+        :returns: iterator of dicts with content for all messages.
+        :rtype: iterable
+        """
+        return (LeapMessage(self._soledad, docuid, self.mbox)
+                for docuid in self.all_msg_iter())
+
+    def __repr__(self):
+        """
+        Representation string for this object.
+        """
+        return u"<MessageCollection: mbox '%s' (%s)>" % (
+            self.mbox, self.count())
+
+    # XXX should implement __eq__ also !!!
+    # --- use the content hash for that, will be used for dedup.
diff --git a/mail/src/leap/mail/imap/parser.py b/mail/src/leap/mail/imap/parser.py
new file mode 100644
index 0000000..306dcf0
--- /dev/null
+++ b/mail/src/leap/mail/imap/parser.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# parser.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Mail parser mixins.
+"""
+import cStringIO
+import StringIO
+import hashlib
+import re
+
+from email.message import Message
+from email.parser import Parser
+
+from leap.common.check import leap_assert_type
+
+
+class MailParser(object):
+    """
+    Mixin with utility methods to parse raw messages.
+    """
+    def __init__(self):
+        """
+        Initializes the mail parser.
+        """
+        self._parser = Parser()
+
+    def _get_parsed_msg(self, raw, headersonly=False):
+        """
+        Return a parsed Message.
+
+        :param raw: the raw string to parse
+        :type raw: basestring, or StringIO object
+
+        :param headersonly: True for parsing only the headers.
+        :type headersonly: bool
+        """
+        msg = self._get_parser_fun(raw)(raw, headersonly=headersonly)
+        return msg
+
+    def _get_hash(self, msg):
+        """
+        Returns a hash of the string representation of the raw message,
+        suitable for indexing the inmutable pieces.
+
+        :param msg: a Message object
+        :type msg: Message
+        """
+        leap_assert_type(msg, Message)
+        return hashlib.sha256(msg.as_string()).hexdigest()
+
+    def _get_parser_fun(self, o):
+        """
+        Retunn the proper parser function for an object.
+
+        :param o: object
+        :type o: object
+        :param parser: an instance of email.parser.Parser
+        :type parser: email.parser.Parser
+        """
+        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+            return self._parser.parse
+        if isinstance(o, basestring):
+            return self._parser.parsestr
+        # fallback
+        return self._parser.parsestr
+
+    def _stringify(self, o):
+        """
+        Return a string object.
+
+        :param o: object
+        :type o: object
+        """
+        # XXX Maybe we don't need no more, we're using
+        # msg.as_string()
+        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+            return o.getvalue()
+        else:
+            return o
+
+
+class MBoxParser(object):
+    """
+    Utility function to parse mailbox names.
+    """
+    INBOX_NAME = "INBOX"
+    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
+    def _parse_mailbox_name(self, name):
+        """
+        :param name: the name of the mailbox
+        :type name: unicode
+
+        :rtype: unicode
+        """
+        if self.INBOX_RE.match(name):
+            # ensure inital INBOX is uppercase
+            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
+        return name
diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py
deleted file mode 100644
index b9b72d0..0000000
--- a/mail/src/leap/mail/imap/server.py
+++ /dev/null
@@ -1,1807 +0,0 @@
-# -*- coding: utf-8 -*-
-# server.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-"""
-Soledad-backed IMAP Server.
-"""
-import copy
-import logging
-import StringIO
-import cStringIO
-import time
-import re
-
-from collections import defaultdict
-from email.parser import Parser
-
-from zope.interface import implements
-from zope.proxy import sameProxiedObjects
-
-from twisted.mail import imap4
-from twisted.internet import defer
-from twisted.internet.threads import deferToThread
-from twisted.python import log
-
-
-from leap.common import events as leap_events
-from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.mail import get_email_charset
-from leap.mail.messageflow import IMessageConsumer, MessageProducer
-from leap.soledad.client import Soledad
-
-logger = logging.getLogger(__name__)
-
-
-class MissingIndexError(Exception):
-    """
-    Raises when tried to access a non existent index document.
-    """
-
-
-class BadIndexError(Exception):
-    """
-    Raises when index is malformed or has the wrong cardinality.
-    """
-
-
-class WithMsgFields(object):
-    """
-    Container class for class-attributes to be shared by
-    several message-related classes.
-    """
-    # Internal representation of Message
-    DATE_KEY = "date"
-    HEADERS_KEY = "headers"
-    FLAGS_KEY = "flags"
-    MBOX_KEY = "mbox"
-    RAW_KEY = "raw"
-    SUBJECT_KEY = "subject"
-    UID_KEY = "uid"
-
-    # Mailbox specific keys
-    CLOSED_KEY = "closed"
-    CREATED_KEY = "created"
-    SUBSCRIBED_KEY = "subscribed"
-    RW_KEY = "rw"
-    LAST_UID_KEY = "lastuid"
-
-    # Document Type, for indexing
-    TYPE_KEY = "type"
-    TYPE_MESSAGE_VAL = "msg"
-    TYPE_MBOX_VAL = "mbox"
-
-    INBOX_VAL = "inbox"
-
-    # Flags for SoledadDocument for indexing.
-    SEEN_KEY = "seen"
-    RECENT_KEY = "recent"
-
-    # Flags in Mailbox and Message
-    SEEN_FLAG = "\\Seen"
-    RECENT_FLAG = "\\Recent"
-    ANSWERED_FLAG = "\\Answered"
-    FLAGGED_FLAG = "\\Flagged"  # yo dawg
-    DELETED_FLAG = "\\Deleted"
-    DRAFT_FLAG = "\\Draft"
-    NOSELECT_FLAG = "\\Noselect"
-    LIST_FLAG = "List"  # is this OK? (no \. ie, no system flag)
-
-    # Fields in mail object
-    SUBJECT_FIELD = "Subject"
-    DATE_FIELD = "Date"
-
-
-class IndexedDB(object):
-    """
-    Methods dealing with the index.
-
-    This is a MixIn that needs access to the soledad instance,
-    and also assumes that a INDEXES attribute is accessible to the instance.
-
-    INDEXES must be a dictionary of type:
-    {'index-name': ['field1', 'field2']}
-    """
-    # TODO we might want to move this to soledad itself, check
-
-    def initialize_db(self):
-        """
-        Initialize the database.
-        """
-        leap_assert(self._soledad,
-                    "Need a soledad attribute accesible in the instance")
-        leap_assert_type(self.INDEXES, dict)
-
-        # Ask the database for currently existing indexes.
-        if not self._soledad:
-            logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
-            return
-        db_indexes = dict()
-        if self._soledad is not None:
-            db_indexes = dict(self._soledad.list_indexes())
-        for name, expression in SoledadBackedAccount.INDEXES.items():
-            if name not in db_indexes:
-                # The index does not yet exist.
-                self._soledad.create_index(name, *expression)
-                continue
-
-            if expression == db_indexes[name]:
-                # The index exists and is up to date.
-                continue
-            # The index exists but the definition is not what expected, so we
-            # delete it and add the proper index expression.
-            self._soledad.delete_index(name)
-            self._soledad.create_index(name, *expression)
-
-
-#######################################
-# Soledad Account
-#######################################
-
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB):
-    """
-    An implementation of IAccount and INamespacePresenteer
-    that is backed by Soledad Encrypted Documents.
-    """
-
-    implements(imap4.IAccount, imap4.INamespacePresenter)
-
-    _soledad = None
-    selected = None
-
-    TYPE_IDX = 'by-type'
-    TYPE_MBOX_IDX = 'by-type-and-mbox'
-    TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
-    TYPE_SUBS_IDX = 'by-type-and-subscribed'
-    TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
-    TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
-    # Tomas created the `recent and seen index`, but the semantic is not too
-    # correct since the recent flag is volatile.
-    TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
-    KTYPE = WithMsgFields.TYPE_KEY
-    MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL
-
-    INDEXES = {
-        # generic
-        TYPE_IDX: [KTYPE],
-        TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
-        TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY],
-
-        # mailboxes
-        TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
-        # messages
-        TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
-        TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
-        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
-                                  'bool(recent)', 'bool(seen)'],
-    }
-
-    INBOX_NAME = "INBOX"
-    MBOX_KEY = MBOX_VAL
-
-    EMPTY_MBOX = {
-        WithMsgFields.TYPE_KEY: MBOX_KEY,
-        WithMsgFields.TYPE_MBOX_VAL: INBOX_NAME,
-        WithMsgFields.SUBJECT_KEY: "",
-        WithMsgFields.FLAGS_KEY: [],
-        WithMsgFields.CLOSED_KEY: False,
-        WithMsgFields.SUBSCRIBED_KEY: False,
-        WithMsgFields.RW_KEY: 1,
-        WithMsgFields.LAST_UID_KEY: 0
-    }
-
-    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
-    def __init__(self, account_name, soledad=None):
-        """
-        Creates a SoledadAccountIndex that keeps track of the mailboxes
-        and subscriptions handled by this account.
-
-        :param acct_name: The name of the account (user id).
-        :type acct_name: str
-
-        :param soledad: a Soledad instance.
-        :param soledad: Soledad
-        """
-        leap_assert(soledad, "Need a soledad instance to initialize")
-        leap_assert_type(soledad, Soledad)
-
-        # XXX SHOULD assert too that the name matches the user/uuid with which
-        # soledad has been initialized.
-
-        self._account_name = self._parse_mailbox_name(account_name)
-        self._soledad = soledad
-
-        self.initialize_db()
-
-        # every user should have the right to an inbox folder
-        # at least, so let's make one!
-
-        if not self.mailboxes:
-            self.addMailbox(self.INBOX_NAME)
-
-    def _get_empty_mailbox(self):
-        """
-        Returns an empty mailbox.
-
-        :rtype: dict
-        """
-        return copy.deepcopy(self.EMPTY_MBOX)
-
-    def _parse_mailbox_name(self, name):
-        """
-        :param name: the name of the mailbox
-        :type name: unicode
-
-        :rtype: unicode
-        """
-        if self.INBOX_RE.match(name):
-            # ensure inital INBOX is uppercase
-            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
-        return name
-
-    def _get_mailbox_by_name(self, name):
-        """
-        Return an mbox document by name.
-
-        :param name: the name of the mailbox
-        :type name: str
-
-        :rtype: SoledadDocument
-        """
-        doc = self._soledad.get_from_index(
-            self.TYPE_MBOX_IDX, self.MBOX_KEY,
-            self._parse_mailbox_name(name))
-        return doc[0] if doc else None
-
-    @property
-    def mailboxes(self):
-        """
-        A list of the current mailboxes for this account.
-        """
-        return [doc.content[self.MBOX_KEY]
-                for doc in self._soledad.get_from_index(
-                    self.TYPE_IDX, self.MBOX_KEY)]
-
-    @property
-    def subscriptions(self):
-        """
-        A list of the current subscriptions for this account.
-        """
-        return [doc.content[self.MBOX_KEY]
-                for doc in self._soledad.get_from_index(
-                    self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
-
-    def getMailbox(self, name):
-        """
-        Returns a Mailbox with that name, without selecting it.
-
-        :param name: name of the mailbox
-        :type name: str
-
-        :returns: a a SoledadMailbox instance
-        :rtype: SoledadMailbox
-        """
-        name = self._parse_mailbox_name(name)
-
-        if name not in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox")
-
-        return SoledadMailbox(name, soledad=self._soledad)
-
-    ##
-    ## IAccount
-    ##
-
-    def addMailbox(self, name, creation_ts=None):
-        """
-        Add a mailbox to the account.
-
-        :param name: the name of the mailbox
-        :type name: str
-
-        :param creation_ts: an optional creation timestamp to be used as
-                            mailbox id. A timestamp will be used if no
-                            one is provided.
-        :type creation_ts: int
-
-        :returns: True if successful
-        :rtype: bool
-        """
-        name = self._parse_mailbox_name(name)
-
-        if name in self.mailboxes:
-            raise imap4.MailboxCollision, name
-
-        if not creation_ts:
-            # by default, we pass an int value
-            # taken from the current time
-            # we make sure to take enough decimals to get a unique
-            # mailbox-uidvalidity.
-            creation_ts = int(time.time() * 10E2)
-
-        mbox = self._get_empty_mailbox()
-        mbox[self.MBOX_KEY] = name
-        mbox[self.CREATED_KEY] = creation_ts
-
-        doc = self._soledad.create_doc(mbox)
-        return bool(doc)
-
-    def create(self, pathspec):
-        """
-        Create a new mailbox from the given hierarchical name.
-
-        :param pathspec: The full hierarchical name of a new mailbox to create.
-                         If any of the inferior hierarchical names to this one
-                         do not exist, they are created as well.
-        :type pathspec: str
-
-        :return: A true value if the creation succeeds.
-        :rtype: bool
-
-        :raise MailboxException: Raised if this mailbox cannot be added.
-        """
-        # TODO raise MailboxException
-        paths = filter(None,
-            self._parse_mailbox_name(pathspec).split('/'))
-        for accum in range(1, len(paths)):
-            try:
-                self.addMailbox('/'.join(paths[:accum]))
-            except imap4.MailboxCollision:
-                pass
-        try:
-            self.addMailbox('/'.join(paths))
-        except imap4.MailboxCollision:
-            if not pathspec.endswith('/'):
-                return False
-        return True
-
-    def select(self, name, readwrite=1):
-        """
-        Selects a mailbox.
-
-        :param name: the mailbox to select
-        :type name: str
-
-        :param readwrite: 1 for readwrite permissions.
-        :type readwrite: int
-
-        :rtype: bool
-        """
-        name = self._parse_mailbox_name(name)
-
-        if name not in self.mailboxes:
-            return None
-
-        self.selected = name
-
-        return SoledadMailbox(
-            name, rw=readwrite,
-            soledad=self._soledad)
-
-    def delete(self, name, force=False):
-        """
-        Deletes a mailbox.
-
-        Right now it does not purge the messages, but just removes the mailbox
-        name from the mailboxes list!!!
-
-        :param name: the mailbox to be deleted
-        :type name: str
-
-        :param force: if True, it will not check for noselect flag or inferior
-                      names. use with care.
-        :type force: bool
-        """
-        name = self._parse_mailbox_name(name)
-
-        if not name in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox")
-
-        mbox = self.getMailbox(name)
-
-        if force is False:
-            # See if this box is flagged \Noselect
-            # XXX use mbox.flags instead?
-            if self.NOSELECT_FLAG in mbox.getFlags():
-                # Check for hierarchically inferior mailboxes with this one
-                # as part of their root.
-                for others in self.mailboxes:
-                    if others != name and others.startswith(name):
-                        raise imap4.MailboxException, (
-                            "Hierarchically inferior mailboxes "
-                            "exist and \\Noselect is set")
-        mbox.destroy()
-
-        # XXX FIXME --- not honoring the inferior names...
-
-        # if there are no hierarchically inferior names, we will
-        # delete it from our ken.
-        #if self._inferiorNames(name) > 1:
-            # ??! -- can this be rite?
-            #self._index.removeMailbox(name)
-
-    def rename(self, oldname, newname):
-        """
-        Renames a mailbox.
-
-        :param oldname: old name of the mailbox
-        :type oldname: str
-
-        :param newname: new name of the mailbox
-        :type newname: str
-        """
-        oldname = self._parse_mailbox_name(oldname)
-        newname = self._parse_mailbox_name(newname)
-
-        if oldname not in self.mailboxes:
-            raise imap4.NoSuchMailbox, oldname
-
-        inferiors = self._inferiorNames(oldname)
-        inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
-
-        for (old, new) in inferiors:
-            if new in self.mailboxes:
-                raise imap4.MailboxCollision, new
-
-        for (old, new) in inferiors:
-            mbox = self._get_mailbox_by_name(old)
-            mbox.content[self.MBOX_KEY] = new
-            self._soledad.put_doc(mbox)
-
-        # XXX ---- FIXME!!!! ------------------------------------
-        # until here we just renamed the index...
-        # We have to rename also the occurrence of this
-        # mailbox on ALL the messages that are contained in it!!!
-        # ... we maybe could use a reference to the doc_id
-        # in each msg, instead of the "mbox" field in msgs
-        # -------------------------------------------------------
-
-    def _inferiorNames(self, name):
-        """
-        Return hierarchically inferior mailboxes.
-
-        :param name: name of the mailbox
-        :rtype: list
-        """
-        # XXX use wildcard query instead
-        inferiors = []
-        for infname in self.mailboxes:
-            if infname.startswith(name):
-                inferiors.append(infname)
-        return inferiors
-
-    def isSubscribed(self, name):
-        """
-        Returns True if user is subscribed to this mailbox.
-
-        :param name: the mailbox to be checked.
-        :type name: str
-
-        :rtype: bool
-        """
-        mbox = self._get_mailbox_by_name(name)
-        return mbox.content.get('subscribed', False)
-
-    def _set_subscription(self, name, value):
-        """
-        Sets the subscription value for a given mailbox
-
-        :param name: the mailbox
-        :type name: str
-
-        :param value: the boolean value
-        :type value: bool
-        """
-        # maybe we should store subscriptions in another
-        # document...
-        if not name in self.mailboxes:
-            self.addMailbox(name)
-        mbox = self._get_mailbox_by_name(name)
-
-        if mbox:
-            mbox.content[self.SUBSCRIBED_KEY] = value
-            self._soledad.put_doc(mbox)
-
-    def subscribe(self, name):
-        """
-        Subscribe to this mailbox
-
-        :param name: name of the mailbox
-        :type name: str
-        """
-        name = self._parse_mailbox_name(name)
-        if name not in self.subscriptions:
-            self._set_subscription(name, True)
-
-    def unsubscribe(self, name):
-        """
-        Unsubscribe from this mailbox
-
-        :param name: name of the mailbox
-        :type name: str
-        """
-        name = self._parse_mailbox_name(name)
-        if name not in self.subscriptions:
-            raise imap4.MailboxException, "Not currently subscribed to " + name
-        self._set_subscription(name, False)
-
-    def listMailboxes(self, ref, wildcard):
-        """
-        List the mailboxes.
-
-        from rfc 3501:
-        returns a subset of names from the complete set
-        of all names available to the client.  Zero or more untagged LIST
-        replies are returned, containing the name attributes, hierarchy
-        delimiter, and name.
-
-        :param ref: reference name
-        :type ref: str
-
-        :param wildcard: mailbox name with possible wildcards
-        :type wildcard: str
-        """
-        # XXX use wildcard in index query
-        ref = self._inferiorNames(
-            self._parse_mailbox_name(ref))
-        wildcard = imap4.wildcardToRegexp(wildcard, '/')
-        return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
-
-    ##
-    ## INamespacePresenter
-    ##
-
-    def getPersonalNamespaces(self):
-        return [["", "/"]]
-
-    def getSharedNamespaces(self):
-        return None
-
-    def getOtherNamespaces(self):
-        return None
-
-    # extra, for convenience
-
-    def deleteAllMessages(self, iknowhatiamdoing=False):
-        """
-        Deletes all messages from all mailboxes.
-        Danger! high voltage!
-
-        :param iknowhatiamdoing: confirmation parameter, needs to be True
-                                 to proceed.
-        """
-        if iknowhatiamdoing is True:
-            for mbox in self.mailboxes:
-                self.delete(mbox, force=True)
-
-    def __repr__(self):
-        """
-        Representation string for this object.
-        """
-        return "<SoledadBackedAccount (%s)>" % self._account_name
-
-#######################################
-# LeapMessage, MessageCollection
-# and Mailbox
-#######################################
-
-
-class LeapMessage(WithMsgFields):
-
-    implements(imap4.IMessage, imap4.IMessageFile)
-
-    def __init__(self, doc):
-        """
-        Initializes a LeapMessage.
-
-        :param doc: A SoledadDocument containing the internal
-                    representation of the message
-        :type doc: SoledadDocument
-        """
-        self._doc = doc
-
-    def getUID(self):
-        """
-        Retrieve the unique identifier associated with this message
-
-        :return: uid for this message
-        :rtype: int
-        """
-        # XXX debug, to remove after a while...
-        if not self._doc:
-            log.msg('BUG!!! ---- message has no doc!')
-            return
-        return self._doc.content[self.UID_KEY]
-
-    def getFlags(self):
-        """
-        Retrieve the flags associated with this message
-
-        :return: The flags, represented as strings
-        :rtype: tuple
-        """
-        if self._doc is None:
-            return []
-        flags = self._doc.content.get(self.FLAGS_KEY, None)
-        if flags:
-            flags = map(str, flags)
-        return tuple(flags)
-
-    # setFlags, addFlags, removeFlags are not in the interface spec
-    # but we use them with store command.
-
-    def setFlags(self, flags):
-        """
-        Sets the flags for this message
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to update in the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        log.msg('setting flags')
-        doc = self._doc
-        doc.content[self.FLAGS_KEY] = flags
-        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-        doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
-        return doc
-
-    def addFlags(self, flags):
-        """
-        Adds flags to this message.
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to add to the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        oldflags = self.getFlags()
-        return self.setFlags(tuple(set(flags + oldflags)))
-
-    def removeFlags(self, flags):
-        """
-        Remove flags from this message.
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to be removed from the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        oldflags = self.getFlags()
-        return self.setFlags(tuple(set(oldflags) - set(flags)))
-
-    def getInternalDate(self):
-        """
-        Retrieve the date internally associated with this message
-
-        :rtype: C{str}
-        :return: An RFC822-formatted date string.
-        """
-        return str(self._doc.content.get(self.DATE_KEY, ''))
-
-    #
-    # IMessageFile
-    #
-
-    """
-    Optional message interface for representing messages as files.
-
-    If provided by message objects, this interface will be used instead
-    the more complex MIME-based interface.
-    """
-
-    def open(self):
-        """
-        Return an file-like object opened for reading.
-
-        Reading from the returned file will return all the bytes
-        of which this message consists.
-
-        :return: file-like object opened fore reading.
-        :rtype: StringIO
-        """
-        fd = cStringIO.StringIO()
-        content = self._doc.content.get(self.RAW_KEY, '')
-        charset = get_email_charset(
-            unicode(self._doc.content.get(self.RAW_KEY, '')))
-        try:
-            content = content.encode(charset)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            content = content.encode(charset, 'replace')
-        fd.write(content)
-        fd.seek(0)
-        return fd
-
-    #
-    # IMessagePart
-    #
-
-    # XXX should implement the rest of IMessagePart interface:
-    # (and do not use the open above)
-
-    def getBodyFile(self):
-        """
-        Retrieve a file object containing only the body of this message.
-
-        :return: file-like object opened for reading
-        :rtype: StringIO
-        """
-        fd = StringIO.StringIO()
-        content = self._doc.content.get(self.RAW_KEY, '')
-        charset = get_email_charset(
-            unicode(self._doc.content.get(self.RAW_KEY, '')))
-        try:
-            content = content.encode(charset)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            content = content.encode(charset, 'replace')
-        fd.write(content)
-        # XXX SHOULD use a separate BODY FIELD ...
-        fd.seek(0)
-        return fd
-
-    def getSize(self):
-        """
-        Return the total size, in octets, of this message.
-
-        :return: size of the message, in octets
-        :rtype: int
-        """
-        return self.getBodyFile().len
-
-    def _get_headers(self):
-        """
-        Return the headers dict stored in this message document.
-        """
-        return self._doc.content.get(self.HEADERS_KEY, {})
-
-    def getHeaders(self, negate, *names):
-        """
-        Retrieve a group of message headers.
-
-        :param names: The names of the headers to retrieve or omit.
-        :type names: tuple of str
-
-        :param negate: If True, indicates that the headers listed in names
-                       should be omitted from the return value, rather
-                       than included.
-        :type negate: bool
-
-        :return: A mapping of header field names to header field values
-        :rtype: dict
-        """
-        headers = self._get_headers()
-        names = map(lambda s: s.upper(), names)
-        if negate:
-            cond = lambda key: key.upper() not in names
-        else:
-            cond = lambda key: key.upper() in names
-
-        # unpack and filter original dict by negate-condition
-        filter_by_cond = [
-            map(str, (key, val)) for
-            key, val in headers.items()
-            if cond(key)]
-        return dict(filter_by_cond)
-
-    # --- no multipart for now
-    # XXX Fix MULTIPART SUPPORT!
-
-    def isMultipart(self):
-        return False
-
-    def getSubPart(part):
-        return None
-
-    #
-    # accessors
-    #
-
-    def __getitem__(self, key):
-        """
-        Return the content of the message document.
-
-        @param key: The key
-        @type key: str
-
-        @return: The content value indexed by C{key} or None
-        @rtype: str
-        """
-        return self._doc.content.get(key, None)
-
-
-class SoledadDocWriter(object):
-    """
-    This writer will create docs serially in the local soledad database.
-    """
-
-    implements(IMessageConsumer)
-
-    def __init__(self, soledad):
-        """
-        Initialize the writer.
-
-        :param soledad: the soledad instance
-        :type soledad: Soledad
-        """
-        self._soledad = soledad
-
-    def consume(self, queue):
-        """
-        Creates a new document in soledad db.
-
-        :param queue: queue to get item from, with content of the document
-                      to be inserted.
-        :type queue: Queue
-        """
-        empty = queue.empty()
-        while not empty:
-            item = queue.get()
-            payload = item['payload']
-            mode = item['mode']
-            if mode == "create":
-                self._soledad.create_doc(payload)
-            elif mode == "put":
-                self._soledad.put_doc(payload)
-            empty = queue.empty()
-
-
-class MessageCollection(WithMsgFields, IndexedDB):
-    """
-    A collection of messages, surprisingly.
-
-    It is tied to a selected mailbox name that is passed to constructor.
-    Implements a filter query over the messages contained in a soledad
-    database.
-    """
-    # XXX this should be able to produce a MessageSet methinks
-
-    EMPTY_MSG = {
-        WithMsgFields.TYPE_KEY: WithMsgFields.TYPE_MESSAGE_VAL,
-        WithMsgFields.UID_KEY: 1,
-        WithMsgFields.MBOX_KEY: WithMsgFields.INBOX_VAL,
-        WithMsgFields.SUBJECT_KEY: "",
-        WithMsgFields.DATE_KEY: "",
-        WithMsgFields.SEEN_KEY: False,
-        WithMsgFields.RECENT_KEY: True,
-        WithMsgFields.FLAGS_KEY: [],
-        WithMsgFields.HEADERS_KEY: {},
-        WithMsgFields.RAW_KEY: "",
-    }
-
-    # get from SoledadBackedAccount the needed index-related constants
-    INDEXES = SoledadBackedAccount.INDEXES
-    TYPE_IDX = SoledadBackedAccount.TYPE_IDX
-
-    def __init__(self, mbox=None, soledad=None):
-        """
-        Constructor for MessageCollection.
-
-        :param mbox: the name of the mailbox. It is the name
-                     with which we filter the query over the
-                     messages database
-        :type mbox: str
-
-        :param soledad: Soledad database
-        :type soledad: Soledad instance
-        """
-        # XXX pass soledad directly
-
-        leap_assert(mbox, "Need a mailbox name to initialize")
-        leap_assert(mbox.strip() != "", "mbox cannot be blank space")
-        leap_assert(isinstance(mbox, (str, unicode)),
-                    "mbox needs to be a string")
-        leap_assert(soledad, "Need a soledad instance to initialize")
-
-        # This is a wrapper now!...
-        # should move assertion there...
-        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    #"soledad._db must be an instance of SQLCipherDatabase")
-
-        # okay, all in order, keep going...
-
-        self.mbox = mbox.upper()
-        self._soledad = soledad
-        self.initialize_db()
-        self._parser = Parser()
-
-        # I think of someone like nietzsche when reading this
-
-        # this will be the producer that will enqueue the content
-        # to be processed serially by the consumer (the writer). We just
-        # need to `put` the new material on its plate.
-
-        self.soledad_writer = MessageProducer(
-            SoledadDocWriter(soledad),
-            period=0.1)
-
-    def _get_empty_msg(self):
-        """
-        Returns an empty message.
-
-        :return: a dict containing a default empty message
-        :rtype: dict
-        """
-        return copy.deepcopy(self.EMPTY_MSG)
-
-    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
-        """
-        Creates a new message document.
-
-        :param raw: the raw message
-        :type raw: str
-
-        :param subject: subject of the message.
-        :type subject: str
-
-        :param flags: flags
-        :type flags: list
-
-        :param date: the received date for the message
-        :type date: str
-
-        :param uid: the message uid for this mailbox
-        :type uid: int
-        """
-        logger.debug('adding message')
-        if flags is None:
-            flags = tuple()
-        leap_assert_type(flags, tuple)
-
-        def stringify(o):
-            if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-                return o.getvalue()
-            else:
-                return o
-
-        content = self._get_empty_msg()
-        content[self.MBOX_KEY] = self.mbox
-
-        if flags:
-            content[self.FLAGS_KEY] = map(stringify, flags)
-            content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-
-        def _get_parser_fun(o):
-            if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-                return self._parser.parse
-            if isinstance(o, (str, unicode)):
-                return self._parser.parsestr
-
-        msg = _get_parser_fun(raw)(raw, True)
-        headers = dict(msg)
-
-        # XXX get lower case for keys?
-        content[self.HEADERS_KEY] = headers
-        # set subject based on message headers and eventually replace by
-        # subject given as param
-        if self.SUBJECT_FIELD in headers:
-            content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
-        if subject is not None:
-            content[self.SUBJECT_KEY] = subject
-        content[self.RAW_KEY] = stringify(raw)
-
-        if not date and self.DATE_FIELD in headers:
-            content[self.DATE_KEY] = headers[self.DATE_FIELD]
-        else:
-            content[self.DATE_KEY] = date
-
-        # ...should get a sanity check here.
-        content[self.UID_KEY] = uid
-
-        logger.debug('enqueuing message for write')
-
-        # XXX create namedtuple
-        self.soledad_writer.put({"mode": "create",
-                                 "payload": content})
-        # XXX have to decide what shall we do with errors with this change...
-        #return self._soledad.create_doc(content)
-
-    def remove(self, msg):
-        """
-        Removes a message.
-
-        :param msg: a u1db doc containing the message
-        :type msg: SoledadDocument
-        """
-        self._soledad.delete_doc(msg)
-
-    # getters
-
-    def get_by_uid(self, uid):
-        """
-        Retrieves a message document by UID.
-
-        :param uid: the message uid to query by
-        :type uid: int
-
-        :return: A SoledadDocument instance matching the query,
-                 or None if not found.
-        :rtype: SoledadDocument
-        """
-        docs = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox, str(uid))
-
-        return docs[0] if docs else None
-
-    def get_msg_by_uid(self, uid):
-        """
-        Retrieves a LeapMessage by UID.
-
-        :param uid: the message uid to query by
-        :type uid: int
-
-        :return: A LeapMessage instance matching the query,
-                 or None if not found.
-        :rtype: LeapMessage
-        """
-        doc = self.get_by_uid(uid)
-        if doc:
-            return LeapMessage(doc)
-
-    def get_by_index(self, index):
-        """
-        Retrieves a mesage document by mailbox index.
-
-        :param index: the index of the sequence (zero-indexed)
-        :type index: int
-        """
-        # XXX inneficient! ---- we should keep an index document
-        # with uid -- doc_uuid :)
-        try:
-            return self.get_all()[index]
-        except IndexError:
-            return None
-
-    def get_msg_by_index(self, index):
-        """
-        Retrieves a LeapMessage by sequence index.
-
-        :param index: the index of the sequence (zero-indexed)
-        :type index: int
-        """
-        doc = self.get_by_index(index)
-        if doc:
-            return LeapMessage(doc)
-
-    def is_deleted(self, doc):
-        """
-        Returns whether a given doc is deleted or not.
-
-        :param doc: the document to check
-        :rtype: bool
-        """
-        return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
-
-    def get_all(self):
-        """
-        Get all message documents for the selected mailbox.
-        If you want acess to the content, use __iter__ instead
-
-        :return: a list of u1db documents
-        :rtype: list of SoledadDocument
-        """
-        if sameProxiedObjects(self._soledad, None):
-            logger.warning('Tried to get messages but soledad is None!')
-            return []
-
-        #f XXX this should return LeapMessage instances
-        all_docs = [doc for doc in self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox)]
-        # highly inneficient, but first let's grok it and then
-        # let's worry about efficiency.
-
-        # XXX FIXINDEX
-        return sorted(all_docs, key=lambda item: item.content['uid'])
-
-    def count(self):
-        """
-        Return the count of messages for this mailbox.
-
-        :rtype: int
-        """
-        count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox)
-        return count
-
-    # unseen messages
-
-    def unseen_iter(self):
-        """
-        Get an iterator for the message docs with no `seen` flag
-
-        :return: iterator through unseen message docs
-        :rtype: iterable
-        """
-        return (doc for doc in
-                self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-                    self.TYPE_MESSAGE_VAL, self.mbox, '0'))
-
-    def count_unseen(self):
-        """
-        Count all messages with the `Unseen` flag.
-
-        :returns: count
-        :rtype: int
-        """
-        count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox, '0')
-        return count
-
-    def get_unseen(self):
-        """
-        Get all messages with the `Unseen` flag
-
-        :returns: a list of LeapMessages
-        :rtype: list
-        """
-        return [LeapMessage(doc) for doc in self.unseen_iter()]
-
-    # recent messages
-
-    def recent_iter(self):
-        """
-        Get an iterator for the message docs with `recent` flag.
-
-        :return: iterator through recent message docs
-        :rtype: iterable
-        """
-        return (doc for doc in
-                self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-                    self.TYPE_MESSAGE_VAL, self.mbox, '1'))
-
-    def get_recent(self):
-        """
-        Get all messages with the `Recent` flag.
-
-        :returns: a list of LeapMessages
-        :rtype: list
-        """
-        return [LeapMessage(doc) for doc in self.recent_iter()]
-
-    def count_recent(self):
-        """
-        Count all messages with the `Recent` flag.
-
-        :returns: count
-        :rtype: int
-        """
-        count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox, '1')
-        return count
-
-    def __len__(self):
-        """
-        Returns the number of messages on this mailbox.
-
-        :rtype: int
-        """
-        return self.count()
-
-    def __iter__(self):
-        """
-        Returns an iterator over all messages.
-
-        :returns: iterator of dicts with content for all messages.
-        :rtype: iterable
-        """
-        # XXX return LeapMessage instead?! (change accordingly)
-        return (m.content for m in self.get_all())
-
-    def __getitem__(self, uid):
-        """
-        Allows indexing as a list, with msg uid as the index.
-
-        :param uid: an integer index
-        :type uid: int
-
-        :return: LeapMessage or None if not found.
-        :rtype: LeapMessage
-        """
-        # XXX FIXME inneficcient, we are evaulating.
-        try:
-            return [doc
-                    for doc in self.get_all()][uid - 1]
-        except IndexError:
-            return None
-
-    def __repr__(self):
-        """
-        Representation string for this object.
-        """
-        return u"<MessageCollection: mbox '%s' (%s)>" % (
-            self.mbox, self.count())
-
-    # XXX should implement __eq__ also
-
-
-class SoledadMailbox(WithMsgFields):
-    """
-    A Soledad-backed IMAP mailbox.
-
-    Implements the high-level method needed for the Mailbox interfaces.
-    The low-level database methods are contained in MessageCollection class,
-    which we instantiate and make accessible in the `messages` attribute.
-    """
-    implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
-    # XXX should finish the implementation of IMailboxListener
-
-    messages = None
-    _closed = False
-
-    INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
-                  WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
-                  WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
-                  WithMsgFields.LIST_FLAG)
-    flags = None
-
-    CMD_MSG = "MESSAGES"
-    CMD_RECENT = "RECENT"
-    CMD_UIDNEXT = "UIDNEXT"
-    CMD_UIDVALIDITY = "UIDVALIDITY"
-    CMD_UNSEEN = "UNSEEN"
-
-    _listeners = defaultdict(set)
-
-    def __init__(self, mbox, soledad=None, rw=1):
-        """
-        SoledadMailbox constructor. Needs to get passed a name, plus a
-        Soledad instance.
-
-        :param mbox: the mailbox name
-        :type mbox: str
-
-        :param soledad: a Soledad instance.
-        :type soledad: Soledad
-
-        :param rw: read-and-write flags
-        :type rw: int
-        """
-        leap_assert(mbox, "Need a mailbox name to initialize")
-        leap_assert(soledad, "Need a soledad instance to initialize")
-
-        # XXX should move to wrapper
-        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    #"soledad._db must be an instance of SQLCipherDatabase")
-
-        self.mbox = mbox
-        self.rw = rw
-
-        self._soledad = soledad
-
-        self.messages = MessageCollection(
-            mbox=mbox, soledad=self._soledad)
-
-        if not self.getFlags():
-            self.setFlags(self.INIT_FLAGS)
-
-    @property
-    def listeners(self):
-        """
-        Returns listeners for this mbox.
-
-        The server itself is a listener to the mailbox.
-        so we can notify it (and should!) after changes in flags
-        and number of messages.
-
-        :rtype: set
-        """
-        return self._listeners[self.mbox]
-
-    def addListener(self, listener):
-        """
-        Rdds a listener to the listeners queue.
-
-        :param listener: listener to add
-        :type listener: an object that implements IMailboxListener
-        """
-        logger.debug('adding mailbox listener: %s' % listener)
-        self.listeners.add(listener)
-
-    def removeListener(self, listener):
-        """
-        Removes a listener from the listeners queue.
-
-        :param listener: listener to remove
-        :type listener: an object that implements IMailboxListener
-        """
-        self.listeners.remove(listener)
-
-    def _get_mbox(self):
-        """
-        Returns mailbox document.
-
-        :return: A SoledadDocument containing this mailbox, or None if
-                 the query failed.
-        :rtype: SoledadDocument or None.
-        """
-        try:
-            query = self._soledad.get_from_index(
-                SoledadBackedAccount.TYPE_MBOX_IDX,
-                self.TYPE_MBOX_VAL, self.mbox)
-            if query:
-                return query.pop()
-        except Exception as exc:
-            logger.error("Unhandled error %r" % exc)
-
-    def getFlags(self):
-        """
-        Returns the flags defined for this mailbox.
-
-        :returns: tuple of flags for this mailbox
-        :rtype: tuple of str
-        """
-        #return map(str, self.INIT_FLAGS)
-
-        # XXX CHECK against thunderbird XXX
-        # XXX I think this is slightly broken.. :/
-
-        mbox = self._get_mbox()
-        if not mbox:
-            return None
-        flags = mbox.content.get(self.FLAGS_KEY, [])
-        return map(str, flags)
-
-    def setFlags(self, flags):
-        """
-        Sets flags for this mailbox.
-
-        :param flags: a tuple with the flags
-        :type flags: tuple of str
-        """
-        # TODO -- fix also getFlags
-        leap_assert(isinstance(flags, tuple),
-                    "flags expected to be a tuple")
-        mbox = self._get_mbox()
-        if not mbox:
-            return None
-        mbox.content[self.FLAGS_KEY] = map(str, flags)
-        self._soledad.put_doc(mbox)
-
-    # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
-
-    def _get_closed(self):
-        """
-        Return the closed attribute for this mailbox.
-
-        :return: True if the mailbox is closed
-        :rtype: bool
-        """
-        mbox = self._get_mbox()
-        return mbox.content.get(self.CLOSED_KEY, False)
-
-    def _set_closed(self, closed):
-        """
-        Set the closed attribute for this mailbox.
-
-        :param closed: the state to be set
-        :type closed: bool
-        """
-        leap_assert(isinstance(closed, bool), "closed needs to be boolean")
-        mbox = self._get_mbox()
-        mbox.content[self.CLOSED_KEY] = closed
-        self._soledad.put_doc(mbox)
-
-    closed = property(
-        _get_closed, _set_closed, doc="Closed attribute.")
-
-    def _get_last_uid(self):
-        """
-        Return the last uid for this mailbox.
-
-        :return: the last uid for messages in this mailbox
-        :rtype: bool
-        """
-        mbox = self._get_mbox()
-        return mbox.content.get(self.LAST_UID_KEY, 1)
-
-    def _set_last_uid(self, uid):
-        """
-        Sets the last uid for this mailbox.
-
-        :param uid: the uid to be set
-        :type uid: int
-        """
-        leap_assert(isinstance(uid, int), "uid has to be int")
-        mbox = self._get_mbox()
-        key = self.LAST_UID_KEY
-
-        count = self.getMessageCount()
-
-        # XXX safety-catch. If we do get duplicates,
-        # we want to avoid further duplication.
-
-        if uid >= count:
-            value = uid
-        else:
-            # something is wrong,
-            # just set the last uid
-            # beyond the max msg count.
-            logger.debug("WRONG uid < count. Setting last uid to ", count)
-            value = count
-
-        mbox.content[key] = value
-        self._soledad.put_doc(mbox)
-
-    last_uid = property(
-        _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
-
-    def getUIDValidity(self):
-        """
-        Return the unique validity identifier for this mailbox.
-
-        :return: unique validity identifier
-        :rtype: int
-        """
-        mbox = self._get_mbox()
-        return mbox.content.get(self.CREATED_KEY, 1)
-
-    def getUID(self, message):
-        """
-        Return the UID of a message in the mailbox
-
-        .. note:: this implementation does not make much sense RIGHT NOW,
-        but in the future will be useful to get absolute UIDs from
-        message sequence numbers.
-
-        :param message: the message uid
-        :type message: int
-
-        :rtype: int
-        """
-        msg = self.messages.get_msg_by_uid(message)
-        return msg.getUID()
-
-    def getUIDNext(self):
-        """
-        Return the likely UID for the next message added to this
-        mailbox. Currently it returns the higher UID incremented by
-        one.
-
-        We increment the next uid *each* time this function gets called.
-        In this way, there will be gaps if the message with the allocated
-        uid cannot be saved. But that is preferable to having race conditions
-        if we get to parallel message adding.
-
-        :rtype: int
-        """
-        self.last_uid += 1
-        return self.last_uid
-
-    def getMessageCount(self):
-        """
-        Returns the total count of messages in this mailbox.
-
-        :rtype: int
-        """
-        return self.messages.count()
-
-    def getUnseenCount(self):
-        """
-        Returns the number of messages with the 'Unseen' flag.
-
-        :return: count of messages flagged `unseen`
-        :rtype: int
-        """
-        return self.messages.count_unseen()
-
-    def getRecentCount(self):
-        """
-        Returns the number of messages with the 'Recent' flag.
-
-        :return: count of messages flagged `recent`
-        :rtype: int
-        """
-        return self.messages.count_recent()
-
-    def isWriteable(self):
-        """
-        Get the read/write status of the mailbox.
-
-        :return: 1 if mailbox is read-writeable, 0 otherwise.
-        :rtype: int
-        """
-        return self.rw
-
-    def getHierarchicalDelimiter(self):
-        """
-        Returns the character used to delimite hierarchies in mailboxes.
-
-        :rtype: str
-        """
-        return '/'
-
-    def requestStatus(self, names):
-        """
-        Handles a status request by gathering the output of the different
-        status commands.
-
-        :param names: a list of strings containing the status commands
-        :type names: iter
-        """
-        r = {}
-        if self.CMD_MSG in names:
-            r[self.CMD_MSG] = self.getMessageCount()
-        if self.CMD_RECENT in names:
-            r[self.CMD_RECENT] = self.getRecentCount()
-        if self.CMD_UIDNEXT in names:
-            r[self.CMD_UIDNEXT] = self.getMessageCount() + 1
-        if self.CMD_UIDVALIDITY in names:
-            r[self.CMD_UIDVALIDITY] = self.getUID()
-        if self.CMD_UNSEEN in names:
-            r[self.CMD_UNSEEN] = self.getUnseenCount()
-        return defer.succeed(r)
-
-    def addMessage(self, message, flags, date=None):
-        """
-        Adds a message to this mailbox.
-
-        :param message: the raw message
-        :type message: str
-
-        :param flags: flag list
-        :type flags: list of str
-
-        :param date: timestamp
-        :type date: str
-
-        :return: a deferred that evals to None
-        """
-        # XXX we should treat the message as an IMessage from here
-        uid_next = self.getUIDNext()
-        logger.debug('Adding msg with UID :%s' % uid_next)
-        if flags is None:
-            flags = tuple()
-        else:
-            flags = tuple(str(flag) for flag in flags)
-
-        self.messages.add_msg(message, flags=flags, date=date,
-                              uid=uid_next)
-
-        exists = self.getMessageCount()
-        recent = self.getRecentCount()
-        logger.debug("there are %s messages, %s recent" % (
-            exists,
-            recent))
-        for listener in self.listeners:
-            listener.newMessages(exists, recent)
-        return defer.succeed(None)
-
-    # commands, do not rename methods
-
-    def destroy(self):
-        """
-        Called before this mailbox is permanently deleted.
-
-        Should cleanup resources, and set the \\Noselect flag
-        on the mailbox.
-        """
-        self.setFlags((self.NOSELECT_FLAG,))
-        self.deleteAllDocs()
-
-        # XXX removing the mailbox in situ for now,
-        # we should postpone the removal
-        self._soledad.delete_doc(self._get_mbox())
-
-    def expunge(self):
-        """
-        Remove all messages flagged \\Deleted
-        """
-        if not self.isWriteable():
-            raise imap4.ReadOnlyMailbox
-        delete = []
-        deleted = []
-
-        for m in self.messages.get_all():
-            if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
-                delete.append(m)
-        for m in delete:
-            deleted.append(m.content)
-            self.messages.remove(m)
-
-        # XXX should return the UIDs of the deleted messages
-        # more generically
-        return [x for x in range(len(deleted))]
-
-    def fetch(self, messages, uid):
-        """
-        Retrieve one or more messages in this mailbox.
-
-        from rfc 3501: The data items to be fetched can be either a single atom
-        or a parenthesized list.
-
-        :param messages: IDs of the messages to retrieve information about
-        :type messages: MessageSet
-
-        :param uid: If true, the IDs are UIDs. They are message sequence IDs
-                    otherwise.
-        :type uid: bool
-
-        :rtype: A tuple of two-tuples of message sequence numbers and
-                LeapMessage
-        """
-        result = []
-        sequence = True if uid == 0 else False
-
-        if not messages.last:
-            try:
-                iter(messages)
-            except TypeError:
-                # looks like we cannot iterate
-                messages.last = self.last_uid
-
-        # for sequence numbers (uid = 0)
-        if sequence:
-            for msg_id in messages:
-                msg = self.messages.get_msg_by_index(msg_id - 1)
-                if msg:
-                    result.append((msg.getUID(), msg))
-                else:
-                    print "fetch %s, no msg found!!!" % msg_id
-
-        else:
-            for msg_id in messages:
-                msg = self.messages.get_msg_by_uid(msg_id)
-                if msg:
-                    result.append((msg_id, msg))
-                else:
-                    print "fetch %s, no msg found!!!" % msg_id
-
-        if self.isWriteable():
-            self._unset_recent_flag()
-
-        return tuple(result[:100])
-
-    def _unset_recent_flag(self):
-        """
-        Unsets `Recent` flag from a tuple of messages.
-        Called from fetch.
-
-        From RFC, about `Recent`:
-
-        Message is "recently" arrived in this mailbox.  This session
-        is the first session to have been notified about this
-        message; if the session is read-write, subsequent sessions
-        will not see \Recent set for this message.  This flag can not
-        be altered by the client.
-
-        If it is not possible to determine whether or not this
-        session is the first session to be notified about a message,
-        then that message SHOULD be considered recent.
-        """
-        log.msg('unsetting recent flags...')
-        for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
-            newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
-            self._update(newflags)
-
-    def _signal_unread_to_ui(self):
-        """
-        Sends unread event to ui.
-        """
-        unseen = self.getUnseenCount()
-        leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
-
-    def store(self, messages, flags, mode, uid):
-        """
-        Sets the flags of one or more messages.
-
-        :param messages: The identifiers of the messages to set the flags
-        :type messages: A MessageSet object with the list of messages requested
-
-        :param flags: The flags to set, unset, or add.
-        :type flags: sequence of str
-
-        :param mode: If mode is -1, these flags should be removed from the
-                     specified messages.  If mode is 1, these flags should be
-                     added to the specified messages.  If mode is 0, all
-                     existing flags should be cleared and these flags should be
-                     added.
-        :type mode: -1, 0, or 1
-
-        :param uid: If true, the IDs specified in the query are UIDs;
-                    otherwise they are message sequence IDs.
-        :type uid: bool
-
-        :return: A dict mapping message sequence numbers to sequences of
-                 str representing the flags set on the message after this
-                 operation has been performed.
-        :rtype: dict
-
-        :raise ReadOnlyMailbox: Raised if this mailbox is not open for
-                                read-write.
-        """
-        # XXX implement also sequence (uid = 0)
-        # XXX we should prevent cclient from setting Recent flag.
-        leap_assert(not isinstance(flags, basestring),
-                    "flags cannot be a string")
-        flags = tuple(flags)
-
-        if not self.isWriteable():
-            log.msg('read only mailbox!')
-            raise imap4.ReadOnlyMailbox
-
-        if not messages.last:
-            messages.last = self.messages.count()
-
-        result = {}
-        for msg_id in messages:
-            print "MSG ID = %s" % msg_id
-            msg = self.messages.get_msg_by_uid(msg_id)
-            if mode == 1:
-                self._update(msg.addFlags(flags))
-            elif mode == -1:
-                self._update(msg.removeFlags(flags))
-            elif mode == 0:
-                self._update(msg.setFlags(flags))
-            result[msg_id] = msg.getFlags()
-
-        self._signal_unread_to_ui()
-        return result
-
-    def close(self):
-        """
-        Expunge and mark as closed
-        """
-        self.expunge()
-        self.closed = True
-
-    # convenience fun
-
-    def deleteAllDocs(self):
-        """
-        Deletes all docs in this mailbox
-        """
-        docs = self.messages.get_all()
-        for doc in docs:
-            self.messages._soledad.delete_doc(doc)
-
-    def _update(self, doc):
-        """
-        Updates document in u1db database
-        """
-        # XXX create namedtuple
-        self.messages.soledad_writer.put({"mode": "put",
-                                          "payload": doc})
-
-    def __repr__(self):
-        """
-        Representation string for this mailbox.
-        """
-        return u"<SoledadMailbox: mbox '%s' (%s)>" % (
-            self.mbox, self.messages.count())
diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py
index 8756ddc..234996d 100644
--- a/mail/src/leap/mail/imap/service/imap.py
+++ b/mail/src/leap/mail/imap/service/imap.py
@@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
 from leap.common import events as leap_events
 from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import KeyManager
-from leap.mail.imap.server import SoledadBackedAccount
+from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
 from leap.soledad.client import Soledad
 
@@ -87,6 +87,8 @@ class LeapIMAPServer(imap4.IMAP4Server):
         :param line: the line from the server, without the line delimiter.
         :type line: str
         """
+        print "RECV: STATE (%s)" % self.state
+
         if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
diff --git a/mail/src/leap/mail/imap/tests/getmail b/mail/src/leap/mail/imap/tests/getmail
new file mode 100755
index 0000000..17e195c
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/getmail
@@ -0,0 +1,282 @@
+#!/usr/bin/env python
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE in twisted for details.
+
+# Modifications by LEAP Developers 2014 to fit
+# Bitmask configuration settings.
+
+
+"""
+Simple IMAP4 client which displays the subjects of all messages in a
+particular mailbox.
+"""
+
+import sys
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+
+class TrivialPrompter(basic.LineReceiver):
+    from os import linesep as delimiter
+
+    promptDeferred = None
+
+    def prompt(self, msg):
+        assert self.promptDeferred is None
+        self.display(msg)
+        self.promptDeferred = defer.Deferred()
+        return self.promptDeferred
+
+    def display(self, msg):
+        self.transport.write(msg)
+
+    def lineReceived(self, line):
+        if self.promptDeferred is None:
+            return
+        d, self.promptDeferred = self.promptDeferred, None
+        d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+    """
+    A client with callbacks for greeting messages from an IMAP server.
+    """
+    greetDeferred = None
+
+    def serverGreeting(self, caps):
+        self.serverCapabilities = caps
+        if self.greetDeferred is not None:
+            d, self.greetDeferred = self.greetDeferred, None
+            d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+    usedUp = False
+
+    protocol = SimpleIMAP4Client
+
+    def __init__(self, username, onConn):
+        self.ctx = ssl.ClientContextFactory()
+
+        self.username = username
+        self.onConn = onConn
+
+    def buildProtocol(self, addr):
+        """
+        Initiate the protocol instance. Since we are building a simple IMAP
+        client, we don't bother checking what capabilities the server has. We
+        just add all the authenticators twisted.mail has.  Note: Gmail no
+        longer uses any of the methods below, it's been using XOAUTH since
+        2010.
+        """
+        assert not self.usedUp
+        self.usedUp = True
+
+        p = self.protocol(self.ctx)
+        p.factory = self
+        p.greetDeferred = self.onConn
+
+        p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+        p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+        p.registerAuthenticator(
+            imap4.CramMD5ClientAuthenticator(self.username))
+
+        return p
+
+    def clientConnectionFailed(self, connector, reason):
+        d, self.onConn = self.onConn, None
+        d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+    """
+    Initial callback - invoked after the server sends us its greet message.
+    """
+    # Hook up stdio
+    tp = TrivialPrompter()
+    stdio.StandardIO(tp)
+
+    # And make it easily accessible
+    proto.prompt = tp.prompt
+    proto.display = tp.display
+
+    # Try to authenticate securely
+    return proto.authenticate(
+        password).addCallback(
+        cbAuthentication,
+        proto).addErrback(
+        ebAuthentication, proto, username, password
+    )
+
+
+def ebConnection(reason):
+    """
+    Fallback error-handler. If anything goes wrong, log it and quit.
+    """
+    log.startLogging(sys.stdout)
+    log.err(reason)
+    return reason
+
+
+def cbAuthentication(result, proto):
+    """
+    Callback after authentication has succeeded.
+
+    Lists a bunch of mailboxes.
+    """
+    return proto.list("", "*"
+        ).addCallback(cbMailboxList, proto
+        )
+
+
+def ebAuthentication(failure, proto, username, password):
+    """
+    Errback invoked when authentication fails.
+
+    If it failed because no SASL mechanisms match, offer the user the choice
+    of logging in insecurely.
+
+    If you are trying to connect to your Gmail account, you will be here!
+    """
+    failure.trap(imap4.NoSupportedAuthentication)
+    return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+    """
+    insecure-login.
+    """
+    return proto.login(username, password
+        ).addCallback(cbAuthentication, proto
+        )
+
+
+def cbMailboxList(result, proto):
+    """
+    Callback invoked when a list of mailboxes has been retrieved.
+    """
+    result = [e[2] for e in result]
+    s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)])
+    if not s:
+        return defer.fail(Exception("No mailboxes exist on server!"))
+    return proto.prompt(s + "\nWhich mailbox? [1] "
+        ).addCallback(cbPickMailbox, proto, result
+        )
+
+
+def cbPickMailbox(result, proto, mboxes):
+    """
+    When the user selects a mailbox, "examine" it.
+    """
+    mbox = mboxes[int(result or '1') - 1]
+    return proto.examine(mbox
+        ).addCallback(cbExamineMbox, proto
+        )
+
+
+def cbExamineMbox(result, proto):
+    """
+    Callback invoked when examine command completes.
+
+    Retrieve the subject header of every message in the mailbox.
+    """
+    return proto.fetchSpecific('1:*',
+                               headerType='HEADER.FIELDS',
+                               headerArgs=['SUBJECT'],
+        ).addCallback(cbFetch, proto,
+        )
+
+
+def cbFetch(result, proto):
+    """
+    Display headers.
+    """
+    if result:
+        keys = result.keys()
+        keys.sort()
+        for k in keys:
+            proto.display('%s %s' % (k, result[k][0][2]))
+    else:
+        print "Hey, an empty mailbox!"
+
+    return proto.prompt("\nWhich message? [1] (Q quits) "
+                        ).addCallback(cbPickMessage, proto)
+
+
+def cbPickMessage(result, proto):
+    """
+    Pick a message.
+    """
+    if result == "Q":
+        print "Bye!"
+        return proto.logout()
+
+    return proto.fetchSpecific(
+        '%s' % result,
+        headerType='',
+        headerArgs=['BODY.PEEK[]'],
+        ).addCallback(cbShowmessage, proto)
+
+
+def cbShowmessage(result, proto):
+    """
+    Display message.
+    """
+    if result:
+        keys = result.keys()
+        keys.sort()
+        for k in keys:
+            proto.display('%s %s' % (k, result[k][0][2]))
+    else:
+        print "Hey, an empty message!"
+
+    return proto.logout()
+
+
+def cbClose(result):
+    """
+    Close the connection when we finish everything.
+    """
+    from twisted.internet import reactor
+    reactor.stop()
+
+
+def main():
+    import sys
+
+    if len(sys.argv) != 3:
+        print "Usage: getmail <user> <pass>"
+        sys.exit()
+
+    hostname = "localhost"
+    port = "1984"
+    username = sys.argv[1]
+    password = sys.argv[2]
+
+    onConn = defer.Deferred(
+        ).addCallback(cbServerGreeting, username, password
+        ).addErrback(ebConnection
+        ).addBoth(cbClose)
+
+    factory = SimpleIMAP4ClientFactory(username, onConn)
+
+    from twisted.internet import reactor
+    if port == '993':
+        reactor.connectSSL(
+            hostname, int(port), factory, ssl.ClientContextFactory())
+    else:
+        if not port:
+            port = 143
+        reactor.connectTCP(hostname, int(port), factory)
+    reactor.run()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message
new file mode 100644
index 0000000..582297c
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message
@@ -0,0 +1,16 @@
+Content-Type: multipart/mixed; boundary="===============6203542367371144092=="
+MIME-Version: 1.0
+Subject: [TEST] 010 - Inceptos cum lorem risus congue
+From: testmailbitmaskspam@gmail.com
+To: test_c5@dev.bitmask.net
+
+--===============6203542367371144092==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+Howdy from python!
+The subject: [TEST] 010 - Inceptos cum lorem risus congue
+Current date & time: Wed Jan  8 16:36:21 2014
+Trying to attach: []
+--===============6203542367371144092==--
diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message
new file mode 100644
index 0000000..9907c2d
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message
@@ -0,0 +1,238 @@
+Date: Mon, 6 Jan 2014 04:40:47 -0400
+From: Kali Kaneko <kali@leap.se>
+To: penguin@example.com
+Subject: signed message
+Message-ID: <20140106084047.GA21317@samsara.lan>
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha1;
+	protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.21 (2012-12-30)
+
+
+--z9ECzHErBrwFF8sy
+Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l"
+Content-Disposition: inline
+
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+
+This is an example of a signed message,
+with attachments.
+
+
+--=20
+Nihil sine chao! =E2=88=B4
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: attachment; filename="attach.txt"
+
+this is attachment in plain text.
+
+--z0eOaCaDLjvTGF2l
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="hack.ico"
+Content-Transfer-Encoding: base64
+
+AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA
+KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG
+RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA
+PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl
+5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA
+/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ
+yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A
+Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK
+ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK
+LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP
+QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy
+AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs
+AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA
+AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA
+gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d
+HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA
+x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7
++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA
+AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5
++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA
+OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK
+igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA
+JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra
+2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA
+xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj
+owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB
+AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA
+AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d
+XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d
+XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA
+AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB
+AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm
+X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC
+AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B
+bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ
+S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu
+J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y
+AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N
+KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB
+XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A
+AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA
+AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d
+XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA
+AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr
+RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA
+AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A
+Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI
+yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA
+CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys
+rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA
+vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d
+HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA
+urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx
+cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA
+CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo
+6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA
+2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7
+OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA
+UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp
+qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA
+lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa
+WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB
+AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA
+ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA
+AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB
+AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB
+AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA
+tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA
+AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB
+AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB
+AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA
+AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd
+AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB
+AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB
+AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8
+ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2
+NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF
+RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB
+lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA
+AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa
+WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA
+AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX
+AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB
+AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA
+AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA
+AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA
+AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA
+AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB
+AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB
+AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA
+ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA
+AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7
+LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA
+AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA
+ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4
+RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi
+JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2
+NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK
+T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN
+UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA
+AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA
+W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA
+AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB
+l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB
+AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+
+WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA
+AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv
+RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA
+AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj
+AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB
+AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA
+AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA
+AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA
+dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A
+AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB
+AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW
+pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+
+--z0eOaCaDLjvTGF2l--
+
+--z9ECzHErBrwFF8sy
+Content-Type: application/pgp-signature
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv
+kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl
+vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK
+PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC
+w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw
+sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr
+BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN
+QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt
+mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ
+jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8
+gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X
+sSdfcAhT7Tno7PB/Acoh
+=+okv
+-----END PGP SIGNATURE-----
+
+--z9ECzHErBrwFF8sy--
diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi.message b/mail/src/leap/mail/imap/tests/rfc822.multi.message
new file mode 100644
index 0000000..30f74e5
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.multi.message
@@ -0,0 +1,96 @@
+Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)

+From: Doug Sauder <doug@penguin.example.com>

+To: Joe Blow <blow@example.com>

+Subject: Test message from PINE

+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>

+MIME-Version: 1.0

+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"

+

+  This message is in MIME format.  The first part should be readable text,

+  while the remaining parts are likely unreadable without MIME-aware tools.

+  Send mail to mime@docserver.cac.washington.edu for more info.

+

+---1463757054-952513540-958744548=:8452

+Content-Type: TEXT/PLAIN; charset=US-ASCII

+

+This is a test message from PINE MUA.

+

+

+---1463757054-952513540-958744548=:8452

+Content-Type: APPLICATION/octet-stream; name="redball.png"

+Content-Transfer-Encoding: BASE64

+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>

+Content-Description: A PNG graphic file

+Content-Disposition: attachment; filename="redball.png"

+

+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A

+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj

+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv

+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0

+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf

+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl

+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL

+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS

+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP

+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP

+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI

+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR

+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6

+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO

+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8

+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8

+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu

+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF

+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp

+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow

+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G

+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn

+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1

+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T

+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8

+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f

+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/

+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms

+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA

+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7

+vAAAAABJRU5ErkJggg==

+---1463757054-952513540-958744548=:8452

+Content-Type: APPLICATION/octet-stream; name="blueball.png"

+Content-Transfer-Encoding: BASE64

+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>

+Content-Description: A PNG graphic file

+Content-Disposition: attachment; filename="blueball.png"

+

+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A

+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI

+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y

+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S

+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM

+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB

+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu

+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b

+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo

+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp

+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX

+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M

+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8

++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE

+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42

+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY

+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp

+Z3VldDZzO7wAAAAASUVORK5CYII=

+---1463757054-952513540-958744548=:8452--

diff --git a/mail/src/leap/mail/imap/tests/rfc822.plain.message b/mail/src/leap/mail/imap/tests/rfc822.plain.message
new file mode 100644
index 0000000..fc627c3
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.plain.message
@@ -0,0 +1,66 @@
+From pyar-bounces@python.org.ar Wed Jan  8 14:46:02 2014
+Return-Path: <pyar-bounces@python.org.ar>
+X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net
+X-Spam-Level: **
+X-Spam-Pyzor: Reported 0 times.
+X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE,
+	CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP,
+	NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled
+	version=3.3.2
+Delivered-To: kali@leap.se
+Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33])
+	(using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))
+	(Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified))
+	by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F
+	for <kali@leap.se>; Wed,  8 Jan 2014 18:46:02 +0000 (UTC)
+Received: from pyar.usla.org.ar (unknown [190.228.30.157])
+	by mx1.riseup.net (Postfix) with ESMTP id F244C533F4
+	for <kali@leap.se>; Wed,  8 Jan 2014 10:46:01 -0800 (PST)
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+	by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F
+	for <kali@leap.se>; Wed,  8 Jan 2014 15:46:00 -0300 (ART)
+MIME-Version: 1.0
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+From: pyar-request@python.org.ar
+To: kali@leap.se
+Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+Reply-To: pyar-request@python.org.ar
+Auto-Submitted: auto-replied
+Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar>
+Date: Wed, 08 Jan 2014 15:45:59 -0300
+Precedence: bulk
+X-BeenThere: pyar@python.org.ar
+X-Mailman-Version: 2.1.15
+List-Id: Python Argentina <pyar.python.org.ar>
+X-List-Administrivia: yes
+Errors-To: pyar-bounces@python.org.ar
+Sender: "pyar" <pyar-bounces@python.org.ar>
+X-Virus-Scanned: clamav-milter 0.97.8 at mx1
+X-Virus-Status: Clean
+
+Mailing list subscription confirmation notice for mailing list pyar
+
+We have received a request de kaliyuga@riseup.net for subscription of
+your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar
+mailing list.  To confirm that you want to be added to this mailing
+list, simply reply to this message, keeping the Subject: header
+intact.  Or visit this web page:
+
+    http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b=
+3377148ac2
+
+
+Or include the following line -- and only the following line -- in a
+message to pyar-request@python.org.ar:
+
+    confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+
+Note that simply sending a `reply' to this message should work from
+most mail readers, since that usually leaves the Subject: line in the
+right form (additional "Re:" text in the Subject: is okay).
+
+If you do not wish to be subscribed to this list, please simply
+disregard this message.  If you think you are being maliciously
+subscribed to the list, or have any other questions, send them to
+pyar-owner@python.org.ar.
diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py
index ea75854..8c1cf20 100644
--- a/mail/src/leap/mail/imap/tests/test_imap.py
+++ b/mail/src/leap/mail/imap/tests/test_imap.py
@@ -25,7 +25,7 @@ XXX add authors from the original twisted tests.
 @license: GPLv3, see included LICENSE file
 """
 # XXX review license of the original tests!!!
-from nose.twistedtools import deferred
+from email import parser
 
 try:
     from cStringIO import StringIO
@@ -36,9 +36,13 @@ import os
 import types
 import tempfile
 import shutil
+import time
+
+from itertools import chain
 
 
 from mock import Mock
+from nose.twistedtools import deferred, stop_reactor
 
 
 from twisted.mail import imap4
@@ -58,9 +62,9 @@ import twisted.cred.portal
 # import u1db
 
 from leap.common.testing.basetest import BaseLeapTest
-from leap.mail.imap.server import SoledadMailbox
-from leap.mail.imap.server import SoledadBackedAccount
-from leap.mail.imap.server import MessageCollection
+from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.messages import MessageCollection
 
 from leap.soledad.client import Soledad
 from leap.soledad.client import SoledadCrypto
@@ -321,6 +325,9 @@ class IMAP4HelperMixin(BaseLeapTest):
         for mb in self.server.theAccount.mailboxes:
             self.server.theAccount.delete(mb)
 
+        # email parser
+        self.parser = parser.Parser()
+
     def tearDown(self):
         """
         tearDown method called after each test.
@@ -350,11 +357,11 @@ class IMAP4HelperMixin(BaseLeapTest):
 
         # XXX we also should put this in a mailbox!
 
-        self._soledad.messages.add_msg('', subject="test1")
-        self._soledad.messages.add_msg('', subject="test2")
-        self._soledad.messages.add_msg('', subject="test3")
+        self._soledad.messages.add_msg('', uid=1, subject="test1")
+        self._soledad.messages.add_msg('', uid=2, subject="test2")
+        self._soledad.messages.add_msg('', uid=3, subject="test3")
         # XXX should change Flags too
-        self._soledad.messages.add_msg('', subject="test4")
+        self._soledad.messages.add_msg('', uid=4, subject="test4")
 
     def delete_all_docs(self):
         """
@@ -389,6 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
     """
     Tests for the MessageCollection class
     """
+    count = 0
 
     def setUp(self):
         """
@@ -396,34 +404,35 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         We override mixin method since we are only testing
         MessageCollection interface in this particular TestCase
         """
-        self.messages = MessageCollection("testmbox", self._soledad)
-        for m in self.messages.get_all():
-            self.messages.remove(m)
+        self.messages = MessageCollection("testmbox%s" % (self.count,),
+                                          self._soledad)
+        MessageCollectionTestCase.count += 1
 
     def tearDown(self):
         """
         tearDown method for each test
-        Delete the message collection
         """
         del self.messages
 
+    def wait(self):
+        time.sleep(2)
+
     def testEmptyMessage(self):
         """
         Test empty message and collection
         """
-        em = self.messages._get_empty_msg()
+        em = self.messages._get_empty_doc()
         self.assertEqual(
             em,
             {
-                "date": '',
                 "flags": [],
-                "headers": {},
                 "mbox": "inbox",
-                "raw": "",
                 "recent": True,
                 "seen": False,
-                "subject": "",
-                "type": "msg",
+                "deleted": False,
+                "multi": False,
+                "size": 0,
+                "type": "flags",
                 "uid": 1,
             })
         self.assertEqual(self.messages.count(), 0)
@@ -432,23 +441,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Add multiple messages
         """
+        # TODO really profile addition
         mc = self.messages
+        print "messages", self.messages
         self.assertEqual(self.messages.count(), 0)
-        mc.add_msg('Stuff', subject="test1")
-        self.assertEqual(self.messages.count(), 1)
-        mc.add_msg('Stuff', subject="test2")
-        self.assertEqual(self.messages.count(), 2)
-        mc.add_msg('Stuff', subject="test3")
-        self.assertEqual(self.messages.count(), 3)
-        mc.add_msg('Stuff', subject="test4")
+        mc.add_msg('Stuff', uid=1, subject="test1")
+        mc.add_msg('Stuff', uid=2, subject="test2")
+        mc.add_msg('Stuff', uid=3, subject="test3")
+        mc.add_msg('Stuff', uid=4, subject="test4")
+        self.wait()
         self.assertEqual(self.messages.count(), 4)
-        mc.add_msg('Stuff', subject="test5")
-        mc.add_msg('Stuff', subject="test6")
-        mc.add_msg('Stuff', subject="test7")
-        mc.add_msg('Stuff', subject="test8")
-        mc.add_msg('Stuff', subject="test9")
-        mc.add_msg('Stuff', subject="test10")
-        self.assertEqual(self.messages.count(), 10)
+        mc.add_msg('Stuff', uid=5, subject="test5")
+        mc.add_msg('Stuff', uid=6, subject="test6")
+        mc.add_msg('Stuff', uid=7, subject="test7")
+        self.wait()
+        self.assertEqual(self.messages.count(), 7)
+        self.wait()
 
     def testRecentCount(self):
         """
@@ -456,45 +464,48 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         mc = self.messages
         self.assertEqual(self.messages.count_recent(), 0)
-        mc.add_msg('Stuff', subject="test1", uid=1)
+        mc.add_msg('Stuff', uid=1, subject="test1")
         # For the semantics defined in the RFC, we auto-add the
         # recent flag by default.
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 1)
-        mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',))
+        mc.add_msg('Stuff', subject="test2", uid=2,
+                   flags=('\\Deleted',))
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 2)
-        mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',))
+        mc.add_msg('Stuff', subject="test3", uid=3,
+                   flags=('\\Recent',))
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 3)
         mc.add_msg('Stuff', subject="test4", uid=4,
                    flags=('\\Deleted', '\\Recent'))
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 4)
 
-        for m in mc:
-            msg = self.messages.get_msg_by_uid(m.get('uid'))
-            msg_newflags = msg.removeFlags(('\\Recent',))
-            self._soledad.put_doc(msg_newflags)
-
+        for msg in mc:
+            msg.removeFlags(('\\Recent',))
         self.assertEqual(mc.count_recent(), 0)
 
     def testFilterByMailbox(self):
         """
         Test that queries filter by selected mailbox
         """
+        def wait():
+            time.sleep(1)
+
         mc = self.messages
         self.assertEqual(self.messages.count(), 0)
-        mc.add_msg('', subject="test1")
-        self.assertEqual(self.messages.count(), 1)
-        mc.add_msg('', subject="test2")
-        self.assertEqual(self.messages.count(), 2)
-        mc.add_msg('', subject="test3")
+        mc.add_msg('', uid=1, subject="test1")
+        mc.add_msg('', uid=2, subject="test2")
+        mc.add_msg('', uid=3, subject="test3")
+        wait()
         self.assertEqual(self.messages.count(), 3)
-
-        newmsg = mc._get_empty_msg()
+        newmsg = mc._get_empty_doc()
         newmsg['mailbox'] = "mailbox/foo"
-        newmsg['subject'] = "test another mailbox"
         mc._soledad.create_doc(newmsg)
         self.assertEqual(mc.count(), 3)
         self.assertEqual(
-            len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4)
+            len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
 
 
 class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
@@ -1174,16 +1185,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(0.5)
+
         def append():
             return self.client.append(
                 'root/subthing',
                 message,
-                ['\\SEEN', '\\DELETED'],
+                ('\\SEEN', '\\DELETED'),
                 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
             )
 
         d1 = self.connected.addCallback(strip(login))
         d1.addCallbacks(strip(append), self._ebGeneral)
+        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
@@ -1191,17 +1206,31 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestFullAppend(self, ignored, infile):
         mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing')
+        time.sleep(0.5)
         self.assertEqual(1, len(mb.messages))
 
+        msg = mb.messages.get_msg_by_uid(1)
         self.assertEqual(
-            ['\\SEEN', '\\DELETED'],
-            mb.messages[1].content['flags'])
+            ('\\SEEN', '\\DELETED'),
+            msg.getFlags())
 
         self.assertEqual(
             'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
-            mb.messages[1].content['date'])
+            msg.getInternalDate())
+
+        parsed = self.parser.parse(open(infile))
+        body = parsed.get_payload()
+        headers = parsed.items()
+        self.assertEqual(
+            body,
+            msg.getBodyFile().read())
+
+        msg_headers = msg.getHeaders(True, "",)
+        gotheaders = list(chain(
+            *[[(k, item) for item in v] for (k, v) in msg_headers.items()]))
 
-        self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+        self.assertItemsEqual(
+            headers, gotheaders)
 
     @deferred(timeout=None)
     def testPartialAppend(self):
@@ -1209,12 +1238,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test partially appending a message to the mailbox
         """
         infile = util.sibpath(__file__, 'rfc822.message')
-        message = open(infile)
         SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
 
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(1)
+
         def append():
             message = file(infile)
             return self.client.sendCommand(
@@ -1226,6 +1257,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                 )
             )
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(strip(append), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -1235,15 +1267,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestPartialAppend(self, ignored, infile):
         mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
-
+        time.sleep(1)
         self.assertEqual(1, len(mb.messages))
+        msg = mb.messages.get_msg_by_uid(1)
         self.assertEqual(
-            ['\\SEEN', ],
-            mb.messages[1].content['flags']
+            ('\\SEEN', ),
+            msg.getFlags()
         )
+        #self.assertEqual(
+            #'Right now', msg.getInternalDate())
+        parsed = self.parser.parse(open(infile))
+        body = parsed.get_payload()
         self.assertEqual(
-            'Right now', mb.messages[1].content['date'])
-        self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+            body,
+            msg.getBodyFile().read())
 
     @deferred(timeout=None)
     def testCheck(self):
@@ -1279,14 +1316,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.server.theAccount.addMailbox(name)
 
         m = SimpleLEAPServer.theAccount.getMailbox(name)
-        m.messages.add_msg('', subject="Message 1",
+        m.messages.add_msg('test 1', uid=1, subject="Message 1",
                            flags=('\\Deleted', 'AnotherFlag'))
-        m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
-        m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
+        m.messages.add_msg('test 2', uid=2, subject="Message 2",
+                           flags=('AnotherFlag',))
+        m.messages.add_msg('test 3', uid=3, subject="Message 3",
+                           flags=('\\Deleted',))
 
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(1)
+
         def select():
             return self.client.select(name)
 
@@ -1294,6 +1336,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return self.client.close()
 
         d = self.connected.addCallback(strip(login))
+        d.addCallbacks(strip(wait), self._ebGeneral)
         d.addCallbacks(strip(select), self._ebGeneral)
         d.addCallbacks(strip(close), self._ebGeneral)
         d.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -1302,8 +1345,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestClose(self, ignored, m):
         self.assertEqual(len(m.messages), 1)
+        messages = [msg for msg in m.messages]
+        self.assertFalse(messages[0] is None)
         self.assertEqual(
-            m.messages[1].content['subject'],
+            messages[0]._hdoc.content['subject'],
             'Message 2')
         self.failUnless(m.closed)
 
@@ -1315,17 +1360,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         name = 'mailbox-expunge'
         SimpleLEAPServer.theAccount.addMailbox(name)
         m = SimpleLEAPServer.theAccount.getMailbox(name)
-        m.messages.add_msg('', subject="Message 1",
+        m.messages.add_msg('test 1', uid=1, subject="Message 1",
                            flags=('\\Deleted', 'AnotherFlag'))
-        self.failUnless(m.messages.count() == 1)
-        m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
-        self.failUnless(m.messages.count() == 2)
-        m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
-        self.failUnless(m.messages.count() == 3)
+        m.messages.add_msg('test 2', uid=2, subject="Message 2",
+                           flags=('AnotherFlag',))
+        m.messages.add_msg('test 3', uid=3, subject="Message 3",
+                           flags=('\\Deleted',))
 
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(2)
+
         def select():
             return self.client.select('mailbox-expunge')
 
@@ -1338,6 +1385,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.results = None
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(strip(select), self._ebGeneral)
         d1.addCallbacks(strip(expunge), self._ebGeneral)
         d1.addCallbacks(expunged, self._ebGeneral)
@@ -1348,18 +1396,94 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestExpunge(self, ignored, m):
         # we only left 1 mssage with no deleted flag
-        self.assertEqual(m.messages.count(), 1)
+        self.assertEqual(len(m.messages), 1)
+        messages = [msg for msg in m.messages]
         self.assertEqual(
-            m.messages[1].content['subject'],
+            messages[0]._hdoc.content['subject'],
             'Message 2')
-        self.assertEqual(self.results, [0, 1])
-        # XXX fix this thing with the indexes...
+        # the uids of the deleted messages
+        self.assertItemsEqual(self.results, [1, 3])
+
+
+class StoreAndFetchTestCase(unittest.TestCase, IMAP4HelperMixin):
+    """
+    Several tests to check that the internal storage representation
+    is able to render the message structures as we expect them.
+    """
+    # TODO get rid of the fucking sleeps with a proper defer
+    # management.
+
+    def setUp(self):
+        IMAP4HelperMixin.setUp(self)
+        MBOX_NAME = "multipart/SIGNED"
+        self.received_messages = self.received_uid = None
+        self.result = None
+
+        self.server.state = 'select'
+
+        infile = util.sibpath(__file__, 'rfc822.multi-signed.message')
+        raw = open(infile).read()
+
+        self.server.theAccount.addMailbox(MBOX_NAME)
+        mbox = self.server.theAccount.getMailbox(MBOX_NAME)
+        time.sleep(1)
+        self.server.mbox = mbox
+        self.server.mbox.messages.add_msg(raw, uid=1)
+        time.sleep(1)
+
+    def addListener(self, x):
+        pass
+
+    def removeListener(self, x):
+        pass
+
+    def _fetchWork(self, uids):
+
+        def result(R):
+            self.result = R
+
+        self.connected.addCallback(
+            lambda _: self.function(
+                uids, uid=1)  # do NOT use seq numbers!
+            ).addCallback(result).addCallback(
+            self._cbStopClient).addErrback(self._ebGeneral)
+
+        d = loopback.loopbackTCP(self.server, self.client, noisy=False)
+        d.addCallback(lambda x: self.assertEqual(self.result, self.expected))
+        return d
+
+    @deferred(timeout=None)
+    def testMultiBody(self):
+        """
+        Test that a multipart signed message is retrieved the same
+        as we stored it.
+        """
+        time.sleep(1)
+        self.function = self.client.fetchBody
+        messages = '1'
+
+        # XXX review. This probably should give everything?
+
+        self.expected = {1: {
+            'RFC822.TEXT': 'This is an example of a signed message,\n'
+                           'with attachments.\n\n\n--=20\n'
+                           'Nihil sine chao! =E2=88=B4\n',
+            'UID': '1'}}
+        print "test multi: fetch uid", messages
+        return self._fetchWork(messages)
 
 
 class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     """
-    Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}.
+    Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}.
     """
     # XXX coming soon to your screens!
     pass
+
+
+def tearDownModule():
+    """
+    Tear down functions for module level
+    """
+    stop_reactor()
diff --git a/mail/src/leap/mail/imap/tests/walktree.py b/mail/src/leap/mail/imap/tests/walktree.py
new file mode 100644
index 0000000..1626f65
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/walktree.py
@@ -0,0 +1,117 @@
+#t -*- coding: utf-8 -*-
+# walktree.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the walktree module.
+"""
+import os
+from email import parser
+
+from leap.mail import walk as W
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+p = parser.Parser()
+
+# TODO pass an argument of the type of message
+
+##################################################
+# Input from hell
+
+#msg = p.parse(open('rfc822.multi-signed.message'))
+#msg = p.parse(open('rfc822.plain.message'))
+msg = p.parse(open('rfc822.multi-minimal.message'))
+DO_CHECK = False
+#################################################
+
+parts = W.get_parts(msg)
+
+if DEBUG:
+    def trim(item):
+        item = item[:10]
+    [trim(part["phash"]) for part in parts if part.get('phash', None)]
+
+raw_docs = list(W.get_raw_docs(msg, parts))
+
+body_phash_fun = [W.get_body_phash_simple,
+                  W.get_body_phash_multi][int(msg.is_multipart())]
+body_phash = body_phash_fun(W.get_payloads(msg))
+parts_map = W.walk_msg_tree(parts, body_phash=body_phash)
+
+
+# TODO add missing headers!
+expected = {
+    'body': '1ddfa80485',
+    'multi': True,
+    'part_map': {
+        1: {
+            'headers': {'Content-Disposition': 'inline',
+                        'Content-Type': 'multipart/mixed; '
+                        'boundary="z0eOaCaDLjvTGF2l"'},
+            'multi': True,
+            'part_map': {1: {'ctype': 'text/plain',
+                             'headers': [
+                                 ('Content-Type',
+                                  'text/plain; charset=utf-8'),
+                                 ('Content-Disposition',
+                                  'inline'),
+                                 ('Content-Transfer-Encoding',
+                                  'quoted-printable')],
+                             'multi': False,
+                             'parts': 1,
+                             'phash': '1ddfa80485',
+                             'size': 206},
+                         2: {'ctype': 'text/plain',
+                             'headers': [('Content-Type',
+                                          'text/plain; charset=us-ascii'),
+                                         ('Content-Disposition',
+                                          'attachment; '
+                                          'filename="attach.txt"')],
+                             'multi': False,
+                             'parts': 1,
+                             'phash': '7a94e4d769',
+                             'size': 133},
+                         3: {'ctype': 'application/octet-stream',
+                             'headers': [('Content-Type',
+                                          'application/octet-stream'),
+                                         ('Content-Disposition',
+                                          'attachment; filename="hack.ico"'),
+                                         ('Content-Transfer-Encoding',
+                                          'base64')],
+                             'multi': False,
+                             'parts': 1,
+                             'phash': 'c42cccebbd',
+                             'size': 12736}}},
+        2: {'ctype': 'application/pgp-signature',
+            'headers': [('Content-Type', 'application/pgp-signature')],
+            'multi': False,
+            'parts': 1,
+            'phash': '8f49fbf749',
+            'size': 877}}}
+
+if DEBUG and DO_CHECK:
+    # TODO turn this into a proper unittest
+    assert(parts_map == expected)
+    print "Structure: OK"
+
+
+import pprint
+print
+print "RAW DOCS"
+pprint.pprint(raw_docs)
+print
+print "PARTS MAP"
+pprint.pprint(parts_map)
diff --git a/mail/src/leap/mail/utils.py b/mail/src/leap/mail/utils.py
new file mode 100644
index 0000000..2480efc
--- /dev/null
+++ b/mail/src/leap/mail/utils.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Small utilities.
+"""
+
+
+def first(things):
+    """
+    Return the head of a collection.
+    """
+    try:
+        return things[0]
+    except (IndexError, TypeError):
+        return None
diff --git a/mail/src/leap/mail/walk.py b/mail/src/leap/mail/walk.py
new file mode 100644
index 0000000..820b8c7
--- /dev/null
+++ b/mail/src/leap/mail/walk.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# walk.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for walking along a message tree.
+"""
+import hashlib
+import os
+
+from leap.mail.utils import first
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+if DEBUG:
+    get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10]
+else:
+    get_hash = lambda s: hashlib.sha256(s).hexdigest()
+
+
+"""
+Get interesting message parts
+"""
+get_parts = lambda msg: [
+    {'multi': part.is_multipart(),
+     'ctype': part.get_content_type(),
+     'size': len(part.as_string()),
+     'parts': len(part.get_payload())
+        if isinstance(part.get_payload(), list)
+        else 1,
+     'headers': part.items(),
+     'phash': get_hash(part.get_payload())
+        if not part.is_multipart() else None}
+    for part in msg.walk()]
+
+"""
+Utility lambda functions for getting the parts vector and the
+payloads from the original message.
+"""
+
+get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts)
+get_payloads = lambda msg: ((x.get_payload(),
+                             dict(((str.lower(k), v) for k, v in (x.items()))))
+                            for x in msg.walk())
+
+get_body_phash_simple = lambda payloads: first(
+    [get_hash(payload) for payload, headers in payloads
+     if "text/plain" in headers.get('content-type')])
+
+get_body_phash_multi = lambda payloads: (first(
+    [get_hash(payload) for payload, headers in payloads
+     if "text/plain" in headers.get('content-type')])
+    or get_body_phash_simple(payloads))
+
+"""
+On getting the raw docs, we get also some of the headers to be able to
+index the content. Here we remove any mutable part, as the the filename
+in the content disposition.
+"""
+
+get_raw_docs = lambda msg, parts: (
+    {"type": "cnt",  # type content they'll be
+     "raw": payload if not DEBUG else payload[:100],
+     "phash": get_hash(payload),
+     "content-disposition": first(headers.get(
+         'content-disposition', '').split(';')),
+     "content-type": headers.get(
+         'content-type', ''),
+     "content-transfer-encoding": headers.get(
+         'content-transfer-type', '')}
+    for payload, headers in get_payloads(msg)
+    if not isinstance(payload, list))
+
+
+def walk_msg_tree(parts, body_phash=None):
+    """
+    Take a list of interesting items of a message subparts structure,
+    and return a dict of dicts almost ready to be written to the content
+    documents that will be stored in Soledad.
+
+    It walks down the subparts in the parsed message tree, and collapses
+    the leaf docuents into a wrapper document until no multipart submessages
+    are left. To achieve this, it iteratively calculates a wrapper vector of
+    all documents in the sequence that have more than one part and have unitary
+    documents to their right. To collapse a multipart, take as many
+    unitary documents as parts the submessage contains, and replace the object
+    in the sequence with the new wrapper document.
+
+    :param parts: A list of dicts containing the interesting properties for
+                  the message structure. Normally this has been generated by
+                  doing a message walk.
+    :type parts: list of dicts.
+    :param body_phash: the payload hash of the body part, to be included
+                       in the outer content doc for convenience.
+    :type body_phash: basestring or None
+    """
+    # parts vector
+    pv = list(get_parts_vector(parts))
+
+    if len(parts) == 2:
+        inner_headers = parts[1].get("headers", None)
+
+    if DEBUG:
+        print "parts vector: ", pv
+        print
+
+    # wrappers vector
+    getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False
+                        for i in range(len(pv) - 1)]
+    wv = getwv(pv)
+
+    # do until no wrapper document is left
+    while any(wv):
+        wind = wv.index(True)  # wrapper index
+        nsub = pv[wind]  # number of subparts to pick
+        slic = parts[wind + 1:wind + 1 + nsub]  # slice with subparts
+
+        cwra = {
+            "multi": True,
+            "part_map": dict((index + 1, part)  # content wrapper
+                             for index, part in enumerate(slic)),
+            "headers": dict(parts[wind]['headers'])
+        }
+
+        # remove subparts and substitue wrapper
+        map(lambda i: parts.remove(i), slic)
+        parts[wind] = cwra
+
+        # refresh vectors for this iteration
+        pv = list(get_parts_vector(parts))
+        wv = getwv(pv)
+
+    outer = parts[0]
+    outer.pop('headers')
+    if not "part_map" in outer:
+        # we have a multipart with 1 part only, so kind of fix it
+        # although it would be prettier if I take this special case at
+        # the beginning of the walk.
+        pdoc = {"multi": True,
+                "part_map": {1: outer}}
+        pdoc["part_map"][1]["multi"] = False
+        if not pdoc["part_map"][1].get("phash", None):
+            pdoc["part_map"][1]["phash"] = body_phash
+        pdoc["part_map"][1]["headers"] = inner_headers
+    else:
+        pdoc = outer
+    pdoc["body"] = body_phash
+    return pdoc