cleanup imap implementation

1 files changed, 181 insertions, 826 deletions

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index d47c8eb..7e0f973 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-# messages.py
-# Copyright (C) 2013, 2014 LEAP
+# imap/messages.py
+# Copyright (C) 2013-2015 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,85 +15,41 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-LeapMessage and MessageCollection.
+IMAPMessage and IMAPMessageCollection.
 """
-import copy
 import logging
-import threading
-import StringIO
-
-from collections import defaultdict
-from functools import partial
-
+# import StringIO
 from twisted.mail import imap4
-from twisted.internet import reactor
 from zope.interface import implements
-from zope.proxy import sameProxiedObjects
 
 from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
-from leap.mail.adaptors import soledad_indexes as indexes
-from leap.mail.constants import INBOX_NAME
-from leap.mail.utils import find_charset, empty
-from leap.mail.imap.index import IndexedDB
-from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
-from leap.mail.imap.parser import MBoxParser
-
-logger = logging.getLogger(__name__)
-
-# TODO ------------------------------------------------------------
 
-# [ ] Add ref to incoming message during add_msg
-# [ ] Add linked-from info.
-#     * Need a new type of documents: linkage info.
-#     * HDOCS are linked from FDOCs (ref to chash)
-#     * CDOCS are linked from HDOCS (ref to chash)
+from leap.mail.utils import find_charset
 
-# [ ] Delete incoming mail only after successful write!
-# [ ] Remove UID from syncable db. Store only those indexes locally.
+from leap.mail.imap.messageparts import MessagePart
+# from leap.mail.imap.messagepargs import MessagePartDoc
 
+logger = logging.getLogger(__name__)
 
-def try_unique_query(curried):
-    """
-    Try to execute a query that is expected to have a
-    single outcome, and log a warning if more than one document found.
-
-    :param curried: a curried function
-    :type curried: callable
-    """
-    # XXX FIXME ---------- convert to deferreds
-    leap_assert(callable(curried), "A callable is expected")
-    try:
-        query = curried()
-        if query:
-            if len(query) > 1:
-                # TODO we could take action, like trigger a background
-                # process to kill dupes.
-                name = getattr(curried, 'expected', 'doc')
-                logger.warning(
-                    "More than one %s found for this mbox, "
-                    "we got a duplicate!!" % (name,))
-            return query.pop()
-        else:
-            return None
-    except Exception as exc:
-        logger.exception("Unhandled error %r" % exc)
-
+# TODO ------------------------------------------------------------
 
-# FIXME remove-me
-#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+# [ ] Add ref to incoming message during add_msg.
+# [ ] Delete incoming mail only after successful write.
 
 
-class IMAPMessage(fields, MBoxParser):
+class IMAPMessage(object):
     """
     The main representation of a message.
     """
 
     implements(imap4.IMessage)
 
-    def __init__(self, soledad, uid, mbox):
+    # TODO ---- see what should we pass here instead
+    # where's UID added to the message?
+    # def __init__(self, soledad, uid, mbox):
+    def __init__(self, message, collection):
         """
         Initializes a LeapMessage.
 
@@ -103,81 +59,14 @@ class IMAPMessage(fields, MBoxParser):
         :type uid: int or basestring
         :param mbox: the mbox this message belongs to
         :type mbox: str or unicode
-        :param collection: a reference to the parent collection object
-        :type collection: MessageCollection
-        :param container: a IMessageContainer implementor instance
-        :type container: IMessageContainer
         """
-        self._soledad = soledad
-        self._uid = int(uid) if uid is not None else None
-        self._mbox = self._parse_mailbox_name(mbox)
-
-        self.__chash = None
-        self.__bdoc = None
-
-    # TODO collection and container are deprecated.
-
-    # TODO move to adaptor
-
-    #@property
-    #def fdoc(self):
-        #"""
-        #An accessor to the flags document.
-        #"""
-        #if all(map(bool, (self._uid, self._mbox))):
-            #fdoc = None
-            #if self._container is not None:
-                #fdoc = self._container.fdoc
-            #if not fdoc:
-                #fdoc = self._get_flags_doc()
-            #if fdoc:
-                #fdoc_content = fdoc.content
-                #self.__chash = fdoc_content.get(
-                    #fields.CONTENT_HASH_KEY, None)
-            #return fdoc
-#
-    #@property
-    #def hdoc(self):
-        #"""
-        #An accessor to the headers document.
-        #"""
-        #container = self._container
-        #if container is not None:
-            #hdoc = self._container.hdoc
-            #if hdoc and not empty(hdoc.content):
-                #return hdoc
-        #hdoc = self._get_headers_doc()
-#
-        #if container and not empty(hdoc.content):
-            # mem-cache it
-            #hdoc_content = hdoc.content
-            #chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
-            #hdocs = {chash: hdoc_content}
-            #container.memstore.load_header_docs(hdocs)
-        #return hdoc
-#
-    #@property
-    #def chash(self):
-        #"""
-        #An accessor to the content hash for this message.
-        #"""
-        #if not self.fdoc:
-            #return None
-        #if not self.__chash and self.fdoc:
-            #self.__chash = self.fdoc.content.get(
-                #fields.CONTENT_HASH_KEY, None)
-        #return self.__chash
-
-    #@property
-    #def bdoc(self):
-        #"""
-        #An accessor to the body document.
-        #"""
-        #if not self.hdoc:
-            #return None
-        #if not self.__bdoc:
-            #self.__bdoc = self._get_body_doc()
-        #return self.__bdoc
+        #self._uid = int(uid) if uid is not None else None
+        #self._mbox = normalize_mailbox(mbox)
+
+        self.message = message
+
+        # TODO maybe not needed, see setFlags below
+        self.collection = collection
 
     # IMessage implementation
 
@@ -188,12 +77,7 @@ class IMAPMessage(fields, MBoxParser):
         :return: uid for this message
         :rtype: int
         """
-        # TODO ----> return lookup in local sqlcipher table.
-        return self._uid
-
-    # --------------------------------------------------------------
-    # TODO -- from here on, all the methods should be proxied to the
-    # instance of leap.mail.mail.Message
+        return self.message.get_uid()
 
     def getFlags(self):
         """
@@ -202,24 +86,14 @@ class IMAPMessage(fields, MBoxParser):
         :return: The flags, represented as strings
         :rtype: tuple
         """
-        uid = self._uid
-
-        flags = set([])
-        fdoc = self.fdoc
-        if fdoc:
-            flags = set(fdoc.content.get(self.FLAGS_KEY, None))
+        return self.message.get_flags()
 
-        msgcol = self._collection
+    # setFlags not in the interface spec but we use it with store command.
 
-        # We treat the recent flag specially: gotten from
-        # a mailbox-level document.
-        if msgcol and uid in msgcol.recent_flags:
-            flags.add(fields.RECENT_FLAG)
-        if flags:
-            flags = map(str, flags)
-        return tuple(flags)
+    # XXX if we can move it to a collection method, we don't need to pass
+    # collection to the IMAPMessage
 
-    # setFlags not in the interface spec but we use it with store command.
+    # lookup method? IMAPMailbox?
 
     def setFlags(self, flags, mode):
         """
@@ -231,32 +105,11 @@ class IMAPMessage(fields, MBoxParser):
         :type mode: int
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        mbox, uid = self._mbox, self._uid
-
-        APPEND = 1
-        REMOVE = -1
-        SET = 0
-
-        doc = self.fdoc
-        if not doc:
-            logger.warning(
-                "Could not find FDOC for %r:%s while setting flags!" %
-                (mbox, uid))
-            return
-        current = doc.content[self.FLAGS_KEY]
-        if mode == APPEND:
-            newflags = tuple(set(tuple(current) + flags))
-        elif mode == REMOVE:
-            newflags = tuple(set(current).difference(set(flags)))
-        elif mode == SET:
-            newflags = flags
-        new_fdoc = {
-            self.FLAGS_KEY: newflags,
-            self.SEEN_KEY: self.SEEN_FLAG in newflags,
-            self.DEL_KEY: self.DELETED_FLAG in newflags}
-        self._collection.memstore.update_flags(mbox, uid, new_fdoc)
-
-        return map(str, newflags)
+        # XXX
+        # return new flags
+        # map to str
+        #self.message.set_flags(flags, mode)
+        self.collection.update_flags(self.message, flags, mode)
 
     def getInternalDate(self):
         """
@@ -273,8 +126,7 @@ class IMAPMessage(fields, MBoxParser):
         :return: An RFC822-formatted date string.
         :rtype: str
         """
-        date = self.hdoc.content.get(fields.DATE_KEY, '')
-        return date
+        return self.message.get_internal_date()
 
     #
     # IMessagePart
@@ -290,42 +142,40 @@ class IMAPMessage(fields, MBoxParser):
         :return: file-like object opened for reading
         :rtype: StringIO
         """
-        def write_fd(body):
-            fd.write(body)
-            fd.seek(0)
-            return fd
-
+        #def write_fd(body):
+            #fd.write(body)
+            #fd.seek(0)
+            #return fd
+#
         # TODO refactor with getBodyFile in MessagePart
-
-        fd = StringIO.StringIO()
-
-        if self.bdoc is not None:
-            bdoc_content = self.bdoc.content
-            if empty(bdoc_content):
-                logger.warning("No BDOC content found for message!!!")
-                return write_fd("")
-
-            body = bdoc_content.get(self.RAW_KEY, "")
-            content_type = bdoc_content.get('content-type', "")
-            charset = find_charset(content_type)
-            if charset is None:
-                charset = self._get_charset(body)
-            try:
-                if isinstance(body, unicode):
-                    body = body.encode(charset)
-            except UnicodeError as exc:
-                logger.error(
-                    "Unicode error, using 'replace'. {0!r}".format(exc))
-                logger.debug("Attempted to encode with: %s" % charset)
-                body = body.encode(charset, 'replace')
-            finally:
-                return write_fd(body)
-
-        # We are still returning funky characters from here.
-        else:
-            logger.warning("No BDOC found for message.")
-            return write_fd("")
-
+#
+        #fd = StringIO.StringIO()
+#
+        #if self.bdoc is not None:
+            #bdoc_content = self.bdoc.content
+            #if empty(bdoc_content):
+                #logger.warning("No BDOC content found for message!!!")
+                #return write_fd("")
+#
+            #body = bdoc_content.get(self.RAW_KEY, "")
+            #content_type = bdoc_content.get('content-type', "")
+            #charset = find_charset(content_type)
+            #if charset is None:
+                #charset = self._get_charset(body)
+            #try:
+                #if isinstance(body, unicode):
+                    #body = body.encode(charset)
+            #except UnicodeError as exc:
+                #logger.error(
+                    #"Unicode error, using 'replace'. {0!r}".format(exc))
+                #logger.debug("Attempted to encode with: %s" % charset)
+                #body = body.encode(charset, 'replace')
+            #finally:
+                #return write_fd(body)
+
+        return self.message.get_body_file()
+
+    # TODO move to mail.mail
     @memoized_method
     def _get_charset(self, stuff):
         """
@@ -337,7 +187,7 @@ class IMAPMessage(fields, MBoxParser):
         """
         # XXX shouldn't we make the scope
         # of the decorator somewhat more persistent?
-        # ah! yes! and put memory bounds.
+        # and put memory bounds.
         return get_email_charset(stuff)
 
     def getSize(self):
@@ -347,17 +197,11 @@ class IMAPMessage(fields, MBoxParser):
         :return: size of the message, in octets
         :rtype: int
         """
-        size = None
-        if self.fdoc is not None:
-            fdoc_content = self.fdoc.content
-            size = fdoc_content.get(self.SIZE_KEY, False)
-        else:
-            logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
-                                                       self._uid))
-        #if not size:
-            # XXX fallback, should remove when all migrated.
-            #size = self.getBodyFile().len
-        return size
+        #size = None
+        #fdoc_content = self.fdoc.content
+        #size = fdoc_content.get(self.SIZE_KEY, False)
+        #return size
+        return self.message.get_size()
 
     def getHeaders(self, negate, *names):
         """
@@ -374,10 +218,10 @@ class IMAPMessage(fields, MBoxParser):
         :return: A mapping of header field names to header field values
         :rtype: dict
         """
-        # TODO split in smaller methods
+        # TODO split in smaller methods -- format_headers()?
         # XXX refactor together with MessagePart method
 
-        headers = self._get_headers()
+        headers = self.message.get_headers()
 
         # XXX keep this in the imap imessage implementation,
         # because the server impl. expects content-type to be present.
@@ -417,34 +261,15 @@ class IMAPMessage(fields, MBoxParser):
                 headers2[key] = value
         return headers2
 
-    def _get_headers(self):
-        """
-        Return the headers dict for this message.
-        """
-        if self.hdoc is not None:
-            hdoc_content = self.hdoc.content
-            headers = hdoc_content.get(self.HEADERS_KEY, {})
-            return headers
-
-        else:
-            logger.warning(
-                "No HEADERS doc for msg %s:%s" % (
-                    self._mbox,
-                    self._uid))
-
     def isMultipart(self):
         """
         Return True if this message is multipart.
         """
-        if self.fdoc:
-            fdoc_content = self.fdoc.content
-            is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
-            return is_multipart
-        else:
-            logger.warning(
-                "No FLAGS doc for msg %s:%s" % (
-                    self._mbox,
-                    self._uid))
+        #fdoc_content = self.fdoc.content
+        #is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
+        #return is_multipart
+
+        return self.message.fdoc.is_multi
 
     def getSubPart(self, part):
         """
@@ -463,12 +288,16 @@ class IMAPMessage(fields, MBoxParser):
             pmap_dict = self._get_part_from_parts_map(part + 1)
         except KeyError:
             raise IndexError
+
+        # TODO move access to adaptor ----
         return MessagePart(self._soledad, pmap_dict)
 
     #
     # accessors
     #
 
+    # FIXME
+    # -- move to wrapper/adaptor
     def _get_part_from_parts_map(self, part):
         """
         Get a part map from the headers doc
@@ -476,100 +305,44 @@ class IMAPMessage(fields, MBoxParser):
         :raises: KeyError if key does not exist
         :rtype: dict
         """
-        if not self.hdoc:
-            logger.warning("Tried to get part but no HDOC found!")
-            return None
-
-        hdoc_content = self.hdoc.content
-        pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
+        raise NotImplementedError()
 
+        #hdoc_content = self.hdoc.content
+        #pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
+#
         # remember, lads, soledad is using strings in its keys,
         # not integers!
-        return pmap[str(part)]
+        #return pmap[str(part)]
 
-    # XXX moved to memory store
-    # move the rest too. ------------------------------------------
-    def _get_flags_doc(self):
-        """
-        Return the document that keeps the flags for this
-        message.
-        """
-        def get_first_if_any(docs):
-            result = first(docs)
-            return result if result else {}
-
-        d = self._soledad.get_from_index(
-            fields.TYPE_MBOX_UID_IDX,
-            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
-        d.addCallback(get_first_if_any)
-        return d
-
-    # TODO move to soledadstore instead of accessing soledad directly
-    def _get_headers_doc(self):
-        """
-        Return the document that keeps the headers for this
-        message.
-        """
-        d = self._soledad.get_from_index(
-            fields.TYPE_C_HASH_IDX,
-            fields.TYPE_HEADERS_VAL, str(self.chash))
-        d.addCallback(lambda docs: first(docs))
-        return d
-
-    # TODO move to soledadstore instead of accessing soledad directly
+    # TODO move to wrapper/adaptor
     def _get_body_doc(self):
         """
         Return the document that keeps the body for this
         message.
         """
-        # XXX FIXME --- this might need a maybedeferred
-        # on the receiving side...
-        hdoc_content = self.hdoc.content
-        body_phash = hdoc_content.get(
-            fields.BODY_KEY, None)
-        if not body_phash:
-            logger.warning("No body phash for this document!")
-            return None
-
-        # XXX get from memstore too...
-        # if memstore: memstore.get_phrash
-        # memstore should keep a dict with weakrefs to the
-        # phash doc...
-
-        if self._container is not None:
-            bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
-            if not empty(bdoc) and not empty(bdoc.content):
-                return bdoc
-
+        # FIXME
+        # -- just get the body and retrieve the cdoc P-<phash>
+        #hdoc_content = self.hdoc.content
+        #body_phash = hdoc_content.get(
+            #fields.BODY_KEY, None)
+        #if not body_phash:
+            #logger.warning("No body phash for this document!")
+            #return None
+#
+        #if self._container is not None:
+            #bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
+            #if not empty(bdoc) and not empty(bdoc.content):
+                #return bdoc
+#
         # no memstore, or no body doc found there
-        d = self._soledad.get_from_index(
-            fields.TYPE_P_HASH_IDX,
-            fields.TYPE_CONTENT_VAL, str(body_phash))
-        d.addCallback(lambda docs: first(docs))
-        return d
-
-    def __getitem__(self, key):
-        """
-        Return an item from the content of the flags document,
-        for convenience.
-
-        :param key: The key
-        :type key: str
-
-        :return: The content value indexed by C{key} or None
-        :rtype: str
-        """
-        return self.fdoc.content.get(key, None)
-
-    def does_exist(self):
-        """
-        Return True if there is actually a flags document for this
-        UID and mbox.
-        """
-        return not empty(self.fdoc)
+        #d = self._soledad.get_from_index(
+            #fields.TYPE_P_HASH_IDX,
+            #fields.TYPE_CONTENT_VAL, str(body_phash))
+        #d.addCallback(lambda docs: first(docs))
+        #return d
 
 
-class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
+class IMAPMessageCollection(object):
     """
     A collection of messages, surprisingly.
 
@@ -578,9 +351,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
     database.
     """
 
-    # XXX this should be able to produce a MessageSet methinks
-    # could validate these kinds of objects turning them
-    # into a template for the class.
+    messageklass = IMAPMessage
+
+    # TODO
+    # [ ] Add RECENT flags docs to mailbox-doc attributes (list-of-uids)
+    # [ ] move Query for all the headers documents to Collection
+
+    # TODO this should be able to produce a MessageSet methinks
+    # TODO --- reimplement, review and prune documentation below.
+
     FLAGS_DOC = "FLAGS"
     HEADERS_DOC = "HEADERS"
     CONTENT_DOC = "CONTENT"
@@ -604,145 +383,40 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
     """
     HDOCS_SET_DOC = "HDOCS_SET"
 
-    templates = {
-
-        # Mailbox Level
-
-        RECENT_DOC: {
-            "type": indexes.RECENT,
-            "mbox": INBOX_NAME,
-            fields.RECENTFLAGS_KEY: [],
-        },
-
-        HDOCS_SET_DOC: {
-            "type": indexes.HDOCS_SET,
-            "mbox": INBOX_NAME,
-            fields.HDOCS_SET_KEY: [],
-        }
-
-
-    }
-
-    # Different locks for wrapping both the u1db document getting/setting
-    # and the property getting/settting in an atomic operation.
-
-    # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks
-    _rdoc_lock = defaultdict(lambda: threading.Lock())
-    _rdoc_write_lock = defaultdict(lambda: threading.Lock())
-    _rdoc_read_lock = defaultdict(lambda: threading.Lock())
-    _rdoc_property_lock = defaultdict(lambda: threading.Lock())
-
-    _initialized = {}
-
-    def __init__(self, mbox=None, soledad=None, memstore=None):
-        """
-        Constructor for MessageCollection.
-
-        On initialization, we ensure that we have a document for
-        storing the recent flags. The nature of this flag make us wanting
-        to store the set of the UIDs with this flag at the level of the
-        MessageCollection for each mailbox, instead of treating them
-        as a property of each message.
-
-        We are passed an instance of MemoryStore, the same for the
-        SoledadBackedAccount, that we use as a read cache and a buffer
-        for writes.
-
-        :param mbox: the name of the mailbox. It is the name
-                     with which we filter the query over the
-                     messages database.
-        :type mbox: str
-        :param soledad: Soledad database
-        :type soledad: Soledad instance
-        :param memstore: a MemoryStore instance
-        :type memstore: MemoryStore
+    def __init__(self, collection):
         """
-        leap_assert(mbox, "Need a mailbox name to initialize")
-        leap_assert(mbox.strip() != "", "mbox cannot be blank space")
-        leap_assert(isinstance(mbox, (str, unicode)),
-                    "mbox needs to be a string")
-        leap_assert(soledad, "Need a soledad instance to initialize")
+        Constructor for IMAPMessageCollection.
 
-        # okay, all in order, keep going...
-
-        self.mbox = self._parse_mailbox_name(mbox)
-
-        # XXX get a SoledadStore passed instead
-        self._soledad = soledad
-        self.memstore = memstore
-
-        self.__rflags = None
-
-        if not self._initialized.get(mbox, False):
-            try:
-                self.initialize_db()
-                # ensure that we have a recent-flags doc
-                self._get_or_create_rdoc()
-            except Exception:
-                logger.debug("Error initializing %r" % (mbox,))
-            else:
-                self._initialized[mbox] = True
-
-    def _get_empty_doc(self, _type=FLAGS_DOC):
-        """
-        Returns an empty doc for storing different message parts.
-        Defaults to returning a template for a flags document.
-        :return: a dict with the template
-        :rtype: dict
-        """
-        if _type not in self.templates.keys():
-            raise TypeError("Improper type passed to _get_empty_doc")
-        return copy.deepcopy(self.templates[_type])
-
-    def _get_or_create_rdoc(self):
-        """
-        Try to retrieve the recent-flags doc for this MessageCollection,
-        and create one if not found.
+        :param collection: an instance of a MessageCollection
+        :type collection: MessageCollection
         """
-        # XXX should move this to memstore too
-        with self._rdoc_write_lock[self.mbox]:
-            rdoc = self._get_recent_doc_from_soledad()
-            if rdoc is None:
-                rdoc = self._get_empty_doc(self.RECENT_DOC)
-                if self.mbox != fields.INBOX_VAL:
-                    rdoc[fields.MBOX_KEY] = self.mbox
-                self._soledad.create_doc(rdoc)
-
-    # --------------------------------------------------------------------
+        leap_assert(
+            collection.is_mailbox_collection(),
+            "Need a mailbox name to initialize")
+        mbox_name = collection.mbox_name
+        leap_assert(mbox_name.strip() != "", "mbox cannot be blank space")
+        leap_assert(isinstance(mbox_name, (str, unicode)),
+                    "mbox needs to be a string")
+        self.collection = collection
 
-    # -----------------------------------------------------------------------
+        # XXX this has to be done in IMAPAccount
+        # (Where the collection must be instantiated and passed to us)
+        # self.mbox = normalize_mailbox(mbox)
 
-    def _fdoc_already_exists(self, chash):
+    @property
+    def mbox_name(self):
         """
-        Check whether we can find a flags doc for this mailbox with the
-        given content-hash. It enforces that we can only have the same maessage
-        listed once for a a given mailbox.
-
-        :param chash: the content-hash to check about.
-        :type chash: basestring
-        :return: False, if it does not exist, or UID.
+        Return the string that identifies this mailbox.
         """
-        exist = False
-        exist = self.memstore.get_fdoc_from_chash(chash, self.mbox)
+        return self.collection.mbox_name
 
-        if not exist:
-            exist = self._get_fdoc_from_chash(chash)
-        if exist and exist.content is not None:
-            return exist.content.get(fields.UID_KEY, "unknown-uid")
-        else:
-            return False
-
-    def add_msg(self, raw, subject=None, flags=None, date=None,
-                notify_on_disk=False):
+    def add_msg(self, raw, flags=None, date=None):
         """
         Creates a new message document.
 
         :param raw: the raw message
         :type raw: str
 
-        :param subject: subject of the message.
-        :type subject: str
-
         :param flags: flags
         :type flags: list
 
@@ -756,212 +430,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         if flags is None:
             flags = tuple()
         leap_assert_type(flags, tuple)
+        return self.collection.add_msg(raw, flags, date)
 
-        # TODO ---- proxy to MessageCollection addMessage
-
-        #observer = defer.Deferred()
-        #d = self._do_parse(raw)
-        #d.addCallback(lambda result: reactor.callInThread(
-            #self._do_add_msg, result, flags, subject, date,
-            #notify_on_disk, observer))
-        #return observer
-
-    # TODO ---------------------------------------------------
-    # move this to leap.mail.adaptors.soledad
-
-    def _do_add_msg(self, parse_result, flags, subject,
-                    date, notify_on_disk, observer):
-        """
-        """
-        msg, parts, chash, size, multi = parse_result
-
-        # XXX move to SoledadAdaptor write operation ... ???
-        # check for uniqueness --------------------------------
-        # Watch out! We're reserving a UID right after this!
-        existing_uid = self._fdoc_already_exists(chash)
-        if existing_uid:
-            msg = self.get_msg_by_uid(existing_uid)
-            reactor.callFromThread(observer.callback, existing_uid)
-            msg.setFlags((fields.DELETED_FLAG,), -1)
-            return
-
-        # TODO move UID autoincrement to MessageCollection.addMessage(mailbox)
-        # TODO S2 -- get FUCKING UID from autoincremental table
-        #uid = self.memstore.increment_last_soledad_uid(self.mbox)
-        #self.set_recent_flag(uid)
-
-
-    # ------------------------------------------------------------
-
-    #
-    # getters: specific queries
-    #
-
-    # recent flags
-
-    def _get_recent_flags(self):
-        """
-        An accessor for the recent-flags set for this mailbox.
+    def get_msg_by_uid(self, uid, absolute=True):
         """
-        # XXX check if we should remove this
-        if self.__rflags is not None:
-            return self.__rflags
-
-        if self.memstore is not None:
-            with self._rdoc_lock[self.mbox]:
-                rflags = self.memstore.get_recent_flags(self.mbox)
-                if not rflags:
-                    # not loaded in the memory store yet.
-                    # let's fetch them from soledad...
-                    rdoc = self._get_recent_doc_from_soledad()
-                    if rdoc is None:
-                        return set([])
-                    rflags = set(rdoc.content.get(
-                        fields.RECENTFLAGS_KEY, []))
-                    # ...and cache them now.
-                    self.memstore.load_recent_flags(
-                        self.mbox,
-                        {'doc_id': rdoc.doc_id, 'set': rflags})
-            return rflags
-
-    def _set_recent_flags(self, value):
-        """
-        Setter for the recent-flags set for this mailbox.
-        """
-        if self.memstore is not None:
-            self.memstore.set_recent_flags(self.mbox, value)
-
-    recent_flags = property(
-        _get_recent_flags, _set_recent_flags,
-        doc="Set of UIDs with the recent flag for this mailbox.")
-
-    def _get_recent_doc_from_soledad(self):
-        """
-        Get recent-flags document from Soledad for this mailbox.
-        :rtype: SoledadDocument or None
-        """
-        # FIXME ----- use deferreds.
-        curried = partial(
-            self._soledad.get_from_index,
-            fields.TYPE_MBOX_IDX,
-            fields.TYPE_RECENT_VAL, self.mbox)
-        curried.expected = "rdoc"
-        with self._rdoc_read_lock[self.mbox]:
-            return try_unique_query(curried)
-
-    # Property-set modification (protected by a different
-    # lock to give atomicity to the read/write operation)
-
-    def unset_recent_flags(self, uids):
-        """
-        Unset Recent flag for a sequence of uids.
-
-        :param uids: the uids to unset
-        :type uid: sequence
-        """
-        # FIXME ----- use deferreds.
-        with self._rdoc_property_lock[self.mbox]:
-            self.recent_flags.difference_update(
-                set(uids))
-
-    # Individual flags operations
-
-    def unset_recent_flag(self, uid):
-        """
-        Unset Recent flag for a given uid.
-
-        :param uid: the uid to unset
-        :type uid: int
-        """
-        # FIXME ----- use deferreds.
-        with self._rdoc_property_lock[self.mbox]:
-            self.recent_flags.difference_update(
-                set([uid]))
-
-    def set_recent_flag(self, uid):
-        """
-        Set Recent flag for a given uid.
+        Retrieves a IMAPMessage by UID.
+        This is used primarity in the Mailbox fetch and store methods.
 
-        :param uid: the uid to set
+        :param uid: the message uid to query by
         :type uid: int
-        """
-        # FIXME ----- use deferreds.
-        with self._rdoc_property_lock[self.mbox]:
-            self.recent_flags = self.recent_flags.union(
-                set([uid]))
-
-    # individual doc getters, message layer.
 
-    def _get_fdoc_from_chash(self, chash):
+        :rtype: IMAPMessage
         """
-        Return a flags document for this mailbox with a given chash.
+        def make_imap_msg(msg):
+            kls = self.messageklass
+            # TODO --- remove ref to collection
+            return kls(msg, self.collection)
 
-        :return: A SoledadDocument containing the Flags Document, or None if
-                 the query failed.
-        :rtype: SoledadDocument or None.
-        """
-        # USED from:
-        # [ ] duplicated fdoc detection
-        # [ ] _get_uid_from_msgidCb
-
-        # FIXME ----- use deferreds.
-        curried = partial(
-            self._soledad.get_from_index,
-            fields.TYPE_MBOX_C_HASH_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox, chash)
-        curried.expected = "fdoc"
-        fdoc = try_unique_query(curried)
-        if fdoc is not None:
-            return fdoc
-        else:
-            # probably this should be the other way round,
-            # ie, try fist on memstore...
-            cf = self.memstore._chash_fdoc_store
-            fdoc = cf[chash][self.mbox]
-            # hey, I just needed to wrap fdoc thing into
-            # a "content" attribute, look a better way...
-            if not empty(fdoc):
-                return MessagePartDoc(
-                    new=None, dirty=None, part=None,
-                    store=None, doc_id=None,
-                    content=fdoc)
-
-    def _get_uid_from_msgidCb(self, msgid):
-        hdoc = None
-        curried = partial(
-            self._soledad.get_from_index,
-            fields.TYPE_MSGID_IDX,
-            fields.TYPE_HEADERS_VAL, msgid)
-        curried.expected = "hdoc"
-        hdoc = try_unique_query(curried)
-
-        # XXX this is only a quick hack to avoid regression
-        # on the "multiple copies of the draft" issue, but
-        # this is currently broken since  it's not efficient to
-        # look for this. Should lookup better.
-        # FIXME!
-
-        if hdoc is not None:
-            hdoc_dict = hdoc.content
+        d = self.collection.get_msg_by_uid(uid, absolute=absolute)
+        d.addCalback(make_imap_msg)
+        return d
 
-        else:
-            hdocstore = self.memstore._hdoc_store
-            match = [x for _, x in hdocstore.items() if x['msgid'] == msgid]
-            hdoc_dict = first(match)
-
-        if hdoc_dict is None:
-            logger.warning("Could not find hdoc for msgid %s"
-                           % (msgid,))
-            return None
-        msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY)
-
-        fdoc = self._get_fdoc_from_chash(msg_chash)
-        if not fdoc:
-            logger.warning("Could not find fdoc for msgid %s"
-                           % (msgid,))
-            return None
-        return fdoc.content.get(fields.UID_KEY, None)
 
+    # TODO -- move this to collection too
+    # Used for the Search (Drafts) queries?
     def _get_uid_from_msgid(self, msgid):
         """
         Return a UID for a given message-id.
@@ -972,15 +464,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
 
         :return: A UID, or None
         """
-        # We need to wait a little bit, cause in some of the cases
-        # the query is received right after we've saved the document,
-        # and we cannot find it otherwise. This seems to be enough.
-
-        # XXX do a deferLater instead ??
-        # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
-    def set_flags(self, mbox, messages, flags, mode, observer):
+    # TODO handle deferreds
+    def set_flags(self, messages, flags, mode):
         """
         Set flags for a sequence of messages.
 
@@ -1000,142 +487,27 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         getmsg = self.get_msg_by_uid
 
         def set_flags(uid, flags, mode):
-            msg = getmsg(uid, mem_only=True, flags_only=True)
+            msg = getmsg(uid)
             if msg is not None:
+                # XXX IMAPMessage needs access to the collection
+                # to be able to set flags. Better if we make use
+                # of collection... here.
                 return uid, msg.setFlags(flags, mode)
 
         setted_flags = [set_flags(uid, flags, mode) for uid in messages]
         result = dict(filter(None, setted_flags))
+        # XXX return gatherResults or something
+        return result
 
-        # TODO -- remove
-        reactor.callFromThread(observer.callback, result)
-
-    # getters: generic for a mailbox
-
-    def get_msg_by_uid(self, uid, mem_only=False, flags_only=False):
-        """
-        Retrieves a LeapMessage by UID.
-        This is used primarity in the Mailbox fetch and store methods.
-
-        :param uid: the message uid to query by
-        :type uid: int
-        :param mem_only: a flag that indicates whether this Message should
-                         pass a reference to soledad to retrieve missing pieces
-                         or not.
-        :type mem_only: bool
-        :param flags_only: whether the message should carry only a reference
-                           to the flags document.
-        :type flags_only: bool
-
-        :return: A LeapMessage instance matching the query,
-                 or None if not found.
-        :rtype: LeapMessage
-        """
-        msg_container = self.memstore.get_message(
-            self.mbox, uid, flags_only=flags_only)
-
-        if msg_container is not None:
-            if mem_only:
-                msg = IMAPMessage(None, uid, self.mbox, collection=self,
-                                  container=msg_container)
-            else:
-                # We pass a reference to soledad just to be able to retrieve
-                # missing parts that cannot be found in the container, like
-                # the content docs after a copy.
-                msg = IMAPMessage(self._soledad, uid, self.mbox,
-                                  collection=self, container=msg_container)
-        else:
-            msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self)
-
-        if not msg.does_exist():
-            return None
-        return msg
-
-    # FIXME --- used where ? ---------------------------------------------
-    #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
-        #"""
-        #Get all documents for the selected mailbox of the
-        #passed type. By default, it returns the flag docs.
-#
-        #If you want acess to the content, use __iter__ instead
-#
-        #:return: a Deferred, that will fire with a list of u1db documents
-        #:rtype: Deferred (promise of list of SoledadDocument)
-        #"""
-        #if _type not in fields.__dict__.values():
-            #raise TypeError("Wrong type passed to get_all_docs")
-#
-        # FIXME ----- either raise or return a deferred wrapper.
-        #if sameProxiedObjects(self._soledad, None):
-            #logger.warning('Tried to get messages but soledad is None!')
-            #return []
-#
-        #def get_sorted_docs(docs):
-            #all_docs = [doc for doc in docs]
-            # inneficient, but first let's grok it and then
-            # let's worry about efficiency.
-            # XXX FIXINDEX -- should implement order by in soledad
-            # FIXME ----------------------------------------------
-            #return sorted(all_docs, key=lambda item: item.content['uid'])
-#
-        #d = self._soledad.get_from_index(
-            #fields.TYPE_MBOX_IDX, _type, self.mbox)
-        #d.addCallback(get_sorted_docs)
-        #return d
-
-    def all_soledad_uid_iter(self):
-        """
-        Return an iterator through the UIDs of all messages, sorted in
-        ascending order.
-        """
-        # XXX FIXME ------ sorted???
-
-        def get_uids(docs):
-            return set([
-                doc.content[self.UID_KEY] for doc in docs if not empty(doc)])
-
-        d = self._soledad.get_from_index(
-            fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox)
-        d.addCallback(get_uids)
-        return d
-
-    def all_uid_iter(self):
-        """
-        Return an iterator through the UIDs of all messages, from memory.
+    def count(self):
         """
-        mem_uids = self.memstore.get_uids(self.mbox)
-        soledad_known_uids = self.memstore.get_soledad_known_uids(
-            self.mbox)
-        combined = tuple(set(mem_uids).union(soledad_known_uids))
-        return combined
+        Return the count of messages for this mailbox.
 
-    def get_all_soledad_flag_docs(self):
+        :rtype: int
         """
-        Return a dict with the content of all the flag documents
-        in soledad store for the given mbox.
+        return self.collection.count()
 
-        :param mbox: the mailbox
-        :type mbox: str or unicode
-        :rtype: dict
-        """
-        # XXX we really could return a reduced version with
-        # just {'uid': (flags-tuple,) since the prefetch is
-        # only oriented to get the flag tuples.
-
-        def get_content(docs):
-            all_docs = [(
-                doc.content[self.UID_KEY],
-                dict(doc.content))
-                for doc in docs
-                if not empty(doc.content)]
-            all_flags = dict(all_docs)
-            return all_flags
-
-        d = self._soledad.get_from_index(
-            fields.TYPE_MBOX_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox)
-        d.addCallback(get_content)
-        return d
+    # headers query
 
     def all_headers(self):
         """
@@ -1144,15 +516,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
 
         :rtype: dict
         """
-        return self.memstore.all_headers(self.mbox)
-
-    def count(self):
-        """
-        Return the count of messages for this mailbox.
-
-        :rtype: int
-        """
-        return self.memstore.count(self.mbox)
+        # Use self.collection.mbox_indexer
+        # and derive all the doc_ids for the hdocs
+        raise NotImplementedError()
 
     # unseen messages
 
@@ -1164,7 +530,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :return: iterator through unseen message doc UIDs
         :rtype: iterable
         """
-        return self.memstore.unseen_iter(self.mbox)
+        raise NotImplementedError()
 
     def count_unseen(self):
         """
@@ -1182,13 +548,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :returns: a list of LeapMessages
         :rtype: list
         """
-        return [IMAPMessage(self._soledad, docid, self.mbox, collection=self)
-                for docid in self.unseen_iter()]
+        raise NotImplementedError()
+        #return [self.messageklass(self._soledad, doc_id, self.mbox)
+                #for doc_id in self.unseen_iter()]
 
     # recent messages
 
-    # XXX take it from memstore
-    # XXX Used somewhere?
     def count_recent(self):
         """
         Count all messages with the `Recent` flag.
@@ -1199,32 +564,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :returns: count
         :rtype: int
         """
-        return len(self.recent_flags)
+        raise NotImplementedError()
+
+    # magic
 
     def __len__(self):
         """
         Returns the number of messages on this mailbox.
-
         :rtype: int
         """
         return self.count()
 
-    def __iter__(self):
-        """
-        Returns an iterator over all messages.
-
-        :returns: iterator of dicts with content for all messages.
-        :rtype: iterable
-        """
-        return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self)
-                for docuid in self.all_uid_iter())
-
     def __repr__(self):
         """
         Representation string for this object.
         """
-        return u"<MessageCollection: mbox '%s' (%s)>" % (
-            self.mbox, self.count())
+        return u"<IMAPMessageCollection: mbox '%s' (%s)>" % (
+            self.mbox_name, self.count())
 
-    # XXX should implement __eq__ also !!!
-    # use chash...
+    # TODO implement __iter__ ?