Serializable Models + Soledad Adaptor

9 files changed, 301 insertions, 768 deletions

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index fe466cb..7dfbbd1 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -28,10 +28,10 @@ from twisted.python import log
 from zope.interface import implements
 
 from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.imap.index import IndexedDB
+
+from leap.mail.mail import Account
 from leap.mail.imap.fields import WithMsgFields
-from leap.mail.imap.parser import MBoxParser
-from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox
 from leap.soledad.client import Soledad
 
 logger = logging.getLogger(__name__)
@@ -39,7 +39,6 @@ logger = logging.getLogger(__name__)
 PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
 
 if PROFILE_CMD:
-
     def _debugProfiling(result, cmdname, start):
         took = (time.time() - start) * 1000
         log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
@@ -47,96 +46,43 @@ if PROFILE_CMD:
 
 
 #######################################
-# Soledad Account
+# Soledad IMAP Account
 #######################################
 
+# TODO remove MsgFields too
 
-# TODO change name to LeapIMAPAccount, since we're using
-# the memstore.
-# IndexedDB should also not be here anymore.
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+class IMAPAccount(WithMsgFields):
     """
-    An implementation of IAccount and INamespacePresenteer
+    An implementation of an imap4 Account
     that is backed by Soledad Encrypted Documents.
     """
 
     implements(imap4.IAccount, imap4.INamespacePresenter)
 
-    _soledad = None
     selected = None
     closed = False
-    _initialized = False
 
-    def __init__(self, account_name, soledad, memstore=None):
+    def __init__(self, user_id, store):
         """
-        Creates a SoledadAccountIndex that keeps track of the mailboxes
-        and subscriptions handled by this account.
+        Keeps track of the mailboxes and subscriptions handled by this account.
 
-        :param acct_name: The name of the account (user id).
-        :type acct_name: str
+        :param account: The name of the account (user id).
+        :type account: str
 
-        :param soledad: a Soledad instance.
-        :type soledad: Soledad
-        :param memstore: a MemoryStore instance.
-        :type memstore: MemoryStore
+        :param store: a Soledad instance.
+        :type store: Soledad
         """
-        leap_assert(soledad, "Need a soledad instance to initialize")
-        leap_assert_type(soledad, Soledad)
+        # XXX assert a generic store interface instead, so that we
+        # can plug the memory store wrapper seamlessly.
+        leap_assert(store, "Need a store instance to initialize")
+        leap_assert_type(store, Soledad)
 
         # XXX SHOULD assert too that the name matches the user/uuid with which
         # soledad has been initialized.
+        self.user_id = user_id
+        self.account = Account(store)
 
-        # XXX ??? why is this parsing mailbox name??? it's account...
-        # userid? homogenize.
-        self._account_name = self._parse_mailbox_name(account_name)
-        self._soledad = soledad
-        self._memstore = memstore
-
-        self.__mailboxes = set([])
-
-        self._deferred_initialization = defer.Deferred()
-        self._initialize_storage()
-
-    def _initialize_storage(self):
-
-        def add_mailbox_if_none(result):
-            # every user should have the right to an inbox folder
-            # at least, so let's make one!
-            if not self.mailboxes:
-                self.addMailbox(self.INBOX_NAME)
-
-        def finish_initialization(result):
-            self._initialized = True
-            self._deferred_initialization.callback(None)
-
-        def load_mbox_cache(result):
-            d = self._load_mailboxes()
-            d.addCallback(lambda _: result)
-            return d
-
-        d = self.initialize_db()
-
-        d.addCallback(load_mbox_cache)
-        d.addCallback(add_mailbox_if_none)
-        d.addCallback(finish_initialization)
-
-    def callWhenReady(self, cb):
-        if self._initialized:
-            cb(self)
-            return defer.succeed(None)
-        else:
-            self._deferred_initialization.addCallback(cb)
-            return self._deferred_initialization
-
-    def _get_empty_mailbox(self):
-        """
-        Returns an empty mailbox.
-
-        :rtype: dict
-        """
-        return copy.deepcopy(self.EMPTY_MBOX)
-
+    # XXX should hide this in the adaptor...
     def _get_mailbox_by_name(self, name):
         """
         Return an mbox document by name.
@@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         :rtype: SoledadDocument
         """
-        # XXX use soledadstore instead ...;
         def get_first_if_any(docs):
             return docs[0] if docs else None
 
-        d = self._soledad.get_from_index(
+        d = self._store.get_from_index(
             self.TYPE_MBOX_IDX, self.MBOX_KEY,
-            self._parse_mailbox_name(name))
+            normalize_mailbox(name))
         d.addCallback(get_first_if_any)
         return d
 
-    @property
-    def mailboxes(self):
-        """
-        A list of the current mailboxes for this account.
-        :rtype: set
-        """
-        return sorted(self.__mailboxes)
-
-    def _load_mailboxes(self):
-        def update_mailboxes(db_indexes):
-            self.__mailboxes.update(
-                [doc.content[self.MBOX_KEY] for doc in db_indexes])
-        d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY)
-        d.addCallback(update_mailboxes)
-        return d
-
+    # XXX move to Account?
+    # XXX needed?
     def getMailbox(self, name):
         """
         Return a Mailbox with that name, without selecting it.
@@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :returns: a a SoledadMailbox instance
         :rtype: SoledadMailbox
         """
-        name = self._parse_mailbox_name(name)
+        name = normalize_mailbox(name)
 
-        if name not in self.mailboxes:
+        if name not in self.account.mailboxes:
             raise imap4.MailboxException("No such mailbox: %r" % name)
 
-        return SoledadMailbox(name, self._soledad,
-                              memstore=self._memstore)
+        # XXX Does mailbox really need reference to soledad?
+        return SoledadMailbox(name, self._store)
 
     #
     # IAccount
     #
 
+    def _get_empty_mailbox(self):
+        """
+        Returns an empty mailbox.
+
+        :rtype: dict
+        """
+        # XXX move to mailbox module
+        return copy.deepcopy(mailbox.EMPTY_MBOX)
+
+    # TODO use mail.Account.add_mailbox
     def addMailbox(self, name, creation_ts=None):
         """
         Add a mailbox to the account.
@@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :returns: a Deferred that will contain the document if successful.
         :rtype: bool
         """
-        name = self._parse_mailbox_name(name)
+        name = normalize_mailbox(name)
 
         leap_assert(name, "Need a mailbox name to create a mailbox")
 
@@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
             d.addCallback(lambda _: result)
             return d
 
-        d = self._soledad.create_doc(mbox)
+        d = self._store.create_doc(mbox)
         d.addCallback(load_mbox_cache)
         return d
 
+    # TODO use mail.Account.create_mailbox?
+    # Watch out, imap specific exceptions raised here.
     def create(self, pathspec):
         """
         Create a new mailbox from the given hierarchical name.
@@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :raise MailboxException: Raised if this mailbox cannot be added.
         """
         # TODO raise MailboxException
-        paths = filter(
-            None,
-            self._parse_mailbox_name(pathspec).split('/'))
+        paths = filter(None, normalize_mailbox(pathspec).split('/'))
 
         subs = []
         sep = '/'
@@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
             d1.addCallback(load_mbox_cache)
         return d1
 
+    # TODO use mail.Account.get_collection_by_mailbox
     def select(self, name, readwrite=1):
         """
         Selects a mailbox.
@@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         :rtype: SoledadMailbox
         """
-        if PROFILE_CMD:
-            start = time.time()
-
-        name = self._parse_mailbox_name(name)
+        name = normalize_mailbox(name)
         if name not in self.mailboxes:
             logger.warning("No such mailbox!")
             return None
         self.selected = name
 
-        sm = SoledadMailbox(
-            name, self._soledad, self._memstore, readwrite)
-        if PROFILE_CMD:
-            _debugProfiling(None, "SELECT", start)
+        sm = SoledadMailbox(name, self._store, readwrite)
         return sm
 
+    # TODO use mail.Account.delete_mailbox
     def delete(self, name, force=False):
         """
         Deletes a mailbox.
@@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :type force: bool
         :rtype: Deferred
         """
-        name = self._parse_mailbox_name(name)
+        name = normalize_mailbox(name)
 
         if name not in self.mailboxes:
             err = imap4.MailboxException("No such mailbox: %r" % name)
@@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         #  ??! -- can this be rite?
         # self._index.removeMailbox(name)
 
+    # TODO use mail.Account.rename_mailbox
     def rename(self, oldname, newname):
         """
         Renames a mailbox.
@@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :param newname: new name of the mailbox
         :type newname: str
         """
-        oldname = self._parse_mailbox_name(oldname)
-        newname = self._parse_mailbox_name(newname)
+        oldname = normalize_mailbox(oldname)
+        newname = normalize_mailbox(newname)
 
         if oldname not in self.mailboxes:
             raise imap4.NoSuchMailbox(repr(oldname))
@@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
                 inferiors.append(infname)
         return inferiors
 
+    # TODO use mail.Account.list_mailboxes
+    def listMailboxes(self, ref, wildcard):
+        """
+        List the mailboxes.
+
+        from rfc 3501:
+        returns a subset of names from the complete set
+        of all names available to the client.  Zero or more untagged LIST
+        replies are returned, containing the name attributes, hierarchy
+        delimiter, and name.
+
+        :param ref: reference name
+        :type ref: str
+
+        :param wildcard: mailbox name with possible wildcards
+        :type wildcard: str
+        """
+        # XXX use wildcard in index query
+        ref = self._inferiorNames(normalize_mailbox(ref))
+        wildcard = imap4.wildcardToRegexp(wildcard, '/')
+        return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+
+    #
+    # The rest of the methods are specific for leap.mail.imap.account.Account
+    #
+
     # TODO ------------------ can we preserve the attr?
     # maybe add to memory store.
     def isSubscribed(self, name):
@@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         :rtype: Deferred (will fire with bool)
         """
+        # TODO use Flags class
         subscribed = self.SUBSCRIBED_KEY
 
         def is_subscribed(mbox):
@@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         def get_docs_content(docs):
             return [doc.content[self.MBOX_KEY] for doc in docs]
 
-        d = self._soledad.get_from_index(
+        d = self._store.get_from_index(
             self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')
         d.addCallback(get_docs_content)
         return d
@@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         def update_subscribed_value(mbox):
             mbox.content[subscribed] = value
-            return self._soledad.put_doc(mbox)
+            return self._store.put_doc(mbox)
 
         # maybe we should store subscriptions in another
         # document...
@@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :type name: str
         :rtype: Deferred
         """
-        name = self._parse_mailbox_name(name)
+        name = normalize_mailbox(name)
 
         def check_and_subscribe(subscriptions):
             if name not in subscriptions:
@@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :type name: str
         :rtype: Deferred
         """
-        name = self._parse_mailbox_name(name)
+        name = normalize_mailbox(name)
 
         def check_and_unsubscribe(subscriptions):
             if name not in subscriptions:
@@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     def getSubscriptions(self):
         return self._get_subscriptions()
 
-    def listMailboxes(self, ref, wildcard):
-        """
-        List the mailboxes.
-
-        from rfc 3501:
-        returns a subset of names from the complete set
-        of all names available to the client.  Zero or more untagged LIST
-        replies are returned, containing the name attributes, hierarchy
-        delimiter, and name.
-
-        :param ref: reference name
-        :type ref: str
-
-        :param wildcard: mailbox name with possible wildcards
-        :type wildcard: str
-        """
-        # XXX use wildcard in index query
-        ref = self._inferiorNames(
-            self._parse_mailbox_name(ref))
-        wildcard = imap4.wildcardToRegexp(wildcard, '/')
-        return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
-
     #
     # INamespacePresenter
     #
@@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         """
         Representation string for this object.
         """
-        return "<SoledadBackedAccount (%s)>" % self._account_name
+        return "<IMAPAccount (%s)>" % self.user_id
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 4576939..a751c6d 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -17,7 +17,9 @@
 """
 Fields for Mailbox and Message.
 """
-from leap.mail.imap.parser import MBoxParser
+
+# TODO deprecate !!! (move all to constants maybe?)
+# Flags -> foo
 
 
 class WithMsgFields(object):
@@ -25,55 +27,12 @@ class WithMsgFields(object):
     Container class for class-attributes to be shared by
     several message-related classes.
     """
-    # indexing
-    CONTENT_HASH_KEY = "chash"
-    PAYLOAD_HASH_KEY = "phash"
-
-    # Internal representation of Message
-
-    # flags doc
-    UID_KEY = "uid"
-    MBOX_KEY = "mbox"
-    SEEN_KEY = "seen"
-    DEL_KEY = "deleted"
-    RECENT_KEY = "recent"
-    FLAGS_KEY = "flags"
-    MULTIPART_KEY = "multi"
-    SIZE_KEY = "size"
-
-    # headers
-    HEADERS_KEY = "headers"
-    DATE_KEY = "date"
-    SUBJECT_KEY = "subject"
-    PARTS_MAP_KEY = "part_map"
-    BODY_KEY = "body"  # link to phash of body
-    MSGID_KEY = "msgid"
-
-    # content
-    LINKED_FROM_KEY = "lkf"  # XXX not implemented yet!
-    RAW_KEY = "raw"
-    CTYPE_KEY = "ctype"
-
     # Mailbox specific keys
-    CLOSED_KEY = "closed"
-    CREATED_KEY = "created"
-    SUBSCRIBED_KEY = "subscribed"
-    RW_KEY = "rw"
-    LAST_UID_KEY = "lastuid"
+    CREATED_KEY = "created"  # used???
+
     RECENTFLAGS_KEY = "rct"
     HDOCS_SET_KEY = "hdocset"
 
-    # Document Type, for indexing
-    TYPE_KEY = "type"
-    TYPE_MBOX_VAL = "mbox"
-    TYPE_FLAGS_VAL = "flags"
-    TYPE_HEADERS_VAL = "head"
-    TYPE_CONTENT_VAL = "cnt"
-    TYPE_RECENT_VAL = "rct"
-    TYPE_HDOCS_SET_VAL = "hdocset"
-
-    INBOX_VAL = "inbox"
-
     # Flags in Mailbox and Message
     SEEN_FLAG = "\\Seen"
     RECENT_FLAG = "\\Recent"
@@ -88,86 +47,5 @@ class WithMsgFields(object):
     SUBJECT_FIELD = "Subject"
     DATE_FIELD = "Date"
 
-    # Index  types
-    # --------------
-
-    TYPE_IDX = 'by-type'
-    TYPE_MBOX_IDX = 'by-type-and-mbox'
-    TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
-    TYPE_SUBS_IDX = 'by-type-and-subscribed'
-    TYPE_MSGID_IDX = 'by-type-and-message-id'
-    TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
-    TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
-    TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
-    TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
-    TYPE_C_HASH_IDX = 'by-type-and-contenthash'
-    TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
-    TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
-
-    # Tomas created the `recent and seen index`, but the semantic is not too
-    # correct since the recent flag is volatile.
-    TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
-    # Soledad index for incoming mail, without decrypting errors.
-    JUST_MAIL_IDX = "just-mail"
-    # XXX the backward-compatible index, will be deprecated at 0.7
-    JUST_MAIL_COMPAT_IDX = "just-mail-compat"
-
-    INCOMING_KEY = "incoming"
-    ERROR_DECRYPTING_KEY = "errdecr"
-
-    KTYPE = TYPE_KEY
-    MBOX_VAL = TYPE_MBOX_VAL
-    CHASH_VAL = CONTENT_HASH_KEY
-    PHASH_VAL = PAYLOAD_HASH_KEY
-
-    INDEXES = {
-        # generic
-        TYPE_IDX: [KTYPE],
-        TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
-        TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
-
-        # mailboxes
-        TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
-        # fdocs uniqueness
-        TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
-
-        # headers doc - search by msgid.
-        TYPE_MSGID_IDX: [KTYPE, MSGID_KEY],
-
-        # content, headers doc
-        TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
-
-        # attachment payload dedup
-        TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
-
-        # messages
-        TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
-        TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
-        TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
-        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
-                                  'bool(recent)', 'bool(seen)'],
-
-        # incoming queue
-        JUST_MAIL_IDX: [INCOMING_KEY,
-                        "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
-
-        # the backward-compatible index, will be deprecated at 0.7
-        JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
-    }
-
-    MBOX_KEY = MBOX_VAL
-
-    EMPTY_MBOX = {
-        TYPE_KEY: MBOX_KEY,
-        TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
-        SUBJECT_KEY: "",
-        FLAGS_KEY: [],
-        CLOSED_KEY: False,
-        SUBSCRIBED_KEY: False,
-        RW_KEY: 1,
-        LAST_UID_KEY: 0
-    }
 
 fields = WithMsgFields  # alias for convenience
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
deleted file mode 100644
index ea35fff..0000000
--- a/src/leap/mail/imap/index.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# -*- coding: utf-8 -*-
-# index.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-"""
-Index for SoledadBackedAccount, Mailbox and Messages.
-"""
-import logging
-
-from twisted.internet import defer
-
-from leap.common.check import leap_assert, leap_assert_type
-
-from leap.mail.imap.fields import fields
-
-
-logger = logging.getLogger(__name__)
-
-
-class IndexedDB(object):
-    """
-    Methods dealing with the index.
-
-    This is a MixIn that needs access to the soledad instance,
-    and also assumes that a INDEXES attribute is accessible to the instance.
-
-    INDEXES must be a dictionary of type:
-    {'index-name': ['field1', 'field2']}
-    """
-    # TODO we might want to move this to soledad itself, check
-
-    _index_creation_deferreds = []
-    index_ready = False
-
-    def initialize_db(self):
-        """
-        Initialize the database.
-        """
-        leap_assert(self._soledad,
-                    "Need a soledad attribute accesible in the instance")
-        leap_assert_type(self.INDEXES, dict)
-        self._index_creation_deferreds = []
-
-        def _on_indexes_created(ignored):
-            self.index_ready = True
-
-        def _create_index(name, expression):
-            d = self._soledad.create_index(name, *expression)
-            self._index_creation_deferreds.append(d)
-
-        def _create_indexes(db_indexes):
-            db_indexes = dict(db_indexes)
-            for name, expression in fields.INDEXES.items():
-                if name not in db_indexes:
-                    # The index does not yet exist.
-                    _create_index(name, expression)
-                    continue
-
-                if expression == db_indexes[name]:
-                    # The index exists and is up to date.
-                    continue
-                # The index exists but the definition is not what expected, so
-                # we delete it and add the proper index expression.
-                d1 = self._soledad.delete_index(name)
-                d1.addCallback(lambda _: _create_index(name, expression))
-
-            all_created = defer.gatherResults(self._index_creation_deferreds)
-            all_created.addCallback(_on_indexes_created)
-            return all_created
-
-        # Ask the database for currently existing indexes.
-        if not self._soledad:
-            logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
-            return
-        if self._soledad is not None:
-            d = self._soledad.list_indexes()
-            d.addCallback(_create_indexes)
-            return d
diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py
index c906278..f8f25fa 100644
--- a/src/leap/mail/imap/interfaces.py
+++ b/src/leap/mail/imap/interfaces.py
@@ -20,6 +20,7 @@ Interfaces for the IMAP module.
 from zope.interface import Interface, Attribute
 
 
+# TODO remove  ----------------
 class IMessageContainer(Interface):
     """
     I am a container around the different documents that a message
@@ -38,6 +39,7 @@ class IMessageContainer(Interface):
         """
 
 
+# TODO remove --------------------
 class IMessageStore(Interface):
     """
     I represent a generic storage for LEAP Messages.
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 3c1769a..ea54d33 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -1,6 +1,6 @@
 # *- coding: utf-8 -*-
 # mailbox.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,6 +18,7 @@
 Soledad Mailbox.
 """
 import copy
+import re
 import threading
 import logging
 import StringIO
@@ -27,6 +28,7 @@ import os
 from collections import defaultdict
 
 from twisted.internet import defer
+from twisted.internet import reactor
 from twisted.internet.task import deferLater
 from twisted.python import log
 
@@ -36,15 +38,18 @@ from zope.interface import implements
 from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.constants import INBOX_NAME
 from leap.mail.decorators import deferred_to_thread
 from leap.mail.utils import empty
 from leap.mail.imap.fields import WithMsgFields, fields
 from leap.mail.imap.messages import MessageCollection
 from leap.mail.imap.messageparts import MessageWrapper
-from leap.mail.imap.parser import MBoxParser
 
 logger = logging.getLogger(__name__)
 
+# TODO
+# [ ] Restore profile_cmd instrumentation
+
 """
 If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
 notifying clients of new messages. Use during stress tests.
@@ -71,7 +76,9 @@ if PROFILE_CMD:
         d.addErrback(lambda f: log.msg(f.getTraceback()))
 
 
-class SoledadMailbox(WithMsgFields, MBoxParser):
+# TODO Rename to Mailbox
+# TODO Remove WithMsgFields
+class SoledadMailbox(WithMsgFields):
     """
     A Soledad-backed IMAP mailbox.
 
@@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     _last_uid_primed = {}
     _known_uids_primed = {}
 
-    def __init__(self, mbox, soledad, memstore, rw=1):
+    # TODO pass the collection to the constructor
+    # TODO pass the mbox_doc too
+    def __init__(self, mbox, store, rw=1):
         """
         SoledadMailbox constructor. Needs to get passed a name, plus a
         Soledad instance.
@@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param mbox: the mailbox name
         :type mbox: str
 
-        :param soledad: a Soledad instance.
-        :type soledad: Soledad
-
-        :param memstore: a MemoryStore instance
-        :type memstore: MemoryStore
+        :param store:
+        :type store: Soledad
 
         :param rw: read-and-write flag for this mailbox
         :type rw: int
         """
         leap_assert(mbox, "Need a mailbox name to initialize")
-        leap_assert(soledad, "Need a soledad instance to initialize")
+        leap_assert(store, "Need a store instance to initialize")
 
-        from twisted.internet import reactor
-        self.reactor = reactor
-
-        self.mbox = self._parse_mailbox_name(mbox)
+        self.mbox = normalize_mailbox(mbox)
         self.rw = rw
 
-        self._soledad = soledad
-        self._memstore = memstore
-
-        self.messages = MessageCollection(
-            mbox=mbox, soledad=self._soledad, memstore=self._memstore)
+        self.store = store
 
+        self.messages = MessageCollection(mbox=mbox, soledad=store)
         self._uidvalidity = None
 
         # XXX careful with this get/set (it would be
@@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         return self._memstore.get_mbox_doc(self.mbox)
 
-    # XXX the memstore->soledadstore method in memstore is not complete
     def getFlags(self):
         """
         Returns the flags defined for this mailbox.
@@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             flags = self.INIT_FLAGS
         return map(str, flags)
 
-    # XXX the memstore->soledadstore method in memstore is not complete
     def setFlags(self, flags):
         """
         Sets flags for this mailbox.
@@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         d = self._do_add_message(message, flags=flags, date=date,
                                  notify_on_disk=notify_on_disk)
-        if PROFILE_CMD:
-            do_profile_cmd(d, "APPEND")
+        #if PROFILE_CMD:
+            #do_profile_cmd(d, "APPEND")
 
         # XXX should review now that we're not using qtreactor.
         # A better place for this would be  the COPY/APPEND dispatcher
@@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # to work fine for now.
 
         def notifyCallback(x):
-            self.reactor.callLater(0, self.notify_new)
+            reactor.callLater(0, self.notify_new)
             return x
 
         d.addCallback(notifyCallback)
@@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: deferred
         """
         d = defer.Deferred()
-        self.reactor.callInThread(self._do_fetch, messages_asked, uid, d)
-        if PROFILE_CMD:
-            do_profile_cmd(d, "FETCH")
+
+        # XXX do not need no thread...
+        reactor.callInThread(self._do_fetch, messages_asked, uid, d)
         d.addCallback(self.cb_signal_unread_to_ui)
         return d
 
@@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d.addCallback(self.__cb_signal_unread_to_ui)
         return result
 
-    @deferred_to_thread
     def _get_unseen_deferred(self):
         return self.getUnseenCount()
 
@@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: C{list} or C{Deferred}
         """
         # TODO see if we can raise w/o interrupting flow
-        #:raise IllegalQueryError: Raised when query is not valid.
+        # :raise IllegalQueryError: Raised when query is not valid.
         # example query:
         #  ['UNDELETED', 'HEADER', 'Message-ID',
         #   '52D44F11.9060107@dev.bitmask.net']
@@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d.addCallback(createCopy)
         d.addErrback(lambda f: log.msg(f.getTraceback()))
 
-    @deferred_to_thread
+    #@deferred_to_thread
     def _get_msg_copy(self, message):
         """
         Get a copy of the fdoc for this message, and check whether
@@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         return u"<SoledadMailbox: mbox '%s' (%s)>" % (
             self.mbox, self.messages.count())
+
+
+def normalize_mailbox(name):
+    """
+    Return a normalized representation of the mailbox ``name``.
+
+    This method ensures that an eventual initial 'inbox' part of a
+    mailbox name is made uppercase.
+
+    :param name: the name of the mailbox
+    :type name: unicode
+
+    :rtype: unicode
+    """
+    _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+    if _INBOX_RE.match(name):
+        # ensure inital INBOX is uppercase
+        return INBOX_NAME + name[len(INBOX_NAME):]
+    return name
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index c761091..d47c8eb 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # messages.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,30 +19,25 @@ LeapMessage and MessageCollection.
 """
 import copy
 import logging
-import re
 import threading
 import StringIO
 
 from collections import defaultdict
-from email import message_from_string
 from functools import partial
 
-from pycryptopp.hash import sha256
 from twisted.mail import imap4
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
 from zope.interface import implements
 from zope.proxy import sameProxiedObjects
 
 from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
-from leap.mail import walk
-from leap.mail.utils import first, find_charset, lowerdict, empty
-from leap.mail.utils import stringify_parts_map
-from leap.mail.decorators import deferred_to_thread
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.utils import find_charset, empty
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.memorystore import MessageWrapper
 from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
 from leap.mail.imap.parser import MBoxParser
 
@@ -59,9 +54,6 @@ logger = logging.getLogger(__name__)
 # [ ] Delete incoming mail only after successful write!
 # [ ] Remove UID from syncable db. Store only those indexes locally.
 
-MSGID_PATTERN = r"""<([\w@.]+)>"""
-MSGID_RE = re.compile(MSGID_PATTERN)
-
 
 def try_unique_query(curried):
     """
@@ -90,28 +82,18 @@ def try_unique_query(curried):
         logger.exception("Unhandled error %r" % exc)
 
 
-"""
-A dictionary that keeps one lock per mbox and uid.
-"""
-# XXX too much overhead?
-fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+# FIXME remove-me
+#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
 
 
-class LeapMessage(fields, MBoxParser):
+class IMAPMessage(fields, MBoxParser):
     """
     The main representation of a message.
-
-    It indexes the messages in one mailbox by a combination
-    of uid+mailbox name.
     """
 
-    # TODO this has to change.
-    # Should index primarily by chash, and keep a local-only
-    # UID table.
-
     implements(imap4.IMessage)
 
-    def __init__(self, soledad, uid, mbox, collection=None, container=None):
+    def __init__(self, soledad, uid, mbox):
         """
         Initializes a LeapMessage.
 
@@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser):
         self._soledad = soledad
         self._uid = int(uid) if uid is not None else None
         self._mbox = self._parse_mailbox_name(mbox)
-        self._collection = collection
-        self._container = container
 
         self.__chash = None
         self.__bdoc = None
 
-    # XXX make these properties public
-
-    # XXX FIXME ------ the documents can be
-    # deferreds too.... niice.
-
-    @property
-    def fdoc(self):
-        """
-        An accessor to the flags document.
-        """
-        if all(map(bool, (self._uid, self._mbox))):
-            fdoc = None
-            if self._container is not None:
-                fdoc = self._container.fdoc
-            if not fdoc:
-                fdoc = self._get_flags_doc()
-            if fdoc:
-                fdoc_content = fdoc.content
-                self.__chash = fdoc_content.get(
-                    fields.CONTENT_HASH_KEY, None)
-            return fdoc
-
-    @property
-    def hdoc(self):
-        """
-        An accessor to the headers document.
-        """
-        container = self._container
-        if container is not None:
-            hdoc = self._container.hdoc
-            if hdoc and not empty(hdoc.content):
-                return hdoc
-        hdoc = self._get_headers_doc()
-
-        if container and not empty(hdoc.content):
+    # TODO collection and container are deprecated.
+
+    # TODO move to adaptor
+
+    #@property
+    #def fdoc(self):
+        #"""
+        #An accessor to the flags document.
+        #"""
+        #if all(map(bool, (self._uid, self._mbox))):
+            #fdoc = None
+            #if self._container is not None:
+                #fdoc = self._container.fdoc
+            #if not fdoc:
+                #fdoc = self._get_flags_doc()
+            #if fdoc:
+                #fdoc_content = fdoc.content
+                #self.__chash = fdoc_content.get(
+                    #fields.CONTENT_HASH_KEY, None)
+            #return fdoc
+#
+    #@property
+    #def hdoc(self):
+        #"""
+        #An accessor to the headers document.
+        #"""
+        #container = self._container
+        #if container is not None:
+            #hdoc = self._container.hdoc
+            #if hdoc and not empty(hdoc.content):
+                #return hdoc
+        #hdoc = self._get_headers_doc()
+#
+        #if container and not empty(hdoc.content):
             # mem-cache it
-            hdoc_content = hdoc.content
-            chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
-            hdocs = {chash: hdoc_content}
-            container.memstore.load_header_docs(hdocs)
-        return hdoc
-
-    @property
-    def chash(self):
-        """
-        An accessor to the content hash for this message.
-        """
-        if not self.fdoc:
-            return None
-        if not self.__chash and self.fdoc:
-            self.__chash = self.fdoc.content.get(
-                fields.CONTENT_HASH_KEY, None)
-        return self.__chash
-
-    @property
-    def bdoc(self):
-        """
-        An accessor to the body document.
-        """
-        if not self.hdoc:
-            return None
-        if not self.__bdoc:
-            self.__bdoc = self._get_body_doc()
-        return self.__bdoc
+            #hdoc_content = hdoc.content
+            #chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
+            #hdocs = {chash: hdoc_content}
+            #container.memstore.load_header_docs(hdocs)
+        #return hdoc
+#
+    #@property
+    #def chash(self):
+        #"""
+        #An accessor to the content hash for this message.
+        #"""
+        #if not self.fdoc:
+            #return None
+        #if not self.__chash and self.fdoc:
+            #self.__chash = self.fdoc.content.get(
+                #fields.CONTENT_HASH_KEY, None)
+        #return self.__chash
+
+    #@property
+    #def bdoc(self):
+        #"""
+        #An accessor to the body document.
+        #"""
+        #if not self.hdoc:
+            #return None
+        #if not self.__bdoc:
+            #self.__bdoc = self._get_body_doc()
+        #return self.__bdoc
 
     # IMessage implementation
 
@@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser):
         :return: uid for this message
         :rtype: int
         """
+        # TODO ----> return lookup in local sqlcipher table.
         return self._uid
 
+    # --------------------------------------------------------------
+    # TODO -- from here on, all the methods should be proxied to the
+    # instance of leap.mail.mail.Message
+
     def getFlags(self):
         """
         Retrieve the flags associated with this Message.
@@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser):
         REMOVE = -1
         SET = 0
 
-        with fdoc_locks[mbox][uid]:
-            doc = self.fdoc
-            if not doc:
-                logger.warning(
-                    "Could not find FDOC for %r:%s while setting flags!" %
-                    (mbox, uid))
-                return
-            current = doc.content[self.FLAGS_KEY]
-            if mode == APPEND:
-                newflags = tuple(set(tuple(current) + flags))
-            elif mode == REMOVE:
-                newflags = tuple(set(current).difference(set(flags)))
-            elif mode == SET:
-                newflags = flags
-            new_fdoc = {
-                self.FLAGS_KEY: newflags,
-                self.SEEN_KEY: self.SEEN_FLAG in newflags,
-                self.DEL_KEY: self.DELETED_FLAG in newflags}
-            self._collection.memstore.update_flags(mbox, uid, new_fdoc)
+        doc = self.fdoc
+        if not doc:
+            logger.warning(
+                "Could not find FDOC for %r:%s while setting flags!" %
+                (mbox, uid))
+            return
+        current = doc.content[self.FLAGS_KEY]
+        if mode == APPEND:
+            newflags = tuple(set(tuple(current) + flags))
+        elif mode == REMOVE:
+            newflags = tuple(set(current).difference(set(flags)))
+        elif mode == SET:
+            newflags = flags
+        new_fdoc = {
+            self.FLAGS_KEY: newflags,
+            self.SEEN_KEY: self.SEEN_FLAG in newflags,
+            self.DEL_KEY: self.DELETED_FLAG in newflags}
+        self._collection.memstore.update_flags(mbox, uid, new_fdoc)
 
         return map(str, newflags)
 
@@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser):
         else:
             logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
                                                        self._uid))
-        if not size:
+        #if not size:
             # XXX fallback, should remove when all migrated.
-            size = self.getBodyFile().len
+            #size = self.getBodyFile().len
         return size
 
     def getHeaders(self, negate, *names):
@@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser):
         # XXX refactor together with MessagePart method
 
         headers = self._get_headers()
+
+        # XXX keep this in the imap imessage implementation,
+        # because the server impl. expects content-type to be present.
         if not headers:
             logger.warning("No headers found")
             return {str('content-type'): str('')}
@@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
     (the u1db index) for all the headers documents for a given mailbox.
     We use it to prefetch massively all the headers for a mailbox.
     This is the second massive query, after fetching all the FLAGS,  that
-    a MUA will do in a case where we do not have local disk cache.
+    a typical IMAP MUA will do in a case where we do not have local disk cache.
     """
     HDOCS_SET_DOC = "HDOCS_SET"
 
     templates = {
 
-        # Message Level
-
-        FLAGS_DOC: {
-            fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
-            fields.UID_KEY: 1,  # XXX moe to a local table
-            fields.MBOX_KEY: fields.INBOX_VAL,
-            fields.CONTENT_HASH_KEY: "",
-
-            fields.SEEN_KEY: False,
-            fields.DEL_KEY: False,
-            fields.FLAGS_KEY: [],
-            fields.MULTIPART_KEY: False,
-            fields.SIZE_KEY: 0
-        },
-
-        HEADERS_DOC: {
-            fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
-            fields.CONTENT_HASH_KEY: "",
-
-            fields.DATE_KEY: "",
-            fields.SUBJECT_KEY: "",
-
-            fields.HEADERS_KEY: {},
-            fields.PARTS_MAP_KEY: {},
-        },
-
-        CONTENT_DOC: {
-            fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
-            fields.PAYLOAD_HASH_KEY: "",
-            fields.LINKED_FROM_KEY: [],
-            fields.CTYPE_KEY: "",  # should index by this too
-
-            # should only get inmutable headers parts
-            # (for indexing)
-            fields.HEADERS_KEY: {},
-            fields.RAW_KEY: "",
-            fields.PARTS_MAP_KEY: {},
-            fields.HEADERS_KEY: {},
-            fields.MULTIPART_KEY: False,
-        },
-
         # Mailbox Level
 
         RECENT_DOC: {
-            fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
-            fields.MBOX_KEY: fields.INBOX_VAL,
+            "type": indexes.RECENT,
+            "mbox": INBOX_NAME,
             fields.RECENTFLAGS_KEY: [],
         },
 
         HDOCS_SET_DOC: {
-            fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL,
-            fields.MBOX_KEY: fields.INBOX_VAL,
+            "type": indexes.HDOCS_SET,
+            "mbox": INBOX_NAME,
             fields.HDOCS_SET_KEY: [],
         }
 
@@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
     # Different locks for wrapping both the u1db document getting/setting
     # and the property getting/settting in an atomic operation.
 
-    # TODO we would abstract this to a SoledadProperty class
-
+    # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks
     _rdoc_lock = defaultdict(lambda: threading.Lock())
     _rdoc_write_lock = defaultdict(lambda: threading.Lock())
     _rdoc_read_lock = defaultdict(lambda: threading.Lock())
@@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
                     rdoc[fields.MBOX_KEY] = self.mbox
                 self._soledad.create_doc(rdoc)
 
-    @deferred_to_thread
-    def _do_parse(self, raw):
-        """
-        Parse raw message and return it along with
-        relevant information about its outer level.
-
-        This is done in a separate thread, and the callback is passed
-        to `_do_add_msg` method.
+    # --------------------------------------------------------------------
 
-        :param raw: the raw message
-        :type raw: StringIO or basestring
-        :return: msg, parts, chash, size, multi
-        :rtype: tuple
-        """
-        msg = message_from_string(raw)
-        parts = walk.get_parts(msg)
-        size = len(raw)
-        chash = sha256.SHA256(raw).hexdigest()
-        multi = msg.is_multipart()
-        return msg, parts, chash, size, multi
-
-    def _populate_flags(self, flags, uid, chash, size, multi):
-        """
-        Return a flags doc.
-
-        XXX Missing DOC -----------
-        """
-        fd = self._get_empty_doc(self.FLAGS_DOC)
-
-        fd[self.MBOX_KEY] = self.mbox
-        fd[self.UID_KEY] = uid
-        fd[self.CONTENT_HASH_KEY] = chash
-        fd[self.SIZE_KEY] = size
-        fd[self.MULTIPART_KEY] = multi
-        if flags:
-            fd[self.FLAGS_KEY] = flags
-            fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
-            fd[self.DEL_KEY] = self.DELETED_FLAG in flags
-            fd[self.RECENT_KEY] = True  # set always by default
-        return fd
-
-    def _populate_headr(self, msg, chash, subject, date):
-        """
-        Return a headers doc.
-
-        XXX Missing DOC -----------
-        """
-        headers = defaultdict(list)
-        for k, v in msg.items():
-            headers[k].append(v)
-
-        # "fix" for repeated headers.
-        for k, v in headers.items():
-            newline = "\n%s: " % (k,)
-            headers[k] = newline.join(v)
-
-        lower_headers = lowerdict(headers)
-        msgid = first(MSGID_RE.findall(
-            lower_headers.get('message-id', '')))
-
-        hd = self._get_empty_doc(self.HEADERS_DOC)
-        hd[self.CONTENT_HASH_KEY] = chash
-        hd[self.HEADERS_KEY] = headers
-        hd[self.MSGID_KEY] = msgid
-
-        if not subject and self.SUBJECT_FIELD in headers:
-            hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
-        else:
-            hd[self.SUBJECT_KEY] = subject
-
-        if not date and self.DATE_FIELD in headers:
-            hd[self.DATE_KEY] = headers[self.DATE_FIELD]
-        else:
-            hd[self.DATE_KEY] = date
-        return hd
+    # -----------------------------------------------------------------------
 
     def _fdoc_already_exists(self, chash):
         """
@@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             flags = tuple()
         leap_assert_type(flags, tuple)
 
-        # TODO return soledad deferred instead
-        observer = defer.Deferred()
-        d = self._do_parse(raw)
-        d.addCallback(lambda result: reactor.callInThread(
-            self._do_add_msg, result, flags, subject, date,
-            notify_on_disk, observer))
-        return observer
+        # TODO ---- proxy to MessageCollection addMessage
+
+        #observer = defer.Deferred()
+        #d = self._do_parse(raw)
+        #d.addCallback(lambda result: reactor.callInThread(
+            #self._do_add_msg, result, flags, subject, date,
+            #notify_on_disk, observer))
+        #return observer
+
+    # TODO ---------------------------------------------------
+    # move this to leap.mail.adaptors.soledad
 
-    # Called in thread
     def _do_add_msg(self, parse_result, flags, subject,
                     date, notify_on_disk, observer):
         """
-        Helper that creates a new message document.
-        Here lives the magic of the leap mail. Well, in soledad, really.
-
-        See `add_msg` docstring for parameter info.
-
-        :param parse_result: a tuple with the results of `self._do_parse`
-        :type parse_result: tuple
-        :param observer: a deferred that will be fired with the message
-                         uid when the adding succeed.
-        :type observer: deferred
         """
-        # TODO signal that we can delete the original message!-----
-        # when all the processing is done.
-
-        # TODO add the linked-from info !
-        # TODO add reference to the original message
-
         msg, parts, chash, size, multi = parse_result
 
+        # XXX move to SoledadAdaptor write operation ... ???
         # check for uniqueness --------------------------------
         # Watch out! We're reserving a UID right after this!
         existing_uid = self._fdoc_already_exists(chash)
         if existing_uid:
             msg = self.get_msg_by_uid(existing_uid)
-
-            # We can say the observer that we're done
-            # TODO return soledad deferred instead
             reactor.callFromThread(observer.callback, existing_uid)
             msg.setFlags((fields.DELETED_FLAG,), -1)
             return
 
+        # TODO move UID autoincrement to MessageCollection.addMessage(mailbox)
         # TODO S2 -- get FUCKING UID from autoincremental table
-        uid = self.memstore.increment_last_soledad_uid(self.mbox)
-
-        # We can say the observer that we're done at this point, but
-        # before that we should make sure it has no serious consequences
-        # if we're issued, for instance, a fetch command right after...
-        # reactor.callFromThread(observer.callback, uid)
-        # if we did the notify, we need to invalidate the deferred
-        # so not to try to fire it twice.
-        # observer = None
-
-        fd = self._populate_flags(flags, uid, chash, size, multi)
-        hd = self._populate_headr(msg, chash, subject, date)
-
-        body_phash_fun = [walk.get_body_phash_simple,
-                          walk.get_body_phash_multi][int(multi)]
-        body_phash = body_phash_fun(walk.get_payloads(msg))
-        parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
-
-        # add parts map to header doc
-        # (body, multi, part_map)
-        for key in parts_map:
-            hd[key] = parts_map[key]
-        del parts_map
+        #uid = self.memstore.increment_last_soledad_uid(self.mbox)
+        #self.set_recent_flag(uid)
 
-        hd = stringify_parts_map(hd)
 
-        # The MessageContainer expects a dict, one-indexed
-        cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
-
-        self.set_recent_flag(uid)
-        msg_container = MessageWrapper(fd, hd, cdocs)
-
-        # TODO S1 -- just pass this to memstore and return that deferred.
-        self.memstore.create_message(
-            self.mbox, uid, msg_container,
-            observer=observer, notify_on_disk=notify_on_disk)
+    # ------------------------------------------------------------
 
     #
     # getters: specific queries
@@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
                  the query failed.
         :rtype: SoledadDocument or None.
         """
+        # USED from:
+        # [ ] duplicated fdoc detection
+        # [ ] _get_uid_from_msgidCb
+
         # FIXME ----- use deferreds.
         curried = partial(
             self._soledad.get_from_index,
@@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
 
         if msg_container is not None:
             if mem_only:
-                msg = LeapMessage(None, uid, self.mbox, collection=self,
+                msg = IMAPMessage(None, uid, self.mbox, collection=self,
                                   container=msg_container)
             else:
                 # We pass a reference to soledad just to be able to retrieve
                 # missing parts that cannot be found in the container, like
                 # the content docs after a copy.
-                msg = LeapMessage(self._soledad, uid, self.mbox,
+                msg = IMAPMessage(self._soledad, uid, self.mbox,
                                   collection=self, container=msg_container)
         else:
-            msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
+            msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self)
 
         if not msg.does_exist():
             return None
         return msg
 
-    def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
-        """
-        Get all documents for the selected mailbox of the
-        passed type. By default, it returns the flag docs.
-
-        If you want acess to the content, use __iter__ instead
-
-        :return: a Deferred, that will fire with a list of u1db documents
-        :rtype: Deferred (promise of list of SoledadDocument)
-        """
-        if _type not in fields.__dict__.values():
-            raise TypeError("Wrong type passed to get_all_docs")
-
+    # FIXME --- used where ? ---------------------------------------------
+    #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
+        #"""
+        #Get all documents for the selected mailbox of the
+        #passed type. By default, it returns the flag docs.
+#
+        #If you want acess to the content, use __iter__ instead
+#
+        #:return: a Deferred, that will fire with a list of u1db documents
+        #:rtype: Deferred (promise of list of SoledadDocument)
+        #"""
+        #if _type not in fields.__dict__.values():
+            #raise TypeError("Wrong type passed to get_all_docs")
+#
         # FIXME ----- either raise or return a deferred wrapper.
-        if sameProxiedObjects(self._soledad, None):
-            logger.warning('Tried to get messages but soledad is None!')
-            return []
-
-        def get_sorted_docs(docs):
-            all_docs = [doc for doc in docs]
+        #if sameProxiedObjects(self._soledad, None):
+            #logger.warning('Tried to get messages but soledad is None!')
+            #return []
+#
+        #def get_sorted_docs(docs):
+            #all_docs = [doc for doc in docs]
             # inneficient, but first let's grok it and then
             # let's worry about efficiency.
             # XXX FIXINDEX -- should implement order by in soledad
             # FIXME ----------------------------------------------
-            return sorted(all_docs, key=lambda item: item.content['uid'])
-
-        d = self._soledad.get_from_index(
-            fields.TYPE_MBOX_IDX, _type, self.mbox)
-        d.addCallback(get_sorted_docs)
-        return d
+            #return sorted(all_docs, key=lambda item: item.content['uid'])
+#
+        #d = self._soledad.get_from_index(
+            #fields.TYPE_MBOX_IDX, _type, self.mbox)
+        #d.addCallback(get_sorted_docs)
+        #return d
 
     def all_soledad_uid_iter(self):
         """
@@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :returns: a list of LeapMessages
         :rtype: list
         """
-        return [LeapMessage(self._soledad, docid, self.mbox, collection=self)
+        return [IMAPMessage(self._soledad, docid, self.mbox, collection=self)
                 for docid in self.unseen_iter()]
 
     # recent messages
@@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         :returns: iterator of dicts with content for all messages.
         :rtype: iterable
         """
-        return (LeapMessage(self._soledad, docuid, self.mbox, collection=self)
+        return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self)
                 for docuid in self.all_uid_iter())
 
     def __repr__(self):
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
deleted file mode 100644
index 4a801b0..0000000
--- a/src/leap/mail/imap/parser.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# parser.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-"""
-Mail parser mixin.
-"""
-import re
-
-
-class MBoxParser(object):
-    """
-    Utility function to parse mailbox names.
-    """
-    INBOX_NAME = "INBOX"
-    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
-    def _parse_mailbox_name(self, name):
-        """
-        Return a normalized representation of the mailbox C{name}.
-
-        This method ensures that an eventual initial 'inbox' part of a
-        mailbox name is made uppercase.
-
-        :param name: the name of the mailbox
-        :type name: unicode
-
-        :rtype: unicode
-        """
-        if self.INBOX_RE.match(name):
-            # ensure inital INBOX is uppercase
-            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
-        return name
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index dd4294c..5af499f 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         MessageCollection interface in this particular TestCase
         """
         super(MessageCollectionTestCase, self).setUp()
+
+        # TODO deprecate memstore
         memstore = MemoryStore()
         self.messages = MessageCollection("testmbox%s" % (self.count,),
                                           self._soledad, memstore=memstore)
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index 9a3868c..920eeb0 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client):
         self.transport.loseConnection()
 
 
+# XXX move to common helper
 def initialize_soledad(email, gnupg_home, tempdir):
     """
     Initializes soledad by hand
@@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest):
         """
         Setup method for each test.
 
-        Initializes and run a LEAP IMAP4 Server,
-        but passing the same Soledad instance (it's costly to initialize),
-        so we have to be sure to restore state across tests.
+        Initializes and run a LEAP IMAP4 Server.
         """
         self.old_path = os.environ['PATH']
         self.old_home = os.environ['HOME']
@@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest):
     def tearDown(self):
         """
         tearDown method called after each test.
-
-        Deletes all documents in the Index, and deletes
-        instances of server and client.
         """
         try:
             self._soledad.close()
+        except Exception:
+            print "ERROR WHILE CLOSING SOLEDAD"
+        finally:
             os.environ["PATH"] = self.old_path
             os.environ["HOME"] = self.old_home
             # safety check
             assert 'leap_tests-' in self.tempdir
             shutil.rmtree(self.tempdir)
-        except Exception:
-            print "ERROR WHILE CLOSING SOLEDAD"
 
     def populateMessages(self):
         """
@@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest):
 
     def loopback(self):
         return loopback.loopbackAsync(self.server, self.client)
-
-