Serializable Models + Soledad Adaptor

8 files changed, 1732 insertions, 0 deletions

diff --git a/src/leap/mail/adaptors/__init__.py b/src/leap/mail/adaptors/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/adaptors/__init__.py
diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py
new file mode 100644
index 0000000..1648059
--- /dev/null
+++ b/src/leap/mail/adaptors/models.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+# models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Models to be used by the Document Adaptors.
+"""
+import copy
+
+
+class SerializableModel(object):
+    """
+    A Generic document model, that can be serialized into a dictionary.
+
+    Subclasses of this `SerializableModel` are meant to be added as class
+    attributes of classes inheriting from DocumentWrapper.
+
+    A subclass __meta__ of this SerializableModel might exist, and contain info
+    relative to particularities of this model.
+
+    For instance, the use of `__meta__.index` marks the existence of a primary
+    index in the model, which will be used to do unique queries (in which case
+    all the other indexed fields in the underlying document will be filled with
+    the default info contained in the model definition).
+    """
+
+    @classmethod
+    def serialize(klass):
+        """
+        Get a dictionary representation of the public attributes in the model
+        class. To avoid collisions with builtin functions, any occurrence of an
+        attribute ended in '_' (like 'type_') will be normalized by removing
+        the trailing underscore.
+
+        This classmethod is used from within the serialized method of a
+        DocumentWrapper instance: it provides defaults for the
+        empty document.
+        """
+        assert isinstance(klass, type)
+        return _normalize_dict(klass.__dict__)
+
+
+class DocumentWrapper(object):
+    """
+    A Wrapper object that can be manipulated, passed around, and serialized in
+    a format that the store understands.
+    It is related to a SerializableModel, which must be specified as the
+    ``model`` class attribute.  The instance of this DocumentWrapper will not
+    allow any other *public* attributes than those defined in the corresponding
+    model.
+    """
+    # TODO we could do some very basic type checking here
+    # TODO set a dirty flag (on __setattr__, whenever the value is != from
+    # before)
+    # TODO we could enforce the existence of a correct "model" attribute
+    # in some other way (other than in the initializer)
+
+    def __init__(self, **kwargs):
+        if not getattr(self, 'model', None):
+            raise RuntimeError(
+                'DocumentWrapper class needs a model attribute')
+
+        defaults = self.model.serialize()
+
+        if kwargs:
+            values = copy.deepcopy(defaults)
+            values.update(kwargs)
+        else:
+            values = defaults
+
+        for k, v in values.items():
+            k = k.replace('-', '_')
+            setattr(self, k, v)
+
+    def __setattr__(self, attr, value):
+        normalized = _normalize_dict(self.model.__dict__)
+        if not attr.startswith('_') and attr not in normalized:
+            raise RuntimeError(
+                "Cannot set attribute because it's not defined "
+                "in the model: %s" % attr)
+        object.__setattr__(self, attr, value)
+
+    def serialize(self):
+        return _normalize_dict(self.__dict__)
+
+    def create(self):
+        raise NotImplementedError()
+
+    def update(self):
+        raise NotImplementedError()
+
+    def delete(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def get_or_create(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def get_all(self):
+        raise NotImplementedError()
+
+
+def _normalize_dict(_dict):
+    items = _dict.items()
+    not_callable = lambda (k, v): not callable(v)
+    not_private = lambda(k, v): not k.startswith('_')
+    for cond in not_callable, not_private:
+        items = filter(cond, items)
+    items = [(k, v) if not k.endswith('_') else (k[:-1], v)
+             for (k, v) in items]
+    items = [(k.replace('-', '_'), v) for (k, v) in items]
+    return dict(items)
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py
new file mode 100644
index 0000000..2e25f04
--- /dev/null
+++ b/src/leap/mail/adaptors/soledad.py
@@ -0,0 +1,723 @@
+# -*- coding: utf-8 -*-
+# soledad.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledadad MailAdaptor module.
+"""
+import re
+from collections import defaultdict
+from email import message_from_string
+
+from pycryptopp.hash import sha256
+from twisted.internet import defer
+from zope.interface import implements
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail import walk
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.adaptors import models
+from leap.mail.imap.mailbox import normalize_mailbox
+from leap.mail.utils import lowerdict, first
+from leap.mail.utils import stringify_parts_map
+from leap.mail.interfaces import IMailAdaptor, IMessageWrapper
+
+
+# TODO
+# [ ] Convenience function to create mail specifying subject, date, etc?
+
+
+_MSGID_PATTERN = r"""<([\w@.]+)>"""
+_MSGID_RE = re.compile(_MSGID_PATTERN)
+
+
+class DuplicatedDocumentError(Exception):
+    """
+    Raised when a duplicated document is detected.
+    """
+    pass
+
+
+class SoledadDocumentWrapper(models.DocumentWrapper):
+    """
+    A Wrapper object that can be manipulated, passed around, and serialized in
+    a format that the Soledad Store understands.
+
+    It ensures atomicity of the document operations on creation, update and
+    deletion.
+    """
+
+    # TODO we could also use a _dirty flag (in models)
+
+    # We keep a dictionary with DeferredLocks, that will be
+    # unique to every subclass of SoledadDocumentWrapper.
+    _k_locks = defaultdict(defer.DeferredLock)
+
+    @classmethod
+    def _get_klass_lock(cls):
+        """
+        Get a DeferredLock that is unique for this subclass name.
+        Used to lock the access to indexes in the `get_or_create` call
+        for a particular DocumentWrapper.
+        """
+        return cls._k_locks[cls.__name__]
+
+    def __init__(self, **kwargs):
+        doc_id = kwargs.pop('doc_id', None)
+        self._doc_id = doc_id
+        self._lock = defer.DeferredLock()
+        super(SoledadDocumentWrapper, self).__init__(**kwargs)
+
+    @property
+    def doc_id(self):
+        return self._doc_id
+
+    def create(self, store):
+        """
+        Create the documents for this wrapper.
+        Since this method will not check for duplication, the
+        responsibility of avoiding duplicates is left to the caller.
+
+        You might be interested in using `get_or_create` classmethod
+        instead (that's the preferred way of creating documents from
+        the wrapper object).
+
+        :return: a deferred that will fire when the underlying
+                 Soledad document has been created.
+        :rtype: Deferred
+        """
+        leap_assert(self._doc_id is None,
+                    "This document already has a doc_id!")
+
+        def update_doc_id(doc):
+            self._doc_id = doc.doc_id
+            return doc
+        d = store.create_doc(self.serialize())
+        d.addCallback(update_doc_id)
+        return d
+
+    def update(self, store):
+        """
+        Update the documents for this wrapper.
+
+        :return: a deferred that will fire when the underlying
+                 Soledad document has been updated.
+        :rtype: Deferred
+        """
+        # the deferred lock guards against revision conflicts
+        return self._lock.run(self._update, store)
+
+    def _update(self, store):
+        leap_assert(self._doc_id is not None,
+                    "Need to create doc before updating")
+
+        def update_and_put_doc(doc):
+            doc.content.update(self.serialize())
+            return store.put_doc(doc)
+
+        d = store.get_doc(self._doc_id)
+        d.addCallback(update_and_put_doc)
+        return d
+
+    def delete(self, store):
+        """
+        Delete the documents for this wrapper.
+
+        :return: a deferred that will fire when the underlying
+                 Soledad document has been deleted.
+        :rtype: Deferred
+        """
+        # the deferred lock guards against conflicts while updating
+        return self._lock.run(self._delete, store)
+
+    def _delete(self, store):
+        leap_assert(self._doc_id is not None,
+                    "Need to create doc before deleting")
+        # XXX might want to flag this DocumentWrapper to avoid
+        # updating it by mistake. This could go in models.DocumentWrapper
+
+        def delete_doc(doc):
+            return store.delete_doc(doc)
+
+        d = store.get_doc(self._doc_id)
+        d.addCallback(delete_doc)
+        return d
+
+    @classmethod
+    def get_or_create(cls, store, index, value):
+        """
+        Get a unique DocumentWrapper by index, or create a new one if the
+        matching query does not exist.
+
+        :param index: the primary index for the model.
+        :type index: str
+        :param value: the value to query the primary index.
+        :type value: str
+
+        :return: a deferred that will be fired with the SoledadDocumentWrapper
+                 matching the index query, either existing or just created.
+        :rtype: Deferred
+        """
+        return cls._get_klass_lock().run(
+            cls._get_or_create, store, index, value)
+
+    @classmethod
+    def _get_or_create(cls, store, index, value):
+        assert store is not None
+        assert index is not None
+        assert value is not None
+
+        def get_main_index():
+            try:
+                return cls.model.__meta__.index
+            except AttributeError:
+                raise RuntimeError("The model is badly defined")
+
+        def try_to_get_doc_from_index(indexes):
+            values = []
+            idx_def = dict(indexes)[index]
+            if len(idx_def) == 1:
+                values = [value]
+            else:
+                main_index = get_main_index()
+                fields = cls.model.serialize()
+                for field in idx_def:
+                    if field == main_index:
+                        values.append(value)
+                    else:
+                        values.append(fields[field])
+            d = store.get_from_index(index, *values)
+            return d
+
+        def get_first_doc_if_any(docs):
+            if not docs:
+                return None
+            if len(docs) > 1:
+                raise DuplicatedDocumentError
+            return docs[0]
+
+        def wrap_existing_or_create_new(doc):
+            if doc:
+                return cls(doc_id=doc.doc_id, **doc.content)
+            else:
+                return create_and_wrap_new_doc()
+
+        def create_and_wrap_new_doc():
+            # XXX use closure to store indexes instead of
+            # querying for them again.
+            d = store.list_indexes()
+            d.addCallback(get_wrapper_instance_from_index)
+            d.addCallback(return_wrapper_when_created)
+            return d
+
+        def get_wrapper_instance_from_index(indexes):
+            init_values = {}
+            idx_def = dict(indexes)[index]
+            if len(idx_def) == 1:
+                init_value = {idx_def[0]: value}
+                return cls(**init_value)
+            main_index = get_main_index()
+            fields = cls.model.serialize()
+            for field in idx_def:
+                if field == main_index:
+                    init_values[field] = value
+                else:
+                    init_values[field] = fields[field]
+            return cls(**init_values)
+
+        def return_wrapper_when_created(wrapper):
+            d = wrapper.create(store)
+            d.addCallback(lambda doc: wrapper)
+            return d
+
+        d = store.list_indexes()
+        d.addCallback(try_to_get_doc_from_index)
+        d.addCallback(get_first_doc_if_any)
+        d.addCallback(wrap_existing_or_create_new)
+        return d
+
+    @classmethod
+    def get_all(cls, store):
+        """
+        Get a collection of wrappers around all the documents belonging
+        to this kind.
+
+        For this to work, the model.__meta__ needs to include a tuple with
+        the index to be used for listing purposes, and which is the field to be
+        used to query the index.
+
+        Note that this method only supports indexes of a single field at the
+        moment. It also might be too expensive to return all the documents
+        matching the query, so handle with care.
+
+        class __meta__(object):
+            index = "name"
+            list_index = ("by-type", "type_")
+
+        :return: a deferred that will be fired with an iterable containing
+                 as many SoledadDocumentWrapper are matching the index defined
+                 in the model as the `list_index`.
+        :rtype: Deferred
+        """
+        # TODO
+        # [ ] extend support to indexes with n-ples
+        # [ ] benchmark the cost of querying and returning indexes in a big
+        #     database. This might badly need pagination before being put to
+        #     serious use.
+        return cls._get_klass_lock().run(cls._get_all, store)
+
+    @classmethod
+    def _get_all(cls, store):
+        try:
+            list_index, list_attr = cls.model.__meta__.list_index
+        except AttributeError:
+            raise RuntimeError("The model is badly defined: no list_index")
+        try:
+            index_value = getattr(cls.model, list_attr)
+        except AttributeError:
+            raise RuntimeError("The model is badly defined: "
+                               "no attribute matching list_index")
+
+        def wrap_docs(docs):
+            return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs)
+
+        d = store.get_from_index(list_index, index_value)
+        d.addCallback(wrap_docs)
+        return d
+
+    # TODO
+    # [ ] get_count() ???
+
+    def __repr__(self):
+        try:
+            idx = getattr(self, self.model.__meta__.index)
+        except AttributeError:
+            idx = ""
+        return "<%s: %s (%s)>" % (self.__class__.__name__,
+                                  idx, self._doc_id)
+
+
+#
+# Message documents
+#
+
+class FlagsDocWrapper(SoledadDocumentWrapper):
+
+    class model(models.SerializableModel):
+        type_ = "flags"
+        chash = ""
+
+        mbox = "inbox"
+        seen = False
+        deleted = False
+        recent = False
+        multi = False
+        flags = []
+        tags = []
+        size = 0
+
+        class __meta__(object):
+            index = "mbox"
+
+
+class HeaderDocWrapper(SoledadDocumentWrapper):
+
+    class model(models.SerializableModel):
+        type_ = "head"
+        chash = ""
+
+        date = ""
+        subject = ""
+        headers = {}
+        part_map = {}
+        body = ""  # link to phash of body
+        msgid = ""
+        multi = False
+
+        class __meta__(object):
+            index = "chash"
+
+
+class ContentDocWrapper(SoledadDocumentWrapper):
+
+    class model(models.SerializableModel):
+        type_ = "cnt"
+        phash = ""
+
+        ctype = ""  # XXX index by ctype too?
+        lkf = []  # XXX not implemented yet!
+        raw = ""
+
+        content_disposition = ""
+        content_transfer_encoding = ""
+        content_type = ""
+
+        class __meta__(object):
+            index = "phash"
+
+
+class MessageWrapper(object):
+
+    # TODO generalize wrapper composition?
+    # This could benefit of a DeferredLock to create/update all the
+    # documents at the same time maybe, and defend against concurrent updates?
+
+    implements(IMessageWrapper)
+
+    def __init__(self, fdoc, hdoc, cdocs=None):
+        """
+        Need at least a flag-document and a header-document to instantiate a
+        MessageWrapper. Content-documents can be retrieved lazily.
+
+        cdocs, if any, should be a dictionary in which the keys are ascending
+        integers, beginning at one, and the values are dictionaries with the
+        content of the content-docs.
+        """
+        self.fdoc = FlagsDocWrapper(**fdoc)
+        self.hdoc = HeaderDocWrapper(**hdoc)
+        if cdocs is None:
+            cdocs = {}
+        cdocs_keys = cdocs.keys()
+        assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1)
+        self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in
+                           cdocs.items()])
+
+    def create(self, store):
+        """
+        Create all the parts for this message in the store.
+        """
+        leap_assert(self.cdocs,
+                    "Need non empty cdocs to create the "
+                    "MessageWrapper documents")
+        leap_assert(self.fdoc.doc_id is None,
+                    "Cannot create: fdoc has a doc_id")
+
+        # TODO I think we need to tolerate the no hdoc.doc_id case, for when we
+        # are doing a copy to another mailbox.
+        leap_assert(self.hdoc.doc_id is None,
+                    "Cannot create: hdoc has a doc_id")
+        d = []
+        d.append(self.fdoc.create(store))
+        d.append(self.hdoc.create(store))
+        for cdoc in self.cdocs.values():
+            if cdoc.doc_id is not None:
+                # we could be just linking to an existing
+                # content-doc.
+                continue
+            d.append(cdoc.create(store))
+        return defer.gatherResults(d)
+
+    def update(self, store):
+        """
+        Update the only mutable parts, which are within the flags document.
+        """
+        return self.fdoc.update(store)
+
+    def delete(self, store):
+        # Eventually this would have to do the duplicate search or send for the
+        # garbage collector. At least the fdoc can be unlinked.
+        raise NotImplementedError()
+
+#
+# Mailboxes
+#
+
+
+class MailboxWrapper(SoledadDocumentWrapper):
+
+    class model(models.SerializableModel):
+        type_ = "mbox"
+        mbox = INBOX_NAME
+        flags = []
+        closed = False
+        subscribed = False
+        rw = True
+
+        class __meta__(object):
+            index = "mbox"
+            list_index = (indexes.TYPE_IDX, 'type_')
+
+
+#
+# Soledad Adaptor
+#
+
+# TODO make this an interface?
+class SoledadIndexMixin(object):
+    """
+    this will need a class attribute `indexes`, that is a dictionary containing
+    the index definitions for the underlying u1db store underlying soledad.
+
+    It needs to be in the following format:
+    {'index-name': ['field1', 'field2']}
+    """
+    # TODO could have a wrapper class for indexes, supporting introspection
+    # and __getattr__
+    indexes = {}
+
+    store_ready = False
+    _index_creation_deferreds = []
+
+    # TODO we might want to move this logic to soledad itself
+    # so that each application can pass a set of indexes for their data model.
+    # TODO check also the decorator used in keymanager for waiting for indexes
+    # to be ready.
+
+    def initialize_store(self, store):
+        """
+        Initialize the indexes in the database.
+
+        :param store: store
+        :returns: a Deferred that will fire when the store is correctly
+                  initialized.
+        :rtype: deferred
+        """
+        # TODO I think we *should* get another deferredLock in here, but
+        # global to the soledad namespace, to protect from several points
+        # initializing soledad indexes at the same time.
+
+        leap_assert(store, "Need a store")
+        leap_assert_type(self.indexes, dict)
+        self._index_creation_deferreds = []
+
+        def _on_indexes_created(ignored):
+            self.store_ready = True
+
+        def _create_index(name, expression):
+            d = store.create_index(name, *expression)
+            self._index_creation_deferreds.append(d)
+
+        def _create_indexes(db_indexes):
+            db_indexes = dict(db_indexes)
+
+            for name, expression in self.indexes.items():
+                if name not in db_indexes:
+                    # The index does not yet exist.
+                    _create_index(name, expression)
+                    continue
+
+                if expression == db_indexes[name]:
+                    # The index exists and is up to date.
+                    continue
+                # The index exists but the definition is not what expected, so
+                # we delete it and add the proper index expression.
+                d1 = store.delete_index(name)
+                d1.addCallback(lambda _: _create_index(name, expression))
+
+            all_created = defer.gatherResults(self._index_creation_deferreds)
+            all_created.addCallback(_on_indexes_created)
+            return all_created
+
+        # Ask the database for currently existing indexes, and create them
+        # if not found.
+        d = store.list_indexes()
+        d.addCallback(_create_indexes)
+        return d
+
+
+class SoledadMailAdaptor(SoledadIndexMixin):
+
+    implements(IMailAdaptor)
+    store = None
+
+    indexes = indexes.MAIL_INDEXES
+
+    # Message handling
+
+    def get_msg_from_string(self, MessageClass, raw_msg):
+        """
+        Get an instance of a MessageClass initialized with a MessageWrapper
+        that contains all the parts obtained from parsing the raw string for
+        the message.
+
+        :param MessageClass: any Message class that can be initialized passing
+                             an instance of an IMessageWrapper implementor.
+        :type MessageClass: type
+        :param raw_msg: a string containing the raw email message.
+        :type raw_msg: str
+        :rtype: MessageClass instance.
+        """
+        assert(MessageClass is not None)
+        fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
+        return self.get_msg_from_docs(
+            MessageClass, fdoc, hdoc, cdocs)
+
+    def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None):
+        """
+        Get an instance of a MessageClass initialized with a MessageWrapper
+        that contains the passed part documents.
+
+        This is not the recommended way of obtaining a message, unless you know
+        how to take care of ensuring the internal consistency between the part
+        documents, or unless you are glueing together the part documents that
+        have been previously generated by `get_msg_from_string`.
+
+        :param MessageClass: any Message class that can be initialized passing
+                             an instance of an IMessageWrapper implementor.
+        :type MessageClass: type
+        :param fdoc: a dictionary containing values from which a
+                     FlagsDocWrapper can be initialized
+        :type fdoc: dict
+        :param hdoc: a dictionary containing values from which a
+                     HeaderDocWrapper can be initialized
+        :type hdoc: dict
+        :param cdocs: None, or a dictionary mapping integers (1-indexed) to
+                      dicts from where a ContentDocWrapper can be initialized.
+        :type cdocs: dict, or None
+
+        :rtype: MessageClass instance.
+        """
+        assert(MessageClass is not None)
+        return MessageClass(MessageWrapper(fdoc, hdoc, cdocs))
+
+    def create_msg(self, store, msg):
+        """
+        :param store: an instance of soledad, or anything that behaves alike
+        :type store:
+        :param msg: a Message object.
+
+        :return: a Deferred that is fired when all the underlying documents
+                 have been created.
+        :rtype: defer.Deferred
+        """
+        wrapper = msg.get_wrapper()
+        return wrapper.create(store)
+
+    def update_msg(self, store, msg):
+        """
+        :param msg: a Message object.
+        :param store: an instance of soledad, or anything that behaves alike
+        :type store:
+        :param msg: a Message object.
+        :return: a Deferred that is fired when all the underlying documents
+                 have been updated (actually, it's only the fdoc that's allowed
+                 to update).
+        :rtype: defer.Deferred
+        """
+        wrapper = msg.get_wrapper()
+        return wrapper.update(store)
+
+    # Mailbox handling
+
+    def get_or_create_mbox(self, store, name):
+        """
+        Get the mailbox with the given name, or creatre one if it does not
+        exist.
+
+        :param name: the name of the mailbox
+        :type name: str
+        """
+        index = indexes.TYPE_MBOX_IDX
+        mbox = normalize_mailbox(name)
+        return MailboxWrapper.get_or_create(store, index, mbox)
+
+    def update_mbox(self, store, mbox_wrapper):
+        """
+        Update the documents for a given mailbox.
+        :param mbox_wrapper: MailboxWrapper instance
+        :type mbox_wrapper: MailboxWrapper
+        :return: a Deferred that will be fired when the mailbox documents
+                 have been updated.
+        :rtype: defer.Deferred
+        """
+        return mbox_wrapper.update(store)
+
+    def get_all_mboxes(self, store):
+        """
+        Retrieve a list with wrappers for all the mailboxes.
+
+        :return: a deferred that will be fired with a list of all the
+                 MailboxWrappers found.
+        :rtype: defer.Deferred
+        """
+        return MailboxWrapper.get_all(store)
+
+
+def _split_into_parts(raw):
+    # TODO signal that we can delete the original message!-----
+    # when all the processing is done.
+    # TODO add the linked-from info !
+    # TODO add reference to the original message?
+    # TODO populate Default FLAGS/TAGS (unseen?)
+    # TODO seed propely the content_docs with defaults??
+
+    msg, parts, chash, size, multi = _parse_msg(raw)
+    body_phash_fun = [walk.get_body_phash_simple,
+                      walk.get_body_phash_multi][int(multi)]
+    body_phash = body_phash_fun(walk.get_payloads(msg))
+    parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+
+    fdoc = _build_flags_doc(chash, size, multi)
+    hdoc = _build_headers_doc(msg, chash, parts_map)
+
+    # The MessageWrapper expects a dict, one-indexed
+    cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
+
+    # XXX convert each to_dicts...
+    return fdoc, hdoc, cdocs
+
+
+def _parse_msg(raw):
+    msg = message_from_string(raw)
+    parts = walk.get_parts(msg)
+    size = len(raw)
+    chash = sha256.SHA256(raw).hexdigest()
+    multi = msg.is_multipart()
+    return msg, parts, chash, size, multi
+
+
+def _build_flags_doc(chash, size, multi):
+    _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi)
+    return _fdoc.serialize()
+
+
+def _build_headers_doc(msg, chash, parts_map):
+    """
+    Assemble a headers document from the original parsed message, the
+    content-hash, and the parts map.
+
+    It takes into account possibly repeated headers.
+    """
+    headers = defaultdict(list)
+    for k, v in msg.items():
+        headers[k].append(v)
+
+    # "fix" for repeated headers.
+    for k, v in headers.items():
+        newline = "\n%s: " % (k,)
+        headers[k] = newline.join(v)
+
+    lower_headers = lowerdict(headers)
+    msgid = first(_MSGID_RE.findall(
+        lower_headers.get('message-id', '')))
+
+    _hdoc = HeaderDocWrapper(
+        chash=chash, headers=lower_headers, msgid=msgid)
+
+    def copy_attr(headers, key, doc):
+        if key in headers:
+            setattr(doc, key, headers[key])
+
+    copy_attr(lower_headers, "subject", _hdoc)
+    copy_attr(lower_headers, "date", _hdoc)
+
+    hdoc = _hdoc.serialize()
+    # add parts map to header doc
+    # (body, multi, part_map)
+    for key in parts_map:
+        hdoc[key] = parts_map[key]
+    return stringify_parts_map(hdoc)
diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py
new file mode 100644
index 0000000..f3e990d
--- /dev/null
+++ b/src/leap/mail/adaptors/soledad_indexes.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+# soledad_indexes.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Indexes for Mail Documents.
+"""
+
+# TODO
+# [ ] hide most of the constants here
+
+# Document Type, for indexing
+
+TYPE = "type"
+MBOX = "mbox"
+FLAGS = "flags"
+HEADERS = "head"
+CONTENT = "cnt"
+RECENT = "rct"
+HDOCS_SET = "hdocset"
+
+INCOMING_KEY = "incoming"
+ERROR_DECRYPTING_KEY = "errdecr"
+
+# indexing keys
+CONTENT_HASH = "chash"
+PAYLOAD_HASH = "phash"
+MSGID = "msgid"
+UID = "uid"
+
+
+# Index  types
+# --------------
+
+TYPE_IDX = 'by-type'
+TYPE_MBOX_IDX = 'by-type-and-mbox'
+#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
+TYPE_SUBS_IDX = 'by-type-and-subscribed'
+TYPE_MSGID_IDX = 'by-type-and-message-id'
+TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
+TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
+
+# Soledad index for incoming mail, without decrypting errors.
+# and the backward-compatible index, will be deprecated at 0.7
+JUST_MAIL_IDX = "just-mail"
+JUST_MAIL_COMPAT_IDX = "just-mail-compat"
+
+# Tomas created the `recent and seen index`, but the semantic is not too
+# correct since the recent flag is volatile --- XXX review and delete.
+#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
+
+# TODO
+# it would be nice to measure the cost of indexing
+# by many fields.
+
+# TODO
+# make the indexes dict more readable!
+
+MAIL_INDEXES = {
+    # generic
+    TYPE_IDX: [TYPE],
+    TYPE_MBOX_IDX: [TYPE, MBOX],
+
+    # XXX deprecate 0.4.0
+    # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID],
+
+    # mailboxes
+    TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'],
+
+    # fdocs uniqueness
+    TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH],
+
+    # headers doc - search by msgid.
+    TYPE_MSGID_IDX: [TYPE, MSGID],
+
+    # content, headers doc
+    TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH],
+
+    # attachment payload dedup
+    TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH],
+
+    # messages
+    TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'],
+    TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'],
+    TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'],
+    #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX,
+                              #'bool(recent)', 'bool(seen)'],
+
+    # incoming queue
+    JUST_MAIL_IDX: [INCOMING_KEY,
+                    "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
+
+    # the backward-compatible index, will be deprecated at 0.7
+    JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
+}
diff --git a/src/leap/mail/adaptors/tests/__init__.py b/src/leap/mail/adaptors/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/__init__.py
diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message
new file mode 100644
index 0000000..ee97ab9
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+	by localhost with POP3 (fetchmail-6.2.1)
+	for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+	by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+	for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+	by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+	id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+	id 18w63j-0007VK-00
+	for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+	<mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+	<mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19	Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py	Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+             clazz.__dict__.clear()
+             clazz.__getattr__ = __getattr__
+             clazz.__module__ = module.__name__
++    if newclasses:
++        import gc
++        if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++            hasBrokenRebuild = 1
++            gc_objects = gc.get_objects()
++        else:
++            hasBrokenRebuild = 0
+     for nclass in newclasses:
+         ga = getattr(module, nclass.__name__)
+         if ga is nclass:
+             log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+         else:
+-            import gc
+-            for r in gc.get_referrers(nclass):
+-                if isinstance(r, nclass):
++            if hasBrokenRebuild:
++                for r in gc_objects:
++                    if not getattr(r, '__class__', None) is nclass:
++                        continue
+                     r.__class__ = ga
++            else:
++                for r in gc.get_referrers(nclass):
++                    if getattr(r, '__class__', None) is nclass:
++                        r.__class__ = ga
+     if doLog:
+         log.msg('')
+         log.msg('  (fixing   %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py
new file mode 100644
index 0000000..efe0bf2
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/test_models.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+# test_models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the leap.mail.adaptors.models module.
+"""
+from twisted.trial import unittest
+
+from leap.mail.adaptors import models
+
+
+class SerializableModelsTestCase(unittest.TestCase):
+
+    def test_good_serialized_model(self):
+
+        class M(models.SerializableModel):
+            foo = 42
+            bar = 33
+            baaz_ = None
+            _nope = 0
+            __nope = 0
+
+            def not_today(self):
+                pass
+
+            class IgnoreMe(object):
+                pass
+
+            killmeplease = lambda x: x
+
+        serialized = M.serialize()
+        expected = {'foo': 42, 'bar': 33, 'baaz': None}
+        self.assertEqual(serialized, expected)
+
+
+class DocumentWrapperTestCase(unittest.TestCase):
+
+    def test_wrapper_defaults(self):
+
+        class Wrapper(models.DocumentWrapper):
+            class model(models.SerializableModel):
+                foo = 42
+                bar = 11
+
+        wrapper = Wrapper()
+        wrapper._ignored = True
+        serialized = wrapper.serialize()
+        expected = {'foo': 42, 'bar': 11}
+        self.assertEqual(serialized, expected)
+
+    def test_initialized_wrapper(self):
+
+        class Wrapper(models.DocumentWrapper):
+            class model(models.SerializableModel):
+                foo = 42
+                bar_ = 11
+
+        wrapper = Wrapper(foo=0, bar=-1)
+        serialized = wrapper.serialize()
+        expected = {'foo': 0, 'bar': -1}
+        self.assertEqual(serialized, expected)
+
+        wrapper.foo = 23
+        serialized = wrapper.serialize()
+        expected = {'foo': 23, 'bar': -1}
+        self.assertEqual(serialized, expected)
+
+        wrapper = Wrapper(foo=0)
+        serialized = wrapper.serialize()
+        expected = {'foo': 0, 'bar': 11}
+        self.assertEqual(serialized, expected)
+
+    def test_invalid_initialized_wrapper(self):
+
+        class Wrapper(models.DocumentWrapper):
+            class model(models.SerializableModel):
+                foo = 42
+        getwrapper = lambda: Wrapper(bar=1)
+        self.assertRaises(RuntimeError, getwrapper)
+
+    def test_no_model_wrapper(self):
+
+        class Wrapper(models.DocumentWrapper):
+            pass
+
+        def getwrapper():
+            w = Wrapper()
+            w.foo = None
+
+        self.assertRaises(RuntimeError, getwrapper)
diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
new file mode 100644
index 0000000..657a602
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
@@ -0,0 +1,583 @@
+# -*- coding: utf-8 -*-
+# test_soledad_adaptor.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad
+"""
+import os
+import shutil
+import tempfile
+
+from functools import partial
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from leap.common.testing.basetest import BaseLeapTest
+from leap.mail.adaptors import models
+from leap.mail.adaptors.soledad import SoledadDocumentWrapper
+from leap.mail.adaptors.soledad import SoledadIndexMixin
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.soledad.client import Soledad
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+# DEBUG
+# import logging
+# logging.basicConfig(level=logging.DEBUG)
+
+
+def initialize_soledad(email, gnupg_home, tempdir):
+    """
+    Initializes soledad by hand
+
+    :param email: ID for the user
+    :param gnupg_home: path to home used by gnupg
+    :param tempdir: path to temporal dir
+    :rtype: Soledad instance
+    """
+
+    uuid = "foobar-uuid"
+    passphrase = u"verysecretpassphrase"
+    secret_path = os.path.join(tempdir, "secret.gpg")
+    local_db_path = os.path.join(tempdir, "soledad.u1db")
+    server_url = "https://provider"
+    cert_file = ""
+
+    soledad = Soledad(
+        uuid,
+        passphrase,
+        secret_path,
+        local_db_path,
+        server_url,
+        cert_file,
+        syncable=False)
+
+    return soledad
+
+
+# TODO move to common module
+# XXX remove duplication
+class SoledadTestMixin(BaseLeapTest):
+    """
+    It is **VERY** important that this base is added *AFTER* unittest.TestCase
+    """
+
+    def setUp(self):
+        self.results = []
+
+        self.old_path = os.environ['PATH']
+        self.old_home = os.environ['HOME']
+        self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+        self.home = self.tempdir
+        bin_tdir = os.path.join(
+            self.tempdir,
+            'bin')
+        os.environ["PATH"] = bin_tdir
+        os.environ["HOME"] = self.tempdir
+
+        # Soledad: config info
+        self.gnupg_home = "%s/gnupg" % self.tempdir
+        self.email = 'leap@leap.se'
+
+        # initialize soledad by hand so we can control keys
+        self._soledad = initialize_soledad(
+            self.email,
+            self.gnupg_home,
+            self.tempdir)
+
+    def tearDown(self):
+        """
+        tearDown method called after each test.
+        """
+        self.results = []
+        try:
+            self._soledad.close()
+        except Exception as exc:
+            print "ERROR WHILE CLOSING SOLEDAD"
+            # logging.exception(exc)
+        finally:
+            os.environ["PATH"] = self.old_path
+            os.environ["HOME"] = self.old_home
+            # safety check
+            assert 'leap_tests-' in self.tempdir
+            shutil.rmtree(self.tempdir)
+
+
+class CounterWrapper(SoledadDocumentWrapper):
+    class model(models.SerializableModel):
+        counter = 0
+        flag = None
+
+
+class CharacterWrapper(SoledadDocumentWrapper):
+    class model(models.SerializableModel):
+        name = ""
+        age = 20
+
+
+class ActorWrapper(SoledadDocumentWrapper):
+    class model(models.SerializableModel):
+        type_ = "actor"
+        name = None
+
+        class __meta__(object):
+            index = "name"
+            list_index = ("by-type", "type_")
+
+
+class TestAdaptor(SoledadIndexMixin):
+    indexes = {'by-name': ['name'],
+               'by-type-and-name': ['type', 'name'],
+               'by-type': ['type']}
+
+
+class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin):
+    """
+    Tests for the SoledadDocumentWrapper.
+    """
+    def assert_num_docs(self, num, docs):
+        self.assertEqual(len(docs[1]), num)
+
+    def test_create_single(self):
+
+        store = self._soledad
+        wrapper = CounterWrapper()
+
+        def assert_one_doc(docs):
+            self.assertEqual(docs[0], 1)
+
+        d = wrapper.create(store)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(assert_one_doc)
+        return d
+
+    def test_create_many(self):
+
+        store = self._soledad
+        w1 = CounterWrapper()
+        w2 = CounterWrapper(counter=1)
+        w3 = CounterWrapper(counter=2)
+        w4 = CounterWrapper(counter=3)
+        w5 = CounterWrapper(counter=4)
+
+        d1 = [w1.create(store),
+              w2.create(store),
+              w3.create(store),
+              w4.create(store),
+              w5.create(store)]
+
+        d = defer.gatherResults(d1)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 5))
+        return d
+
+    def test_multiple_updates(self):
+
+        store = self._soledad
+        wrapper = CounterWrapper(counter=1)
+        MAX = 100
+
+        def assert_doc_id(doc):
+            self.assertTrue(wrapper._doc_id is not None)
+            return doc
+
+        def assert_counter_initial_ok(doc):
+            self.assertEqual(wrapper.counter, 1)
+
+        def increment_counter(ignored):
+            d1 = []
+
+            def record_revision(revision):
+                rev = int(revision.split(':')[1])
+                self.results.append(rev)
+
+            for i in list(range(MAX)):
+                wrapper.counter += 1
+                wrapper.flag = i % 2 == 0
+                d = wrapper.update(store)
+                d.addCallback(record_revision)
+                d1.append(d)
+
+            return defer.gatherResults(d1)
+
+        def assert_counter_final_ok(doc):
+            self.assertEqual(doc.content['counter'], MAX + 1)
+            self.assertEqual(doc.content['flag'], False)
+
+        def assert_results_ordered_list(ignored):
+            self.assertEqual(self.results, sorted(range(2, MAX + 2)))
+
+        d = wrapper.create(store)
+        d.addCallback(assert_doc_id)
+        d.addCallback(assert_counter_initial_ok)
+        d.addCallback(increment_counter)
+        d.addCallback(lambda _: store.get_doc(wrapper._doc_id))
+        d.addCallback(assert_counter_final_ok)
+        d.addCallback(assert_results_ordered_list)
+        return d
+
+    def test_delete(self):
+        adaptor = TestAdaptor()
+        store = self._soledad
+
+        wrapper_list = []
+
+        def get_or_create_bob(ignored):
+            def add_to_list(wrapper):
+                wrapper_list.append(wrapper)
+                return wrapper
+            wrapper = CharacterWrapper.get_or_create(
+                store, 'by-name', 'bob')
+            wrapper.addCallback(add_to_list)
+            return wrapper
+
+        def delete_bob(ignored):
+            wrapper = wrapper_list[0]
+            return wrapper.delete(store)
+
+        d = adaptor.initialize_store(store)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 0))
+
+        # this should create bob document
+        d.addCallback(get_or_create_bob)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 1))
+
+        d.addCallback(delete_bob)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 0))
+        return d
+
+    def test_get_or_create(self):
+        adaptor = TestAdaptor()
+        store = self._soledad
+
+        def get_or_create_bob(ignored):
+            wrapper = CharacterWrapper.get_or_create(
+                store, 'by-name', 'bob')
+            return wrapper
+
+        d = adaptor.initialize_store(store)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 0))
+
+        # this should create bob document
+        d.addCallback(get_or_create_bob)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 1))
+
+        # this should get us bob document
+        d.addCallback(get_or_create_bob)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 1))
+        return d
+
+    def test_get_or_create_multi_index(self):
+        adaptor = TestAdaptor()
+        store = self._soledad
+
+        def get_or_create_actor_harry(ignored):
+            wrapper = ActorWrapper.get_or_create(
+                store, 'by-type-and-name', 'harrison')
+            return wrapper
+
+        def create_director_harry(ignored):
+            wrapper = ActorWrapper(name="harrison", type="director")
+            return wrapper.create(store)
+
+        d = adaptor.initialize_store(store)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 0))
+
+        # this should create harrison document
+        d.addCallback(get_or_create_actor_harry)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 1))
+
+        # this should get us harrison document
+        d.addCallback(get_or_create_actor_harry)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 1))
+
+        # create director harry, should create new doc
+        d.addCallback(create_director_harry)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 2))
+
+        # this should get us harrison document, still 2 docs
+        d.addCallback(get_or_create_actor_harry)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 2))
+        return d
+
+    def test_get_all(self):
+        adaptor = TestAdaptor()
+        store = self._soledad
+        actor_names = ["harry", "carrie", "mark", "david"]
+
+        def create_some_actors(ignored):
+            deferreds = []
+            for name in actor_names:
+                dw = ActorWrapper.get_or_create(
+                    store, 'by-type-and-name', name)
+                deferreds.append(dw)
+            return defer.gatherResults(deferreds)
+
+        d = adaptor.initialize_store(store)
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 0))
+
+        d.addCallback(create_some_actors)
+
+        d.addCallback(lambda _: store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 4))
+
+        def assert_actor_list_is_expected(res):
+            got = set([actor.name for actor in res])
+            expected = set(actor_names)
+            self.assertEqual(got, expected)
+
+        d.addCallback(lambda _: ActorWrapper.get_all(store))
+        d.addCallback(assert_actor_list_is_expected)
+        return d
+
+here = os.path.split(os.path.abspath(__file__))[0]
+
+
+class TestMessageClass(object):
+    def __init__(self, wrapper):
+        self.wrapper = wrapper
+
+    def get_wrapper(self):
+        return self.wrapper
+
+
+class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
+    """
+    Tests for the SoledadMailAdaptor.
+    """
+
+    def get_adaptor(self):
+        adaptor = SoledadMailAdaptor()
+        adaptor.store = self._soledad
+        return adaptor
+
+    def assert_num_docs(self, num, docs):
+        self.assertEqual(len(docs[1]), num)
+
+    def test_mail_adaptor_init(self):
+        adaptor = self.get_adaptor()
+        self.assertTrue(isinstance(adaptor.indexes, dict))
+        self.assertTrue(len(adaptor.indexes) != 0)
+
+    # Messages
+
+    def test_get_msg_from_string(self):
+        adaptor = self.get_adaptor()
+
+        with open(os.path.join(here, "rfc822.message")) as f:
+            raw = f.read()
+
+        msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+
+        chash = ("D27B2771C0DCCDCB468EE65A4540438"
+                 "09DBD11588E87E951545BE0CBC321C308")
+        phash = ("64934534C1C80E0D4FA04BE1CCBA104"
+                 "F07BCA5F469C86E2C0ABE1D41310B7299")
+        subject = ("[Twisted-commits] rebuild now works on "
+                   "python versions from 2.2.0 and up.")
+        self.assertTrue(msg.wrapper.fdoc is not None)
+        self.assertTrue(msg.wrapper.hdoc is not None)
+        self.assertTrue(msg.wrapper.cdocs is not None)
+        self.assertEquals(len(msg.wrapper.cdocs), 1)
+        self.assertEquals(msg.wrapper.fdoc.chash, chash)
+        self.assertEquals(msg.wrapper.fdoc.size, 3834)
+        self.assertEquals(msg.wrapper.hdoc.chash, chash)
+        self.assertEqual(msg.wrapper.hdoc.headers['subject'],
+                         subject)
+        self.assertEqual(msg.wrapper.hdoc.subject, subject)
+        self.assertEqual(msg.wrapper.cdocs[1].phash, phash)
+
+    def test_get_msg_from_docs(self):
+        adaptor = self.get_adaptor()
+        fdoc = dict(
+            mbox="Foobox",
+            flags=('\Seen', '\Nice'),
+            tags=('Personal', 'TODO'),
+            seen=False, deleted=False,
+            recent=False, multi=False)
+        hdoc = dict(
+            subject="Test Msg")
+        cdocs = {
+            1: dict(
+                raw='This is a test message')}
+
+        msg = adaptor.get_msg_from_docs(
+            TestMessageClass, fdoc, hdoc, cdocs=cdocs)
+        self.assertEqual(msg.wrapper.fdoc.flags,
+                         ('\Seen', '\Nice'))
+        self.assertEqual(msg.wrapper.fdoc.tags,
+                         ('Personal', 'TODO'))
+        self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox")
+        self.assertEqual(msg.wrapper.hdoc.multi, False)
+        self.assertEqual(msg.wrapper.hdoc.subject,
+                         "Test Msg")
+        self.assertEqual(msg.wrapper.cdocs[1].raw,
+                         "This is a test message")
+
+    def test_create_msg(self):
+        adaptor = self.get_adaptor()
+
+        with open(os.path.join(here, "rfc822.message")) as f:
+            raw = f.read()
+        msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+
+        def check_create_result(created):
+            self.assertEqual(len(created), 3)
+            for doc in created:
+                self.assertTrue(
+                    doc.__class__.__name__,
+                    "SoledadDocument")
+
+        d = adaptor.create_msg(adaptor.store, msg)
+        d.addCallback(check_create_result)
+        return d
+
+    def test_update_msg(self):
+        adaptor = self.get_adaptor()
+        with open(os.path.join(here, "rfc822.message")) as f:
+            raw = f.read()
+
+        def assert_msg_has_doc_id(ignored, msg):
+            wrapper = msg.get_wrapper()
+            self.assertTrue(wrapper.fdoc.doc_id is not None)
+
+        def assert_msg_has_no_flags(ignored, msg):
+            wrapper = msg.get_wrapper()
+            self.assertEqual(wrapper.fdoc.flags, [])
+
+        def update_msg_flags(ignored, msg):
+            wrapper = msg.get_wrapper()
+            wrapper.fdoc.flags = ["This", "That"]
+            return wrapper.update(adaptor.store)
+
+        def assert_msg_has_flags(ignored, msg):
+            wrapper = msg.get_wrapper()
+            self.assertEqual(wrapper.fdoc.flags, ["This", "That"])
+
+        def get_fdoc_and_check_flags(ignored):
+            def assert_doc_has_flags(doc):
+                self.assertEqual(doc.content['flags'],
+                                 ['This', 'That'])
+            wrapper = msg.get_wrapper()
+            d = adaptor.store.get_doc(wrapper.fdoc.doc_id)
+            d.addCallback(assert_doc_has_flags)
+            return d
+
+        msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+        d = adaptor.create_msg(adaptor.store, msg)
+        d.addCallback(lambda _: adaptor.store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 3))
+        d.addCallback(assert_msg_has_doc_id, msg)
+        d.addCallback(assert_msg_has_no_flags, msg)
+
+        # update it!
+        d.addCallback(update_msg_flags, msg)
+        d.addCallback(assert_msg_has_flags, msg)
+        d.addCallback(get_fdoc_and_check_flags)
+        return d
+
+    # Mailboxes
+
+    def test_get_or_create_mbox(self):
+        adaptor = self.get_adaptor()
+
+        def get_or_create_mbox(ignored):
+            d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+            return d
+
+        def assert_good_doc(mbox_wrapper):
+            self.assertTrue(mbox_wrapper.doc_id is not None)
+            self.assertEqual(mbox_wrapper.mbox, "Trash")
+            self.assertEqual(mbox_wrapper.type, "mbox")
+            self.assertEqual(mbox_wrapper.closed, False)
+            self.assertEqual(mbox_wrapper.subscribed, False)
+
+        d = adaptor.initialize_store(adaptor.store)
+        d.addCallback(get_or_create_mbox)
+        d.addCallback(assert_good_doc)
+        d.addCallback(lambda _: adaptor.store.get_all_docs())
+        d.addCallback(partial(self.assert_num_docs, 1))
+        return d
+
+    def test_update_mbox(self):
+        adaptor = self.get_adaptor()
+
+        wrapper_ref = []
+
+        def get_or_create_mbox(ignored):
+            d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+            return d
+
+        def update_wrapper(wrapper, wrapper_ref):
+            wrapper_ref.append(wrapper)
+            wrapper.subscribed = True
+            wrapper.closed = True
+            d = adaptor.update_mbox(adaptor.store, wrapper)
+            return d
+
+        def get_mbox_doc_and_check_flags(res, wrapper_ref):
+            wrapper = wrapper_ref[0]
+
+            def assert_doc_has_flags(doc):
+                self.assertEqual(doc.content['subscribed'], True)
+                self.assertEqual(doc.content['closed'], True)
+            d = adaptor.store.get_doc(wrapper.doc_id)
+            d.addCallback(assert_doc_has_flags)
+            return d
+
+        d = adaptor.initialize_store(adaptor.store)
+        d.addCallback(get_or_create_mbox)
+        d.addCallback(update_wrapper, wrapper_ref)
+        d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref)
+        return d
+
+    def test_get_all_mboxes(self):
+        adaptor = self.get_adaptor()
+        mboxes = ("Sent", "Trash", "Personal", "ListFoo")
+
+        def get_or_create_mboxes(ignored):
+            d = []
+            for mbox in mboxes:
+                d.append(adaptor.get_or_create_mbox(
+                    adaptor.store, mbox))
+            return defer.gatherResults(d)
+
+        def get_all_mboxes(ignored):
+            return adaptor.get_all_mboxes(adaptor.store)
+
+        def assert_mboxes_match_expected(wrappers):
+            names = [m.mbox for m in wrappers]
+            self.assertEqual(set(names), set(mboxes))
+
+        d = adaptor.initialize_store(adaptor.store)
+        d.addCallback(get_or_create_mboxes)
+        d.addCallback(get_all_mboxes)
+        d.addCallback(assert_mboxes_match_expected)
+        return d