MessageCollections + MailboxIndexer

10 files changed, 1204 insertions, 261 deletions

diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py
index 2e25f04..0b97869 100644
--- a/src/leap/mail/adaptors/soledad.py
+++ b/src/leap/mail/adaptors/soledad.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # soledad.py
 # Copyright (C) 2014 LEAP
 #
@@ -20,6 +19,7 @@ Soledadad MailAdaptor module.
 import re
 from collections import defaultdict
 from email import message_from_string
+from functools import partial
 
 from pycryptopp.hash import sha256
 from twisted.internet import defer
@@ -27,6 +27,7 @@ from zope.interface import implements
 
 from leap.common.check import leap_assert, leap_assert_type
 
+from leap.mail import constants
 from leap.mail import walk
 from leap.mail.adaptors import soledad_indexes as indexes
 from leap.mail.constants import INBOX_NAME
@@ -60,7 +61,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
     It ensures atomicity of the document operations on creation, update and
     deletion.
     """
-
     # TODO we could also use a _dirty flag (in models)
 
     # We keep a dictionary with DeferredLocks, that will be
@@ -79,6 +79,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
     def __init__(self, **kwargs):
         doc_id = kwargs.pop('doc_id', None)
         self._doc_id = doc_id
+        self._future_doc_id = kwargs.pop('future_doc_id', None)
         self._lock = defer.DeferredLock()
         super(SoledadDocumentWrapper, self).__init__(**kwargs)
 
@@ -86,6 +87,13 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
     def doc_id(self):
         return self._doc_id
 
+    @property
+    def future_doc_id(self):
+        return self._future_doc_id
+
+    def set_future_doc_id(self, doc_id):
+        self._future_doc_id = doc_id
+
     def create(self, store):
         """
         Create the documents for this wrapper.
@@ -105,8 +113,14 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
 
         def update_doc_id(doc):
             self._doc_id = doc.doc_id
+            self._future_doc_id = None
             return doc
-        d = store.create_doc(self.serialize())
+
+        if self.future_doc_id is None:
+            d = store.create_doc(self.serialize())
+        else:
+            d = store.create_doc(self.serialize(),
+                                 doc_id=self.future_doc_id)
         d.addCallback(update_doc_id)
         return d
 
@@ -333,6 +347,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper):
         class __meta__(object):
             index = "mbox"
 
+    def set_mbox(self, mbox):
+        # XXX raise error if already created, should use copy instead
+        new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash)
+        self._future_doc_id = new_id
+        self.mbox = mbox
+
 
 class HeaderDocWrapper(SoledadDocumentWrapper):
 
@@ -370,6 +390,23 @@ class ContentDocWrapper(SoledadDocumentWrapper):
             index = "phash"
 
 
+class MetaMsgDocWrapper(SoledadDocumentWrapper):
+
+    class model(models.SerializableModel):
+        type_ = "meta"
+        fdoc = ""
+        hdoc = ""
+        cdocs = []
+
+    def set_mbox(self, mbox):
+        # XXX raise error if already created, should use copy instead
+        chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0]
+        new_id = constants.METAMSGID.format(mbox=mbox, chash=chash)
+        new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash)
+        self._future_doc_id = new_id
+        self.fdoc = new_fdoc_id
+
+
 class MessageWrapper(object):
 
     # TODO generalize wrapper composition?
@@ -378,23 +415,32 @@ class MessageWrapper(object):
 
     implements(IMessageWrapper)
 
-    def __init__(self, fdoc, hdoc, cdocs=None):
+    def __init__(self, mdoc, fdoc, hdoc, cdocs=None):
         """
-        Need at least a flag-document and a header-document to instantiate a
-        MessageWrapper. Content-documents can be retrieved lazily.
+        Need at least a metamsg-document, a flag-document and a header-document
+        to instantiate a MessageWrapper. Content-documents can be retrieved
+        lazily.
 
         cdocs, if any, should be a dictionary in which the keys are ascending
         integers, beginning at one, and the values are dictionaries with the
         content of the content-docs.
         """
+        self.mdoc = MetaMsgDocWrapper(**mdoc)
+
         self.fdoc = FlagsDocWrapper(**fdoc)
+        self.fdoc.set_future_doc_id(self.mdoc.fdoc)
+
         self.hdoc = HeaderDocWrapper(**hdoc)
+        self.hdoc.set_future_doc_id(self.mdoc.hdoc)
+
         if cdocs is None:
             cdocs = {}
         cdocs_keys = cdocs.keys()
         assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1)
         self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in
                            cdocs.items()])
+        for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()):
+            cdoc.set_future_doc_id(doc_id)
 
     def create(self, store):
         """
@@ -403,16 +449,21 @@ class MessageWrapper(object):
         leap_assert(self.cdocs,
                     "Need non empty cdocs to create the "
                     "MessageWrapper documents")
+        leap_assert(self.mdoc.doc_id is None,
+                    "Cannot create: mdoc has a doc_id")
         leap_assert(self.fdoc.doc_id is None,
                     "Cannot create: fdoc has a doc_id")
 
+        # TODO check that the doc_ids in the mdoc are coherent
         # TODO I think we need to tolerate the no hdoc.doc_id case, for when we
         # are doing a copy to another mailbox.
-        leap_assert(self.hdoc.doc_id is None,
-                    "Cannot create: hdoc has a doc_id")
+        # leap_assert(self.hdoc.doc_id is None,
+        # "Cannot create: hdoc has a doc_id")
         d = []
+        d.append(self.mdoc.create(store))
         d.append(self.fdoc.create(store))
-        d.append(self.hdoc.create(store))
+        if self.hdoc.doc_id is None:
+            d.append(self.hdoc.create(store))
         for cdoc in self.cdocs.values():
             if cdoc.doc_id is not None:
                 # we could be just linking to an existing
@@ -432,6 +483,25 @@ class MessageWrapper(object):
         # garbage collector. At least the fdoc can be unlinked.
         raise NotImplementedError()
 
+    def copy(self, store, newmailbox):
+        """
+        Return a copy of this MessageWrapper in a new mailbox.
+        """
+        # 1. copy the fdoc, mdoc
+        # 2. remove the doc_id of that fdoc
+        # 3. create it (with new doc_id)
+        # 4. return new wrapper (new meta too!)
+        raise NotImplementedError()
+
+    def set_mbox(self, mbox):
+        """
+        Set the mailbox for this wrapper.
+        This method should only be used before the Documents for the
+        MessageWrapper have been created, will raise otherwise.
+        """
+        self.mdoc.set_mbox(mbox)
+        self.fdoc.set_mbox(mbox)
+
 #
 # Mailboxes
 #
@@ -535,6 +605,7 @@ class SoledadMailAdaptor(SoledadIndexMixin):
     store = None
 
     indexes = indexes.MAIL_INDEXES
+    mboxwrapper_klass = MailboxWrapper
 
     # Message handling
 
@@ -552,11 +623,11 @@ class SoledadMailAdaptor(SoledadIndexMixin):
         :rtype: MessageClass instance.
         """
         assert(MessageClass is not None)
-        fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
+        mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
         return self.get_msg_from_docs(
-            MessageClass, fdoc, hdoc, cdocs)
+            MessageClass, mdoc, fdoc, hdoc, cdocs)
 
-    def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None):
+    def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None):
         """
         Get an instance of a MessageClass initialized with a MessageWrapper
         that contains the passed part documents.
@@ -582,7 +653,62 @@ class SoledadMailAdaptor(SoledadIndexMixin):
         :rtype: MessageClass instance.
         """
         assert(MessageClass is not None)
-        return MessageClass(MessageWrapper(fdoc, hdoc, cdocs))
+        return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs))
+
+    def _get_msg_from_variable_doc_list(self, doc_list, msg_class):
+        if len(doc_list) == 2:
+            fdoc, hdoc = doc_list
+            cdocs = None
+        elif len(doc_list) > 2:
+            fdoc, hdoc = doc_list[:2]
+            cdocs = dict(enumerate(doc_list[2:], 1))
+        return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs)
+
+    def get_msg_from_mdoc_id(self, MessageClass, store, doc_id,
+                             get_cdocs=False):
+        metamsg_id = doc_id
+
+        def wrap_meta_doc(doc):
+            cls = MetaMsgDocWrapper
+            return cls(doc_id=doc.doc_id, **doc.content)
+
+        def get_part_docs_from_mdoc_wrapper(wrapper):
+            d_docs = []
+            d_docs.append(store.get_doc(wrapper.fdoc))
+            d_docs.append(store.get_doc(wrapper.hdoc))
+            for cdoc in wrapper.cdocs:
+                d_docs.append(store.get_doc(cdoc))
+            d = defer.gatherResults(d_docs)
+            return d
+
+        def get_parts_doc_from_mdoc_id():
+            mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0]
+            chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0]
+
+            def _get_fdoc_id_from_mdoc_id():
+                return constants.FDOCID.format(mbox=mbox, chash=chash)
+
+            def _get_hdoc_id_from_mdoc_id():
+                return constants.FDOCID.format(mbox=mbox, chash=chash)
+
+            d_docs = []
+            fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id)
+            hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id)
+            d_docs.append(store.get_doc(fdoc_id))
+            d_docs.append(store.get_doc(hdoc_id))
+            d = defer.gatherResults(d_docs)
+            return d
+
+        if get_cdocs:
+            d = store.get_doc(metamsg_id)
+            d.addCallback(wrap_meta_doc)
+            d.addCallback(get_part_docs_from_mdoc_wrapper)
+        else:
+            d = get_parts_doc_from_mdoc_id()
+
+        d.addCallback(partial(self._get_msg_from_variable_doc_list,
+                              msg_class=MessageClass))
+        return d
 
     def create_msg(self, store, msg):
         """
@@ -615,7 +741,7 @@ class SoledadMailAdaptor(SoledadIndexMixin):
 
     def get_or_create_mbox(self, store, name):
         """
-        Get the mailbox with the given name, or creatre one if it does not
+        Get the mailbox with the given name, or create one if it does not
         exist.
 
         :param name: the name of the mailbox
@@ -636,6 +762,9 @@ class SoledadMailAdaptor(SoledadIndexMixin):
         """
         return mbox_wrapper.update(store)
 
+    def delete_mbox(self, store, mbox_wrapper):
+        return mbox_wrapper.delete(store)
+
     def get_all_mboxes(self, store):
         """
         Retrieve a list with wrappers for all the mailboxes.
@@ -660,15 +789,17 @@ def _split_into_parts(raw):
                       walk.get_body_phash_multi][int(multi)]
     body_phash = body_phash_fun(walk.get_payloads(msg))
     parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+    cdocs_list = list(walk.get_raw_docs(msg, parts))
+    cdocs_phashes = [c['phash'] for c in cdocs_list]
 
+    mdoc = _build_meta_doc(chash, cdocs_phashes)
     fdoc = _build_flags_doc(chash, size, multi)
     hdoc = _build_headers_doc(msg, chash, parts_map)
 
     # The MessageWrapper expects a dict, one-indexed
-    cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
+    cdocs = dict(enumerate(cdocs_list, 1))
 
-    # XXX convert each to_dicts...
-    return fdoc, hdoc, cdocs
+    return mdoc, fdoc, hdoc, cdocs
 
 
 def _parse_msg(raw):
@@ -680,6 +811,14 @@ def _parse_msg(raw):
     return msg, parts, chash, size, multi
 
 
+def _build_meta_doc(chash, cdocs_phashes):
+    _mdoc = MetaMsgDocWrapper()
+    _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash)
+    _mdoc.hdoc = constants.HDOCID.format(chash=chash)
+    _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes]
+    return _mdoc.serialize()
+
+
 def _build_flags_doc(chash, size, multi):
     _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi)
     return _fdoc.serialize()
diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message
index ee97ab9..b19cc28 100644..120000
--- a/src/leap/mail/adaptors/tests/rfc822.message
+++ b/src/leap/mail/adaptors/tests/rfc822.message
@@ -1,86 +1 @@
-Return-Path: <twisted-commits-admin@twistedmatrix.com>
-Delivered-To: exarkun@meson.dyndns.org
-Received: from localhost [127.0.0.1]
-	by localhost with POP3 (fetchmail-6.2.1)
-	for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
-Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
-	by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
-	for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
-Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
-	by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
-	id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
-Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
-	id 18w63j-0007VK-00
-	for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
-To: twisted-commits@twistedmatrix.com
-From: etrepum CVS <etrepum@twistedmatrix.com>
-Reply-To: twisted-python@twistedmatrix.com
-X-Mailer: CVSToys
-Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
-Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
-Sender: twisted-commits-admin@twistedmatrix.com
-Errors-To: twisted-commits-admin@twistedmatrix.com
-X-BeenThere: twisted-commits@twistedmatrix.com
-X-Mailman-Version: 2.0.11
-Precedence: bulk
-List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
-List-Post: <mailto:twisted-commits@twistedmatrix.com>
-List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
-	<mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
-List-Id: <twisted-commits.twistedmatrix.com>
-List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
-	<mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
-List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
-Date: Thu, 20 Mar 2003 13:50:39 -0600
-
-Modified files:
-Twisted/twisted/python/rebuild.py 1.19 1.20
-
-Log message:
-rebuild now works on python versions from 2.2.0 and up.
-
-
-ViewCVS links:
-http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
-
-Index: Twisted/twisted/python/rebuild.py
-diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
---- Twisted/twisted/python/rebuild.py:1.19	Fri Jan 17 13:50:49 2003
-+++ Twisted/twisted/python/rebuild.py	Thu Mar 20 11:50:08 2003
-@@ -206,15 +206,27 @@
-             clazz.__dict__.clear()
-             clazz.__getattr__ = __getattr__
-             clazz.__module__ = module.__name__
-+    if newclasses:
-+        import gc
-+        if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
-+            hasBrokenRebuild = 1
-+            gc_objects = gc.get_objects()
-+        else:
-+            hasBrokenRebuild = 0
-     for nclass in newclasses:
-         ga = getattr(module, nclass.__name__)
-         if ga is nclass:
-             log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
-         else:
--            import gc
--            for r in gc.get_referrers(nclass):
--                if isinstance(r, nclass):
-+            if hasBrokenRebuild:
-+                for r in gc_objects:
-+                    if not getattr(r, '__class__', None) is nclass:
-+                        continue
-                     r.__class__ = ga
-+            else:
-+                for r in gc.get_referrers(nclass):
-+                    if getattr(r, '__class__', None) is nclass:
-+                        r.__class__ = ga
-     if doLog:
-         log.msg('')
-         log.msg('  (fixing   %s): ' % str(module.__name__))
-
-
-_______________________________________________
-Twisted-commits mailing list
-Twisted-commits@twistedmatrix.com
-http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
+../../tests/rfc822.message
\ No newline at end of file
diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
index 657a602..0cca5ef 100644
--- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
+++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
@@ -18,106 +18,22 @@
 Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad
 """
 import os
-import shutil
-import tempfile
-
 from functools import partial
 
 from twisted.internet import defer
 from twisted.trial import unittest
 
-from leap.common.testing.basetest import BaseLeapTest
 from leap.mail.adaptors import models
 from leap.mail.adaptors.soledad import SoledadDocumentWrapper
 from leap.mail.adaptors.soledad import SoledadIndexMixin
 from leap.mail.adaptors.soledad import SoledadMailAdaptor
-from leap.soledad.client import Soledad
-
-TEST_USER = "testuser@leap.se"
-TEST_PASSWD = "1234"
+from leap.mail.tests.common import SoledadTestMixin
 
 # DEBUG
 # import logging
 # logging.basicConfig(level=logging.DEBUG)
 
 
-def initialize_soledad(email, gnupg_home, tempdir):
-    """
-    Initializes soledad by hand
-
-    :param email: ID for the user
-    :param gnupg_home: path to home used by gnupg
-    :param tempdir: path to temporal dir
-    :rtype: Soledad instance
-    """
-
-    uuid = "foobar-uuid"
-    passphrase = u"verysecretpassphrase"
-    secret_path = os.path.join(tempdir, "secret.gpg")
-    local_db_path = os.path.join(tempdir, "soledad.u1db")
-    server_url = "https://provider"
-    cert_file = ""
-
-    soledad = Soledad(
-        uuid,
-        passphrase,
-        secret_path,
-        local_db_path,
-        server_url,
-        cert_file,
-        syncable=False)
-
-    return soledad
-
-
-# TODO move to common module
-# XXX remove duplication
-class SoledadTestMixin(BaseLeapTest):
-    """
-    It is **VERY** important that this base is added *AFTER* unittest.TestCase
-    """
-
-    def setUp(self):
-        self.results = []
-
-        self.old_path = os.environ['PATH']
-        self.old_home = os.environ['HOME']
-        self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
-        self.home = self.tempdir
-        bin_tdir = os.path.join(
-            self.tempdir,
-            'bin')
-        os.environ["PATH"] = bin_tdir
-        os.environ["HOME"] = self.tempdir
-
-        # Soledad: config info
-        self.gnupg_home = "%s/gnupg" % self.tempdir
-        self.email = 'leap@leap.se'
-
-        # initialize soledad by hand so we can control keys
-        self._soledad = initialize_soledad(
-            self.email,
-            self.gnupg_home,
-            self.tempdir)
-
-    def tearDown(self):
-        """
-        tearDown method called after each test.
-        """
-        self.results = []
-        try:
-            self._soledad.close()
-        except Exception as exc:
-            print "ERROR WHILE CLOSING SOLEDAD"
-            # logging.exception(exc)
-        finally:
-            os.environ["PATH"] = self.old_path
-            os.environ["HOME"] = self.old_home
-            # safety check
-            assert 'leap_tests-' in self.tempdir
-            shutil.rmtree(self.tempdir)
-
-
 class CounterWrapper(SoledadDocumentWrapper):
     class model(models.SerializableModel):
         counter = 0
@@ -357,7 +273,7 @@ class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin):
         d.addCallback(assert_actor_list_is_expected)
         return d
 
-here = os.path.split(os.path.abspath(__file__))[0]
+HERE = os.path.split(os.path.abspath(__file__))[0]
 
 
 class TestMessageClass(object):
@@ -391,7 +307,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
     def test_get_msg_from_string(self):
         adaptor = self.get_adaptor()
 
-        with open(os.path.join(here, "rfc822.message")) as f:
+        with open(os.path.join(HERE, "rfc822.message")) as f:
             raw = f.read()
 
         msg = adaptor.get_msg_from_string(TestMessageClass, raw)
@@ -416,6 +332,10 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
 
     def test_get_msg_from_docs(self):
         adaptor = self.get_adaptor()
+        mdoc = dict(
+            fdoc="F-Foobox-deadbeef",
+            hdoc="H-deadbeef",
+            cdocs=["C-deadabad"])
         fdoc = dict(
             mbox="Foobox",
             flags=('\Seen', '\Nice'),
@@ -423,13 +343,14 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
             seen=False, deleted=False,
             recent=False, multi=False)
         hdoc = dict(
+            chash="deadbeef",
             subject="Test Msg")
         cdocs = {
             1: dict(
                 raw='This is a test message')}
 
         msg = adaptor.get_msg_from_docs(
-            TestMessageClass, fdoc, hdoc, cdocs=cdocs)
+            TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs)
         self.assertEqual(msg.wrapper.fdoc.flags,
                          ('\Seen', '\Nice'))
         self.assertEqual(msg.wrapper.fdoc.tags,
@@ -441,15 +362,20 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
         self.assertEqual(msg.wrapper.cdocs[1].raw,
                          "This is a test message")
 
+    def test_get_msg_from_metamsg_doc_id(self):
+        # XXX complete-me!
+        self.fail()
+
     def test_create_msg(self):
         adaptor = self.get_adaptor()
 
-        with open(os.path.join(here, "rfc822.message")) as f:
+        with open(os.path.join(HERE, "rfc822.message")) as f:
             raw = f.read()
         msg = adaptor.get_msg_from_string(TestMessageClass, raw)
 
         def check_create_result(created):
-            self.assertEqual(len(created), 3)
+            # that's one mdoc, one hdoc, one fdoc, one cdoc
+            self.assertEqual(len(created), 4)
             for doc in created:
                 self.assertTrue(
                     doc.__class__.__name__,
@@ -461,7 +387,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
 
     def test_update_msg(self):
         adaptor = self.get_adaptor()
-        with open(os.path.join(here, "rfc822.message")) as f:
+        with open(os.path.join(HERE, "rfc822.message")) as f:
             raw = f.read()
 
         def assert_msg_has_doc_id(ignored, msg):
@@ -493,7 +419,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
         msg = adaptor.get_msg_from_string(TestMessageClass, raw)
         d = adaptor.create_msg(adaptor.store, msg)
         d.addCallback(lambda _: adaptor.store.get_all_docs())
-        d.addCallback(partial(self.assert_num_docs, 3))
+        d.addCallback(partial(self.assert_num_docs, 4))
         d.addCallback(assert_msg_has_doc_id, msg)
         d.addCallback(assert_msg_has_no_flags, msg)
 
diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py
index 55bf1da..bf1db7f 100644
--- a/src/leap/mail/constants.py
+++ b/src/leap/mail/constants.py
@@ -19,3 +19,20 @@ Constants for leap.mail.
 """
 
 INBOX_NAME = "INBOX"
+
+# Regular expressions for the identifiers to be used in the Message Data Layer.
+
+METAMSGID = "M-{mbox}-{chash}"
+METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+"
+METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)"
+METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+"
+
+FDOCID = "F-{mbox}-{chash}"
+FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+"
+FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)"
+
+HDOCID = "H-{chash}"
+HDOCID_RE = "H\-[0-9a-fA-F]+"
+
+CDOCID = "C-{phash}"
+CDOCID_RE = "C\-[0-9a-fA-F]+"
diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py
index ea9c95e..ca07f67 100644
--- a/src/leap/mail/mail.py
+++ b/src/leap/mail/mail.py
@@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API.
 from twisted.internet import defer
 
 from leap.mail.constants import INBOX_NAME
+from leap.mail.mailbox_indexer import MailboxIndexer
 from leap.mail.adaptors.soledad import SoledadMailAdaptor
 
 
@@ -27,8 +28,17 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor
 # [ ] Probably change the name of this module to "api" or "account", mail is
 #     too generic (there's also IncomingMail, and OutgoingMail
 
+def _get_mdoc_id(mbox, chash):
+    """
+    Get the doc_id for the metamsg document.
+    """
+    return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash)
+
 
 class Message(object):
+    """
+    Represents a single message, and gives access to all its attributes.
+    """
 
     def __init__(self, wrapper):
         """
@@ -37,45 +47,56 @@ class Message(object):
         self._wrapper = wrapper
 
     def get_wrapper(self):
+        """
+        Get the wrapper for this message.
+        """
         return self._wrapper
 
     # imap.IMessage methods
 
-    def get_flags():
+    def get_flags(self):
         """
         """
+        return tuple(self._wrapper.fdoc.flags)
 
-    def get_internal_date():
+    def get_internal_date(self):
         """
         """
+        return self._wrapper.fdoc.date
 
     # imap.IMessageParts
 
-    def get_headers():
+    def get_headers(self):
         """
         """
+        # XXX process here? from imap.messages
+        return self._wrapper.hdoc.headers
 
-    def get_body_file():
+    def get_body_file(self):
         """
         """
 
-    def get_size():
+    def get_size(self):
         """
         """
+        return self._wrapper.fdoc.size
 
-    def is_multipart():
+    def is_multipart(self):
         """
         """
+        return self._wrapper.fdoc.multi
 
-    def get_subpart(part):
+    def get_subpart(self, part):
         """
         """
+        # XXX ??? return MessagePart?
 
     # Custom methods.
 
-    def get_tags():
+    def get_tags(self):
         """
         """
+        return tuple(self._wrapper.fdoc.tags)
 
 
 class MessageCollection(object):
@@ -85,43 +106,174 @@ class MessageCollection(object):
     master documents.
 
     Since LEAP Mail is primarily oriented to store mail in Soledad, the default
-    (and, so far, only) implementation of the store is contained in this
-    Soledad Mail Adaptor. If you need to use a different adaptor, change the
+    (and, so far, only) implementation of the store is contained in the
+    Soledad Mail Adaptor, which is passed to every collection on creation by
+    the root Account object. If you need to use a different adaptor, change the
     adaptor class attribute in your Account object.
 
     Store is a reference to a particular instance of the message store (soledad
     instance or proxy, for instance).
     """
 
-    # TODO look at IMessageSet methods
+    # TODO
+    # [ ] look at IMessageSet methods
+    # [ ] make constructor with a per-instance deferredLock to use on
+    #     creation/deletion?
+    # [ ] instead of a mailbox, we could pass an arbitrary container with
+    #     pointers to different doc_ids (type: foo)
+    # [ ] To guarantee synchronicity of the documents sent together during a
+    #     sync, we could get hold of a deferredLock that inhibits
+    #     synchronization while we are updating (think more about this!)
 
     # Account should provide an adaptor instance when creating this collection.
     adaptor = None
     store = None
+    messageklass = Message
 
-    def get_message_by_doc_id(self, doc_id):
-        # ... get from soledad etc
-        # ... but that should be part of adaptor/store too... :/
-        fdoc, hdoc = None
-        return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc)
+    def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None):
+        """
+        """
+        self.adaptor = adaptor
+        self.store = store
 
-    # TODO review if this is the best place for:
+        # TODO I have to think about what to do when there is no mbox passed to
+        # the initialization. We could still get the MetaMsg by index, instead
+        # of by doc_id. See get_message_by_content_hash
+        self.mbox_indexer = mbox_indexer
+        self.mbox_wrapper = mbox_wrapper
 
-    def create_docs():
-        pass
+    def is_mailbox_collection(self):
+        """
+        Return True if this collection represents a Mailbox.
+        :rtype: bool
+        """
+        return bool(self.mbox_wrapper)
+
+    # Get messages
+
+    def get_message_by_content_hash(self, chash, get_cdocs=False):
+        """
+        Retrieve a message by its content hash.
+        :rtype: Deferred
+        """
+
+        if not self.is_mailbox_collection():
+            # instead of getting the metamsg by chash, query by (meta) index
+            # or use the internal collection of pointers-to-docs.
+            raise NotImplementedError()
+
+        metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash)
+
+        return self.adaptor.get_msg_from_mdoc_id(
+            self.messageklass, self.store,
+            metamsg_id, get_cdocs=get_cdocs)
+
+    def get_message_by_uid(self, uid, absolute=True, get_cdocs=False):
+        """
+        Retrieve a message by its Unique Identifier.
+
+        If this is a Mailbox collection, that is the message UID, unique for a
+        given mailbox, or a relative sequence number depending on the absolute
+        flag. For now, only absolute identifiers are supported.
+        :rtype: Deferred
+        """
+        if not absolute:
+            raise NotImplementedError("Does not support relative ids yet")
+
+        def get_msg_from_mdoc_id(doc_id):
+            return self.adaptor.get_msg_from_mdoc_id(
+                self.messageklass, self.store,
+                doc_id, get_cdocs=get_cdocs)
+
+        d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid)
+        d.addCallback(get_msg_from_mdoc_id)
+        return d
+
+    def count(self):
+        """
+        Count the messages in this collection.
+        :rtype: int
+        """
+        if not self.is_mailbox_collection():
+            raise NotImplementedError()
+        return self.mbox_indexer.count(self.mbox_wrapper.mbox)
+
+    # Manipulate messages
+
+    def add_msg(self, raw_msg):
+        """
+        Add a message to this collection.
+        """
+        msg = self.adaptor.get_msg_from_string(Message, raw_msg)
+        wrapper = msg.get_wrapper()
+
+        if self.is_mailbox_collection():
+            mbox = self.mbox_wrapper.mbox
+            wrapper.set_mbox(mbox)
+
+        def insert_mdoc_id(_):
+            # XXX does this work?
+            doc_id = wrapper.mdoc.doc_id
+            return self.mbox_indexer.insert_doc(
+                self.mbox_wrapper.mbox, doc_id)
+
+        d = wrapper.create(self.store)
+        d.addCallback(insert_mdoc_id)
+        return d
+
+    def copy_msg(self, msg, newmailbox):
+        """
+        Copy the message to another collection. (it only makes sense for
+        mailbox collections)
+        """
+        if not self.is_mailbox_collection():
+            raise NotImplementedError()
+
+        def insert_copied_mdoc_id(wrapper):
+            return self.mbox_indexer.insert_doc(
+                newmailbox, wrapper.mdoc.doc_id)
 
-    def udpate_flags():
+        wrapper = msg.get_wrapper()
+        d = wrapper.copy(self.store, newmailbox)
+        d.addCallback(insert_copied_mdoc_id)
+        return d
+
+    def delete_msg(self, msg):
+        """
+        Delete this message.
+        """
+        wrapper = msg.get_wrapper()
+
+        def delete_mdoc_id(_):
+            # XXX does this work?
+            doc_id = wrapper.mdoc.doc_id
+            return self.mbox_indexer.delete_doc_by_hash(
+                self.mbox_wrapper.mbox, doc_id)
+        d = wrapper.delete(self.store)
+        d.addCallback(delete_mdoc_id)
+        return d
+
+    # TODO should add a delete-by-uid to collection?
+
+    def udpate_flags(self, msg, flags, mode):
+        """
+        Update flags for a given message.
+        """
+        wrapper = msg.get_wrapper()
         # 1. update the flags in the message wrapper --- stored where???
-        # 2. call adaptor.update_msg(store)
+        # 2. update the special flags in the wrapper (seen, etc)
+        # 3. call adaptor.update_msg(store)
         pass
 
-    def update_tags():
+    def update_tags(self, msg, tags, mode):
+        """
+        Update tags for a given message.
+        """
+        wrapper = msg.get_wrapper()
         # 1. update the tags in the message wrapper --- stored where???
         # 2. call adaptor.update_msg(store)
         pass
 
-    # TODO add delete methods here?
-
 
 class Account(object):
     """
@@ -147,8 +299,8 @@ class Account(object):
     def __init__(self, store):
         self.store = store
         self.adaptor = self.adaptor_class()
+        self.mbox_indexer = MailboxIndexer(self.store)
 
-        self.__mailboxes = set([])
         self._initialized = False
         self._deferred_initialization = defer.Deferred()
 
@@ -156,23 +308,16 @@ class Account(object):
 
     def _initialize_storage(self):
 
-        def add_mailbox_if_none(result):
-            # every user should have the right to an inbox folder
-            # at least, so let's make one!
-            if not self.mailboxes:
+        def add_mailbox_if_none(mboxes):
+            if not mboxes:
                 self.add_mailbox(INBOX_NAME)
 
         def finish_initialization(result):
             self._initialized = True
             self._deferred_initialization.callback(None)
 
-        def load_mbox_cache(result):
-            d = self._load_mailboxes()
-            d.addCallback(lambda _: result)
-            return d
-
         d = self.adaptor.initialize_store(self.store)
-        d.addCallback(load_mbox_cache)
+        d.addCallback(self.list_all_mailbox_names)
         d.addCallback(add_mailbox_if_none)
         d.addCallback(finish_initialization)
 
@@ -185,64 +330,83 @@ class Account(object):
             self._deferred_initialization.addCallback(cb)
             return self._deferred_initialization
 
-    @property
-    def mailboxes(self):
-        """
-        A list of the current mailboxes for this account.
-        :rtype: set
-        """
-        return sorted(self.__mailboxes)
+    #
+    # Public API Starts
+    #
 
-    def _load_mailboxes(self):
+    def list_all_mailbox_names(self):
+        def filter_names(mboxes):
+            return [m.name for m in mboxes]
 
-        def update_mailboxes(mbox_names):
-            self.__mailboxes.update(mbox_names)
+        d = self.get_all_mailboxes()
+        d.addCallback(filter_names)
+        return d
 
+    def get_all_mailboxes(self):
         d = self.adaptor.get_all_mboxes(self.store)
-        d.addCallback(update_mailboxes)
         return d
 
-    #
-    # Public API Starts
-    #
+    def add_mailbox(self, name):
 
-    # XXX params for IMAP only???
-    def list_mailboxes(self, ref, wildcard):
-        self.adaptor.get_all_mboxes(self.store)
-
-    def add_mailbox(self, name, mbox=None):
-        pass
+        def create_uid_table_cb(res):
+            d = self.mbox_uid.create_table(name)
+            d.addCallback(lambda _: res)
+            return d
 
-    def create_mailbox(self, pathspec):
-        pass
+        d = self.adaptor.__class__.get_or_create(name)
+        d.addCallback(create_uid_table_cb)
+        return d
 
     def delete_mailbox(self, name):
-        pass
+        def delete_uid_table_cb(res):
+            d = self.mbox_uid.delete_table(name)
+            d.addCallback(lambda _: res)
+            return d
+
+        d = self.adaptor.delete_mbox(self.store)
+        d.addCallback(delete_uid_table_cb)
+        return d
 
     def rename_mailbox(self, oldname, newname):
-        pass
+        def _rename_mbox(wrapper):
+            wrapper.mbox = newname
+            return wrapper.update()
 
-    # FIXME yet to be decided if it belongs here...
+        def rename_uid_table_cb(res):
+            d = self.mbox_uid.rename_table(oldname, newname)
+            d.addCallback(lambda _: res)
+            return d
+
+        d = self.adaptor.__class__.get_or_create(oldname)
+        d.addCallback(_rename_mbox)
+        d.addCallback(rename_uid_table_cb)
+        return d
 
     def get_collection_by_mailbox(self, name):
         """
         :rtype: MessageCollection
         """
         # imap select will use this, passing the collection to SoledadMailbox
-        # XXX pass adaptor to MessageCollection
-        pass
+        def get_collection_for_mailbox(mbox_wrapper):
+            return MessageCollection(
+                self.adaptor, self.store, self.mbox_indexer, mbox_wrapper)
+
+        mboxwrapper_klass = self.adaptor.mboxwrapper_klass
+        d = mboxwrapper_klass.get_or_create(name)
+        d.addCallback(get_collection_for_mailbox)
+        return d
 
     def get_collection_by_docs(self, docs):
         """
         :rtype: MessageCollection
         """
         # get a collection of docs by a list of doc_id
-        # XXX pass adaptor to MessageCollection
-        pass
+        # get.docs(...) --> it should be a generator. does it behave in the
+        # threadpool?
+        raise NotImplementedError()
 
     def get_collection_by_tag(self, tag):
         """
         :rtype: MessageCollection
         """
-        # is this a good idea?
-        pass
+        raise NotImplementedError()
diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py
new file mode 100644
index 0000000..bc298ea
--- /dev/null
+++ b/src/leap/mail/mailbox_indexer.py
@@ -0,0 +1,254 @@
+# -*- coding: utf-8 -*-
+# mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Local tables to store the message Unique Identifiers for a given mailbox.
+"""
+import re
+
+from leap.mail.constants import METAMSGID_RE
+
+
+class WrongMetaDocIDError(Exception):
+    pass
+
+
+class MailboxIndexer(object):
+    """
+    This class contains the commands needed to create, modify and alter the
+    local-only UID tables for a given mailbox.
+
+    Its purpouse is to keep a local-only index with the messages in each
+    mailbox, mainly to satisfy the demands of the IMAP specification, but
+    useful too for any effective listing of the messages in a mailbox.
+
+    Since the incoming mail can be processed at any time in any replica, it's
+    preferred not to attempt to maintain a global chronological global index.
+
+    These indexes are Message Attributes needed for the IMAP specification (rfc
+    3501), although they can be useful for other non-imap store
+    implementations.
+    """
+    # The uids are expected to be 32-bits values, but the ROWIDs in sqlite
+    # are 64-bit values. I *don't* think it really matters for any
+    # practical use, but it's good to remmeber we've got that difference going
+    # on.
+
+    store = None
+    table_preffix = "leapmail_uid_"
+
+    def __init__(self, store):
+        self.store = store
+
+    def _query(self, *args, **kw):
+        assert self.store is not None
+        return self.store.raw_sqlcipher_query(*args, **kw)
+
+    def create_table(self, mailbox):
+        """
+        Create the UID table for a given mailbox.
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :rtype: Deferred
+        """
+        assert mailbox
+        sql = ("CREATE TABLE if not exists {preffix}{name}( "
+               "uid  INTEGER PRIMARY KEY AUTOINCREMENT, "
+               "hash TEXT UNIQUE NOT NULL)".format(
+                   preffix=self.table_preffix, name=mailbox))
+        return self._query(sql)
+
+    def delete_table(self, mailbox):
+        """
+        Delete the UID table for a given mailbox.
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :rtype: Deferred
+        """
+        assert mailbox
+        sql = ("DROP TABLE if exists {preffix}{name}".format(
+            preffix=self.table_preffix, name=mailbox))
+        return self._query(sql)
+
+    def rename_table(self, oldmailbox, newmailbox):
+        """
+        Delete the UID table for a given mailbox.
+        :param oldmailbox: the old mailbox name
+        :type oldmailbox: str
+        :param newmailbox: the new mailbox name
+        :type newmailbox: str
+        :rtype: Deferred
+        """
+        assert oldmailbox
+        assert newmailbox
+        assert oldmailbox != newmailbox
+        sql = ("ALTER TABLE {preffix}{old} "
+               "RENAME TO {preffix}{new}".format(
+                   preffix=self.table_preffix,
+                   old=oldmailbox, new=newmailbox))
+        return self._query(sql)
+
+    def insert_doc(self, mailbox, doc_id):
+        """
+        Insert the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+        The doc_id must be in the format:
+
+            M+<mailbox>+<content-hash-of-the-message>
+
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :param doc_id: the doc_id for the MetaMsg
+        :type doc_id: str
+        :return: a deferred that will fire with the uid of the newly inserted
+                 document.
+        :rtype: Deferred
+        """
+        assert mailbox
+        assert doc_id
+
+        if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id):
+            raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id")
+
+        def get_rowid(result):
+            return result[0][0]
+
+        sql = ("INSERT INTO {preffix}{name} VALUES ("
+               "NULL, ?)".format(
+                   preffix=self.table_preffix, name=mailbox))
+        values = (doc_id,)
+
+        sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} "
+                    "LIMIT 1;").format(
+            preffix=self.table_preffix, name=mailbox)
+        d = self._query(sql, values)
+        d.addCallback(lambda _: self._query(sql_last))
+        d.addCallback(get_rowid)
+        return d
+
+    def delete_doc_by_uid(self, mailbox, uid):
+        """
+        Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :param uid: the UID of the message.
+        :type uid: int
+        :rtype: Deferred
+        """
+        assert mailbox
+        assert uid
+        sql = ("DELETE FROM {preffix}{name} "
+               "WHERE uid=?".format(
+                   preffix=self.table_preffix, name=mailbox))
+        values = (uid,)
+        return self._query(sql, values)
+
+    def delete_doc_by_hash(self, mailbox, doc_id):
+        """
+        Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+        The doc_id must be in the format:
+
+            M+<mailbox>+<content-hash-of-the-message>
+
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :param doc_id: the doc_id for the MetaMsg
+        :type doc_id: str
+        :return: a deferred that will fire with the uid of the newly inserted
+                 document.
+        :rtype: Deferred
+        """
+        assert mailbox
+        assert doc_id
+        sql = ("DELETE FROM {preffix}{name} "
+               "WHERE hash=?".format(
+                   preffix=self.table_preffix, name=mailbox))
+        values = (doc_id,)
+        return self._query(sql, values)
+
+    def get_doc_id_from_uid(self, mailbox, uid):
+        """
+        Get the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :param uid: the uid for the MetaMsg for this mailbox
+        :type uid: int
+        :rtype: Deferred
+        """
+        def get_hash(result):
+            return result[0][0]
+
+        sql = ("SELECT hash from {preffix}{name} "
+               "WHERE uid=?".format(
+                   preffix=self.table_preffix, name=mailbox))
+        values = (uid,)
+        d = self._query(sql, values)
+        d.addCallback(get_hash)
+        return d
+
+    def get_doc_ids_from_uids(self, mailbox, uids):
+        # For IMAP relative numbering /sequences.
+        # XXX dereference the range (n,*)
+        raise NotImplementedError()
+
+    def count(self, mailbox):
+        """
+        Get the number of entries in the UID table for a given mailbox.
+
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :return: a deferred that will fire with an integer returning the count.
+        :rtype: Deferred
+        """
+        def get_count(result):
+            return result[0][0]
+
+        sql = ("SELECT Count(*) FROM {preffix}{name};".format(
+            preffix=self.table_preffix, name=mailbox))
+        d = self._query(sql)
+        d.addCallback(get_count)
+        return d
+
+    def get_next_uid(self, mailbox):
+        """
+        Get the next integer beyond the highest UID count for a given mailbox.
+
+        This is expected by the IMAP implementation. There are no guarantees
+        that a document to be inserted in the future gets the returned UID: the
+        only thing that can be assured is that it will be equal or greater than
+        the value returned.
+
+        :param mailbox: the mailbox name
+        :type mailbox: str
+        :return: a deferred that will fire with an integer returning the next
+                 uid.
+        :rtype: Deferred
+        """
+        assert mailbox
+
+        def increment(result):
+            return result[0][0] + 1
+
+        sql = ("SELECT MAX(rowid) FROM {preffix}{name} "
+               "LIMIT 1;").format(
+            preffix=self.table_preffix, name=mailbox)
+
+        d = self._query(sql)
+        d.addCallback(increment)
+        return d
diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py
new file mode 100644
index 0000000..fefa7ee
--- /dev/null
+++ b/src/leap/mail/tests/common.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# common.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Common utilities for testing Soledad.
+"""
+import os
+import shutil
+import tempfile
+
+from leap.common.testing.basetest import BaseLeapTest
+from leap.soledad.client import Soledad
+
+# TODO move to common module, or Soledad itself
+# XXX remove duplication
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+
+def _initialize_soledad(email, gnupg_home, tempdir):
+    """
+    Initializes soledad by hand
+
+    :param email: ID for the user
+    :param gnupg_home: path to home used by gnupg
+    :param tempdir: path to temporal dir
+    :rtype: Soledad instance
+    """
+
+    uuid = "foobar-uuid"
+    passphrase = u"verysecretpassphrase"
+    secret_path = os.path.join(tempdir, "secret.gpg")
+    local_db_path = os.path.join(tempdir, "soledad.u1db")
+    server_url = "https://provider"
+    cert_file = ""
+
+    soledad = Soledad(
+        uuid,
+        passphrase,
+        secret_path,
+        local_db_path,
+        server_url,
+        cert_file,
+        syncable=False)
+
+    return soledad
+
+
+class SoledadTestMixin(BaseLeapTest):
+    """
+    It is **VERY** important that this base is added *AFTER* unittest.TestCase
+    """
+
+    def setUp(self):
+        self.results = []
+
+        self.old_path = os.environ['PATH']
+        self.old_home = os.environ['HOME']
+        self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+        self.home = self.tempdir
+        bin_tdir = os.path.join(
+            self.tempdir,
+            'bin')
+        os.environ["PATH"] = bin_tdir
+        os.environ["HOME"] = self.tempdir
+
+        # Soledad: config info
+        self.gnupg_home = "%s/gnupg" % self.tempdir
+        self.email = 'leap@leap.se'
+
+        # initialize soledad by hand so we can control keys
+        self._soledad = _initialize_soledad(
+            self.email,
+            self.gnupg_home,
+            self.tempdir)
+
+    def tearDown(self):
+        """
+        tearDown method called after each test.
+        """
+        self.results = []
+        try:
+            self._soledad.close()
+        except Exception as exc:
+            print "ERROR WHILE CLOSING SOLEDAD"
+            # logging.exception(exc)
+        finally:
+            os.environ["PATH"] = self.old_path
+            os.environ["HOME"] = self.old_home
+            # safety check
+            assert 'leap_tests-' in self.tempdir
+            shutil.rmtree(self.tempdir)
diff --git a/src/leap/mail/tests/rfc822.message b/src/leap/mail/tests/rfc822.message
new file mode 100644
index 0000000..ee97ab9
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+	by localhost with POP3 (fetchmail-6.2.1)
+	for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+	by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+	for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+	by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+	id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+	id 18w63j-0007VK-00
+	for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+	<mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+	<mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19	Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py	Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+             clazz.__dict__.clear()
+             clazz.__getattr__ = __getattr__
+             clazz.__module__ = module.__name__
++    if newclasses:
++        import gc
++        if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++            hasBrokenRebuild = 1
++            gc_objects = gc.get_objects()
++        else:
++            hasBrokenRebuild = 0
+     for nclass in newclasses:
+         ga = getattr(module, nclass.__name__)
+         if ga is nclass:
+             log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+         else:
+-            import gc
+-            for r in gc.get_referrers(nclass):
+-                if isinstance(r, nclass):
++            if hasBrokenRebuild:
++                for r in gc_objects:
++                    if not getattr(r, '__class__', None) is nclass:
++                        continue
+                     r.__class__ = ga
++            else:
++                for r in gc.get_referrers(nclass):
++                    if getattr(r, '__class__', None) is nclass:
++                        r.__class__ = ga
+     if doLog:
+         log.msg('')
+         log.msg('  (fixing   %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py
new file mode 100644
index 0000000..ce2366c
--- /dev/null
+++ b/src/leap/mail/tests/test_mail.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# test_mail.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mail module.
+"""
+import os
+from functools import partial
+
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.mail import MessageCollection
+from leap.mail.mailbox_indexer import MailboxIndexer
+from leap.mail.tests.common import SoledadTestMixin
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+
+class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin):
+    """
+    Tests for the SoledadDocumentWrapper.
+    """
+
+    def get_collection(self, mbox_collection=True):
+        """
+        Get a collection for tests.
+        """
+        adaptor = SoledadMailAdaptor()
+        store = self._soledad
+        adaptor.store = store
+        if mbox_collection:
+            mbox_indexer = MailboxIndexer(store)
+            mbox_name = "TestMbox"
+        else:
+            mbox_indexer = mbox_name = None
+
+        def get_collection_from_mbox_wrapper(wrapper):
+            return MessageCollection(
+                adaptor, store,
+                mbox_indexer=mbox_indexer, mbox_wrapper=wrapper)
+
+        d = adaptor.initialize_store(store)
+        if mbox_collection:
+            d.addCallback(lambda _: mbox_indexer.create_table(mbox_name))
+        d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name))
+        d.addCallback(get_collection_from_mbox_wrapper)
+        return d
+
+    def test_is_mailbox_collection(self):
+
+        def assert_is_mbox_collection(collection):
+            self.assertTrue(collection.is_mailbox_collection())
+
+        d = self.get_collection()
+        d.addCallback(assert_is_mbox_collection)
+        return d
+
+    def assert_collection_count(self, _, expected, collection):
+
+        def _assert_count(count):
+            self.assertEqual(count, expected)
+        d = collection.count()
+        d.addCallback(_assert_count)
+        return d
+
+    def test_add_msg(self):
+
+        with open(os.path.join(HERE, "rfc822.message")) as f:
+            raw = f.read()
+
+        def add_msg_to_collection_and_assert_count(collection):
+            d = collection.add_msg(raw)
+            d.addCallback(partial(
+                self.assert_collection_count,
+                expected=1, collection=collection))
+            return d
+
+        d = self.get_collection()
+        d.addCallback(add_msg_to_collection_and_assert_count)
+        return d
diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py
new file mode 100644
index 0000000..47a3bdc
--- /dev/null
+++ b/src/leap/mail/tests/test_mailbox_indexer.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+# test_mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mailbox_indexer module.
+"""
+from functools import partial
+
+from twisted.trial import unittest
+
+from leap.mail import mailbox_indexer as mi
+from leap.mail.tests.common import SoledadTestMixin
+
+hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e'
+hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014'
+hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752'
+hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13'
+hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a'
+
+
+def fmt_hash(mailbox, hash):
+    return "M-" + mailbox + "-" + hash
+
+
+class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin):
+    """
+    Tests for the MailboxUID class.
+    """
+    def get_mbox_uid(self):
+        m_uid = mi.MailboxIndexer(self._soledad)
+        return m_uid
+
+    def list_mail_tables_cb(self, ignored):
+        def filter_mailuid_tables(tables):
+            filtered = [
+                table[0] for table in tables if
+                table[0].startswith(mi.MailboxIndexer.table_preffix)]
+            return filtered
+
+        sql = "SELECT name FROM sqlite_master WHERE type='table';"
+        d = self._soledad.raw_sqlcipher_query(sql)
+        d.addCallback(filter_mailuid_tables)
+        return d
+
+    def select_uid_rows(self, mailbox):
+        sql = "SELECT * FROM %s%s;" % (
+            mi.MailboxIndexer.table_preffix, mailbox)
+        d = self._soledad.raw_sqlcipher_query(sql)
+        return d
+
+    def test_create_table(self):
+        def assert_table_created(tables):
+            self.assertEqual(
+                tables, ["leapmail_uid_inbox"])
+
+        m_uid = self.get_mbox_uid()
+        d = m_uid.create_table('inbox')
+        d.addCallback(self.list_mail_tables_cb)
+        d.addCallback(assert_table_created)
+        return d
+
+    def test_create_and_delete_table(self):
+        def assert_table_deleted(tables):
+            self.assertEqual(tables, [])
+
+        m_uid = self.get_mbox_uid()
+        d = m_uid.create_table('inbox')
+        d.addCallback(lambda _: m_uid.delete_table('inbox'))
+        d.addCallback(self.list_mail_tables_cb)
+        d.addCallback(assert_table_deleted)
+        return d
+
+    def test_rename_table(self):
+        def assert_table_renamed(tables):
+            self.assertEqual(
+                tables, ["leapmail_uid_foomailbox"])
+
+        m_uid = self.get_mbox_uid()
+        d = m_uid.create_table('inbox')
+        d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox'))
+        d.addCallback(self.list_mail_tables_cb)
+        d.addCallback(assert_table_renamed)
+        return d
+
+    def test_insert_doc(self):
+        m_uid = self.get_mbox_uid()
+        mbox = 'foomailbox'
+
+        h1 = fmt_hash(mbox, hash_test0)
+        h2 = fmt_hash(mbox, hash_test1)
+        h3 = fmt_hash(mbox, hash_test2)
+        h4 = fmt_hash(mbox, hash_test3)
+        h5 = fmt_hash(mbox, hash_test4)
+
+        def assert_uid_rows(rows):
+            expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)]
+            self.assertEquals(rows, expected)
+
+        d = m_uid.create_table(mbox)
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+        d.addCallback(lambda _: self.select_uid_rows(mbox))
+        d.addCallback(assert_uid_rows)
+        return d
+
+    def test_insert_doc_return(self):
+        m_uid = self.get_mbox_uid()
+        mbox = 'foomailbox'
+
+        def assert_rowid(rowid, expected=None):
+            self.assertEqual(rowid, expected)
+
+        h1 = fmt_hash(mbox, hash_test0)
+        h2 = fmt_hash(mbox, hash_test1)
+        h3 = fmt_hash(mbox, hash_test2)
+
+        d = m_uid.create_table(mbox)
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+        d.addCallback(partial(assert_rowid, expected=1))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+        d.addCallback(partial(assert_rowid, expected=2))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+        d.addCallback(partial(assert_rowid, expected=3))
+        return d
+
+    def test_delete_doc(self):
+        m_uid = self.get_mbox_uid()
+        mbox = 'foomailbox'
+
+        h1 = fmt_hash(mbox, hash_test0)
+        h2 = fmt_hash(mbox, hash_test1)
+        h3 = fmt_hash(mbox, hash_test2)
+        h4 = fmt_hash(mbox, hash_test3)
+        h5 = fmt_hash(mbox, hash_test4)
+
+        def assert_uid_rows(rows):
+            expected = [(4, h4), (5, h5)]
+            self.assertEquals(rows, expected)
+
+        d = m_uid.create_table(mbox)
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+
+        d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1))
+        d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2))
+        d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3))
+
+        d.addCallback(lambda _: self.select_uid_rows(mbox))
+        d.addCallback(assert_uid_rows)
+        return d
+
+    def test_get_doc_id_from_uid(self):
+        m_uid = self.get_mbox_uid()
+        mbox = 'foomailbox'
+
+        h1 = fmt_hash(mbox, hash_test0)
+
+        def assert_doc_hash(res):
+            self.assertEqual(res, h1)
+
+        d = m_uid.create_table(mbox)
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+        d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1))
+        d.addCallback(assert_doc_hash)
+        return d
+
+    def test_count(self):
+        m_uid = self.get_mbox_uid()
+        mbox = 'foomailbox'
+
+        h1 = fmt_hash(mbox, hash_test0)
+        h2 = fmt_hash(mbox, hash_test1)
+        h3 = fmt_hash(mbox, hash_test2)
+        h4 = fmt_hash(mbox, hash_test3)
+        h5 = fmt_hash(mbox, hash_test4)
+
+        d = m_uid.create_table(mbox)
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+
+        def assert_count_after_inserts(count):
+            self.assertEquals(count, 5)
+
+        d.addCallback(lambda _: m_uid.count(mbox))
+        d.addCallback(assert_count_after_inserts)
+
+        d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1))
+        d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2))
+
+        def assert_count_after_deletions(count):
+            self.assertEquals(count, 3)
+
+        d.addCallback(lambda _: m_uid.count(mbox))
+        d.addCallback(assert_count_after_deletions)
+        return d
+
+    def test_get_next_uid(self):
+        m_uid = self.get_mbox_uid()
+        mbox = 'foomailbox'
+
+        h1 = fmt_hash(mbox, hash_test0)
+        h2 = fmt_hash(mbox, hash_test1)
+        h3 = fmt_hash(mbox, hash_test2)
+        h4 = fmt_hash(mbox, hash_test3)
+        h5 = fmt_hash(mbox, hash_test4)
+
+        d = m_uid.create_table(mbox)
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+        d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+
+        def assert_next_uid(result, expected=1):
+            self.assertEquals(result, expected)
+
+        d.addCallback(lambda _: m_uid.get_next_uid(mbox))
+        d.addCallback(partial(assert_next_uid, expected=6))
+        return d