From 541bd8aec1f67834c42bc2e5df14c1f73c569082 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 6 Dec 2013 17:45:21 -0400
Subject: pep8 cleanup

---
 src/leap/mail/_version.py                | 35 +++++++++++++++----------
 src/leap/mail/imap/tests/imapclient.py   |  7 ++---
 src/leap/mail/imap/tests/test_imap.py    | 45 +++++++++++++++++++-------------
 src/leap/mail/smtp/__init__.py           |  6 ++---
 src/leap/mail/smtp/rfc3156.py            |  2 +-
 src/leap/mail/smtp/tests/test_gateway.py | 45 ++++++++++++++++++++++----------
 6 files changed, 87 insertions(+), 53 deletions(-)

diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py
index 8a66c1f..d80ec47 100644
--- a/src/leap/mail/_version.py
+++ b/src/leap/mail/_version.py
@@ -17,6 +17,7 @@ git_full = "$Format:%H$"
 import subprocess
 import sys
 
+
 def run_command(args, cwd=None, verbose=False):
     try:
         # remember shell=False, so use git.cmd on windows, not just git
@@ -41,6 +42,7 @@ import sys
 import re
 import os.path
 
+
 def get_expanded_variables(versionfile_source):
     # the code embedded in _version.py can just fetch the value of these
     # variables. When used from setup.py, we don't want to import
@@ -48,7 +50,7 @@ def get_expanded_variables(versionfile_source):
     # used from _version.py.
     variables = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_source, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
@@ -63,12 +65,13 @@ def get_expanded_variables(versionfile_source):
         pass
     return variables
 
+
 def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
     refnames = variables["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
             print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+        return {}  # unexpanded, so not in an unpacked git-archive tarball
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -84,7 +87,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs-tags))
+            print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):
@@ -93,13 +96,14 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
+            return {"version": r,
+                    "full": variables["full"].strip()}
     # no suitable tags, so we use the full revision id
     if verbose:
         print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
+    return {"version": variables["full"].strip(),
+            "full": variables["full"].strip()}
+
 
 def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
     # this runs 'git' from the root of the source tree. That either means
@@ -116,7 +120,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
         here = os.path.abspath(__file__)
     except NameError:
         # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
+        return {}  # not always correct
 
     # versionfile_source is the relative path from the top of the source tree
     # (where the .git directory might live) to this file. Invert this to find
@@ -141,7 +145,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
         return {}
     if not stdout.startswith(tag_prefix):
         if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
+            print("tag '%s' doesn't start with prefix '%s'" %
+                  (stdout, tag_prefix))
         return {}
     tag = stdout[len(tag_prefix):]
     stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
@@ -153,7 +158,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
     return {"version": tag, "full": full}
 
 
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
+def versions_from_parentdir(parentdir_prefix, versionfile_source,
+                            verbose=False):
     if IN_LONG_VERSION_PY:
         # We're running from _version.py. If it's from a source tree
         # (execute-in-place), we can work upwards to find the root of the
@@ -163,7 +169,7 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False)
             here = os.path.abspath(__file__)
         except NameError:
             # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
+            return {}  # without __file__, we have no hope
         # versionfile_source is the relative path from the top of the source
         # tree to _version.py. Invert this to find the root from __file__.
         root = here
@@ -180,7 +186,8 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False)
     dirname = os.path.basename(root)
     if not dirname.startswith(parentdir_prefix):
         if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
+            print("guessing rootdir is '%s', but '%s' doesn't "
+                  "start with prefix '%s'" %
                   (root, dirname, parentdir_prefix))
         return None
     return {"version": dirname[len(parentdir_prefix):], "full": ""}
@@ -189,8 +196,9 @@ tag_prefix = ""
 parentdir_prefix = "leap-mail"
 versionfile_source = "src/leap/mail/_version.py"
 
+
 def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
+    variables = {"refnames": git_refnames, "full": git_full}
     ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
     if not ver:
         ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
@@ -200,4 +208,3 @@ def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
     if not ver:
         ver = default
     return ver
-
diff --git a/src/leap/mail/imap/tests/imapclient.py b/src/leap/mail/imap/tests/imapclient.py
index 027396c..c353cee 100755
--- a/src/leap/mail/imap/tests/imapclient.py
+++ b/src/leap/mail/imap/tests/imapclient.py
@@ -21,7 +21,7 @@ from twisted.python import log
 
 
 class TrivialPrompter(basic.LineReceiver):
-    #from os import linesep as delimiter
+    # from os import linesep as delimiter
 
     promptDeferred = None
 
@@ -42,6 +42,7 @@ class TrivialPrompter(basic.LineReceiver):
 
 
 class SimpleIMAP4Client(imap4.IMAP4Client):
+
     """
     Add callbacks when the client receives greeting messages from
     an IMAP server.
@@ -98,8 +99,8 @@ def cbServerGreeting(proto, username, password):
     # Try to authenticate securely
     return proto.authenticate(
         password).addCallback(
-            cbAuthentication, proto).addErrback(
-                ebAuthentication, proto, username, password)
+        cbAuthentication, proto).addErrback(
+        ebAuthentication, proto, username, password)
 
 
 def ebConnection(reason):
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index ca73a11..7d26862 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -54,7 +54,7 @@ import twisted.cred.credentials
 import twisted.cred.portal
 
 
-#import u1db
+# import u1db
 
 from leap.common.testing.basetest import BaseLeapTest
 from leap.mail.imap.server import SoledadMailbox
@@ -120,17 +120,19 @@ def initialize_soledad(email, gnupg_home, tempdir):
     return _soledad
 
 
-##########################################
+#
 # Simple LEAP IMAP4 Server for testing
-##########################################
+#
 
 class SimpleLEAPServer(imap4.IMAP4Server):
+
     """
     A Simple IMAP4 Server with mailboxes backed by Soledad.
 
     This should be pretty close to the real LeapIMAP4Server that we
     will be instantiating as a service, minus the authentication bits.
     """
+
     def __init__(self, *args, **kw):
 
         soledad = kw.pop('soledad', None)
@@ -153,7 +155,7 @@ class SimpleLEAPServer(imap4.IMAP4Server):
 
     def lineReceived(self, line):
         if self.timeoutTest:
-            #Do not send a respones
+            # Do not send a respones
             return
 
         imap4.IMAP4Server.lineReceived(self, line)
@@ -168,6 +170,7 @@ class SimpleLEAPServer(imap4.IMAP4Server):
 
 
 class TestRealm:
+
     """
     A minimal auth realm for testing purposes only
     """
@@ -177,12 +180,13 @@ class TestRealm:
         return imap4.IAccount, self.theAccount, lambda: None
 
 
-######################################
+#
 # Simple IMAP4 Client for testing
-######################################
+#
 
 
 class SimpleClient(imap4.IMAP4Client):
+
     """
     A Simple IMAP4 Client to test our
     Soledad-LEAPServer
@@ -210,6 +214,7 @@ class SimpleClient(imap4.IMAP4Client):
 
 
 class IMAP4HelperMixin(BaseLeapTest):
+
     """
     MixIn containing several utilities to be shared across
     different TestCases
@@ -245,13 +250,13 @@ class IMAP4HelperMixin(BaseLeapTest):
         # Soledad: config info
         cls.gnupg_home = "%s/gnupg" % cls.tempdir
         cls.email = 'leap@leap.se'
-        #cls.db1_file = "%s/db1.u1db" % cls.tempdir
-        #cls.db2_file = "%s/db2.u1db" % cls.tempdir
+        # cls.db1_file = "%s/db1.u1db" % cls.tempdir
+        # cls.db2_file = "%s/db2.u1db" % cls.tempdir
         # open test dbs
-        #cls._db1 = u1db.open(cls.db1_file, create=True,
-                              #document_factory=SoledadDocument)
-        #cls._db2 = u1db.open(cls.db2_file, create=True,
-                              #document_factory=SoledadDocument)
+        # cls._db1 = u1db.open(cls.db1_file, create=True,
+                              # document_factory=SoledadDocument)
+        # cls._db2 = u1db.open(cls.db2_file, create=True,
+                              # document_factory=SoledadDocument)
 
         # initialize soledad by hand so we can control keys
         cls._soledad = initialize_soledad(
@@ -261,7 +266,7 @@ class IMAP4HelperMixin(BaseLeapTest):
 
         # now we're passing the mailbox name, so we
         # should get this into a partial or something.
-        #cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad)
+        # cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad)
         # XXX REFACTOR --- self.server (in setUp) is initializing
         # a SoledadBackedAccount
 
@@ -273,8 +278,8 @@ class IMAP4HelperMixin(BaseLeapTest):
         Restores the old path and home environment variables.
         Removes the temporal dir created for tests.
         """
-        #cls._db1.close()
-        #cls._db2.close()
+        # cls._db1.close()
+        # cls._db2.close()
         cls._soledad.close()
 
         os.environ["PATH"] = cls.old_path
@@ -328,8 +333,8 @@ class IMAP4HelperMixin(BaseLeapTest):
             acct.delete(mb)
 
         # FIXME add again
-        #for subs in acct.subscriptions:
-            #acct.unsubscribe(subs)
+        # for subs in acct.subscriptions:
+            # acct.unsubscribe(subs)
 
         del self.server
         del self.client
@@ -375,9 +380,11 @@ class IMAP4HelperMixin(BaseLeapTest):
 #
 
 class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
+
     """
     Tests for the MessageCollection class
     """
+
     def setUp(self):
         """
         setUp method for each test
@@ -434,6 +441,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
 
 
 class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
+
     """
     Tests for the generic behavior of the LeapIMAP4Server
     which, right now, it's just implemented in this test file as
@@ -1149,7 +1157,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
         self.assertEqual(1, len(mb.messages))
         self.assertEqual(
-            ['\\SEEN',],
+            ['\\SEEN', ],
             mb.messages[1].content['flags']
         )
         self.assertEqual(
@@ -1262,6 +1270,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
 
 class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
+
     """
     Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}.
     """
diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py
index d3eb9e8..bbd4064 100644
--- a/src/leap/mail/smtp/__init__.py
+++ b/src/leap/mail/smtp/__init__.py
@@ -30,7 +30,7 @@ from leap.mail.smtp.gateway import SMTPFactory
 
 
 def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
-                     smtp_cert, smtp_key, encrypted_only):
+                       smtp_cert, smtp_key, encrypted_only):
     """
     Setup SMTP gateway to run with Twisted.
 
@@ -52,8 +52,8 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
     :type smtp_cert: str
     :param smtp_key: The client key for authentication.
     :type smtp_key: str
-    :param encrypted_only: Whether the SMTP gateway should send unencrypted mail
-                           or not.
+    :param encrypted_only: Whether the SMTP gateway should send unencrypted
+                           mail or not.
     :type encrypted_only: bool
 
     :returns: tuple of SMTPFactory, twisted.internet.tcp.Port
diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py
index dd48475..b0288b4 100644
--- a/src/leap/mail/smtp/rfc3156.py
+++ b/src/leap/mail/smtp/rfc3156.py
@@ -361,7 +361,7 @@ class PGPSignature(MIMEApplication):
     """
     def __init__(self, _data, name='signature.asc'):
         MIMEApplication.__init__(self, _data, 'pgp-signature',
-            _encoder=lambda x: x, name=name)
+                                 encoder=lambda x: x, name=name)
         self.add_header('Content-Description', 'OpenPGP Digital Signature')
 
 
diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py
index 4c2f04f..5b15b5b 100644
--- a/src/leap/mail/smtp/tests/test_gateway.py
+++ b/src/leap/mail/smtp/tests/test_gateway.py
@@ -101,10 +101,16 @@ class TestSmtpGateway(TestCaseWithKeyManager):
                         '250 Sender address accepted',
                         '250 Recipient address accepted',
                         '354 Continue']
-        proto = SMTPFactory(u'anotheruser@leap.se',
-            self._km, self._config['host'], self._config['port'],
+
+        # XXX this bit can be refactored away in a helper
+        # method...
+        proto = SMTPFactory(
+            u'anotheruser@leap.se',
+            self._km, self._config['host'],
+            self._config['port'],
             self._config['cert'], self._config['key'],
             self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
+        # snip...
         transport = proto_helpers.StringTransport()
         proto.makeConnection(transport)
         for i, line in enumerate(self.EMAIL_DATA):
@@ -118,8 +124,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         """
         Test if message gets encrypted to destination email.
         """
-        proto = SMTPFactory(u'anotheruser@leap.se',
-            self._km, self._config['host'], self._config['port'],
+        proto = SMTPFactory(
+            u'anotheruser@leap.se',
+            self._km, self._config['host'],
+            self._config['port'],
             self._config['cert'], self._config['key'],
             self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
         fromAddr = Address(ADDRESS_2)
@@ -158,8 +166,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         Test if message gets encrypted to destination email and signed with
         sender key.
         """
-        proto = SMTPFactory(u'anotheruser@leap.se',
-            self._km, self._config['host'], self._config['port'],
+        proto = SMTPFactory(
+            u'anotheruser@leap.se',
+            self._km, self._config['host'],
+            self._config['port'],
             self._config['cert'], self._config['key'],
             self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
         user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS)
@@ -202,11 +212,14 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         """
         # mock the key fetching
         self._km.fetch_keys_from_server = Mock(return_value=[])
-        proto = SMTPFactory(u'anotheruser@leap.se',
-            self._km, self._config['host'], self._config['port'],
+        proto = SMTPFactory(
+            u'anotheruser@leap.se',
+            self._km, self._config['host'],
+            self._config['port'],
             self._config['cert'], self._config['key'],
             self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
-        user = User('ihavenopubkey@nonleap.se', 'gateway.leap.se', proto, ADDRESS)
+        user = User('ihavenopubkey@nonleap.se',
+                    'gateway.leap.se', proto, ADDRESS)
         fromAddr = Address(ADDRESS_2)
         m = EncryptedMessage(
             fromAddr, user, self._km, self._config['host'],
@@ -226,7 +239,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         self.assertEqual('pgp-sha512', m._msg.get_param('micalg'))
         # assert content of message
         self.assertEqual(
-            '\r\n'.join(self.EMAIL_DATA[9:13])+'\r\n--\r\n' +
+            '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' +
             'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
             m._msg.get_payload(0).get_payload(decode=True))
         # assert content of signature
@@ -262,8 +275,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         # mock the key fetching
         self._km.fetch_keys_from_server = Mock(return_value=[])
         # prepare the SMTP factory
-        proto = SMTPFactory(u'anotheruser@leap.se',
-            self._km, self._config['host'], self._config['port'],
+        proto = SMTPFactory(
+            u'anotheruser@leap.se',
+            self._km, self._config['host'],
+            self._config['port'],
             self._config['cert'], self._config['key'],
             self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
         transport = proto_helpers.StringTransport()
@@ -291,8 +306,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         # mock the key fetching
         self._km.fetch_keys_from_server = Mock(return_value=[])
         # prepare the SMTP factory with encrypted only equal to false
-        proto = SMTPFactory(u'anotheruser@leap.se',
-            self._km, self._config['host'], self._config['port'],
+        proto = SMTPFactory(
+            u'anotheruser@leap.se',
+            self._km, self._config['host'],
+            self._config['port'],
             self._config['cert'], self._config['key'],
             False).buildProtocol(('127.0.0.1', 0))
         transport = proto_helpers.StringTransport()
-- 
cgit v1.2.3


From 629b37165737dd2fca16410cf64f75a2902f5c20 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 10 Dec 2013 13:08:15 -0400
Subject: add testing reqs

---
 pkg/requirements-testing.pip | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkg/requirements-testing.pip b/pkg/requirements-testing.pip
index 7233634..41222f7 100644
--- a/pkg/requirements-testing.pip
+++ b/pkg/requirements-testing.pip
@@ -1,2 +1,7 @@
 setuptools-trial
 mock
+nose
+rednose
+nose-progressive
+coverage
+pep8>=1.1
-- 
cgit v1.2.3


From b42c58056e6734cc242142b6f1583976ef9247a4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 10 Dec 2013 13:09:29 -0400
Subject: pep8

---
 src/leap/mail/imap/fetch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 14f7a9b..7cecaba 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -511,7 +511,7 @@ class LeapIncomingMail(object):
         if PGP_BEGIN in data:
             begin = data.find(PGP_BEGIN)
             end = data.find(PGP_END)
-            pgp_message = data[begin:end+len(PGP_END)]
+            pgp_message = data[begin:end + len(PGP_END)]
             try:
                 decrdata, valid_sig = self._decrypt_and_verify_data(
                     pgp_message, senderPubkey)
-- 
cgit v1.2.3


From 44b8f5eaaaeeacbb1f9ceca1231cb53ef13f16ab Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 10 Dec 2013 21:00:18 -0400
Subject: make exceptions fail the test.

right now, the exceptions were visible in the stdout,
but the test was not *actually* failing.

using nose deferred decorator for this.
---
 src/leap/mail/imap/tests/test_imap.py | 45 +++++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 7d26862..d78115e 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -25,6 +25,7 @@ XXX add authors from the original twisted tests.
 @license: GPLv3, see included LICENSE file
 """
 # XXX review license of the original tests!!!
+from nose.twistedtools import deferred
 
 try:
     from cStringIO import StringIO
@@ -370,6 +371,7 @@ class IMAP4HelperMixin(BaseLeapTest):
         self.client.transport.loseConnection()
         self.server.transport.loseConnection()
         log.err(failure, "Problem with %r" % (self.function,))
+        failure.trap(Exception)
 
     def loopback(self):
         return loopback.loopbackAsync(self.server, self.client)
@@ -426,8 +428,11 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test that queries filter by selected mailbox
         """
         mc = self.messages
+        self.assertEqual(self.messages.count(), 0)
         mc.add_msg('', subject="test1")
+        self.assertEqual(self.messages.count(), 1)
         mc.add_msg('', subject="test2")
+        self.assertEqual(self.messages.count(), 2)
         mc.add_msg('', subject="test3")
         self.assertEqual(self.messages.count(), 3)
 
@@ -459,6 +464,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     # mailboxes operations
     #
 
+    @deferred(timeout=None)
     def testCreate(self):
         """
         Test whether we can create mailboxes
@@ -497,6 +503,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         answers.sort()
         self.assertEqual(mbox, [a.upper() for a in answers])
 
+    @deferred(timeout=None)
     def testDelete(self):
         """
         Test whether we can delete mailboxes
@@ -545,6 +552,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                                                            failure.Failure)))
         return d
 
+    @deferred(timeout=None)
     def testNonExistentDelete(self):
         """
         Test what happens if we try to delete a non-existent mailbox.
@@ -570,6 +578,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                                                  'No such mailbox'))
         return d
 
+    @deferred(timeout=None)
     def testIllegalDelete(self):
         """
         Try deleting a mailbox with sub-folders, and \NoSelect flag set.
@@ -604,6 +613,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                       self.assertEqual(str(self.failure.value), expected))
         return d
 
+    @deferred(timeout=None)
     def testRename(self):
         """
         Test whether we can rename a mailbox
@@ -627,6 +637,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                           ['NEWNAME']))
         return d
 
+    @deferred(timeout=None)
     def testIllegalInboxRename(self):
         """
         Try to rename inbox. We expect it to fail. Then it would be not
@@ -654,6 +665,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                           self.stashed, failure.Failure)))
         return d
 
+    @deferred(timeout=None)
     def testHierarchicalRename(self):
         """
         Try to rename hierarchical mailboxes
@@ -680,6 +692,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         mboxes.sort()
         self.assertEqual(mboxes, [s.upper() for s in expected])
 
+    @deferred(timeout=None)
     def testSubscribe(self):
         """
         Test whether we can mark a mailbox as subscribed to
@@ -701,6 +714,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                           ['THIS/MBOX']))
         return d
 
+    @deferred(timeout=None)
     def testUnsubscribe(self):
         """
         Test whether we can unsubscribe from a set of mailboxes
@@ -725,6 +739,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                           ['THAT/MBOX']))
         return d
 
+    @deferred(timeout=None)
     def testSelect(self):
         """
         Try to select a mailbox
@@ -764,6 +779,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     # capabilities
     #
 
+    @deferred(timeout=None)
     def testCapability(self):
         caps = {}
 
@@ -779,6 +795,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         return d.addCallback(lambda _: self.assertEqual(expected, caps))
 
+    @deferred(timeout=None)
     def testCapabilityWithAuth(self):
         caps = {}
         self.server.challengers[
@@ -803,6 +820,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     # authentication
     #
 
+    @deferred(timeout=None)
     def testLogout(self):
         """
         Test log out
@@ -817,6 +835,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d = self.loopback()
         return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1))
 
+    @deferred(timeout=None)
     def testNoop(self):
         """
         Test noop command
@@ -832,6 +851,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d = self.loopback()
         return d.addCallback(lambda _: self.assertEqual(self.responses, []))
 
+    @deferred(timeout=None)
     def testLogin(self):
         """
         Test login
@@ -848,6 +868,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertEqual(self.server.account, SimpleLEAPServer.theAccount)
         self.assertEqual(self.server.state, 'auth')
 
+    @deferred(timeout=None)
     def testFailedLogin(self):
         """
         Test bad login
@@ -866,6 +887,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertEqual(self.server.account, None)
         self.assertEqual(self.server.state, 'unauth')
 
+    @deferred(timeout=None)
     def testLoginRequiringQuoting(self):
         """
         Test login requiring quoting
@@ -890,6 +912,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     # Inspection
     #
 
+    @deferred(timeout=None)
     def testNamespace(self):
         """
         Test retrieving namespace
@@ -914,6 +937,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                                                  [[['', '/']], [], []]))
         return d
 
+    @deferred(timeout=None)
     def testExamine(self):
         """
         L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and
@@ -983,6 +1007,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d2 = self.loopback()
         return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed)
 
+    @deferred(timeout=None)
     def testList(self):
         """
         Test List command
@@ -999,22 +1024,21 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         ))
         return d
 
-    # XXX implement subscriptions
-    '''
+    @deferred(timeout=None)
     def testLSub(self):
         """
         Test LSub command
         """
-        SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL')
+        SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL2')
 
         def lsub():
             return self.client.lsub('root', '%')
         d = self._listSetup(lsub)
         d.addCallback(self.assertEqual,
-                      [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL")])
+                      [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL2")])
         return d
-    '''
 
+    @deferred(timeout=None)
     def testStatus(self):
         """
         Test Status command
@@ -1046,6 +1070,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         ))
         return d
 
+    @deferred(timeout=None)
     def testFailedStatus(self):
         """
         Test failed status command with a non-existent mailbox
@@ -1085,6 +1110,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     # messages
     #
 
+    @deferred(timeout=None)
     def testFullAppend(self):
         """
         Test appending a full message to the mailbox
@@ -1125,6 +1151,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
 
+    @deferred(timeout=None)
     def testPartialAppend(self):
         """
         Test partially appending a message to the mailbox
@@ -1151,10 +1178,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
-        return d.addCallback(self._cbTestPartialAppend, infile)
+        return d.addCallback(
+            self._cbTestPartialAppend, infile)
 
     def _cbTestPartialAppend(self, ignored, infile):
         mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
+
         self.assertEqual(1, len(mb.messages))
         self.assertEqual(
             ['\\SEEN', ],
@@ -1164,6 +1193,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             'Right now', mb.messages[1].content['date'])
         self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
 
+    @deferred(timeout=None)
     def testCheck(self):
         """
         Test check command
@@ -1187,6 +1217,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         # Okay, that was fun
 
+    @deferred(timeout=None)
     def testClose(self):
         """
         Test closing the mailbox. We expect to get deleted all messages flagged
@@ -1222,9 +1253,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertEqual(
             m.messages[1].content['subject'],
             'Message 2')
-
         self.failUnless(m.closed)
 
+    @deferred(timeout=None)
     def testExpunge(self):
         """
         Test expunge command
-- 
cgit v1.2.3


From ddad3391ba8ad611a9bdaaf689b408d44eec9cc6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 11 Dec 2013 12:11:21 -0400
Subject: consume messages eagerly

---
 src/leap/mail/imap/server.py          | 19 +++++++++++++------
 src/leap/mail/imap/tests/test_imap.py | 24 +++++++++++++++++++++++-
 src/leap/mail/messageflow.py          | 25 ++++++++++---------------
 3 files changed, 46 insertions(+), 22 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 6320a51..73ec223 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -834,14 +834,19 @@ class SoledadDocWriter(object):
         """
         self._soledad = soledad
 
-    def consume(self, item):
+    def consume(self, queue):
         """
         Creates a new document in soledad db.
 
-        :param item: object to update. content of the document to be inserted.
-        :type item: dict
+        :param queue: queue to get item from, with content of the document
+                      to be inserted.
+        :type queue: Queue
         """
-        self._soledad.create_doc(item)
+        empty = queue.empty()
+        while not empty:
+            item = queue.get()
+            self._soledad.create_doc(item)
+            empty = queue.empty()
 
 
 class MessageCollection(WithMsgFields, IndexedDB):
@@ -911,7 +916,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
 
         self._soledad_writer = MessageProducer(
             SoledadDocWriter(soledad),
-            period=0.2)
+            period=0.1)
 
     def _get_empty_msg(self):
         """
@@ -941,6 +946,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :param uid: the message uid for this mailbox
         :type uid: int
         """
+        logger.debug('adding message')
         if flags is None:
             flags = tuple()
         leap_assert_type(flags, tuple)
@@ -985,6 +991,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         # ...should get a sanity check here.
         content[self.UID_KEY] = uid
 
+        logger.debug('enqueuing message for write')
         self._soledad_writer.put(content)
         # XXX have to decide what shall we do with errors with this change...
         #return self._soledad.create_doc(content)
@@ -1518,9 +1525,9 @@ class SoledadMailbox(WithMsgFields):
         """
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
-
         delete = []
         deleted = []
+
         for m in self.messages.get_all():
             if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
                 delete.append(m)
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index d78115e..9989989 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -394,6 +394,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         MessageCollection interface in this particular TestCase
         """
         self.messages = MessageCollection("testmbox", self._soledad._db)
+        for m in self.messages.get_all():
+            self.messages.remove(m)
 
     def tearDown(self):
         """
@@ -423,6 +425,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
             })
         self.assertEqual(self.messages.count(), 0)
 
+    def testMultipleAdd(self):
+        """
+        Add multiple messages
+        """
+        # XXX watch out! we're serializing with a delay...
+        mc = self.messages
+        self.assertEqual(self.messages.count(), 0)
+        mc.add_msg('Stuff', subject="test1")
+        self.assertEqual(self.messages.count(), 1)
+        mc.add_msg('Stuff', subject="test2")
+        self.assertEqual(self.messages.count(), 2)
+        mc.add_msg('Stuff', subject="test3")
+        self.assertEqual(self.messages.count(), 3)
+        mc.add_msg('Stuff', subject="test4")
+        self.assertEqual(self.messages.count(), 4)
+
     def testFilterByMailbox(self):
         """
         Test that queries filter by selected mailbox
@@ -1265,8 +1283,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         m = SimpleLEAPServer.theAccount.getMailbox(name)
         m.messages.add_msg('', subject="Message 1",
                            flags=('\\Deleted', 'AnotherFlag'))
+        self.failUnless(m.messages.count() == 1)
         m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
+        self.failUnless(m.messages.count() == 2)
         m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
+        self.failUnless(m.messages.count() == 3)
 
         def login():
             return self.client.login('testuser', 'password-test')
@@ -1292,7 +1313,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestExpunge, m)
 
     def _cbTestExpunge(self, ignored, m):
-        self.assertEqual(len(m.messages), 1)
+        # we only left 1 mssage with no deleted flag
+        self.assertEqual(m.messages.count(), 1)
         self.assertEqual(
             m.messages[1].content['subject'],
             'Message 2')
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index 21f6d62..a0a571d 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -26,11 +26,11 @@ from zope.interface import Interface, implements
 
 class IMessageConsumer(Interface):
 
-    def consume(self, item):
+    def consume(self, queue):
         """
         Consumes the passed item.
 
-        :param item: an object to be consumed.
+        :param item: q queue where we put the object to be consumed.
         :type item: object
         """
         # TODO we could add an optional type to be passed
@@ -44,11 +44,12 @@ class DummyMsgConsumer(object):
 
     implements(IMessageConsumer)
 
-    def consume(self, item):
+    def consume(self, queue):
         """
         Just prints the passed item.
         """
-        print "got item %s" % item
+        if not queue.empty():
+            print "got item %s" % queue.get()
 
 
 class MessageProducer(object):
@@ -97,14 +98,9 @@ class MessageProducer(object):
         If the queue is found empty, the loop is stopped. It will be started
         again after the addition of new items.
         """
-        # XXX right now I'm assuming that the period is good enough to allow
-        # a right pace of processing. but we could also pass the queue object
-        # to the consumer and let it choose whether process a new item or not.
-
+        self._consumer.consume(self._queue)
         if self._queue.empty():
             self.stop()
-        else:
-            self._consumer.consume(self._queue.get())
 
     # public methods
 
@@ -114,20 +110,19 @@ class MessageProducer(object):
 
         If the queue was empty, we will start the loop again.
         """
-        was_empty = self._queue.empty()
-
         # XXX this might raise if the queue does not accept any new
         # items. what to do then?
         self._queue.put(item)
-        if was_empty:
-            self.start()
+        self.start()
 
     def start(self):
         """
         Starts polling for new items.
         """
         if not self._loop.running:
-            self._loop.start(self._period)
+            self._loop.start(self._period, now=True)
+        else:
+            print "was running..., not starting"
 
     def stop(self):
         """
-- 
cgit v1.2.3


From 56d33617c49fcad58a272b3a85b64dac3e1271ed Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 11 Dec 2013 13:32:49 -0400
Subject: add changes

---
 changes/bug_4715_fix_message_adding | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/bug_4715_fix_message_adding

diff --git a/changes/bug_4715_fix_message_adding b/changes/bug_4715_fix_message_adding
new file mode 100644
index 0000000..53b875c
--- /dev/null
+++ b/changes/bug_4715_fix_message_adding
@@ -0,0 +1 @@
+  o Soledad writer consumes messages eagerly. Fixes failing tests. Closes: #4715
-- 
cgit v1.2.3


From d1719ca40f6c7838fb41915706960c822f081237 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 5 Dec 2013 11:24:23 -0400
Subject: count_foo uses expanded u1db count method.

Other fixes in the commit:

* Correct the semantic for the recent flag (reset)
* Minor unicode fixes.
* Use a field for tracking the last_uid

In general, this tries to squash all the quick and naive methods
that were relying on evaluating all the message objects before returning
a result.

Further work is still needed, planned also for 0.5 release.
get_by_index needs to be indexed too.
---
 changes/VERSION_COMPAT                |   1 +
 changes/feaure_4616_fix_mail_indexing |   1 +
 src/leap/mail/imap/server.py          | 184 +++++++++++++++++++++++++---------
 src/leap/mail/imap/tests/test_imap.py |  42 +++++++-
 4 files changed, 175 insertions(+), 53 deletions(-)
 create mode 100644 changes/feaure_4616_fix_mail_indexing

diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
index 032b26a..1d5643f 100644
--- a/changes/VERSION_COMPAT
+++ b/changes/VERSION_COMPAT
@@ -8,4 +8,5 @@
 #
 # BEGIN DEPENDENCY LIST -------------------------
 # leap.foo.bar>=x.y.z
+leap.soledad.client 0.5.0 # get_count_by_index
 
diff --git a/changes/feaure_4616_fix_mail_indexing b/changes/feaure_4616_fix_mail_indexing
new file mode 100644
index 0000000..6e94100
--- /dev/null
+++ b/changes/feaure_4616_fix_mail_indexing
@@ -0,0 +1 @@
+  o Makes efficient use of indexes and count method. Closes: #4616
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 73ec223..b79e691 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -74,6 +74,7 @@ class WithMsgFields(object):
     CREATED_KEY = "created"
     SUBSCRIBED_KEY = "subscribed"
     RW_KEY = "rw"
+    LAST_UID_KEY = "lastuid"
 
     # Document Type, for indexing
     TYPE_KEY = "type"
@@ -165,6 +166,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
     TYPE_SUBS_IDX = 'by-type-and-subscribed'
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+    # Tomas created the `recent and seen index`, but the semantic is not too
+    # correct since the recent flag is volatile.
     TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
 
     KTYPE = WithMsgFields.TYPE_KEY
@@ -197,6 +200,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         WithMsgFields.CLOSED_KEY: False,
         WithMsgFields.SUBSCRIBED_KEY: False,
         WithMsgFields.RW_KEY: 1,
+        WithMsgFields.LAST_UID_KEY: 0
     }
 
     def __init__(self, account_name, soledad=None):
@@ -618,14 +622,14 @@ class LeapMessage(WithMsgFields):
         Retrieve the flags associated with this message
 
         :return: The flags, represented as strings
-        :rtype: iterable
+        :rtype: tuple
         """
         if self._doc is None:
             return []
         flags = self._doc.content.get(self.FLAGS_KEY, None)
         if flags:
             flags = map(str, flags)
-        return flags
+        return tuple(flags)
 
     # setFlags, addFlags, removeFlags are not in the interface spec
     # but we use them with store command.
@@ -637,11 +641,12 @@ class LeapMessage(WithMsgFields):
         Returns a SoledadDocument that needs to be updated by the caller.
 
         :param flags: the flags to update in the message.
-        :type flags: sequence of str
+        :type flags: tuple of str
 
         :return: a SoledadDocument instance
         :rtype: SoledadDocument
         """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         log.msg('setting flags')
         doc = self._doc
         doc.content[self.FLAGS_KEY] = flags
@@ -656,13 +661,14 @@ class LeapMessage(WithMsgFields):
         Returns a SoledadDocument that needs to be updated by the caller.
 
         :param flags: the flags to add to the message.
-        :type flags: sequence of str
+        :type flags: tuple of str
 
         :return: a SoledadDocument instance
         :rtype: SoledadDocument
         """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         oldflags = self.getFlags()
-        return self.setFlags(list(set(flags + oldflags)))
+        return self.setFlags(tuple(set(flags + oldflags)))
 
     def removeFlags(self, flags):
         """
@@ -671,20 +677,21 @@ class LeapMessage(WithMsgFields):
         Returns a SoledadDocument that needs to be updated by the caller.
 
         :param flags: the flags to be removed from the message.
-        :type flags: sequence of str
+        :type flags: tuple of str
 
         :return: a SoledadDocument instance
         :rtype: SoledadDocument
         """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         oldflags = self.getFlags()
-        return self.setFlags(list(set(oldflags) - set(flags)))
+        return self.setFlags(tuple(set(oldflags) - set(flags)))
 
     def getInternalDate(self):
         """
         Retrieve the date internally associated with this message
 
-        @rtype: C{str}
-        @retur: An RFC822-formatted date string.
+        :rtype: C{str}
+        :return: An RFC822-formatted date string.
         """
         return str(self._doc.content.get(self.DATE_KEY, ''))
 
@@ -710,8 +717,9 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = cStringIO.StringIO()
-        charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
         content = self._doc.content.get(self.RAW_KEY, '')
+        charset = get_email_charset(
+            unicode(self._doc.content.get(self.RAW_KEY, '')))
         try:
             content = content.encode(charset)
         except (UnicodeEncodeError, UnicodeDecodeError) as e:
@@ -736,8 +744,9 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
         content = self._doc.content.get(self.RAW_KEY, '')
+        charset = get_email_charset(
+            unicode(self._doc.content.get(self.RAW_KEY, '')))
         try:
             content = content.encode(charset)
         except (UnicodeEncodeError, UnicodeDecodeError) as e:
@@ -1046,6 +1055,8 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :param index: the index of the sequence (zero-indexed)
         :type index: int
         """
+        # XXX inneficient! ---- we should keep an index document
+        # with uid -- doc_uuid :)
         try:
             return self.get_all()[index]
         except IndexError:
@@ -1071,15 +1082,6 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
 
-    def get_last(self):
-        """
-        Gets the last LeapMessage
-        """
-        _all = self.get_all()
-        if not _all:
-            return None
-        return LeapMessage(_all[-1])
-
     def get_all(self):
         """
         Get all message documents for the selected mailbox.
@@ -1096,11 +1098,25 @@ class MessageCollection(WithMsgFields, IndexedDB):
         all_docs = [doc for doc in self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_IDX,
             self.TYPE_MESSAGE_VAL, self.mbox)]
-            #if not self.is_deleted(doc)]
         # highly inneficient, but first let's grok it and then
         # let's worry about efficiency.
+
+        # XXX FIXINDEX
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
+    def count(self):
+        """
+        Return the count of messages for this mailbox.
+
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            SoledadBackedAccount.TYPE_MBOX_IDX,
+            self.TYPE_MESSAGE_VAL, self.mbox)
+        return count
+
+    # unseen messages
+
     def unseen_iter(self):
         """
         Get an iterator for the message docs with no `seen` flag
@@ -1110,8 +1126,20 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         return (doc for doc in
                 self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_RECT_SEEN_IDX,
-                    self.TYPE_MESSAGE_VAL, self.mbox, '1', '0'))
+                    SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
+                    self.TYPE_MESSAGE_VAL, self.mbox, '0'))
+
+    def count_unseen(self):
+        """
+        Count all messages with the `Unseen` flag.
+
+        :returns: count
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
+            self.TYPE_MESSAGE_VAL, self.mbox, '0')
+        return count
 
     def get_unseen(self):
         """
@@ -1122,6 +1150,8 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         return [LeapMessage(doc) for doc in self.unseen_iter()]
 
+    # recent messages
+
     def recent_iter(self):
         """
         Get an iterator for the message docs with `recent` flag.
@@ -1143,13 +1173,17 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         return [LeapMessage(doc) for doc in self.recent_iter()]
 
-    def count(self):
+    def count_recent(self):
         """
-        Return the count of messages for this mailbox.
+        Count all messages with the `Recent` flag.
 
+        :returns: count
         :rtype: int
         """
-        return len(self.get_all())
+        count = self._soledad.get_count_from_index(
+            SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
+            self.TYPE_MESSAGE_VAL, self.mbox, '1')
+        return count
 
     def __len__(self):
         """
@@ -1179,8 +1213,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :return: LeapMessage or None if not found.
         :rtype: LeapMessage
         """
-        #try:
-            #return self.get_msg_by_uid(uid)
+        # XXX FIXME inneficcient, we are evaulating.
         try:
             return [doc
                     for doc in self.get_all()][uid - 1]
@@ -1252,7 +1285,7 @@ class SoledadMailbox(WithMsgFields):
         self._soledad = soledad
 
         self.messages = MessageCollection(
-            mbox=mbox, soledad=soledad)
+            mbox=mbox, soledad=self._soledad)
 
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
@@ -1367,6 +1400,32 @@ class SoledadMailbox(WithMsgFields):
     closed = property(
         _get_closed, _set_closed, doc="Closed attribute.")
 
+    def _get_last_uid(self):
+        """
+        Return the last uid for this mailbox.
+
+        :return: the last uid for messages in this mailbox
+        :rtype: bool
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.LAST_UID_KEY, 1)
+
+    def _set_last_uid(self, uid):
+        """
+        Sets the last uid for this mailbox.
+
+        :param uid: the uid to be set
+        :type uid: int
+        """
+        leap_assert(isinstance(uid, int), "uid has to be int")
+        mbox = self._get_mbox()
+        key = self.LAST_UID_KEY
+        mbox.content[key] = uid
+        self._soledad.put_doc(mbox)
+
+    last_uid = property(
+        _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
+
     def getUIDValidity(self):
         """
         Return the unique validity identifier for this mailbox.
@@ -1396,17 +1455,18 @@ class SoledadMailbox(WithMsgFields):
     def getUIDNext(self):
         """
         Return the likely UID for the next message added to this
-        mailbox. Currently it returns the current length incremented
-        by one.
+        mailbox. Currently it returns the higher UID incremented by
+        one.
+
+        We increment the next uid *each* time this function gets called.
+        In this way, there will be gaps if the message with the allocated
+        uid cannot be saved. But that is preferable to having race conditions
+        if we get to parallel message adding.
 
         :rtype: int
         """
-        last = self.messages.get_last()
-        if last:
-            nextuid = last.getUID() + 1
-        else:
-            nextuid = 1
-        return nextuid
+        self.last_uid += 1
+        return self.last_uid
 
     def getMessageCount(self):
         """
@@ -1423,7 +1483,7 @@ class SoledadMailbox(WithMsgFields):
         :return: count of messages flagged `unseen`
         :rtype: int
         """
-        return len(self.messages.get_unseen())
+        return self.messages.count_unseen()
 
     def getRecentCount(self):
         """
@@ -1432,7 +1492,7 @@ class SoledadMailbox(WithMsgFields):
         :return: count of messages flagged `recent`
         :rtype: int
         """
-        return len(self.messages.get_recent())
+        return self.messages.count_recent()
 
     def isWriteable(self):
         """
@@ -1489,6 +1549,7 @@ class SoledadMailbox(WithMsgFields):
         """
         # XXX we should treat the message as an IMessage from here
         uid_next = self.getUIDNext()
+        logger.debug('Adding msg with UID :%s' % uid_next)
         if flags is None:
             flags = tuple()
         else:
@@ -1497,8 +1558,11 @@ class SoledadMailbox(WithMsgFields):
         self.messages.add_msg(message, flags=flags, date=date,
                               uid=uid_next)
 
-        exists = len(self.messages)
-        recent = len(self.messages.get_recent())
+        exists = self.getMessageCount()
+        recent = self.getRecentCount()
+        logger.debug("there are %s messages, %s recent" % (
+            exists,
+            recent))
         for listener in self.listeners:
             listener.newMessages(exists, recent)
         return defer.succeed(None)
@@ -1564,12 +1628,7 @@ class SoledadMailbox(WithMsgFields):
                 iter(messages)
             except TypeError:
                 # looks like we cannot iterate
-                last = self.messages.get_last()
-                if last is None:
-                    uid_last = 1
-                else:
-                    uid_last = last.getUID()
-                messages.last = uid_last
+                messages.last = self.last_uid
 
         # for sequence numbers (uid = 0)
         if sequence:
@@ -1588,14 +1647,37 @@ class SoledadMailbox(WithMsgFields):
                 else:
                     print "fetch %s, no msg found!!!" % msg_id
 
+        if self.isWriteable():
+            self._unset_recent_flag()
         return tuple(result)
 
+    def _unset_recent_flag(self):
+        """
+        Unsets `Recent` flag from a tuple of messages.
+        Called from fetch.
+
+        From RFC, about `Recent`:
+
+        Message is "recently" arrived in this mailbox.  This session
+        is the first session to have been notified about this
+        message; if the session is read-write, subsequent sessions
+        will not see \Recent set for this message.  This flag can not
+        be altered by the client.
+
+        If it is not possible to determine whether or not this
+        session is the first session to be notified about a message,
+        then that message SHOULD be considered recent.
+        """
+        for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
+            newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
+            self._update(newflags)
+
     def _signal_unread_to_ui(self):
         """
         Sends unread event to ui.
         """
-        leap_events.signal(
-            IMAP_UNREAD_MAIL, str(self.getUnseenCount()))
+        unseen = self.getUnseenCount()
+        leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
 
     def store(self, messages, flags, mode, uid):
         """
@@ -1627,6 +1709,10 @@ class SoledadMailbox(WithMsgFields):
                                 read-write.
         """
         # XXX implement also sequence (uid = 0)
+        # XXX we should prevent cclient from setting Recent flag.
+        leap_assert(not isinstance(flags, basestring),
+                    "flags cannot be a string")
+        flags = tuple(flags)
 
         if not self.isWriteable():
             log.msg('read only mailbox!')
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 9989989..f87b534 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -370,8 +370,11 @@ class IMAP4HelperMixin(BaseLeapTest):
     def _ebGeneral(self, failure):
         self.client.transport.loseConnection()
         self.server.transport.loseConnection()
-        log.err(failure, "Problem with %r" % (self.function,))
-        failure.trap(Exception)
+        # can we do something similar?
+        # I guess this was ok with trial, but not in noseland...
+        #log.err(failure, "Problem with %r" % (self.function,))
+        raise failure.value
+        #failure.trap(Exception)
 
     def loopback(self):
         return loopback.loopbackAsync(self.server, self.client)
@@ -393,7 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         We override mixin method since we are only testing
         MessageCollection interface in this particular TestCase
         """
-        self.messages = MessageCollection("testmbox", self._soledad._db)
+        self.messages = MessageCollection("testmbox", self._soledad)
         for m in self.messages.get_all():
             self.messages.remove(m)
 
@@ -429,7 +432,6 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Add multiple messages
         """
-        # XXX watch out! we're serializing with a delay...
         mc = self.messages
         self.assertEqual(self.messages.count(), 0)
         mc.add_msg('Stuff', subject="test1")
@@ -440,6 +442,38 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertEqual(self.messages.count(), 3)
         mc.add_msg('Stuff', subject="test4")
         self.assertEqual(self.messages.count(), 4)
+        mc.add_msg('Stuff', subject="test5")
+        mc.add_msg('Stuff', subject="test6")
+        mc.add_msg('Stuff', subject="test7")
+        mc.add_msg('Stuff', subject="test8")
+        mc.add_msg('Stuff', subject="test9")
+        mc.add_msg('Stuff', subject="test10")
+        self.assertEqual(self.messages.count(), 10)
+
+    def testRecentCount(self):
+        """
+        Test the recent count
+        """
+        mc = self.messages
+        self.assertEqual(self.messages.count_recent(), 0)
+        mc.add_msg('Stuff', subject="test1", uid=1)
+        # For the semantics defined in the RFC, we auto-add the
+        # recent flag by default.
+        self.assertEqual(self.messages.count_recent(), 1)
+        mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',))
+        self.assertEqual(self.messages.count_recent(), 2)
+        mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',))
+        self.assertEqual(self.messages.count_recent(), 3)
+        mc.add_msg('Stuff', subject="test4", uid=4,
+                   flags=('\\Deleted', '\\Recent'))
+        self.assertEqual(self.messages.count_recent(), 4)
+
+        for m in mc:
+            msg = self.messages.get_msg_by_uid(m.get('uid'))
+            msg_newflags = msg.removeFlags(('\\Recent',))
+            self._soledad.put_doc(msg_newflags)
+
+        self.assertEqual(mc.count_recent(), 0)
 
     def testFilterByMailbox(self):
         """
-- 
cgit v1.2.3


From e58baf83ddef9072bc9c0e9d4c7b7be1cb542a12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= <chiiph@leap.se>
Date: Tue, 17 Dec 2013 15:49:15 -0300
Subject: Use git.cmd instead of git.exe in windows since we use GitBash

---
 versioneer.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/versioneer.py b/versioneer.py
index 34e4807..4e2c0a5 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -115,7 +115,7 @@ import sys
 
 def run_command(args, cwd=None, verbose=False):
     try:
-        # remember shell=False, so use git.cmd on windows, not just git
+        # remember shell=False, so use git.exe on windows, not just git
         p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
     except EnvironmentError:
         e = sys.exc_info()[1]
@@ -230,7 +230,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
 
     GIT = "git"
     if sys.platform == "win32":
-        GIT = "git.cmd"
+        GIT = "git.exe"
     stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
                          cwd=root)
     if stdout is None:
@@ -305,7 +305,7 @@ import sys
 
 def run_command(args, cwd=None, verbose=False):
     try:
-        # remember shell=False, so use git.cmd on windows, not just git
+        # remember shell=False, so use git.exe on windows, not just git
         p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
     except EnvironmentError:
         e = sys.exc_info()[1]
@@ -420,7 +420,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
 
     GIT = "git"
     if sys.platform == "win32":
-        GIT = "git.cmd"
+        GIT = "git.exe"
     stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
                          cwd=root)
     if stdout is None:
@@ -476,7 +476,7 @@ import sys
 def do_vcs_install(versionfile_source, ipy):
     GIT = "git"
     if sys.platform == "win32":
-        GIT = "git.cmd"
+        GIT = "git.exe"
     run_command([GIT, "add", "versioneer.py"])
     run_command([GIT, "add", versionfile_source])
     run_command([GIT, "add", ipy])
@@ -489,13 +489,13 @@ def do_vcs_install(versionfile_source, ipy):
                     present = True
         f.close()
     except EnvironmentError:
-        pass    
+        pass
     if not present:
         f = open(".gitattributes", "a+")
         f.write("%s export-subst\n" % versionfile_source)
         f.close()
         run_command([GIT, "add", ".gitattributes"])
-    
+
 
 SHORT_VERSION_PY = """
 # This file was generated by 'versioneer.py' (0.7+) from
-- 
cgit v1.2.3


From 63d40a45f97ea663137fdecc55df12fd72ad5d5c Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Tue, 17 Dec 2013 16:26:14 -0300
Subject: Footer url shouldn't end in period.

[Closes #4791]
---
 changes/bug-4791_url-should-not-end-in-period | 1 +
 src/leap/mail/smtp/gateway.py                 | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changes/bug-4791_url-should-not-end-in-period

diff --git a/changes/bug-4791_url-should-not-end-in-period b/changes/bug-4791_url-should-not-end-in-period
new file mode 100644
index 0000000..d4ff29c
--- /dev/null
+++ b/changes/bug-4791_url-should-not-end-in-period
@@ -0,0 +1 @@
+  o Footer url shouldn't end in period. Closes #4791.
diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py
index a78bd55..a24115b 100644
--- a/src/leap/mail/smtp/gateway.py
+++ b/src/leap/mail/smtp/gateway.py
@@ -603,7 +603,7 @@ class EncryptedMessage(object):
         from_address = validate_address(self._fromAddress.addrstr)
         username, domain = from_address.split('@')
         self.lines.append('--')
-        self.lines.append('%s - https://%s/key/%s.' %
+        self.lines.append('%s - https://%s/key/%s' %
                           (self.FOOTER_STRING, domain, username))
         self.lines.append('')
         self._origmsg = self.parseMessage()
-- 
cgit v1.2.3


From f755b8c4aff56378923f1014ec95d7bced64afdf Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 18 Dec 2013 22:44:57 -0400
Subject: memoize the special method

---
 src/leap/mail/imap/fetch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 7cecaba..f69681a 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -392,7 +392,7 @@ class LeapIncomingMail(object):
         if fromHeader is not None:
             _, senderAddress = parseaddr(fromHeader)
             try:
-                senderPubkey = self._keymanager.get_key(
+                senderPubkey = self._keymanager.get_key_from_cache(
                     senderAddress, OpenPGPKey)
             except keymanager_errors.KeyNotFound:
                 pass
-- 
cgit v1.2.3


From 0de2307c11338bf4c5e36dd9fe76f445b700c288 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 19 Dec 2013 13:30:10 -0400
Subject: deferToThread unsetting recent flag

---
 changes/bug_defer-unset-recent | 2 ++
 src/leap/mail/imap/server.py   | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 changes/bug_defer-unset-recent

diff --git a/changes/bug_defer-unset-recent b/changes/bug_defer-unset-recent
new file mode 100644
index 0000000..e651d11
--- /dev/null
+++ b/changes/bug_defer-unset-recent
@@ -0,0 +1,2 @@
+  o deferToThread unsetting of recent flag. this was holding the new
+    mails from being displayed soonish. 
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index b79e691..c79cf85 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -31,8 +31,10 @@ from zope.proxy import sameProxiedObjects
 
 from twisted.mail import imap4
 from twisted.internet import defer
+from twisted.internet.threads import deferToThread
 from twisted.python import log
 
+
 from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
@@ -1648,7 +1650,8 @@ class SoledadMailbox(WithMsgFields):
                     print "fetch %s, no msg found!!!" % msg_id
 
         if self.isWriteable():
-            self._unset_recent_flag()
+            deferToThread(self._unset_recent_flag)
+
         return tuple(result)
 
     def _unset_recent_flag(self):
@@ -1668,6 +1671,7 @@ class SoledadMailbox(WithMsgFields):
         session is the first session to be notified about a message,
         then that message SHOULD be considered recent.
         """
+        log.msg('unsetting recent flags...')
         for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
             newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
             self._update(newflags)
-- 
cgit v1.2.3


From e14cbdd27cfae72c3fa851441853d1bd93a33dda Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Thu, 19 Dec 2013 22:57:07 -0200
Subject: Stop providing hostname for helo in smtp gateway (#4335).

---
 changes/feature_4335_stop-providing-hostname-for-helo | 1 +
 src/leap/mail/smtp/gateway.py                         | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 changes/feature_4335_stop-providing-hostname-for-helo

diff --git a/changes/feature_4335_stop-providing-hostname-for-helo b/changes/feature_4335_stop-providing-hostname-for-helo
new file mode 100644
index 0000000..f4b6c29
--- /dev/null
+++ b/changes/feature_4335_stop-providing-hostname-for-helo
@@ -0,0 +1 @@
+  o Stop providing hostname for helo in smtp gateway (#4335).
diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py
index a24115b..bef5c6d 100644
--- a/src/leap/mail/smtp/gateway.py
+++ b/src/leap/mail/smtp/gateway.py
@@ -52,6 +52,7 @@ from leap.common.events import proto, signal
 from leap.keymanager import KeyManager
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.keymanager.errors import KeyNotFound
+from leap.mail import __version__
 from leap.mail.smtp.rfc3156 import (
     MultipartSigned,
     MultipartEncrypted,
@@ -492,7 +493,7 @@ class EncryptedMessage(object):
             heloFallback=True,
             requireAuthentication=False,
             requireTransportSecurity=True)
-        factory.domain = LOCAL_FQDN
+        factory.domain = __version__
         signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr)
         reactor.connectSSL(
             self._host, self._port, factory,
-- 
cgit v1.2.3


From 05884e9fd9a6131eaff6f86b37af8dc7b0a88217 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Thu, 19 Dec 2013 23:13:43 -0200
Subject: Only try to fetch keys for multipart signed or encrypted messages
 when fetching mail (#4671).

---
 ...ture_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted | 1 +
 src/leap/mail/imap/fetch.py                                           | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted

diff --git a/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted b/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
new file mode 100644
index 0000000..de3bb86
--- /dev/null
+++ b/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
@@ -0,0 +1 @@
+  o Only try to fetch keys for multipart signed or encrypted emails (#4671).
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index f69681a..b1c34ba 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -389,7 +389,9 @@ class LeapIncomingMail(object):
         # try to obtain sender public key
         senderPubkey = None
         fromHeader = msg.get('from', None)
-        if fromHeader is not None:
+        if fromHeader is not None \
+                and (msg.get_content_type() == 'multipart/encrypted' \
+                     or msg.get_content_type() == 'multipart/signed'):
             _, senderAddress = parseaddr(fromHeader)
             try:
                 senderPubkey = self._keymanager.get_key_from_cache(
-- 
cgit v1.2.3


From 8547783e7fe517772d610b0ae73b0b6be6450e98 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Thu, 19 Dec 2013 23:45:39 -0200
Subject: Fix tests and bug introduced in
 541bd8aec1f67834c42bc2e5df14c1f73c569082.

---
 src/leap/mail/smtp/rfc3156.py            |  2 +-
 src/leap/mail/smtp/tests/test_gateway.py | 15 +++++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py
index b0288b4..9739531 100644
--- a/src/leap/mail/smtp/rfc3156.py
+++ b/src/leap/mail/smtp/rfc3156.py
@@ -361,7 +361,7 @@ class PGPSignature(MIMEApplication):
     """
     def __init__(self, _data, name='signature.asc'):
         MIMEApplication.__init__(self, _data, 'pgp-signature',
-                                 encoder=lambda x: x, name=name)
+                                 _encoder=lambda x: x, name=name)
         self.add_header('Content-Description', 'OpenPGP Digital Signature')
 
 
diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py
index 5b15b5b..88ee5f7 100644
--- a/src/leap/mail/smtp/tests/test_gateway.py
+++ b/src/leap/mail/smtp/tests/test_gateway.py
@@ -137,7 +137,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
             self._config['port'], self._config['cert'], self._config['key'])
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
-        m.eomReceived()
+        #m.eomReceived()  # this includes a defer, so we avoid calling it here
+        m.lines.append('')  # add a trailing newline
         # we need to call the following explicitelly because it was deferred
         # inside the previous method
         m._maybe_encrypt_and_sign()
@@ -157,7 +158,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
             m._msg.get_payload(1).get_payload(), privkey)
         self.assertEqual(
             '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' +
-            'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
+            'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
             decrypted,
             'Decrypted text differs from plaintext.')
 
@@ -180,7 +181,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
         # trigger encryption and signing
-        m.eomReceived()
+        #m.eomReceived()  # this includes a defer, so we avoid calling it here
+        m.lines.append('')  # add a trailing newline
         # we need to call the following explicitelly because it was deferred
         # inside the previous method
         m._maybe_encrypt_and_sign()
@@ -202,7 +204,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
             m._msg.get_payload(1).get_payload(), privkey, verify=pubkey)
         self.assertEqual(
             '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' +
-            'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
+            'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
             decrypted,
             'Decrypted text differs from plaintext.')
 
@@ -227,7 +229,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         for line in self.EMAIL_DATA[4:12]:
             m.lineReceived(line)
         # trigger signing
-        m.eomReceived()
+        #m.eomReceived()  # this includes a defer, so we avoid calling it here
+        m.lines.append('')  # add a trailing newline
         # we need to call the following explicitelly because it was deferred
         # inside the previous method
         m._maybe_encrypt_and_sign()
@@ -240,7 +243,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
         # assert content of message
         self.assertEqual(
             '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' +
-            'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
+            'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
             m._msg.get_payload(0).get_payload(decode=True))
         # assert content of signature
         self.assertTrue(
-- 
cgit v1.2.3


From 9460ea1bb8fd7e536aa3dcf3ed746e3765c96fa1 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 20 Dec 2013 13:41:34 -0400
Subject: use soledad_writer for puts also

---
 src/leap/mail/imap/server.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index c79cf85..f77bf2c 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -755,7 +755,7 @@ class LeapMessage(WithMsgFields):
             logger.error("Unicode error {0}".format(e))
             content = content.encode(charset, 'replace')
         fd.write(content)
-        # SHOULD use a separate BODY FIELD ...
+        # XXX SHOULD use a separate BODY FIELD ...
         fd.seek(0)
         return fd
 
@@ -856,7 +856,12 @@ class SoledadDocWriter(object):
         empty = queue.empty()
         while not empty:
             item = queue.get()
-            self._soledad.create_doc(item)
+            payload = item['payload']
+            mode = item['mode']
+            if mode == "create":
+                self._soledad.create_doc(payload)
+            elif mode == "put":
+                self._soledad.put_doc(payload)
             empty = queue.empty()
 
 
@@ -925,7 +930,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         # to be processed serially by the consumer (the writer). We just
         # need to `put` the new material on its plate.
 
-        self._soledad_writer = MessageProducer(
+        self.soledad_writer = MessageProducer(
             SoledadDocWriter(soledad),
             period=0.1)
 
@@ -1003,7 +1008,10 @@ class MessageCollection(WithMsgFields, IndexedDB):
         content[self.UID_KEY] = uid
 
         logger.debug('enqueuing message for write')
-        self._soledad_writer.put(content)
+
+        # XXX create namedtuple
+        self.soledad_writer.put({"mode": "create",
+                                 "payload": content})
         # XXX have to decide what shall we do with errors with this change...
         #return self._soledad.create_doc(content)
 
@@ -1650,7 +1658,7 @@ class SoledadMailbox(WithMsgFields):
                     print "fetch %s, no msg found!!!" % msg_id
 
         if self.isWriteable():
-            deferToThread(self._unset_recent_flag)
+            self._unset_recent_flag()
 
         return tuple(result)
 
@@ -1761,8 +1769,9 @@ class SoledadMailbox(WithMsgFields):
         """
         Updates document in u1db database
         """
-        #log.msg('updating doc... %s ' % doc)
-        self._soledad.put_doc(doc)
+        # XXX create namedtuple
+        self.messages.soledad_writer.put({"mode": "put",
+                                          "payload": doc})
 
     def __repr__(self):
         """
-- 
cgit v1.2.3


From ee1fa7da3bdc2de2bd12c55a4da9ccc291d3e82c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 20 Dec 2013 16:36:21 -0400
Subject: safety catch against wrong last_uid

---
 src/leap/mail/imap/server.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index f77bf2c..d92ab9d 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -1430,7 +1430,22 @@ class SoledadMailbox(WithMsgFields):
         leap_assert(isinstance(uid, int), "uid has to be int")
         mbox = self._get_mbox()
         key = self.LAST_UID_KEY
-        mbox.content[key] = uid
+
+        count = mbox.getMessageCount()
+
+        # XXX safety-catch. If we do get duplicates,
+        # we want to avoid further duplication.
+
+        if uid >= count:
+            value = uid
+        else:
+            # something is wrong,
+            # just set the last uid
+            # beyond the max msg count.
+            logger.debug("WRONG uid < count. Setting last uid to ", count)
+            value = count
+
+        mbox.content[key] = value
         self._soledad.put_doc(mbox)
 
     last_uid = property(
-- 
cgit v1.2.3


From d7157d12829494a600fb1d460c481b916e3f75be Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 20 Dec 2013 16:38:58 -0400
Subject: fix changes files

---
 changes/bug_defer-unset-recent        | 2 --
 changes/bug_enqueue-unset-recent      | 2 ++
 changes/bug_safety-check-for-last-uid | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 changes/bug_defer-unset-recent
 create mode 100644 changes/bug_enqueue-unset-recent
 create mode 100644 changes/bug_safety-check-for-last-uid

diff --git a/changes/bug_defer-unset-recent b/changes/bug_defer-unset-recent
deleted file mode 100644
index e651d11..0000000
--- a/changes/bug_defer-unset-recent
+++ /dev/null
@@ -1,2 +0,0 @@
-  o deferToThread unsetting of recent flag. this was holding the new
-    mails from being displayed soonish. 
diff --git a/changes/bug_enqueue-unset-recent b/changes/bug_enqueue-unset-recent
new file mode 100644
index 0000000..8903804
--- /dev/null
+++ b/changes/bug_enqueue-unset-recent
@@ -0,0 +1,2 @@
+  o Enqueue unsetting of recent flag. this was holding the new
+    mails from being displayed soonish. 
diff --git a/changes/bug_safety-check-for-last-uid b/changes/bug_safety-check-for-last-uid
new file mode 100644
index 0000000..bb0229f
--- /dev/null
+++ b/changes/bug_safety-check-for-last-uid
@@ -0,0 +1 @@
+  o Sanity check on last_uid setter. Avoids incomplete fetches.
-- 
cgit v1.2.3


From aaedde4a6a04a77e06a332e1ab917afd60f72205 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 20 Dec 2013 17:06:57 -0400
Subject: fix wrong object call

---
 src/leap/mail/imap/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index d92ab9d..5672e25 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -1431,7 +1431,7 @@ class SoledadMailbox(WithMsgFields):
         mbox = self._get_mbox()
         key = self.LAST_UID_KEY
 
-        count = mbox.getMessageCount()
+        count = self.getMessageCount()
 
         # XXX safety-catch. If we do get duplicates,
         # we want to avoid further duplication.
-- 
cgit v1.2.3


From 8d4e17a279218de99b495955e96672587cb237e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= <chiiph@leap.se>
Date: Fri, 20 Dec 2013 19:20:50 -0300
Subject: Limit the size of the returned messages from IMAP to MUA to 100

---
 changes/bug_fetch_size       | 4 ++++
 src/leap/mail/imap/server.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 changes/bug_fetch_size

diff --git a/changes/bug_fetch_size b/changes/bug_fetch_size
new file mode 100644
index 0000000..e9e97b9
--- /dev/null
+++ b/changes/bug_fetch_size
@@ -0,0 +1,4 @@
+  o Limit the size of the messages returned to the IMAP client to 100,
+    since Thunderbird hangs with numbers bigger than those. This is a
+    quick fix until we figure out how does Thunderbird want to receive
+    more than 100 mails at a time.
\ No newline at end of file
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 5672e25..2739f8c 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -1675,7 +1675,7 @@ class SoledadMailbox(WithMsgFields):
         if self.isWriteable():
             self._unset_recent_flag()
 
-        return tuple(result)
+        return tuple(result[:100])
 
     def _unset_recent_flag(self):
         """
-- 
cgit v1.2.3


From 6684735dc24ea02649b55e5fd795c7d2f5824d34 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Tue, 24 Dec 2013 09:27:43 -0200
Subject: Fix parsing of IMAP folder names (#4830).

---
 changes/bug_4830_handle-unicode-in-folder-names |  2 +
 src/leap/mail/imap/server.py                    | 67 ++++++++++++++-----------
 src/leap/mail/imap/tests/test_imap.py           | 28 +++++------
 3 files changed, 55 insertions(+), 42 deletions(-)
 create mode 100644 changes/bug_4830_handle-unicode-in-folder-names

diff --git a/changes/bug_4830_handle-unicode-in-folder-names b/changes/bug_4830_handle-unicode-in-folder-names
new file mode 100644
index 0000000..6824745
--- /dev/null
+++ b/changes/bug_4830_handle-unicode-in-folder-names
@@ -0,0 +1,2 @@
+  o Remove conversion of IMAP folder names to string. This makes the IMAP
+    server use twisted's transparent 7bit conversion (#4830).
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 2739f8c..b9b72d0 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -22,6 +22,7 @@ import logging
 import StringIO
 import cStringIO
 import time
+import re
 
 from collections import defaultdict
 from email.parser import Parser
@@ -205,6 +206,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         WithMsgFields.LAST_UID_KEY: 0
     }
 
+    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
     def __init__(self, account_name, soledad=None):
         """
         Creates a SoledadAccountIndex that keeps track of the mailboxes
@@ -222,7 +225,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         # XXX SHOULD assert too that the name matches the user/uuid with which
         # soledad has been initialized.
 
-        self._account_name = account_name.upper()
+        self._account_name = self._parse_mailbox_name(account_name)
         self._soledad = soledad
 
         self.initialize_db()
@@ -241,19 +244,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         """
         return copy.deepcopy(self.EMPTY_MBOX)
 
+    def _parse_mailbox_name(self, name):
+        """
+        :param name: the name of the mailbox
+        :type name: unicode
+
+        :rtype: unicode
+        """
+        if self.INBOX_RE.match(name):
+            # ensure inital INBOX is uppercase
+            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
+        return name
+
     def _get_mailbox_by_name(self, name):
         """
-        Returns an mbox document by name.
+        Return an mbox document by name.
 
         :param name: the name of the mailbox
         :type name: str
 
         :rtype: SoledadDocument
         """
-        # XXX only upper for INBOX ---
-        name = name.upper()
         doc = self._soledad.get_from_index(
-            self.TYPE_MBOX_IDX, self.MBOX_KEY, name)
+            self.TYPE_MBOX_IDX, self.MBOX_KEY,
+            self._parse_mailbox_name(name))
         return doc[0] if doc else None
 
     @property
@@ -261,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         """
         A list of the current mailboxes for this account.
         """
-        return [str(doc.content[self.MBOX_KEY])
+        return [doc.content[self.MBOX_KEY]
                 for doc in self._soledad.get_from_index(
                     self.TYPE_IDX, self.MBOX_KEY)]
 
@@ -270,7 +284,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         """
         A list of the current subscriptions for this account.
         """
-        return [str(doc.content[self.MBOX_KEY])
+        return [doc.content[self.MBOX_KEY]
                 for doc in self._soledad.get_from_index(
                     self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
 
@@ -284,8 +298,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :returns: a a SoledadMailbox instance
         :rtype: SoledadMailbox
         """
-        # XXX only upper for INBOX
-        name = name.upper()
+        name = self._parse_mailbox_name(name)
+
         if name not in self.mailboxes:
             raise imap4.MailboxException("No such mailbox")
 
@@ -297,12 +311,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 
     def addMailbox(self, name, creation_ts=None):
         """
-        Adds a mailbox to the account.
+        Add a mailbox to the account.
 
         :param name: the name of the mailbox
         :type name: str
 
-        :param creation_ts: a optional creation timestamp to be used as
+        :param creation_ts: an optional creation timestamp to be used as
                             mailbox id. A timestamp will be used if no
                             one is provided.
         :type creation_ts: int
@@ -310,9 +324,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :returns: True if successful
         :rtype: bool
         """
-        # XXX only upper for INBOX
-        name = name.upper()
-        # XXX should check mailbox name for RFC-compliant form
+        name = self._parse_mailbox_name(name)
 
         if name in self.mailboxes:
             raise imap4.MailboxCollision, name
@@ -321,7 +333,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
             # by default, we pass an int value
             # taken from the current time
             # we make sure to take enough decimals to get a unique
-            # maibox-uidvalidity.
+            # mailbox-uidvalidity.
             creation_ts = int(time.time() * 10E2)
 
         mbox = self._get_empty_mailbox()
@@ -346,8 +358,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :raise MailboxException: Raised if this mailbox cannot be added.
         """
         # TODO raise MailboxException
-
-        paths = filter(None, pathspec.split('/'))
+        paths = filter(None,
+            self._parse_mailbox_name(pathspec).split('/'))
         for accum in range(1, len(paths)):
             try:
                 self.addMailbox('/'.join(paths[:accum]))
@@ -372,13 +384,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 
         :rtype: bool
         """
-        # XXX only upper for INBOX
-        name = name.upper()
+        name = self._parse_mailbox_name(name)
 
         if name not in self.mailboxes:
             return None
 
-        self.selected = str(name)
+        self.selected = name
 
         return SoledadMailbox(
             name, rw=readwrite,
@@ -398,8 +409,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
                       names. use with care.
         :type force: bool
         """
-        # XXX only upper for INBOX
-        name = name.upper()
+        name = self._parse_mailbox_name(name)
+
         if not name in self.mailboxes:
             raise imap4.MailboxException("No such mailbox")
 
@@ -436,9 +447,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :param newname: new name of the mailbox
         :type newname: str
         """
-        # XXX only upper for INBOX
-        oldname = oldname.upper()
-        newname = newname.upper()
+        oldname = self._parse_mailbox_name(oldname)
+        newname = self._parse_mailbox_name(newname)
 
         if oldname not in self.mailboxes:
             raise imap4.NoSuchMailbox, oldname
@@ -516,7 +526,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :param name: name of the mailbox
         :type name: str
         """
-        name = name.upper()
+        name = self._parse_mailbox_name(name)
         if name not in self.subscriptions:
             self._set_subscription(name, True)
 
@@ -527,7 +537,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :param name: name of the mailbox
         :type name: str
         """
-        name = name.upper()
+        name = self._parse_mailbox_name(name)
         if name not in self.subscriptions:
             raise imap4.MailboxException, "Not currently subscribed to " + name
         self._set_subscription(name, False)
@@ -549,7 +559,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :type wildcard: str
         """
         # XXX use wildcard in index query
-        ref = self._inferiorNames(ref.upper())
+        ref = self._inferiorNames(
+            self._parse_mailbox_name(ref))
         wildcard = imap4.wildcardToRegexp(wildcard, '/')
         return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
 
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index f87b534..ea75854 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -521,7 +521,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test whether we can create mailboxes
         """
-        succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'FOOBOX')
+        succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox')
         fail = ('testbox', 'test/box')
 
         def cb():
@@ -553,7 +553,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         answers = ['foobox', 'testbox', 'test/box', 'test', 'test/box/box']
         mbox.sort()
         answers.sort()
-        self.assertEqual(mbox, [a.upper() for a in answers])
+        self.assertEqual(mbox, [a for a in answers])
 
     @deferred(timeout=None)
     def testDelete(self):
@@ -686,7 +686,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d.addCallback(lambda _:
                       self.assertEqual(
                           SimpleLEAPServer.theAccount.mailboxes,
-                          ['NEWNAME']))
+                          ['newname']))
         return d
 
     @deferred(timeout=None)
@@ -742,7 +742,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         mboxes = SimpleLEAPServer.theAccount.mailboxes
         expected = ['newname', 'newname/m1', 'newname/m2']
         mboxes.sort()
-        self.assertEqual(mboxes, [s.upper() for s in expected])
+        self.assertEqual(mboxes, [s for s in expected])
 
     @deferred(timeout=None)
     def testSubscribe(self):
@@ -763,7 +763,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d.addCallback(lambda _:
                       self.assertEqual(
                           SimpleLEAPServer.theAccount.subscriptions,
-                          ['THIS/MBOX']))
+                          ['this/mbox']))
         return d
 
     @deferred(timeout=None)
@@ -771,8 +771,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test whether we can unsubscribe from a set of mailboxes
         """
-        SimpleLEAPServer.theAccount.subscribe('THIS/MBOX')
-        SimpleLEAPServer.theAccount.subscribe('THAT/MBOX')
+        SimpleLEAPServer.theAccount.subscribe('this/mbox')
+        SimpleLEAPServer.theAccount.subscribe('that/mbox')
 
         def login():
             return self.client.login('testuser', 'password-test')
@@ -788,7 +788,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d.addCallback(lambda _:
                       self.assertEqual(
                           SimpleLEAPServer.theAccount.subscriptions,
-                          ['THAT/MBOX']))
+                          ['that/mbox']))
         return d
 
     @deferred(timeout=None)
@@ -1029,7 +1029,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestExamine)
 
     def _cbTestExamine(self, ignored):
-        mbox = self.server.theAccount.getMailbox('TEST-MAILBOX-E')
+        mbox = self.server.theAccount.getMailbox('test-mailbox-e')
         self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox)
         self.assertEqual(self.examinedArgs, {
             'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
@@ -1070,8 +1070,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d.addCallback(lambda listed: self.assertEqual(
             sortNest(listed),
             sortNest([
-                (SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL"),
-                (SoledadMailbox.INIT_FLAGS, "/", "ROOT/ANOTHER-THING")
+                (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"),
+                (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing")
             ])
         ))
         return d
@@ -1081,13 +1081,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test LSub command
         """
-        SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL2')
+        SimpleLEAPServer.theAccount.subscribe('root/subthingl2')
 
         def lsub():
             return self.client.lsub('root', '%')
         d = self._listSetup(lsub)
         d.addCallback(self.assertEqual,
-                      [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL2")])
+                      [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")])
         return d
 
     @deferred(timeout=None)
@@ -1190,7 +1190,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestFullAppend, infile)
 
     def _cbTestFullAppend(self, ignored, infile):
-        mb = SimpleLEAPServer.theAccount.getMailbox('ROOT/SUBTHING')
+        mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing')
         self.assertEqual(1, len(mb.messages))
 
         self.assertEqual(
-- 
cgit v1.2.3


From a65aba63cf90efcf036ed7ba0d515865c5e8457e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 24 Dec 2013 20:28:58 -0400
Subject: defer costly operations

---
 src/leap/mail/imap/server.py | 88 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 85 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index b9b72d0..e97ed2a 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -21,11 +21,13 @@ import copy
 import logging
 import StringIO
 import cStringIO
+import os
 import time
 import re
 
 from collections import defaultdict
 from email.parser import Parser
+from functools import wraps
 
 from zope.interface import implements
 from zope.proxy import sameProxiedObjects
@@ -35,6 +37,7 @@ from twisted.internet import defer
 from twisted.internet.threads import deferToThread
 from twisted.python import log
 
+from u1db import errors as u1db_errors
 
 from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
@@ -46,6 +49,65 @@ from leap.soledad.client import Soledad
 logger = logging.getLogger(__name__)
 
 
+def deferred(f):
+    '''
+    Decorator, for deferring methods to Threads.
+
+    It will do a deferToThread of the decorated method
+    unless the environment variable LEAPMAIL_DEBUG is set.
+
+    It uses a descriptor to delay the definition of the
+    method wrapper.
+    '''
+    class descript(object):
+        def __init__(self, f):
+            self.f = f
+
+        def __get__(self, instance, klass):
+            if instance is None:
+                # Class method was requested
+                return self.make_unbound(klass)
+            return self.make_bound(instance)
+
+        def _errback(self, failure):
+            err = failure.value
+            #logger.error(err)
+            log.err(err)
+
+        def make_unbound(self, klass):
+
+            @wraps(self.f)
+            def wrapper(*args, **kwargs):
+                '''This documentation will vanish :)'''
+                raise TypeError(
+                    'unbound method {}() must be called with {} instance '
+                    'as first argument (got nothing instead)'.format(
+                        self.f.__name__,
+                        klass.__name__)
+                )
+            return wrapper
+
+        def make_bound(self, instance):
+
+            @wraps(self.f)
+            def wrapper(*args, **kwargs):
+                '''This documentation will disapear :)'''
+
+                if not os.environ.get('LEAPMAIL_DEBUG'):
+                    d = deferToThread(self.f, instance, *args, **kwargs)
+                    d.addErrback(self._errback)
+                    return d
+                else:
+                    return self.f(instance, *args, **kwargs)
+
+            # This instance does not need the descriptor anymore,
+            # let it find the wrapper directly next time:
+            setattr(instance, self.f.__name__, wrapper)
+            return wrapper
+
+    return descript(f)
+
+
 class MissingIndexError(Exception):
     """
     Raises when tried to access a non existent index document.
@@ -870,9 +932,19 @@ class SoledadDocWriter(object):
             payload = item['payload']
             mode = item['mode']
             if mode == "create":
-                self._soledad.create_doc(payload)
+                call = self._soledad.create_doc
             elif mode == "put":
-                self._soledad.put_doc(payload)
+                call = self._soledad.put_doc
+
+            # should handle errors
+            try:
+                call(payload)
+            except u1db_errors.RevisionConflict as exc:
+                logger.error("Error: %r" % (exc,))
+                # XXX DEBUG -- remove-me
+                #logger.debug("conflicting doc: %s" % payload)
+                raise exc
+
             empty = queue.empty()
 
 
@@ -954,6 +1026,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         return copy.deepcopy(self.EMPTY_MSG)
 
+    @deferred
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
         """
         Creates a new message document.
@@ -1639,6 +1712,7 @@ class SoledadMailbox(WithMsgFields):
         # more generically
         return [x for x in range(len(deleted))]
 
+    @deferred
     def fetch(self, messages, uid):
         """
         Retrieve one or more messages in this mailbox.
@@ -1668,6 +1742,7 @@ class SoledadMailbox(WithMsgFields):
 
         # for sequence numbers (uid = 0)
         if sequence:
+            logger.debug("Getting msg by index: INEFFICIENT call!")
             for msg_id in messages:
                 msg = self.messages.get_msg_by_index(msg_id - 1)
                 if msg:
@@ -1686,8 +1761,11 @@ class SoledadMailbox(WithMsgFields):
         if self.isWriteable():
             self._unset_recent_flag()
 
-        return tuple(result[:100])
+        # XXX workaround for hangs in thunderbird
+        #return tuple(result[:100])
+        return tuple(result)
 
+    @deferred
     def _unset_recent_flag(self):
         """
         Unsets `Recent` flag from a tuple of messages.
@@ -1706,10 +1784,12 @@ class SoledadMailbox(WithMsgFields):
         then that message SHOULD be considered recent.
         """
         log.msg('unsetting recent flags...')
+
         for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
             newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
             self._update(newflags)
 
+    @deferred
     def _signal_unread_to_ui(self):
         """
         Sends unread event to ui.
@@ -1717,6 +1797,7 @@ class SoledadMailbox(WithMsgFields):
         unseen = self.getUnseenCount()
         leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
 
+    @deferred
     def store(self, messages, flags, mode, uid):
         """
         Sets the flags of one or more messages.
@@ -1774,6 +1855,7 @@ class SoledadMailbox(WithMsgFields):
         self._signal_unread_to_ui()
         return result
 
+    @deferred
     def close(self):
         """
         Expunge and mark as closed
-- 
cgit v1.2.3


From 70fdb37740b0737202ad08b4637bf23c12dab87e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 25 Dec 2013 02:46:27 -0400
Subject: Move flags and other metadata to a separate doc.

This change will allow for quicker access times, and smaller
syncs since the fields that change more often will fall in a
pretty small document. For the big raw message, we only need
to sync once.

Also, implemented multipart interface for messages.
This will need additional migration helper in --repair-mailboxes.
---
 src/leap/mail/imap/server.py | 612 ++++++++++++++++++++++++-------------------
 1 file changed, 338 insertions(+), 274 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index e97ed2a..8758dcb 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -25,7 +25,7 @@ import os
 import time
 import re
 
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 from email.parser import Parser
 from functools import wraps
 
@@ -71,7 +71,7 @@ def deferred(f):
 
         def _errback(self, failure):
             err = failure.value
-            #logger.error(err)
+            logger.warning('error in method: %s' % (self.f.__name__))
             log.err(err)
 
         def make_unbound(self, klass):
@@ -133,6 +133,8 @@ class WithMsgFields(object):
     RAW_KEY = "raw"
     SUBJECT_KEY = "subject"
     UID_KEY = "uid"
+    MULTIPART_KEY = "multi"
+    SIZE_KEY = "size"
 
     # Mailbox specific keys
     CLOSED_KEY = "closed"
@@ -145,6 +147,8 @@ class WithMsgFields(object):
     TYPE_KEY = "type"
     TYPE_MESSAGE_VAL = "msg"
     TYPE_MBOX_VAL = "mbox"
+    TYPE_FLAGS_VAL = "flags"
+    # should add also a headers val
 
     INBOX_VAL = "inbox"
 
@@ -166,6 +170,8 @@ class WithMsgFields(object):
     SUBJECT_FIELD = "Subject"
     DATE_FIELD = "Date"
 
+fields = WithMsgFields  # alias for convenience
+
 
 class IndexedDB(object):
     """
@@ -209,12 +215,79 @@ class IndexedDB(object):
             self._soledad.create_index(name, *expression)
 
 
+class MailParser(object):
+    """
+    Mixin with utility methods to parse raw messages.
+    """
+    def __init__(self):
+        """
+        Initializes the mail parser.
+        """
+        self._parser = Parser()
+
+    def _get_parsed_msg(self, raw):
+        """
+        Return a parsed Message.
+
+        :param raw: the raw string to parse
+        :type raw: basestring, or StringIO object
+        """
+        msg = self._get_parser_fun(raw)(raw, True)
+        return msg
+
+    def _get_parser_fun(self, o):
+        """
+        Retunn the proper parser function for an object.
+
+        :param o: object
+        :type o: object
+        :param parser: an instance of email.parser.Parser
+        :type parser: email.parser.Parser
+        """
+        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+            return self._parser.parse
+        if isinstance(o, basestring):
+            return self._parser.parsestr
+
+    def _stringify(self, o):
+        """
+        Return a string object.
+
+        :param o: object
+        :type o: object
+        """
+        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+            return o.getvalue()
+        else:
+            return o
+
+
+class MBoxParser(object):
+    """
+    Utility function to parse mailbox names.
+    """
+    INBOX_NAME = "INBOX"
+    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
+    def _parse_mailbox_name(self, name):
+        """
+        :param name: the name of the mailbox
+        :type name: unicode
+
+        :rtype: unicode
+        """
+        if self.INBOX_RE.match(name):
+            # ensure inital INBOX is uppercase
+            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
+        return name
+
+
 #######################################
 # Soledad Account
 #######################################
 
 
-class SoledadBackedAccount(WithMsgFields, IndexedDB):
+class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     """
     An implementation of IAccount and INamespacePresenteer
     that is backed by Soledad Encrypted Documents.
@@ -254,12 +327,11 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
                                   'bool(recent)', 'bool(seen)'],
     }
 
-    INBOX_NAME = "INBOX"
     MBOX_KEY = MBOX_VAL
 
     EMPTY_MBOX = {
         WithMsgFields.TYPE_KEY: MBOX_KEY,
-        WithMsgFields.TYPE_MBOX_VAL: INBOX_NAME,
+        WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
         WithMsgFields.SUBJECT_KEY: "",
         WithMsgFields.FLAGS_KEY: [],
         WithMsgFields.CLOSED_KEY: False,
@@ -268,8 +340,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         WithMsgFields.LAST_UID_KEY: 0
     }
 
-    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
     def __init__(self, account_name, soledad=None):
         """
         Creates a SoledadAccountIndex that keeps track of the mailboxes
@@ -306,18 +376,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         """
         return copy.deepcopy(self.EMPTY_MBOX)
 
-    def _parse_mailbox_name(self, name):
-        """
-        :param name: the name of the mailbox
-        :type name: unicode
-
-        :rtype: unicode
-        """
-        if self.INBOX_RE.match(name):
-            # ensure inital INBOX is uppercase
-            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
-        return name
-
     def _get_mailbox_by_name(self, name):
         """
         Return an mbox document by name.
@@ -420,7 +478,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :raise MailboxException: Raised if this mailbox cannot be added.
         """
         # TODO raise MailboxException
-        paths = filter(None,
+        paths = filter(
+            None,
             self._parse_mailbox_name(pathspec).split('/'))
         for accum in range(1, len(paths)):
             try:
@@ -665,19 +724,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
 #######################################
 
 
-class LeapMessage(WithMsgFields):
+class LeapMessage(fields, MailParser, MBoxParser):
 
-    implements(imap4.IMessage, imap4.IMessageFile)
+    implements(imap4.IMessage)
 
-    def __init__(self, doc):
+    def __init__(self, soledad, uid, mbox):
         """
         Initializes a LeapMessage.
 
-        :param doc: A SoledadDocument containing the internal
-                    representation of the message
-        :type doc: SoledadDocument
+        :param soledad: a Soledad instance
+        :type soledad: Soledad
+        :param uid: the UID for the message.
+        :type uid: int or basestring
+        :param mbox: the mbox this message belongs to
+        :type mbox: basestring
         """
-        self._doc = doc
+        MailParser.__init__(self)
+        self._soledad = soledad
+        self._uid = int(uid)
+        self._mbox = self._parse_mailbox_name(mbox)
+
+        self.__cdoc = None
+
+    @property
+    def _fdoc(self):
+        """
+        An accessor to the flags docuemnt
+        """
+        return self._get_flags_doc()
+
+    @property
+    def _cdoc(self):
+        """
+        An accessor to the content docuemnt
+        """
+        if not self.__cdoc:
+            self.__cdoc = self._get_content_doc()
+        return self.__cdoc
 
     def getUID(self):
         """
@@ -686,11 +769,7 @@ class LeapMessage(WithMsgFields):
         :return: uid for this message
         :rtype: int
         """
-        # XXX debug, to remove after a while...
-        if not self._doc:
-            log.msg('BUG!!! ---- message has no doc!')
-            return
-        return self._doc.content[self.UID_KEY]
+        return self._uid
 
     def getFlags(self):
         """
@@ -699,9 +778,13 @@ class LeapMessage(WithMsgFields):
         :return: The flags, represented as strings
         :rtype: tuple
         """
-        if self._doc is None:
+        if self._uid is None:
             return []
-        flags = self._doc.content.get(self.FLAGS_KEY, None)
+
+        flags = []
+        flag_doc = self._fdoc
+        if flag_doc:
+            flags = flag_doc.content.get(self.FLAGS_KEY, None)
         if flags:
             flags = map(str, flags)
         return tuple(flags)
@@ -722,12 +805,13 @@ class LeapMessage(WithMsgFields):
         :rtype: SoledadDocument
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        log.msg('setting flags')
-        doc = self._doc
+        log.msg('setting flags: %s' % (self._uid))
+
+        doc = self._fdoc
         doc.content[self.FLAGS_KEY] = flags
         doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
         doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
-        return doc
+        self._soledad.put_doc(doc)
 
     def addFlags(self, flags):
         """
@@ -743,7 +827,7 @@ class LeapMessage(WithMsgFields):
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         oldflags = self.getFlags()
-        return self.setFlags(tuple(set(flags + oldflags)))
+        self.setFlags(tuple(set(flags + oldflags)))
 
     def removeFlags(self, flags):
         """
@@ -759,7 +843,7 @@ class LeapMessage(WithMsgFields):
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         oldflags = self.getFlags()
-        return self.setFlags(tuple(set(oldflags) - set(flags)))
+        self.setFlags(tuple(set(oldflags) - set(flags)))
 
     def getInternalDate(self):
         """
@@ -768,48 +852,14 @@ class LeapMessage(WithMsgFields):
         :rtype: C{str}
         :return: An RFC822-formatted date string.
         """
-        return str(self._doc.content.get(self.DATE_KEY, ''))
-
-    #
-    # IMessageFile
-    #
-
-    """
-    Optional message interface for representing messages as files.
-
-    If provided by message objects, this interface will be used instead
-    the more complex MIME-based interface.
-    """
-
-    def open(self):
-        """
-        Return an file-like object opened for reading.
-
-        Reading from the returned file will return all the bytes
-        of which this message consists.
-
-        :return: file-like object opened fore reading.
-        :rtype: StringIO
-        """
-        fd = cStringIO.StringIO()
-        content = self._doc.content.get(self.RAW_KEY, '')
-        charset = get_email_charset(
-            unicode(self._doc.content.get(self.RAW_KEY, '')))
-        try:
-            content = content.encode(charset)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            content = content.encode(charset, 'replace')
-        fd.write(content)
-        fd.seek(0)
-        return fd
+        return str(self._cdoc.content.get(self.DATE_KEY, ''))
 
     #
     # IMessagePart
     #
 
-    # XXX should implement the rest of IMessagePart interface:
-    # (and do not use the open above)
+    # XXX we should implement this interface too for the subparts
+    # so we allow nested parts...
 
     def getBodyFile(self):
         """
@@ -819,15 +869,21 @@ class LeapMessage(WithMsgFields):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        content = self._doc.content.get(self.RAW_KEY, '')
+
+        cdoc = self._cdoc
+        content = cdoc.content.get(self.RAW_KEY, '')
         charset = get_email_charset(
-            unicode(self._doc.content.get(self.RAW_KEY, '')))
+            unicode(cdoc.content.get(self.RAW_KEY, '')))
         try:
             content = content.encode(charset)
         except (UnicodeEncodeError, UnicodeDecodeError) as e:
             logger.error("Unicode error {0}".format(e))
             content = content.encode(charset, 'replace')
-        fd.write(content)
+
+        raw = self._get_raw_msg()
+        msg = self._get_parsed_msg(raw)
+        body = msg.get_payload()
+        fd.write(body)
         # XXX SHOULD use a separate BODY FIELD ...
         fd.seek(0)
         return fd
@@ -839,13 +895,18 @@ class LeapMessage(WithMsgFields):
         :return: size of the message, in octets
         :rtype: int
         """
-        return self.getBodyFile().len
+        size = self._cdoc.content.get(self.SIZE_KEY, False)
+        if not size:
+            # XXX fallback, should remove when all migrated.
+            size = self.getBodyFile().len
+        return size
 
     def _get_headers(self):
         """
         Return the headers dict stored in this message document.
         """
-        return self._doc.content.get(self.HEADERS_KEY, {})
+        # XXX get from the headers doc
+        return self._cdoc.content.get(self.HEADERS_KEY, {})
 
     def getHeaders(self, negate, *names):
         """
@@ -876,30 +937,90 @@ class LeapMessage(WithMsgFields):
             if cond(key)]
         return dict(filter_by_cond)
 
-    # --- no multipart for now
-    # XXX Fix MULTIPART SUPPORT!
-
     def isMultipart(self):
-        return False
+        """
+        Return True if this message is multipart.
+        """
+        if self._cdoc:
+            retval = self._cdoc.content.get(self.MULTIPART_KEY, False)
+            print "MULTIPART? ", retval
 
-    def getSubPart(part):
-        return None
+    def getSubPart(self, part):
+        """
+        Retrieve a MIME submessage
+
+        :type part: C{int}
+        :param part: The number of the part to retrieve, indexed from 0.
+        :raise IndexError: Raised if the specified part does not exist.
+        :raise TypeError: Raised if this message is not multipart.
+        :rtype: Any object implementing C{IMessagePart}.
+        :return: The specified sub-part.
+        """
+        if not self.isMultipart():
+            raise TypeError
+
+        msg = self._get_parsed_msg()
+        # XXX should wrap IMessagePart
+        return msg.get_payload()[part]
 
     #
     # accessors
     #
 
+    def _get_flags_doc(self):
+        """
+        Return the document that keeps the flags for this
+        message.
+        """
+        flag_docs = self._soledad.get_from_index(
+            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
+            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+        flag_doc = flag_docs[0] if flag_docs else None
+        return flag_doc
+
+    def _get_content_doc(self):
+        """
+        Return the document that keeps the flags for this
+        message.
+        """
+        cont_docs = self._soledad.get_from_index(
+            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
+            fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid))
+        cont_doc = cont_docs[0] if cont_docs else None
+        return cont_doc
+
+    def _get_raw_msg(self):
+        """
+        Return the raw msg.
+        :rtype: basestring
+        """
+        return self._cdoc.content.get(self.RAW_KEY, '')
+
     def __getitem__(self, key):
         """
         Return the content of the message document.
 
-        @param key: The key
-        @type key: str
+        :param key: The key
+        :type key: str
 
-        @return: The content value indexed by C{key} or None
-        @rtype: str
+        :return: The content value indexed by C{key} or None
+        :rtype: str
         """
-        return self._doc.content.get(key, None)
+        return self._cdoc.content.get(key, None)
+
+    def does_exist(self):
+        """
+        Return True if there is actually a message for this
+        UID and mbox.
+        """
+        return bool(self._fdoc)
+
+
+SoledadWriterPayload = namedtuple(
+    'SoledadWriterPayload', ['mode', 'payload'])
+
+SoledadWriterPayload.CREATE = 1
+SoledadWriterPayload.PUT = 2
 
 
 class SoledadDocWriter(object):
@@ -929,26 +1050,22 @@ class SoledadDocWriter(object):
         empty = queue.empty()
         while not empty:
             item = queue.get()
-            payload = item['payload']
-            mode = item['mode']
-            if mode == "create":
+            if item.mode == SoledadWriterPayload.CREATE:
                 call = self._soledad.create_doc
-            elif mode == "put":
+            elif item.mode == SoledadWriterPayload.PUT:
                 call = self._soledad.put_doc
 
             # should handle errors
             try:
-                call(payload)
+                call(item.payload)
             except u1db_errors.RevisionConflict as exc:
                 logger.error("Error: %r" % (exc,))
-                # XXX DEBUG -- remove-me
-                #logger.debug("conflicting doc: %s" % payload)
                 raise exc
 
             empty = queue.empty()
 
 
-class MessageCollection(WithMsgFields, IndexedDB):
+class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     """
     A collection of messages, surprisingly.
 
@@ -959,16 +1076,27 @@ class MessageCollection(WithMsgFields, IndexedDB):
     # XXX this should be able to produce a MessageSet methinks
 
     EMPTY_MSG = {
-        WithMsgFields.TYPE_KEY: WithMsgFields.TYPE_MESSAGE_VAL,
-        WithMsgFields.UID_KEY: 1,
-        WithMsgFields.MBOX_KEY: WithMsgFields.INBOX_VAL,
-        WithMsgFields.SUBJECT_KEY: "",
-        WithMsgFields.DATE_KEY: "",
-        WithMsgFields.SEEN_KEY: False,
-        WithMsgFields.RECENT_KEY: True,
-        WithMsgFields.FLAGS_KEY: [],
-        WithMsgFields.HEADERS_KEY: {},
-        WithMsgFields.RAW_KEY: "",
+        fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL,
+        fields.UID_KEY: 1,
+        fields.MBOX_KEY: fields.INBOX_VAL,
+
+        fields.SUBJECT_KEY: "",
+        fields.DATE_KEY: "",
+        fields.RAW_KEY: "",
+
+        # XXX should separate headers into another doc
+        fields.HEADERS_KEY: {},
+    }
+
+    EMPTY_FLAGS = {
+        fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
+        fields.UID_KEY: 1,
+        fields.MBOX_KEY: fields.INBOX_VAL,
+
+        fields.FLAGS_KEY: [],
+        fields.SEEN_KEY: False,
+        fields.RECENT_KEY: True,
+        fields.MULTIPART_KEY: False,
     }
 
     # get from SoledadBackedAccount the needed index-related constants
@@ -987,25 +1115,17 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :param soledad: Soledad database
         :type soledad: Soledad instance
         """
-        # XXX pass soledad directly
-
+        MailParser.__init__(self)
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(mbox.strip() != "", "mbox cannot be blank space")
         leap_assert(isinstance(mbox, (str, unicode)),
                     "mbox needs to be a string")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
-        # This is a wrapper now!...
-        # should move assertion there...
-        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    #"soledad._db must be an instance of SQLCipherDatabase")
-
         # okay, all in order, keep going...
-
-        self.mbox = mbox.upper()
+        self.mbox = self._parse_mailbox_name(mbox)
         self._soledad = soledad
         self.initialize_db()
-        self._parser = Parser()
 
         # I think of someone like nietzsche when reading this
 
@@ -1015,7 +1135,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
 
         self.soledad_writer = MessageProducer(
             SoledadDocWriter(soledad),
-            period=0.1)
+            period=0.05)
 
     def _get_empty_msg(self):
         """
@@ -1026,6 +1146,15 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         return copy.deepcopy(self.EMPTY_MSG)
 
+    def _get_empty_flags_doc(self):
+        """
+        Returns an empty doc for storing flags.
+
+        :return:
+        :rtype:
+        """
+        return copy.deepcopy(self.EMPTY_FLAGS)
+
     @deferred
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
         """
@@ -1046,58 +1175,57 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :param uid: the message uid for this mailbox
         :type uid: int
         """
+        # TODO: split in smaller methods
         logger.debug('adding message')
         if flags is None:
             flags = tuple()
         leap_assert_type(flags, tuple)
 
-        def stringify(o):
-            if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-                return o.getvalue()
-            else:
-                return o
+        content_doc = self._get_empty_msg()
+        flags_doc = self._get_empty_flags_doc()
 
-        content = self._get_empty_msg()
-        content[self.MBOX_KEY] = self.mbox
+        content_doc[self.MBOX_KEY] = self.mbox
+        flags_doc[self.MBOX_KEY] = self.mbox
+        # ...should get a sanity check here.
+        content_doc[self.UID_KEY] = uid
+        flags_doc[self.UID_KEY] = uid
 
         if flags:
-            content[self.FLAGS_KEY] = map(stringify, flags)
-            content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+            flags_doc[self.FLAGS_KEY] = map(self._stringify, flags)
+            flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags
 
-        def _get_parser_fun(o):
-            if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-                return self._parser.parse
-            if isinstance(o, (str, unicode)):
-                return self._parser.parsestr
-
-        msg = _get_parser_fun(raw)(raw, True)
+        msg = self._get_parsed_msg(raw)
         headers = dict(msg)
 
+        flags_doc[self.MULTIPART_KEY] = msg.is_multipart()
         # XXX get lower case for keys?
-        content[self.HEADERS_KEY] = headers
+        # XXX get headers doc
+        content_doc[self.HEADERS_KEY] = headers
         # set subject based on message headers and eventually replace by
         # subject given as param
         if self.SUBJECT_FIELD in headers:
-            content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
+            content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
         if subject is not None:
-            content[self.SUBJECT_KEY] = subject
-        content[self.RAW_KEY] = stringify(raw)
+            content_doc[self.SUBJECT_KEY] = subject
+
+        # XXX could separate body into its own doc
+        # but should also separate multiparts
+        # that should be wrapped in MessagePart
+        content_doc[self.RAW_KEY] = self._stringify(raw)
+        content_doc[self.SIZE_KEY] = len(raw)
 
         if not date and self.DATE_FIELD in headers:
-            content[self.DATE_KEY] = headers[self.DATE_FIELD]
+            content_doc[self.DATE_KEY] = headers[self.DATE_FIELD]
         else:
-            content[self.DATE_KEY] = date
-
-        # ...should get a sanity check here.
-        content[self.UID_KEY] = uid
+            content_doc[self.DATE_KEY] = date
 
         logger.debug('enqueuing message for write')
 
-        # XXX create namedtuple
-        self.soledad_writer.put({"mode": "create",
-                                 "payload": content})
-        # XXX have to decide what shall we do with errors with this change...
-        #return self._soledad.create_doc(content)
+        ptuple = SoledadWriterPayload
+        self.soledad_writer.put(ptuple(
+            mode=ptuple.CREATE, payload=content_doc))
+        self.soledad_writer.put(ptuple(
+            mode=ptuple.CREATE, payload=flags_doc))
 
     def remove(self, msg):
         """
@@ -1110,23 +1238,6 @@ class MessageCollection(WithMsgFields, IndexedDB):
 
     # getters
 
-    def get_by_uid(self, uid):
-        """
-        Retrieves a message document by UID.
-
-        :param uid: the message uid to query by
-        :type uid: int
-
-        :return: A SoledadDocument instance matching the query,
-                 or None if not found.
-        :rtype: SoledadDocument
-        """
-        docs = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox, str(uid))
-
-        return docs[0] if docs else None
-
     def get_msg_by_uid(self, uid):
         """
         Retrieves a LeapMessage by UID.
@@ -1138,43 +1249,10 @@ class MessageCollection(WithMsgFields, IndexedDB):
                  or None if not found.
         :rtype: LeapMessage
         """
-        doc = self.get_by_uid(uid)
-        if doc:
-            return LeapMessage(doc)
-
-    def get_by_index(self, index):
-        """
-        Retrieves a mesage document by mailbox index.
-
-        :param index: the index of the sequence (zero-indexed)
-        :type index: int
-        """
-        # XXX inneficient! ---- we should keep an index document
-        # with uid -- doc_uuid :)
-        try:
-            return self.get_all()[index]
-        except IndexError:
+        msg = LeapMessage(self._soledad, uid, self.mbox)
+        if not msg.does_exist():
             return None
-
-    def get_msg_by_index(self, index):
-        """
-        Retrieves a LeapMessage by sequence index.
-
-        :param index: the index of the sequence (zero-indexed)
-        :type index: int
-        """
-        doc = self.get_by_index(index)
-        if doc:
-            return LeapMessage(doc)
-
-    def is_deleted(self, doc):
-        """
-        Returns whether a given doc is deleted or not.
-
-        :param doc: the document to check
-        :rtype: bool
-        """
-        return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
+        return msg
 
     def get_all(self):
         """
@@ -1184,18 +1262,19 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :return: a list of u1db documents
         :rtype: list of SoledadDocument
         """
+        # TODO change to get_all_docs and turn this
+        # into returning messages
         if sameProxiedObjects(self._soledad, None):
             logger.warning('Tried to get messages but soledad is None!')
             return []
 
-        #f XXX this should return LeapMessage instances
         all_docs = [doc for doc in self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox)]
-        # highly inneficient, but first let's grok it and then
-        # let's worry about efficiency.
+            fields.TYPE_FLAGS_VAL, self.mbox)]
 
-        # XXX FIXINDEX
+        # inneficient, but first let's grok it and then
+        # let's worry about efficiency.
+        # XXX FIXINDEX -- should implement order by in soledad
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
     def count(self):
@@ -1206,7 +1285,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         count = self._soledad.get_count_from_index(
             SoledadBackedAccount.TYPE_MBOX_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox)
+            fields.TYPE_FLAGS_VAL, self.mbox)
         return count
 
     # unseen messages
@@ -1215,13 +1294,13 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         Get an iterator for the message docs with no `seen` flag
 
-        :return: iterator through unseen message docs
+        :return: iterator through unseen message doc UIDs
         :rtype: iterable
         """
-        return (doc for doc in
+        return (doc.content[self.UID_KEY] for doc in
                 self._soledad.get_from_index(
                     SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-                    self.TYPE_MESSAGE_VAL, self.mbox, '0'))
+                    self.TYPE_FLAGS_VAL, self.mbox, '0'))
 
     def count_unseen(self):
         """
@@ -1232,7 +1311,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         count = self._soledad.get_count_from_index(
             SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox, '0')
+            self.TYPE_FLAGS_VAL, self.mbox, '0')
         return count
 
     def get_unseen(self):
@@ -1242,7 +1321,8 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :returns: a list of LeapMessages
         :rtype: list
         """
-        return [LeapMessage(doc) for doc in self.unseen_iter()]
+        return [LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.unseen_iter()]
 
     # recent messages
 
@@ -1253,10 +1333,10 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :return: iterator through recent message docs
         :rtype: iterable
         """
-        return (doc for doc in
+        return (doc.content[self.UID_KEY] for doc in
                 self._soledad.get_from_index(
                     SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-                    self.TYPE_MESSAGE_VAL, self.mbox, '1'))
+                    self.TYPE_FLAGS_VAL, self.mbox, '1'))
 
     def get_recent(self):
         """
@@ -1265,7 +1345,8 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :returns: a list of LeapMessages
         :rtype: list
         """
-        return [LeapMessage(doc) for doc in self.recent_iter()]
+        return [LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.recent_iter()]
 
     def count_recent(self):
         """
@@ -1276,7 +1357,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         """
         count = self._soledad.get_count_from_index(
             SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-            self.TYPE_MESSAGE_VAL, self.mbox, '1')
+            self.TYPE_FLAGS_VAL, self.mbox, '1')
         return count
 
     def __len__(self):
@@ -1297,23 +1378,6 @@ class MessageCollection(WithMsgFields, IndexedDB):
         # XXX return LeapMessage instead?! (change accordingly)
         return (m.content for m in self.get_all())
 
-    def __getitem__(self, uid):
-        """
-        Allows indexing as a list, with msg uid as the index.
-
-        :param uid: an integer index
-        :type uid: int
-
-        :return: LeapMessage or None if not found.
-        :rtype: LeapMessage
-        """
-        # XXX FIXME inneficcient, we are evaulating.
-        try:
-            return [doc
-                    for doc in self.get_all()][uid - 1]
-        except IndexError:
-            return None
-
     def __repr__(self):
         """
         Representation string for this object.
@@ -1321,10 +1385,11 @@ class MessageCollection(WithMsgFields, IndexedDB):
         return u"<MessageCollection: mbox '%s' (%s)>" % (
             self.mbox, self.count())
 
-    # XXX should implement __eq__ also
+    # XXX should implement __eq__ also !!! --- use a hash
+    # of content for that, will be used for dedup.
 
 
-class SoledadMailbox(WithMsgFields):
+class SoledadMailbox(WithMsgFields, MBoxParser):
     """
     A Soledad-backed IMAP mailbox.
 
@@ -1373,7 +1438,7 @@ class SoledadMailbox(WithMsgFields):
         #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
                     #"soledad._db must be an instance of SQLCipherDatabase")
 
-        self.mbox = mbox
+        self.mbox = self._parse_mailbox_name(mbox)
         self.rw = rw
 
         self._soledad = soledad
@@ -1440,11 +1505,6 @@ class SoledadMailbox(WithMsgFields):
         :returns: tuple of flags for this mailbox
         :rtype: tuple of str
         """
-        #return map(str, self.INIT_FLAGS)
-
-        # XXX CHECK against thunderbird XXX
-        # XXX I think this is slightly broken.. :/
-
         mbox = self._get_mbox()
         if not mbox:
             return None
@@ -1458,7 +1518,6 @@ class SoledadMailbox(WithMsgFields):
         :param flags: a tuple with the flags
         :type flags: tuple of str
         """
-        # TODO -- fix also getFlags
         leap_assert(isinstance(flags, tuple),
                     "flags expected to be a tuple")
         mbox = self._get_mbox()
@@ -1526,7 +1585,7 @@ class SoledadMailbox(WithMsgFields):
             # something is wrong,
             # just set the last uid
             # beyond the max msg count.
-            logger.debug("WRONG uid < count. Setting last uid to ", count)
+            logger.debug("WRONG uid < count. Setting last uid to %s", count)
             value = count
 
         mbox.content[key] = value
@@ -1634,7 +1693,7 @@ class SoledadMailbox(WithMsgFields):
         if self.CMD_RECENT in names:
             r[self.CMD_RECENT] = self.getRecentCount()
         if self.CMD_UIDNEXT in names:
-            r[self.CMD_UIDNEXT] = self.getMessageCount() + 1
+            r[self.CMD_UIDNEXT] = self.last_uid + 1
         if self.CMD_UIDVALIDITY in names:
             r[self.CMD_UIDVALIDITY] = self.getUID()
         if self.CMD_UNSEEN in names:
@@ -1664,17 +1723,34 @@ class SoledadMailbox(WithMsgFields):
         else:
             flags = tuple(str(flag) for flag in flags)
 
+        d = self._do_add_messages(message, flags, date, uid_next)
+        d.addCallback(self._notify_new)
+
+    @deferred
+    def _do_add_messages(self, message, flags, date, uid_next):
+        """
+        Calls to the messageCollection add_msg method (deferred to thread).
+        Invoked from addMessage.
+        """
         self.messages.add_msg(message, flags=flags, date=date,
                               uid=uid_next)
 
+    def _notify_new(self, *args):
+        """
+        Notify of new messages to all the listeners.
+
+        :param args: ignored.
+        """
         exists = self.getMessageCount()
         recent = self.getRecentCount()
-        logger.debug("there are %s messages, %s recent" % (
+        logger.debug("NOTIFY: there are %s messages, %s recent" % (
             exists,
             recent))
-        for listener in self.listeners:
-            listener.newMessages(exists, recent)
-        return defer.succeed(None)
+
+        logger.debug("listeners: %s", str(self.listeners))
+        for l in self.listeners:
+            logger.debug('notifying...')
+            l.newMessages(exists, recent)
 
     # commands, do not rename methods
 
@@ -1743,15 +1819,11 @@ class SoledadMailbox(WithMsgFields):
         # for sequence numbers (uid = 0)
         if sequence:
             logger.debug("Getting msg by index: INEFFICIENT call!")
-            for msg_id in messages:
-                msg = self.messages.get_msg_by_index(msg_id - 1)
-                if msg:
-                    result.append((msg.getUID(), msg))
-                else:
-                    print "fetch %s, no msg found!!!" % msg_id
+            raise NotImplementedError
 
         else:
             for msg_id in messages:
+                print "getting msg by uid", msg_id
                 msg = self.messages.get_msg_by_uid(msg_id)
                 if msg:
                     result.append((msg_id, msg))
@@ -1760,9 +1832,10 @@ class SoledadMailbox(WithMsgFields):
 
         if self.isWriteable():
             self._unset_recent_flag()
+        self._signal_unread_to_ui()
 
         # XXX workaround for hangs in thunderbird
-        #return tuple(result[:100])
+        #return tuple(result[:100])  # --- doesn't show all!!
         return tuple(result)
 
     @deferred
@@ -1784,10 +1857,9 @@ class SoledadMailbox(WithMsgFields):
         then that message SHOULD be considered recent.
         """
         log.msg('unsetting recent flags...')
-
-        for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
-            newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
-            self._update(newflags)
+        for msg in self.messages.get_recent():
+            msg.removeFlags((fields.RECENT_FLAG,))
+        self._signal_unread_to_ui()
 
     @deferred
     def _signal_unread_to_ui(self):
@@ -1842,14 +1914,14 @@ class SoledadMailbox(WithMsgFields):
 
         result = {}
         for msg_id in messages:
-            print "MSG ID = %s" % msg_id
+            log.msg("MSG ID = %s" % msg_id)
             msg = self.messages.get_msg_by_uid(msg_id)
             if mode == 1:
-                self._update(msg.addFlags(flags))
+                msg.addFlags(flags)
             elif mode == -1:
-                self._update(msg.removeFlags(flags))
+                msg.removeFlags(flags)
             elif mode == 0:
-                self._update(msg.setFlags(flags))
+                msg.setFlags(flags)
             result[msg_id] = msg.getFlags()
 
         self._signal_unread_to_ui()
@@ -1873,14 +1945,6 @@ class SoledadMailbox(WithMsgFields):
         for doc in docs:
             self.messages._soledad.delete_doc(doc)
 
-    def _update(self, doc):
-        """
-        Updates document in u1db database
-        """
-        # XXX create namedtuple
-        self.messages.soledad_writer.put({"mode": "put",
-                                          "payload": doc})
-
     def __repr__(self):
         """
         Representation string for this mailbox.
-- 
cgit v1.2.3


From 59756d93f329ceee925a813bd4137c2cb34e7679 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 25 Dec 2013 11:57:42 -0400
Subject: inlineCallbacks all the things!

---
 src/leap/mail/decorators.py  |  93 +++++++++++++++++
 src/leap/mail/imap/fetch.py  | 235 ++++++++++++++++++++++---------------------
 src/leap/mail/imap/server.py |  78 ++------------
 3 files changed, 222 insertions(+), 184 deletions(-)
 create mode 100644 src/leap/mail/decorators.py

diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py
new file mode 100644
index 0000000..9e49605
--- /dev/null
+++ b/src/leap/mail/decorators.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# decorators.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Useful decorators for mail package.
+"""
+import logging
+import os
+import sys
+import traceback
+
+from functools import wraps
+
+from twisted.internet.threads import deferToThread
+from twisted.python import log
+
+logger = logging.getLogger(__name__)
+
+
+def deferred(f):
+    """
+    Decorator, for deferring methods to Threads.
+
+    It will do a deferToThread of the decorated method
+    unless the environment variable LEAPMAIL_DEBUG is set.
+
+    It uses a descriptor to delay the definition of the
+    method wrapper.
+    """
+    class descript(object):
+        def __init__(self, f):
+            self.f = f
+
+        def __get__(self, instance, klass):
+            if instance is None:
+                # Class method was requested
+                return self.make_unbound(klass)
+            return self.make_bound(instance)
+
+        def _errback(self, failure):
+            err = failure.value
+            logger.warning('error in method: %s' % (self.f.__name__))
+            logger.exception(err)
+            log.err(err)
+
+        def make_unbound(self, klass):
+
+            @wraps(self.f)
+            def wrapper(*args, **kwargs):
+                """
+                this doc will vanish
+                """
+                raise TypeError(
+                    'unbound method {}() must be called with {} instance '
+                    'as first argument (got nothing instead)'.format(
+                        self.f.__name__,
+                        klass.__name__)
+                )
+            return wrapper
+
+        def make_bound(self, instance):
+
+            @wraps(self.f)
+            def wrapper(*args, **kwargs):
+                """
+                This documentation will disapear
+                """
+                if not os.environ.get('LEAPMAIL_DEBUG'):
+                    d = deferToThread(self.f, instance, *args, **kwargs)
+                    d.addErrback(self._errback)
+                    return d
+                else:
+                    return self.f(instance, *args, **kwargs)
+
+            # This instance does not need the descriptor anymore,
+            # let it find the wrapper directly next time:
+            setattr(instance, self.f.__name__, wrapper)
+            return wrapper
+
+    return descript(f)
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index b1c34ba..0b31c3b 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -17,21 +17,24 @@
 """
 Incoming mail fetcher.
 """
-import logging
+import copy
 import json
-import ssl
+import logging
+#import ssl
 import threading
 import time
-import copy
-from StringIO import StringIO
+import sys
+import traceback
 
 from email.parser import Parser
 from email.generator import Generator
 from email.utils import parseaddr
+from StringIO import StringIO
 
 from twisted.python import log
+from twisted.internet import defer
 from twisted.internet.task import LoopingCall
-from twisted.internet.threads import deferToThread
+#from twisted.internet.threads import deferToThread
 from zope.proxy import sameProxiedObjects
 
 from leap.common import events as leap_events
@@ -45,12 +48,18 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
 from leap.keymanager.openpgp import OpenPGPKey
+from leap.mail.decorators import deferred
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
 
 logger = logging.getLogger(__name__)
 
+MULTIPART_ENCRYPTED = "multipart/encrypted"
+MULTIPART_SIGNED = "multipart/signed"
+PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
+PGP_END = "-----END PGP MESSAGE-----"
+
 
 class MalformedMessage(Exception):
     """
@@ -125,6 +134,9 @@ class LeapIncomingMail(object):
 
         self._create_soledad_indexes()
 
+        # initialize a mail parser only once
+        self._parser = Parser()
+
     def _create_soledad_indexes(self):
         """
         Create needed indexes on soledad.
@@ -152,9 +164,10 @@ class LeapIncomingMail(object):
         logger.debug("fetching mail for: %s %s" % (
             self._soledad.uuid, self._userid))
         if not self.fetching_lock.locked():
-            d = deferToThread(self._sync_soledad)
-            d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error)
-            d.addCallbacks(self._process_doclist, self._sync_soledad_error)
+            d1 = self._sync_soledad()
+            d = defer.gatherResults([d1], consumeErrors=True)
+            d.addCallbacks(self._signal_fetch_to_ui, self._errback)
+            d.addCallbacks(self._signal_unread_to_ui, self._errback)
             return d
         else:
             logger.debug("Already fetching mail.")
@@ -184,6 +197,11 @@ class LeapIncomingMail(object):
 
     # synchronize incoming mail
 
+    def _errback(self, failure):
+        logger.exception(failure.value)
+        traceback.print_tb(*sys.exc_info())
+
+    @deferred
     def _sync_soledad(self):
         """
         Synchronizes with remote soledad.
@@ -196,10 +214,9 @@ class LeapIncomingMail(object):
             self._soledad.sync()
             log.msg('soledad synced.')
             doclist = self._soledad.get_from_index("just-mail", "*")
+        self._process_doclist(doclist)
 
-        return doclist
-
-    def _signal_unread_to_ui(self):
+    def _signal_unread_to_ui(self, *args):
         """
         Sends unread event to ui.
         """
@@ -215,53 +232,18 @@ class LeapIncomingMail(object):
         :returns: doclist
         :rtype: iterable
         """
+        doclist = doclist[0]  # gatherResults pass us a list
         fetched_ts = time.mktime(time.gmtime())
-        num_mails = len(doclist)
-        log.msg("there are %s mails" % (num_mails,))
+        num_mails = len(doclist) if doclist is not None else 0
+        if num_mails != 0:
+            log.msg("there are %s mails" % (num_mails,))
         leap_events.signal(
             IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts))
-        self._signal_unread_to_ui()
         return doclist
 
-    def _sync_soledad_error(self, failure):
-        """
-        Errback for sync errors.
-        """
-        # XXX should signal unrecoverable maybe.
-        err = failure.value
-        logger.error("error syncing soledad: %s" % (err,))
-        if failure.check(ssl.SSLError):
-            logger.warning('SSL Error while '
-                           'syncing soledad: %r' % (err,))
-        elif failure.check(Exception):
-            logger.warning('Unknown error while '
-                           'syncing soledad: %r' % (err,))
-
-    def _log_err(self, failure):
-        """
-        Generic errback
-        """
-        err = failure.value
-        logger.exception("error!: %r" % (err,))
-
-    def _decryption_error(self, failure):
-        """
-        Errback for decryption errors.
-        """
-        # XXX should signal unrecoverable maybe.
-        err = failure.value
-        logger.error("error decrypting msg: %s" % (err,))
-
-    def _saving_error(self, failure):
-        """
-        Errback for local save errors.
-        """
-        # XXX should signal unrecoverable maybe.
-        err = failure.value
-        logger.error("error saving msg locally: %s" % (err,))
-
     # process incoming mail.
 
+    @defer.inlineCallbacks
     def _process_doclist(self, doclist):
         """
         Iterates through the doclist, checks if each doc
@@ -278,7 +260,6 @@ class LeapIncomingMail(object):
             return
         num_mails = len(doclist)
 
-        docs_cb = []
         for index, doc in enumerate(doclist):
             logger.debug("processing doc %d of %d" % (index + 1, num_mails))
             leap_events.signal(
@@ -287,35 +268,18 @@ class LeapIncomingMail(object):
             if self._is_msg(keys):
                 # Ok, this looks like a legit msg.
                 # Let's process it!
-                # Deferred chain for individual messages
-
-                # XXX use an IConsumer instead... ?
-                d = deferToThread(self._decrypt_doc, doc)
-                d.addCallback(self._process_decrypted_doc)
-                d.addErrback(self._log_err)
-                d.addCallback(self._add_message_locally)
-                d.addErrback(self._log_err)
-                docs_cb.append(d)
+                decrypted = list(self._decrypt_doc(doc))[0]
+                res = self._add_message_locally(decrypted)
+                yield res
+
             else:
                 # Ooops, this does not.
                 logger.debug('This does not look like a proper msg.')
-        return docs_cb
 
     #
     # operations on individual messages
     #
 
-    def _is_msg(self, keys):
-        """
-        Checks if the keys of a dictionary match the signature
-        of the document type we use for messages.
-
-        :param keys: iterable containing the strings to match.
-        :type keys: iterable of strings.
-        :rtype: bool
-        """
-        return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
-
     def _decrypt_doc(self, doc):
         """
         Decrypt the contents of a document.
@@ -339,7 +303,9 @@ class LeapIncomingMail(object):
             logger.error("Error while decrypting msg: %r" % (exc,))
             decrdata = ""
         leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0")
-        return doc, decrdata
+
+        data = list(self._process_decrypted_doc((doc, decrdata)))
+        yield (doc, data)
 
     def _process_decrypted_doc(self, msgtuple):
         """
@@ -357,16 +323,15 @@ class LeapIncomingMail(object):
         doc, data = msgtuple
         msg = json.loads(data)
         if not isinstance(msg, dict):
-            return False
+            defer.returnValue(False)
         if not msg.get(self.INCOMING_KEY, False):
-            return False
+            defer.returnValue(False)
 
         # ok, this is an incoming message
         rawmsg = msg.get(self.CONTENT_KEY, None)
         if not rawmsg:
             return False
-        data = self._maybe_decrypt_msg(rawmsg)
-        return doc, data
+        return self._maybe_decrypt_msg(rawmsg)
 
     def _maybe_decrypt_msg(self, data):
         """
@@ -381,17 +346,16 @@ class LeapIncomingMail(object):
         leap_assert_type(data, unicode)
 
         # parse the original message
-        parser = Parser()
         encoding = get_email_charset(data)
         data = data.encode(encoding)
-        msg = parser.parsestr(data)
+        msg = self._parser.parsestr(data)
 
         # try to obtain sender public key
         senderPubkey = None
         fromHeader = msg.get('from', None)
-        if fromHeader is not None \
-                and (msg.get_content_type() == 'multipart/encrypted' \
-                     or msg.get_content_type() == 'multipart/signed'):
+        if (fromHeader is not None
+            and (msg.get_content_type() == MULTIPART_ENCRYPTED
+                 or msg.get_content_type() == MULTIPART_SIGNED)):
             _, senderAddress = parseaddr(fromHeader)
             try:
                 senderPubkey = self._keymanager.get_key_from_cache(
@@ -400,11 +364,14 @@ class LeapIncomingMail(object):
                 pass
 
         valid_sig = False  # we will add a header saying if sig is valid
-        if msg.get_content_type() == 'multipart/encrypted':
-            decrmsg, valid_sig = self._decrypt_multipart_encrypted_msg(
+        decrypt_multi = self._decrypt_multipart_encrypted_msg
+        decrypt_inline = self._maybe_decrypt_inline_encrypted_msg
+
+        if msg.get_content_type() == MULTIPART_ENCRYPTED:
+            decrmsg, valid_sig = decrypt_multi(
                 msg, encoding, senderPubkey)
         else:
-            decrmsg, valid_sig = self._maybe_decrypt_inline_encrypted_msg(
+            decrmsg, valid_sig = decrypt_inline(
                 msg, encoding, senderPubkey)
 
         # add x-leap-signature header
@@ -419,7 +386,7 @@ class LeapIncomingMail(object):
                 self.LEAP_SIGNATURE_INVALID,
                 pubkey=senderPubkey.key_id)
 
-        return decrmsg.as_string()
+        yield decrmsg.as_string()
 
     def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey):
         """
@@ -437,43 +404,33 @@ class LeapIncomingMail(object):
         """
         log.msg('decrypting multipart encrypted msg')
         msg = copy.deepcopy(msg)
-        # sanity check
-        payload = msg.get_payload()
-        if len(payload) != 2:
-            raise MalformedMessage(
-                'Multipart/encrypted messages should have exactly 2 body '
-                'parts (instead of %d).' % len(payload))
-        if payload[0].get_content_type() != 'application/pgp-encrypted':
-            raise MalformedMessage(
-                "Multipart/encrypted messages' first body part should "
-                "have content type equal to 'application/pgp-encrypted' "
-                "(instead of %s)." % payload[0].get_content_type())
-        if payload[1].get_content_type() != 'application/octet-stream':
-            raise MalformedMessage(
-                "Multipart/encrypted messages' second body part should "
-                "have content type equal to 'octet-stream' (instead of "
-                "%s)." % payload[1].get_content_type())
+        self._multipart_sanity_check(msg)
+
         # parse message and get encrypted content
         pgpencmsg = msg.get_payload()[1]
         encdata = pgpencmsg.get_payload()
+
         # decrypt or fail gracefully
         try:
-            decrdata, valid_sig = self._decrypt_and_verify_data(
+            decrdata, valid_sig = yield self._decrypt_and_verify_data(
                 encdata, senderPubkey)
         except keymanager_errors.DecryptError as e:
             logger.warning('Failed to decrypt encrypted message (%s). '
                            'Storing message without modifications.' % str(e))
-            return msg, False  # return original message
+            # Bailing out!
+            yield (msg, False)
+
         # decrypted successully, now fix encoding and parse
         try:
             decrdata = decrdata.encode(encoding)
         except (UnicodeEncodeError, UnicodeDecodeError) as e:
             logger.error("Unicode error {0}".format(e))
             decrdata = decrdata.encode(encoding, 'replace')
-        parser = Parser()
-        decrmsg = parser.parsestr(decrdata)
+
+        decrmsg = self._parser.parsestr(decrdata)
         # remove original message's multipart/encrypted content-type
         del(msg['content-type'])
+
         # replace headers back in original message
         for hkey, hval in decrmsg.items():
             try:
@@ -481,9 +438,10 @@ class LeapIncomingMail(object):
                 msg.replace_header(hkey, hval)
             except KeyError:
                 msg[hkey] = hval
-        # replace payload by unencrypted payload
+
+        # all ok, replace payload by unencrypted payload
         msg.set_payload(decrmsg.get_payload())
-        return msg, valid_sig
+        yield (msg, valid_sig)
 
     def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
                                             senderPubkey):
@@ -497,8 +455,9 @@ class LeapIncomingMail(object):
         :param senderPubkey: The key of the sender of the message.
         :type senderPubkey: OpenPGPKey
 
-        :return: A unitary tuple containing a decrypted message.
-        :rtype: (Message)
+        :return: A unitary tuple containing a decrypted message and
+                 a bool indicating wether the signature is valid.
+        :rtype: (Message, bool)
         """
         log.msg('maybe decrypting inline encrypted msg')
         # serialize the original message
@@ -507,8 +466,6 @@ class LeapIncomingMail(object):
         g.flatten(origmsg)
         data = buf.getvalue()
         # handle exactly one inline PGP message
-        PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
-        PGP_END = "-----END PGP MESSAGE-----"
         valid_sig = False
         if PGP_BEGIN in data:
             begin = data.find(PGP_BEGIN)
@@ -522,11 +479,11 @@ class LeapIncomingMail(object):
             except keymanager_errors.DecryptError:
                 logger.warning('Failed to decrypt potential inline encrypted '
                                'message. Storing message as is...')
+
         # if message is not encrypted, return raw data
         if isinstance(data, unicode):
             data = data.encode(encoding, 'replace')
-        parser = Parser()
-        return parser.parsestr(data), valid_sig
+        return (self._parser.parsestr(data), valid_sig)
 
     def _decrypt_and_verify_data(self, data, senderPubkey):
         """
@@ -555,7 +512,7 @@ class LeapIncomingMail(object):
         except keymanager_errors.InvalidSignature:
             decrdata = self._keymanager.decrypt(
                 data, self._pkey)
-        return decrdata, valid_sig
+        return (decrdata, valid_sig)
 
     def _add_message_locally(self, msgtuple):
         """
@@ -570,10 +527,54 @@ class LeapIncomingMail(object):
         """
         log.msg('adding message to local db')
         doc, data = msgtuple
-        self._inbox.addMessage(data, (self.RECENT_FLAG,))
+        if isinstance(data, list):
+            data = data[0]
+
+        self._inbox.addMessage(data, flags=(self.RECENT_FLAG,))
+
         leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
         doc_id = doc.doc_id
         self._soledad.delete_doc(doc)
         log.msg("deleted doc %s from incoming" % doc_id)
         leap_events.signal(IMAP_MSG_DELETED_INCOMING)
         self._signal_unread_to_ui()
+        return True
+
+    #
+    # helpers
+    #
+
+    def _msg_multipart_sanity_check(self, msg):
+        """
+        Performs a sanity check against a multipart encrypted msg
+
+        :param msg: The original encrypted message.
+        :type msg: Message
+        """
+        # sanity check
+        payload = msg.get_payload()
+        if len(payload) != 2:
+            raise MalformedMessage(
+                'Multipart/encrypted messages should have exactly 2 body '
+                'parts (instead of %d).' % len(payload))
+        if payload[0].get_content_type() != 'application/pgp-encrypted':
+            raise MalformedMessage(
+                "Multipart/encrypted messages' first body part should "
+                "have content type equal to 'application/pgp-encrypted' "
+                "(instead of %s)." % payload[0].get_content_type())
+        if payload[1].get_content_type() != 'application/octet-stream':
+            raise MalformedMessage(
+                "Multipart/encrypted messages' second body part should "
+                "have content type equal to 'octet-stream' (instead of "
+                "%s)." % payload[1].get_content_type())
+
+    def _is_msg(self, keys):
+        """
+        Checks if the keys of a dictionary match the signature
+        of the document type we use for messages.
+
+        :param keys: iterable containing the strings to match.
+        :type keys: iterable of strings.
+        :rtype: bool
+        """
+        return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 8758dcb..57587a5 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -21,20 +21,17 @@ import copy
 import logging
 import StringIO
 import cStringIO
-import os
 import time
 import re
 
 from collections import defaultdict, namedtuple
 from email.parser import Parser
-from functools import wraps
 
 from zope.interface import implements
 from zope.proxy import sameProxiedObjects
 
 from twisted.mail import imap4
 from twisted.internet import defer
-from twisted.internet.threads import deferToThread
 from twisted.python import log
 
 from u1db import errors as u1db_errors
@@ -44,70 +41,12 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
 from leap.common.mail import get_email_charset
 from leap.mail.messageflow import IMessageConsumer, MessageProducer
+from leap.mail.decorators import deferred
 from leap.soledad.client import Soledad
 
 logger = logging.getLogger(__name__)
 
 
-def deferred(f):
-    '''
-    Decorator, for deferring methods to Threads.
-
-    It will do a deferToThread of the decorated method
-    unless the environment variable LEAPMAIL_DEBUG is set.
-
-    It uses a descriptor to delay the definition of the
-    method wrapper.
-    '''
-    class descript(object):
-        def __init__(self, f):
-            self.f = f
-
-        def __get__(self, instance, klass):
-            if instance is None:
-                # Class method was requested
-                return self.make_unbound(klass)
-            return self.make_bound(instance)
-
-        def _errback(self, failure):
-            err = failure.value
-            logger.warning('error in method: %s' % (self.f.__name__))
-            log.err(err)
-
-        def make_unbound(self, klass):
-
-            @wraps(self.f)
-            def wrapper(*args, **kwargs):
-                '''This documentation will vanish :)'''
-                raise TypeError(
-                    'unbound method {}() must be called with {} instance '
-                    'as first argument (got nothing instead)'.format(
-                        self.f.__name__,
-                        klass.__name__)
-                )
-            return wrapper
-
-        def make_bound(self, instance):
-
-            @wraps(self.f)
-            def wrapper(*args, **kwargs):
-                '''This documentation will disapear :)'''
-
-                if not os.environ.get('LEAPMAIL_DEBUG'):
-                    d = deferToThread(self.f, instance, *args, **kwargs)
-                    d.addErrback(self._errback)
-                    return d
-                else:
-                    return self.f(instance, *args, **kwargs)
-
-            # This instance does not need the descriptor anymore,
-            # let it find the wrapper directly next time:
-            setattr(instance, self.f.__name__, wrapper)
-            return wrapper
-
-    return descript(f)
-
-
 class MissingIndexError(Exception):
     """
     Raises when tried to access a non existent index document.
@@ -248,6 +187,8 @@ class MailParser(object):
             return self._parser.parse
         if isinstance(o, basestring):
             return self._parser.parsestr
+        # fallback
+        return self._parser.parsestr
 
     def _stringify(self, o):
         """
@@ -942,8 +883,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return True if this message is multipart.
         """
         if self._cdoc:
-            retval = self._cdoc.content.get(self.MULTIPART_KEY, False)
-            print "MULTIPART? ", retval
+            retval = self._fdoc.content.get(self.MULTIPART_KEY, False)
+            return retval
 
     def getSubPart(self, part):
         """
@@ -1197,6 +1138,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         msg = self._get_parsed_msg(raw)
         headers = dict(msg)
 
+        logger.debug("adding. is multipart:%s" % msg.is_multipart())
         flags_doc[self.MULTIPART_KEY] = msg.is_multipart()
         # XXX get lower case for keys?
         # XXX get headers doc
@@ -1464,7 +1406,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def addListener(self, listener):
         """
-        Rdds a listener to the listeners queue.
+        Adds a listener to the listeners queue.
+        The server adds itself as a listener when there is a SELECT,
+        so it can send EXIST commands.
 
         :param listener: listener to add
         :type listener: an object that implements IMailboxListener
@@ -1716,6 +1660,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :return: a deferred that evals to None
         """
         # XXX we should treat the message as an IMessage from here
+        leap_assert_type(message, basestring)
         uid_next = self.getUIDNext()
         logger.debug('Adding msg with UID :%s' % uid_next)
         if flags is None:
@@ -1823,12 +1768,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         else:
             for msg_id in messages:
-                print "getting msg by uid", msg_id
                 msg = self.messages.get_msg_by_uid(msg_id)
                 if msg:
                     result.append((msg_id, msg))
                 else:
-                    print "fetch %s, no msg found!!!" % msg_id
+                    logger.debug("fetch %s, no msg found!!!" % msg_id)
 
         if self.isWriteable():
             self._unset_recent_flag()
-- 
cgit v1.2.3


From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 26 Dec 2013 14:10:14 -0400
Subject: Split the near-2k loc file into more handy modules.

...aaaand not a single fuck was given that day!
---
 src/leap/mail/imap/account.py      |  426 ++++++++
 src/leap/mail/imap/fields.py       |  127 +++
 src/leap/mail/imap/index.py        |   69 ++
 src/leap/mail/imap/mailbox.py      |  617 ++++++++++++
 src/leap/mail/imap/messages.py     |  735 ++++++++++++++
 src/leap/mail/imap/parser.py       |   93 ++
 src/leap/mail/imap/server.py       | 1897 ------------------------------------
 src/leap/mail/imap/service/imap.py |    2 +-
 8 files changed, 2068 insertions(+), 1898 deletions(-)
 create mode 100644 src/leap/mail/imap/account.py
 create mode 100644 src/leap/mail/imap/fields.py
 create mode 100644 src/leap/mail/imap/index.py
 create mode 100644 src/leap/mail/imap/mailbox.py
 create mode 100644 src/leap/mail/imap/messages.py
 create mode 100644 src/leap/mail/imap/parser.py
 delete mode 100644 src/leap/mail/imap/server.py

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
new file mode 100644
index 0000000..fd861e7
--- /dev/null
+++ b/src/leap/mail/imap/account.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+# account.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Backed Account.
+"""
+import copy
+import time
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.imap.index import IndexedDB
+from leap.mail.imap.fields import WithMsgFields
+from leap.mail.imap.parser import MBoxParser
+from leap.mail.imap.mailbox import SoledadMailbox
+from leap.soledad.client import Soledad
+
+
+#######################################
+# Soledad Account
+#######################################
+
+
+class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+    """
+    An implementation of IAccount and INamespacePresenteer
+    that is backed by Soledad Encrypted Documents.
+    """
+
+    implements(imap4.IAccount, imap4.INamespacePresenter)
+
+    _soledad = None
+    selected = None
+
+    def __init__(self, account_name, soledad=None):
+        """
+        Creates a SoledadAccountIndex that keeps track of the mailboxes
+        and subscriptions handled by this account.
+
+        :param acct_name: The name of the account (user id).
+        :type acct_name: str
+
+        :param soledad: a Soledad instance.
+        :param soledad: Soledad
+        """
+        leap_assert(soledad, "Need a soledad instance to initialize")
+        leap_assert_type(soledad, Soledad)
+
+        # XXX SHOULD assert too that the name matches the user/uuid with which
+        # soledad has been initialized.
+
+        self._account_name = self._parse_mailbox_name(account_name)
+        self._soledad = soledad
+
+        self.initialize_db()
+
+        # every user should have the right to an inbox folder
+        # at least, so let's make one!
+
+        if not self.mailboxes:
+            self.addMailbox(self.INBOX_NAME)
+
+    def _get_empty_mailbox(self):
+        """
+        Returns an empty mailbox.
+
+        :rtype: dict
+        """
+        return copy.deepcopy(self.EMPTY_MBOX)
+
+    def _get_mailbox_by_name(self, name):
+        """
+        Return an mbox document by name.
+
+        :param name: the name of the mailbox
+        :type name: str
+
+        :rtype: SoledadDocument
+        """
+        doc = self._soledad.get_from_index(
+            self.TYPE_MBOX_IDX, self.MBOX_KEY,
+            self._parse_mailbox_name(name))
+        return doc[0] if doc else None
+
+    @property
+    def mailboxes(self):
+        """
+        A list of the current mailboxes for this account.
+        """
+        return [doc.content[self.MBOX_KEY]
+                for doc in self._soledad.get_from_index(
+                    self.TYPE_IDX, self.MBOX_KEY)]
+
+    @property
+    def subscriptions(self):
+        """
+        A list of the current subscriptions for this account.
+        """
+        return [doc.content[self.MBOX_KEY]
+                for doc in self._soledad.get_from_index(
+                    self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
+
+    def getMailbox(self, name):
+        """
+        Returns a Mailbox with that name, without selecting it.
+
+        :param name: name of the mailbox
+        :type name: str
+
+        :returns: a a SoledadMailbox instance
+        :rtype: SoledadMailbox
+        """
+        name = self._parse_mailbox_name(name)
+
+        if name not in self.mailboxes:
+            raise imap4.MailboxException("No such mailbox")
+
+        return SoledadMailbox(name, soledad=self._soledad)
+
+    ##
+    ## IAccount
+    ##
+
+    def addMailbox(self, name, creation_ts=None):
+        """
+        Add a mailbox to the account.
+
+        :param name: the name of the mailbox
+        :type name: str
+
+        :param creation_ts: an optional creation timestamp to be used as
+                            mailbox id. A timestamp will be used if no
+                            one is provided.
+        :type creation_ts: int
+
+        :returns: True if successful
+        :rtype: bool
+        """
+        name = self._parse_mailbox_name(name)
+
+        if name in self.mailboxes:
+            raise imap4.MailboxCollision, name
+
+        if not creation_ts:
+            # by default, we pass an int value
+            # taken from the current time
+            # we make sure to take enough decimals to get a unique
+            # mailbox-uidvalidity.
+            creation_ts = int(time.time() * 10E2)
+
+        mbox = self._get_empty_mailbox()
+        mbox[self.MBOX_KEY] = name
+        mbox[self.CREATED_KEY] = creation_ts
+
+        doc = self._soledad.create_doc(mbox)
+        return bool(doc)
+
+    def create(self, pathspec):
+        """
+        Create a new mailbox from the given hierarchical name.
+
+        :param pathspec: The full hierarchical name of a new mailbox to create.
+                         If any of the inferior hierarchical names to this one
+                         do not exist, they are created as well.
+        :type pathspec: str
+
+        :return: A true value if the creation succeeds.
+        :rtype: bool
+
+        :raise MailboxException: Raised if this mailbox cannot be added.
+        """
+        # TODO raise MailboxException
+        paths = filter(
+            None,
+            self._parse_mailbox_name(pathspec).split('/'))
+        for accum in range(1, len(paths)):
+            try:
+                self.addMailbox('/'.join(paths[:accum]))
+            except imap4.MailboxCollision:
+                pass
+        try:
+            self.addMailbox('/'.join(paths))
+        except imap4.MailboxCollision:
+            if not pathspec.endswith('/'):
+                return False
+        return True
+
+    def select(self, name, readwrite=1):
+        """
+        Selects a mailbox.
+
+        :param name: the mailbox to select
+        :type name: str
+
+        :param readwrite: 1 for readwrite permissions.
+        :type readwrite: int
+
+        :rtype: bool
+        """
+        name = self._parse_mailbox_name(name)
+
+        if name not in self.mailboxes:
+            return None
+
+        self.selected = name
+
+        return SoledadMailbox(
+            name, rw=readwrite,
+            soledad=self._soledad)
+
+    def delete(self, name, force=False):
+        """
+        Deletes a mailbox.
+
+        Right now it does not purge the messages, but just removes the mailbox
+        name from the mailboxes list!!!
+
+        :param name: the mailbox to be deleted
+        :type name: str
+
+        :param force: if True, it will not check for noselect flag or inferior
+                      names. use with care.
+        :type force: bool
+        """
+        name = self._parse_mailbox_name(name)
+
+        if not name in self.mailboxes:
+            raise imap4.MailboxException("No such mailbox")
+
+        mbox = self.getMailbox(name)
+
+        if force is False:
+            # See if this box is flagged \Noselect
+            # XXX use mbox.flags instead?
+            if self.NOSELECT_FLAG in mbox.getFlags():
+                # Check for hierarchically inferior mailboxes with this one
+                # as part of their root.
+                for others in self.mailboxes:
+                    if others != name and others.startswith(name):
+                        raise imap4.MailboxException, (
+                            "Hierarchically inferior mailboxes "
+                            "exist and \\Noselect is set")
+        mbox.destroy()
+
+        # XXX FIXME --- not honoring the inferior names...
+
+        # if there are no hierarchically inferior names, we will
+        # delete it from our ken.
+        #if self._inferiorNames(name) > 1:
+            # ??! -- can this be rite?
+            #self._index.removeMailbox(name)
+
+    def rename(self, oldname, newname):
+        """
+        Renames a mailbox.
+
+        :param oldname: old name of the mailbox
+        :type oldname: str
+
+        :param newname: new name of the mailbox
+        :type newname: str
+        """
+        oldname = self._parse_mailbox_name(oldname)
+        newname = self._parse_mailbox_name(newname)
+
+        if oldname not in self.mailboxes:
+            raise imap4.NoSuchMailbox, oldname
+
+        inferiors = self._inferiorNames(oldname)
+        inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
+
+        for (old, new) in inferiors:
+            if new in self.mailboxes:
+                raise imap4.MailboxCollision, new
+
+        for (old, new) in inferiors:
+            mbox = self._get_mailbox_by_name(old)
+            mbox.content[self.MBOX_KEY] = new
+            self._soledad.put_doc(mbox)
+
+        # XXX ---- FIXME!!!! ------------------------------------
+        # until here we just renamed the index...
+        # We have to rename also the occurrence of this
+        # mailbox on ALL the messages that are contained in it!!!
+        # ... we maybe could use a reference to the doc_id
+        # in each msg, instead of the "mbox" field in msgs
+        # -------------------------------------------------------
+
+    def _inferiorNames(self, name):
+        """
+        Return hierarchically inferior mailboxes.
+
+        :param name: name of the mailbox
+        :rtype: list
+        """
+        # XXX use wildcard query instead
+        inferiors = []
+        for infname in self.mailboxes:
+            if infname.startswith(name):
+                inferiors.append(infname)
+        return inferiors
+
+    def isSubscribed(self, name):
+        """
+        Returns True if user is subscribed to this mailbox.
+
+        :param name: the mailbox to be checked.
+        :type name: str
+
+        :rtype: bool
+        """
+        mbox = self._get_mailbox_by_name(name)
+        return mbox.content.get('subscribed', False)
+
+    def _set_subscription(self, name, value):
+        """
+        Sets the subscription value for a given mailbox
+
+        :param name: the mailbox
+        :type name: str
+
+        :param value: the boolean value
+        :type value: bool
+        """
+        # maybe we should store subscriptions in another
+        # document...
+        if not name in self.mailboxes:
+            self.addMailbox(name)
+        mbox = self._get_mailbox_by_name(name)
+
+        if mbox:
+            mbox.content[self.SUBSCRIBED_KEY] = value
+            self._soledad.put_doc(mbox)
+
+    def subscribe(self, name):
+        """
+        Subscribe to this mailbox
+
+        :param name: name of the mailbox
+        :type name: str
+        """
+        name = self._parse_mailbox_name(name)
+        if name not in self.subscriptions:
+            self._set_subscription(name, True)
+
+    def unsubscribe(self, name):
+        """
+        Unsubscribe from this mailbox
+
+        :param name: name of the mailbox
+        :type name: str
+        """
+        name = self._parse_mailbox_name(name)
+        if name not in self.subscriptions:
+            raise imap4.MailboxException, "Not currently subscribed to " + name
+        self._set_subscription(name, False)
+
+    def listMailboxes(self, ref, wildcard):
+        """
+        List the mailboxes.
+
+        from rfc 3501:
+        returns a subset of names from the complete set
+        of all names available to the client.  Zero or more untagged LIST
+        replies are returned, containing the name attributes, hierarchy
+        delimiter, and name.
+
+        :param ref: reference name
+        :type ref: str
+
+        :param wildcard: mailbox name with possible wildcards
+        :type wildcard: str
+        """
+        # XXX use wildcard in index query
+        ref = self._inferiorNames(
+            self._parse_mailbox_name(ref))
+        wildcard = imap4.wildcardToRegexp(wildcard, '/')
+        return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+
+    ##
+    ## INamespacePresenter
+    ##
+
+    def getPersonalNamespaces(self):
+        return [["", "/"]]
+
+    def getSharedNamespaces(self):
+        return None
+
+    def getOtherNamespaces(self):
+        return None
+
+    # extra, for convenience
+
+    def deleteAllMessages(self, iknowhatiamdoing=False):
+        """
+        Deletes all messages from all mailboxes.
+        Danger! high voltage!
+
+        :param iknowhatiamdoing: confirmation parameter, needs to be True
+                                 to proceed.
+        """
+        if iknowhatiamdoing is True:
+            for mbox in self.mailboxes:
+                self.delete(mbox, force=True)
+
+    def __repr__(self):
+        """
+        Representation string for this object.
+        """
+        return "<SoledadBackedAccount (%s)>" % self._account_name
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
new file mode 100644
index 0000000..96b937e
--- /dev/null
+++ b/src/leap/mail/imap/fields.py
@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# fields.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Fields for Mailbox and Message.
+"""
+from leap.mail.imap.parser import MBoxParser
+
+
+class WithMsgFields(object):
+    """
+    Container class for class-attributes to be shared by
+    several message-related classes.
+    """
+    # Internal representation of Message
+    DATE_KEY = "date"
+    HEADERS_KEY = "headers"
+    FLAGS_KEY = "flags"
+    MBOX_KEY = "mbox"
+    CONTENT_HASH_KEY = "chash"
+    RAW_KEY = "raw"
+    SUBJECT_KEY = "subject"
+    UID_KEY = "uid"
+    MULTIPART_KEY = "multi"
+    SIZE_KEY = "size"
+
+    # Mailbox specific keys
+    CLOSED_KEY = "closed"
+    CREATED_KEY = "created"
+    SUBSCRIBED_KEY = "subscribed"
+    RW_KEY = "rw"
+    LAST_UID_KEY = "lastuid"
+
+    # Document Type, for indexing
+    TYPE_KEY = "type"
+    TYPE_MBOX_VAL = "mbox"
+    TYPE_MESSAGE_VAL = "msg"
+    TYPE_FLAGS_VAL = "flags"
+    TYPE_HEADERS_VAL = "head"
+    TYPE_ATTACHMENT_VAL = "attach"
+    # should add also a headers val
+
+    INBOX_VAL = "inbox"
+
+    # Flags for SoledadDocument for indexing.
+    SEEN_KEY = "seen"
+    RECENT_KEY = "recent"
+
+    # Flags in Mailbox and Message
+    SEEN_FLAG = "\\Seen"
+    RECENT_FLAG = "\\Recent"
+    ANSWERED_FLAG = "\\Answered"
+    FLAGGED_FLAG = "\\Flagged"  # yo dawg
+    DELETED_FLAG = "\\Deleted"
+    DRAFT_FLAG = "\\Draft"
+    NOSELECT_FLAG = "\\Noselect"
+    LIST_FLAG = "List"  # is this OK? (no \. ie, no system flag)
+
+    # Fields in mail object
+    SUBJECT_FIELD = "Subject"
+    DATE_FIELD = "Date"
+
+    # Index  types
+    # --------------
+
+    TYPE_IDX = 'by-type'
+    TYPE_MBOX_IDX = 'by-type-and-mbox'
+    TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
+    TYPE_SUBS_IDX = 'by-type-and-subscribed'
+    TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+    TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+    TYPE_HASH_IDX = 'by-type-and-hash'
+
+    # Tomas created the `recent and seen index`, but the semantic is not too
+    # correct since the recent flag is volatile.
+    TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
+
+    KTYPE = TYPE_KEY
+    MBOX_VAL = TYPE_MBOX_VAL
+    HASH_VAL = CONTENT_HASH_KEY
+
+    INDEXES = {
+        # generic
+        TYPE_IDX: [KTYPE],
+        TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
+        TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
+
+        # mailboxes
+        TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
+
+        # content, headers doc
+        TYPE_HASH_IDX: [KTYPE, HASH_VAL],
+
+        # messages
+        TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
+        TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
+        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
+                                  'bool(recent)', 'bool(seen)'],
+    }
+
+    MBOX_KEY = MBOX_VAL
+
+    EMPTY_MBOX = {
+        TYPE_KEY: MBOX_KEY,
+        TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
+        SUBJECT_KEY: "",
+        FLAGS_KEY: [],
+        CLOSED_KEY: False,
+        SUBSCRIBED_KEY: False,
+        RW_KEY: 1,
+        LAST_UID_KEY: 0
+    }
+
+fields = WithMsgFields  # alias for convenience
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
new file mode 100644
index 0000000..2280d86
--- /dev/null
+++ b/src/leap/mail/imap/index.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# index.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Index for SoledadBackedAccount, Mailbox and Messages.
+"""
+import logging
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail.imap.account import SoledadBackedAccount
+
+
+logger = logging.getLogger(__name__)
+
+
+class IndexedDB(object):
+    """
+    Methods dealing with the index.
+
+    This is a MixIn that needs access to the soledad instance,
+    and also assumes that a INDEXES attribute is accessible to the instance.
+
+    INDEXES must be a dictionary of type:
+    {'index-name': ['field1', 'field2']}
+    """
+    # TODO we might want to move this to soledad itself, check
+
+    def initialize_db(self):
+        """
+        Initialize the database.
+        """
+        leap_assert(self._soledad,
+                    "Need a soledad attribute accesible in the instance")
+        leap_assert_type(self.INDEXES, dict)
+
+        # Ask the database for currently existing indexes.
+        if not self._soledad:
+            logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
+            return
+        db_indexes = dict()
+        if self._soledad is not None:
+            db_indexes = dict(self._soledad.list_indexes())
+        for name, expression in SoledadBackedAccount.INDEXES.items():
+            if name not in db_indexes:
+                # The index does not yet exist.
+                self._soledad.create_index(name, *expression)
+                continue
+
+            if expression == db_indexes[name]:
+                # The index exists and is up to date.
+                continue
+            # The index exists but the definition is not what expected, so we
+            # delete it and add the proper index expression.
+            self._soledad.delete_index(name)
+            self._soledad.create_index(name, *expression)
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
new file mode 100644
index 0000000..09c06a2
--- /dev/null
+++ b/src/leap/mail/imap/mailbox.py
@@ -0,0 +1,617 @@
+# *- coding: utf-8 -*-
+# mailbox.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Mailbox.
+"""
+import logging
+from collections import defaultdict
+
+from twisted.internet import defer
+from twisted.python import log
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common import events as leap_events
+from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
+from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.decorators import deferred
+from leap.mail.imap.fields import WithMsgFields, fields
+from leap.mail.imap.messages import MessageCollection
+from leap.mail.imap.parser import MBoxParser
+
+logger = logging.getLogger(__name__)
+
+
+class SoledadMailbox(WithMsgFields, MBoxParser):
+    """
+    A Soledad-backed IMAP mailbox.
+
+    Implements the high-level method needed for the Mailbox interfaces.
+    The low-level database methods are contained in MessageCollection class,
+    which we instantiate and make accessible in the `messages` attribute.
+    """
+    implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
+    # XXX should finish the implementation of IMailboxListener
+    # XXX should implement IMessageCopier too
+
+    messages = None
+    _closed = False
+
+    INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
+                  WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
+                  WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
+                  WithMsgFields.LIST_FLAG)
+    flags = None
+
+    CMD_MSG = "MESSAGES"
+    CMD_RECENT = "RECENT"
+    CMD_UIDNEXT = "UIDNEXT"
+    CMD_UIDVALIDITY = "UIDVALIDITY"
+    CMD_UNSEEN = "UNSEEN"
+
+    _listeners = defaultdict(set)
+
+    def __init__(self, mbox, soledad=None, rw=1):
+        """
+        SoledadMailbox constructor. Needs to get passed a name, plus a
+        Soledad instance.
+
+        :param mbox: the mailbox name
+        :type mbox: str
+
+        :param soledad: a Soledad instance.
+        :type soledad: Soledad
+
+        :param rw: read-and-write flags
+        :type rw: int
+        """
+        leap_assert(mbox, "Need a mailbox name to initialize")
+        leap_assert(soledad, "Need a soledad instance to initialize")
+
+        # XXX should move to wrapper
+        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
+                    #"soledad._db must be an instance of SQLCipherDatabase")
+
+        self.mbox = self._parse_mailbox_name(mbox)
+        self.rw = rw
+
+        self._soledad = soledad
+
+        self.messages = MessageCollection(
+            mbox=mbox, soledad=self._soledad)
+
+        if not self.getFlags():
+            self.setFlags(self.INIT_FLAGS)
+
+    @property
+    def listeners(self):
+        """
+        Returns listeners for this mbox.
+
+        The server itself is a listener to the mailbox.
+        so we can notify it (and should!) after changes in flags
+        and number of messages.
+
+        :rtype: set
+        """
+        return self._listeners[self.mbox]
+
+    def addListener(self, listener):
+        """
+        Adds a listener to the listeners queue.
+        The server adds itself as a listener when there is a SELECT,
+        so it can send EXIST commands.
+
+        :param listener: listener to add
+        :type listener: an object that implements IMailboxListener
+        """
+        logger.debug('adding mailbox listener: %s' % listener)
+        self.listeners.add(listener)
+
+    def removeListener(self, listener):
+        """
+        Removes a listener from the listeners queue.
+
+        :param listener: listener to remove
+        :type listener: an object that implements IMailboxListener
+        """
+        self.listeners.remove(listener)
+
+    def _get_mbox(self):
+        """
+        Returns mailbox document.
+
+        :return: A SoledadDocument containing this mailbox, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
+        """
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MBOX_IDX,
+                fields.TYPE_MBOX_VAL, self.mbox)
+            if query:
+                return query.pop()
+        except Exception as exc:
+            logger.error("Unhandled error %r" % exc)
+
+    def getFlags(self):
+        """
+        Returns the flags defined for this mailbox.
+
+        :returns: tuple of flags for this mailbox
+        :rtype: tuple of str
+        """
+        mbox = self._get_mbox()
+        if not mbox:
+            return None
+        flags = mbox.content.get(self.FLAGS_KEY, [])
+        return map(str, flags)
+
+    def setFlags(self, flags):
+        """
+        Sets flags for this mailbox.
+
+        :param flags: a tuple with the flags
+        :type flags: tuple of str
+        """
+        leap_assert(isinstance(flags, tuple),
+                    "flags expected to be a tuple")
+        mbox = self._get_mbox()
+        if not mbox:
+            return None
+        mbox.content[self.FLAGS_KEY] = map(str, flags)
+        self._soledad.put_doc(mbox)
+
+    # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
+
+    def _get_closed(self):
+        """
+        Return the closed attribute for this mailbox.
+
+        :return: True if the mailbox is closed
+        :rtype: bool
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.CLOSED_KEY, False)
+
+    def _set_closed(self, closed):
+        """
+        Set the closed attribute for this mailbox.
+
+        :param closed: the state to be set
+        :type closed: bool
+        """
+        leap_assert(isinstance(closed, bool), "closed needs to be boolean")
+        mbox = self._get_mbox()
+        mbox.content[self.CLOSED_KEY] = closed
+        self._soledad.put_doc(mbox)
+
+    closed = property(
+        _get_closed, _set_closed, doc="Closed attribute.")
+
+    def _get_last_uid(self):
+        """
+        Return the last uid for this mailbox.
+
+        :return: the last uid for messages in this mailbox
+        :rtype: bool
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.LAST_UID_KEY, 1)
+
+    def _set_last_uid(self, uid):
+        """
+        Sets the last uid for this mailbox.
+
+        :param uid: the uid to be set
+        :type uid: int
+        """
+        leap_assert(isinstance(uid, int), "uid has to be int")
+        mbox = self._get_mbox()
+        key = self.LAST_UID_KEY
+
+        count = self.getMessageCount()
+
+        # XXX safety-catch. If we do get duplicates,
+        # we want to avoid further duplication.
+
+        if uid >= count:
+            value = uid
+        else:
+            # something is wrong,
+            # just set the last uid
+            # beyond the max msg count.
+            logger.debug("WRONG uid < count. Setting last uid to %s", count)
+            value = count
+
+        mbox.content[key] = value
+        self._soledad.put_doc(mbox)
+
+    last_uid = property(
+        _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
+
+    def getUIDValidity(self):
+        """
+        Return the unique validity identifier for this mailbox.
+
+        :return: unique validity identifier
+        :rtype: int
+        """
+        mbox = self._get_mbox()
+        return mbox.content.get(self.CREATED_KEY, 1)
+
+    def getUID(self, message):
+        """
+        Return the UID of a message in the mailbox
+
+        .. note:: this implementation does not make much sense RIGHT NOW,
+        but in the future will be useful to get absolute UIDs from
+        message sequence numbers.
+
+        :param message: the message uid
+        :type message: int
+
+        :rtype: int
+        """
+        msg = self.messages.get_msg_by_uid(message)
+        return msg.getUID()
+
+    def getUIDNext(self):
+        """
+        Return the likely UID for the next message added to this
+        mailbox. Currently it returns the higher UID incremented by
+        one.
+
+        We increment the next uid *each* time this function gets called.
+        In this way, there will be gaps if the message with the allocated
+        uid cannot be saved. But that is preferable to having race conditions
+        if we get to parallel message adding.
+
+        :rtype: int
+        """
+        self.last_uid += 1
+        return self.last_uid
+
+    def getMessageCount(self):
+        """
+        Returns the total count of messages in this mailbox.
+
+        :rtype: int
+        """
+        return self.messages.count()
+
+    def getUnseenCount(self):
+        """
+        Returns the number of messages with the 'Unseen' flag.
+
+        :return: count of messages flagged `unseen`
+        :rtype: int
+        """
+        return self.messages.count_unseen()
+
+    def getRecentCount(self):
+        """
+        Returns the number of messages with the 'Recent' flag.
+
+        :return: count of messages flagged `recent`
+        :rtype: int
+        """
+        return self.messages.count_recent()
+
+    def isWriteable(self):
+        """
+        Get the read/write status of the mailbox.
+
+        :return: 1 if mailbox is read-writeable, 0 otherwise.
+        :rtype: int
+        """
+        return self.rw
+
+    def getHierarchicalDelimiter(self):
+        """
+        Returns the character used to delimite hierarchies in mailboxes.
+
+        :rtype: str
+        """
+        return '/'
+
+    def requestStatus(self, names):
+        """
+        Handles a status request by gathering the output of the different
+        status commands.
+
+        :param names: a list of strings containing the status commands
+        :type names: iter
+        """
+        r = {}
+        if self.CMD_MSG in names:
+            r[self.CMD_MSG] = self.getMessageCount()
+        if self.CMD_RECENT in names:
+            r[self.CMD_RECENT] = self.getRecentCount()
+        if self.CMD_UIDNEXT in names:
+            r[self.CMD_UIDNEXT] = self.last_uid + 1
+        if self.CMD_UIDVALIDITY in names:
+            r[self.CMD_UIDVALIDITY] = self.getUID()
+        if self.CMD_UNSEEN in names:
+            r[self.CMD_UNSEEN] = self.getUnseenCount()
+        return defer.succeed(r)
+
+    def addMessage(self, message, flags, date=None):
+        """
+        Adds a message to this mailbox.
+
+        :param message: the raw message
+        :type message: str
+
+        :param flags: flag list
+        :type flags: list of str
+
+        :param date: timestamp
+        :type date: str
+
+        :return: a deferred that evals to None
+        """
+        # XXX we should treat the message as an IMessage from here
+        leap_assert_type(message, basestring)
+        uid_next = self.getUIDNext()
+        logger.debug('Adding msg with UID :%s' % uid_next)
+        if flags is None:
+            flags = tuple()
+        else:
+            flags = tuple(str(flag) for flag in flags)
+
+        d = self._do_add_messages(message, flags, date, uid_next)
+        d.addCallback(self._notify_new)
+
+    @deferred
+    def _do_add_messages(self, message, flags, date, uid_next):
+        """
+        Calls to the messageCollection add_msg method (deferred to thread).
+        Invoked from addMessage.
+        """
+        self.messages.add_msg(message, flags=flags, date=date,
+                              uid=uid_next)
+
+    def _notify_new(self, *args):
+        """
+        Notify of new messages to all the listeners.
+
+        :param args: ignored.
+        """
+        exists = self.getMessageCount()
+        recent = self.getRecentCount()
+        logger.debug("NOTIFY: there are %s messages, %s recent" % (
+            exists,
+            recent))
+
+        logger.debug("listeners: %s", str(self.listeners))
+        for l in self.listeners:
+            logger.debug('notifying...')
+            l.newMessages(exists, recent)
+
+    # commands, do not rename methods
+
+    def destroy(self):
+        """
+        Called before this mailbox is permanently deleted.
+
+        Should cleanup resources, and set the \\Noselect flag
+        on the mailbox.
+        """
+        self.setFlags((self.NOSELECT_FLAG,))
+        self.deleteAllDocs()
+
+        # XXX removing the mailbox in situ for now,
+        # we should postpone the removal
+        self._soledad.delete_doc(self._get_mbox())
+
+    def expunge(self):
+        """
+        Remove all messages flagged \\Deleted
+        """
+        if not self.isWriteable():
+            raise imap4.ReadOnlyMailbox
+        delete = []
+        deleted = []
+
+        for m in self.messages.get_all_docs():
+            # XXX should operate with LeapMessages instead,
+            # so we don't expose the implementation.
+            # (so, iterate for m in self.messages)
+            if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
+                delete.append(m)
+        for m in delete:
+            deleted.append(m.content)
+            self.messages.remove(m)
+
+        # XXX should return the UIDs of the deleted messages
+        # more generically
+        return [x for x in range(len(deleted))]
+
+    @deferred
+    def fetch(self, messages, uid):
+        """
+        Retrieve one or more messages in this mailbox.
+
+        from rfc 3501: The data items to be fetched can be either a single atom
+        or a parenthesized list.
+
+        :param messages: IDs of the messages to retrieve information about
+        :type messages: MessageSet
+
+        :param uid: If true, the IDs are UIDs. They are message sequence IDs
+                    otherwise.
+        :type uid: bool
+
+        :rtype: A tuple of two-tuples of message sequence numbers and
+                LeapMessage
+        """
+        result = []
+        sequence = True if uid == 0 else False
+
+        if not messages.last:
+            try:
+                iter(messages)
+            except TypeError:
+                # looks like we cannot iterate
+                messages.last = self.last_uid
+
+        # for sequence numbers (uid = 0)
+        if sequence:
+            logger.debug("Getting msg by index: INEFFICIENT call!")
+            raise NotImplementedError
+
+        else:
+            for msg_id in messages:
+                msg = self.messages.get_msg_by_uid(msg_id)
+                if msg:
+                    result.append((msg_id, msg))
+                else:
+                    logger.debug("fetch %s, no msg found!!!" % msg_id)
+
+        if self.isWriteable():
+            self._unset_recent_flag()
+        self._signal_unread_to_ui()
+
+        # XXX workaround for hangs in thunderbird
+        #return tuple(result[:100])  # --- doesn't show all!!
+        return tuple(result)
+
+    @deferred
+    def _unset_recent_flag(self):
+        """
+        Unsets `Recent` flag from a tuple of messages.
+        Called from fetch.
+
+        From RFC, about `Recent`:
+
+        Message is "recently" arrived in this mailbox.  This session
+        is the first session to have been notified about this
+        message; if the session is read-write, subsequent sessions
+        will not see \Recent set for this message.  This flag can not
+        be altered by the client.
+
+        If it is not possible to determine whether or not this
+        session is the first session to be notified about a message,
+        then that message SHOULD be considered recent.
+        """
+        log.msg('unsetting recent flags...')
+        for msg in self.messages.get_recent():
+            msg.removeFlags((fields.RECENT_FLAG,))
+        self._signal_unread_to_ui()
+
+    @deferred
+    def _signal_unread_to_ui(self):
+        """
+        Sends unread event to ui.
+        """
+        unseen = self.getUnseenCount()
+        leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
+
+    @deferred
+    def store(self, messages, flags, mode, uid):
+        """
+        Sets the flags of one or more messages.
+
+        :param messages: The identifiers of the messages to set the flags
+        :type messages: A MessageSet object with the list of messages requested
+
+        :param flags: The flags to set, unset, or add.
+        :type flags: sequence of str
+
+        :param mode: If mode is -1, these flags should be removed from the
+                     specified messages.  If mode is 1, these flags should be
+                     added to the specified messages.  If mode is 0, all
+                     existing flags should be cleared and these flags should be
+                     added.
+        :type mode: -1, 0, or 1
+
+        :param uid: If true, the IDs specified in the query are UIDs;
+                    otherwise they are message sequence IDs.
+        :type uid: bool
+
+        :return: A dict mapping message sequence numbers to sequences of
+                 str representing the flags set on the message after this
+                 operation has been performed.
+        :rtype: dict
+
+        :raise ReadOnlyMailbox: Raised if this mailbox is not open for
+                                read-write.
+        """
+        # XXX implement also sequence (uid = 0)
+        # XXX we should prevent cclient from setting Recent flag.
+        leap_assert(not isinstance(flags, basestring),
+                    "flags cannot be a string")
+        flags = tuple(flags)
+
+        if not self.isWriteable():
+            log.msg('read only mailbox!')
+            raise imap4.ReadOnlyMailbox
+
+        if not messages.last:
+            messages.last = self.messages.count()
+
+        result = {}
+        for msg_id in messages:
+            log.msg("MSG ID = %s" % msg_id)
+            msg = self.messages.get_msg_by_uid(msg_id)
+            if mode == 1:
+                msg.addFlags(flags)
+            elif mode == -1:
+                msg.removeFlags(flags)
+            elif mode == 0:
+                msg.setFlags(flags)
+            result[msg_id] = msg.getFlags()
+
+        self._signal_unread_to_ui()
+        return result
+
+    @deferred
+    def close(self):
+        """
+        Expunge and mark as closed
+        """
+        self.expunge()
+        self.closed = True
+
+    #@deferred
+    #def copy(self, messageObject):
+        #"""
+        #Copy the given message object into this mailbox.
+        #"""
+        # XXX should just:
+        # 1. Get the message._fdoc
+        # 2. Change the UID to UIDNext for this mailbox
+        # 3. Add implements IMessageCopier
+
+    # convenience fun
+
+    def deleteAllDocs(self):
+        """
+        Deletes all docs in this mailbox
+        """
+        docs = self.messages.get_all_docs()
+        for doc in docs:
+            self.messages._soledad.delete_doc(doc)
+
+    def __repr__(self):
+        """
+        Representation string for this mailbox.
+        """
+        return u"<SoledadMailbox: mbox '%s' (%s)>" % (
+            self.mbox, self.messages.count())
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
new file mode 100644
index 0000000..b0d5da2
--- /dev/null
+++ b/src/leap/mail/imap/messages.py
@@ -0,0 +1,735 @@
+# -*- coding: utf-8 -*-
+# messages.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+LeapMessage and MessageCollection.
+"""
+import copy
+import logging
+import StringIO
+from collections import namedtuple
+
+from twisted.mail import imap4
+from twisted.python import log
+from u1db import errors as u1db_errors
+from zope.interface import implements
+from zope.proxy import sameProxiedObjects
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.mail import get_email_charset
+from leap.mail.decorators import deferred
+from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.index import IndexedDB
+from leap.mail.imap.fields import fields, WithMsgFields
+from leap.mail.imap.parser import MailParser, MBoxParser
+from leap.mail.messageflow import IMessageConsumer, MessageProducer
+
+logger = logging.getLogger(__name__)
+
+
+class LeapMessage(fields, MailParser, MBoxParser):
+
+    implements(imap4.IMessage)
+
+    def __init__(self, soledad, uid, mbox):
+        """
+        Initializes a LeapMessage.
+
+        :param soledad: a Soledad instance
+        :type soledad: Soledad
+        :param uid: the UID for the message.
+        :type uid: int or basestring
+        :param mbox: the mbox this message belongs to
+        :type mbox: basestring
+        """
+        MailParser.__init__(self)
+        self._soledad = soledad
+        self._uid = int(uid)
+        self._mbox = self._parse_mailbox_name(mbox)
+        self._chash = None
+
+        self.__cdoc = None
+
+    @property
+    def _fdoc(self):
+        """
+        An accessor to the flags document.
+        """
+        return self._get_flags_doc()
+
+    @property
+    def _cdoc(self):
+        """
+        An accessor to the content document.
+        """
+        if not self.__cdoc:
+            self.__cdoc = self._get_content_doc()
+        return self.__cdoc
+
+    @property
+    def _chash(self):
+        """
+        An accessor to the content hash for this message.
+        """
+        if not self._fdoc:
+            return None
+        return self._fdoc.content.get(fields.CONTENT_HASH_KEY, None)
+
+    # IMessage implementation
+
+    def getUID(self):
+        """
+        Retrieve the unique identifier associated with this message
+
+        :return: uid for this message
+        :rtype: int
+        """
+        return self._uid
+
+    def getFlags(self):
+        """
+        Retrieve the flags associated with this message
+
+        :return: The flags, represented as strings
+        :rtype: tuple
+        """
+        if self._uid is None:
+            return []
+
+        flags = []
+        flag_doc = self._fdoc
+        if flag_doc:
+            flags = flag_doc.content.get(self.FLAGS_KEY, None)
+        if flags:
+            flags = map(str, flags)
+        return tuple(flags)
+
+    # setFlags, addFlags, removeFlags are not in the interface spec
+    # but we use them with store command.
+
+    def setFlags(self, flags):
+        """
+        Sets the flags for this message
+
+        Returns a SoledadDocument that needs to be updated by the caller.
+
+        :param flags: the flags to update in the message.
+        :type flags: tuple of str
+
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
+        """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+        log.msg('setting flags: %s' % (self._uid))
+
+        doc = self._fdoc
+        doc.content[self.FLAGS_KEY] = flags
+        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+        doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
+        self._soledad.put_doc(doc)
+
+    def addFlags(self, flags):
+        """
+        Adds flags to this message.
+
+        Returns a SoledadDocument that needs to be updated by the caller.
+
+        :param flags: the flags to add to the message.
+        :type flags: tuple of str
+
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
+        """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+        oldflags = self.getFlags()
+        self.setFlags(tuple(set(flags + oldflags)))
+
+    def removeFlags(self, flags):
+        """
+        Remove flags from this message.
+
+        Returns a SoledadDocument that needs to be updated by the caller.
+
+        :param flags: the flags to be removed from the message.
+        :type flags: tuple of str
+
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
+        """
+        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+        oldflags = self.getFlags()
+        self.setFlags(tuple(set(oldflags) - set(flags)))
+
+    def getInternalDate(self):
+        """
+        Retrieve the date internally associated with this message
+
+        :rtype: C{str}
+        :return: An RFC822-formatted date string.
+        """
+        return str(self._cdoc.content.get(self.DATE_KEY, ''))
+
+    #
+    # IMessagePart
+    #
+
+    # XXX we should implement this interface too for the subparts
+    # so we allow nested parts...
+
+    def getBodyFile(self):
+        """
+        Retrieve a file object containing only the body of this message.
+
+        :return: file-like object opened for reading
+        :rtype: StringIO
+        """
+        fd = StringIO.StringIO()
+
+        cdoc = self._cdoc
+        content = cdoc.content.get(self.RAW_KEY, '')
+        charset = get_email_charset(
+            unicode(cdoc.content.get(self.RAW_KEY, '')))
+        try:
+            content = content.encode(charset)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            content = content.encode(charset, 'replace')
+
+        raw = self._get_raw_msg()
+        msg = self._get_parsed_msg(raw)
+        body = msg.get_payload()
+        fd.write(body)
+        # XXX SHOULD use a separate BODY FIELD ...
+        fd.seek(0)
+        return fd
+
+    def getSize(self):
+        """
+        Return the total size, in octets, of this message.
+
+        :return: size of the message, in octets
+        :rtype: int
+        """
+        size = self._cdoc.content.get(self.SIZE_KEY, False)
+        if not size:
+            # XXX fallback, should remove when all migrated.
+            size = self.getBodyFile().len
+        return size
+
+    def _get_headers(self):
+        """
+        Return the headers dict stored in this message document.
+        """
+        # XXX get from the headers doc
+        return self._cdoc.content.get(self.HEADERS_KEY, {})
+
+    def getHeaders(self, negate, *names):
+        """
+        Retrieve a group of message headers.
+
+        :param names: The names of the headers to retrieve or omit.
+        :type names: tuple of str
+
+        :param negate: If True, indicates that the headers listed in names
+                       should be omitted from the return value, rather
+                       than included.
+        :type negate: bool
+
+        :return: A mapping of header field names to header field values
+        :rtype: dict
+        """
+        headers = self._get_headers()
+        names = map(lambda s: s.upper(), names)
+        if negate:
+            cond = lambda key: key.upper() not in names
+        else:
+            cond = lambda key: key.upper() in names
+
+        # unpack and filter original dict by negate-condition
+        filter_by_cond = [
+            map(str, (key, val)) for
+            key, val in headers.items()
+            if cond(key)]
+        return dict(filter_by_cond)
+
+    def isMultipart(self):
+        """
+        Return True if this message is multipart.
+        """
+        if self._cdoc:
+            retval = self._fdoc.content.get(self.MULTIPART_KEY, False)
+            return retval
+
+    def getSubPart(self, part):
+        """
+        Retrieve a MIME submessage
+
+        :type part: C{int}
+        :param part: The number of the part to retrieve, indexed from 0.
+        :raise IndexError: Raised if the specified part does not exist.
+        :raise TypeError: Raised if this message is not multipart.
+        :rtype: Any object implementing C{IMessagePart}.
+        :return: The specified sub-part.
+        """
+        if not self.isMultipart():
+            raise TypeError
+
+        msg = self._get_parsed_msg()
+        # XXX should wrap IMessagePart
+        return msg.get_payload()[part]
+
+    #
+    # accessors
+    #
+
+    def _get_flags_doc(self):
+        """
+        Return the document that keeps the flags for this
+        message.
+        """
+        flag_docs = self._soledad.get_from_index(
+            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
+            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+        flag_doc = flag_docs[0] if flag_docs else None
+        return flag_doc
+
+    def _get_content_doc(self):
+        """
+        Return the document that keeps the flags for this
+        message.
+        """
+        cont_docs = self._soledad.get_from_index(
+            SoledadBackedAccount.TYPE_HASH_IDX,
+            fields.TYPE_MESSAGE_VAL, self._content_hash, str(self._uid))
+        cont_doc = cont_docs[0] if cont_docs else None
+        return cont_doc
+
+    def _get_raw_msg(self):
+        """
+        Return the raw msg.
+        :rtype: basestring
+        """
+        return self._cdoc.content.get(self.RAW_KEY, '')
+
+    def __getitem__(self, key):
+        """
+        Return the content of the message document.
+
+        :param key: The key
+        :type key: str
+
+        :return: The content value indexed by C{key} or None
+        :rtype: str
+        """
+        return self._cdoc.content.get(key, None)
+
+    def does_exist(self):
+        """
+        Return True if there is actually a message for this
+        UID and mbox.
+        """
+        return bool(self._fdoc)
+
+
+SoledadWriterPayload = namedtuple(
+    'SoledadWriterPayload', ['mode', 'payload'])
+
+SoledadWriterPayload.CREATE = 1
+SoledadWriterPayload.PUT = 2
+
+
+class SoledadDocWriter(object):
+    """
+    This writer will create docs serially in the local soledad database.
+    """
+
+    implements(IMessageConsumer)
+
+    def __init__(self, soledad):
+        """
+        Initialize the writer.
+
+        :param soledad: the soledad instance
+        :type soledad: Soledad
+        """
+        self._soledad = soledad
+
+    def consume(self, queue):
+        """
+        Creates a new document in soledad db.
+
+        :param queue: queue to get item from, with content of the document
+                      to be inserted.
+        :type queue: Queue
+        """
+        empty = queue.empty()
+        while not empty:
+            item = queue.get()
+            if item.mode == SoledadWriterPayload.CREATE:
+                call = self._soledad.create_doc
+            elif item.mode == SoledadWriterPayload.PUT:
+                call = self._soledad.put_doc
+
+            # should handle errors
+            try:
+                call(item.payload)
+            except u1db_errors.RevisionConflict as exc:
+                logger.error("Error: %r" % (exc,))
+                raise exc
+
+            empty = queue.empty()
+
+
+class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
+    """
+    A collection of messages, surprisingly.
+
+    It is tied to a selected mailbox name that is passed to constructor.
+    Implements a filter query over the messages contained in a soledad
+    database.
+    """
+    # XXX this should be able to produce a MessageSet methinks
+
+    EMPTY_MSG = {
+        fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL,
+        fields.UID_KEY: 1,
+        fields.MBOX_KEY: fields.INBOX_VAL,
+
+        fields.SUBJECT_KEY: "",
+        fields.DATE_KEY: "",
+        fields.RAW_KEY: "",
+
+        # XXX should separate headers into another doc
+        fields.HEADERS_KEY: {},
+    }
+
+    EMPTY_FLAGS = {
+        fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
+        fields.UID_KEY: 1,
+        fields.MBOX_KEY: fields.INBOX_VAL,
+
+        fields.FLAGS_KEY: [],
+        fields.SEEN_KEY: False,
+        fields.RECENT_KEY: True,
+        fields.MULTIPART_KEY: False,
+    }
+
+    # get from SoledadBackedAccount the needed index-related constants
+    INDEXES = SoledadBackedAccount.INDEXES
+    TYPE_IDX = SoledadBackedAccount.TYPE_IDX
+
+    def __init__(self, mbox=None, soledad=None):
+        """
+        Constructor for MessageCollection.
+
+        :param mbox: the name of the mailbox. It is the name
+                     with which we filter the query over the
+                     messages database
+        :type mbox: str
+
+        :param soledad: Soledad database
+        :type soledad: Soledad instance
+        """
+        MailParser.__init__(self)
+        leap_assert(mbox, "Need a mailbox name to initialize")
+        leap_assert(mbox.strip() != "", "mbox cannot be blank space")
+        leap_assert(isinstance(mbox, (str, unicode)),
+                    "mbox needs to be a string")
+        leap_assert(soledad, "Need a soledad instance to initialize")
+
+        # okay, all in order, keep going...
+        self.mbox = self._parse_mailbox_name(mbox)
+        self._soledad = soledad
+        self.initialize_db()
+
+        # I think of someone like nietzsche when reading this
+
+        # this will be the producer that will enqueue the content
+        # to be processed serially by the consumer (the writer). We just
+        # need to `put` the new material on its plate.
+
+        self.soledad_writer = MessageProducer(
+            SoledadDocWriter(soledad),
+            period=0.05)
+
+    def _get_empty_msg(self):
+        """
+        Returns an empty message.
+
+        :return: a dict containing a default empty message
+        :rtype: dict
+        """
+        return copy.deepcopy(self.EMPTY_MSG)
+
+    def _get_empty_flags_doc(self):
+        """
+        Returns an empty doc for storing flags.
+
+        :return:
+        :rtype:
+        """
+        return copy.deepcopy(self.EMPTY_FLAGS)
+
+    @deferred
+    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
+        """
+        Creates a new message document.
+
+        :param raw: the raw message
+        :type raw: str
+
+        :param subject: subject of the message.
+        :type subject: str
+
+        :param flags: flags
+        :type flags: list
+
+        :param date: the received date for the message
+        :type date: str
+
+        :param uid: the message uid for this mailbox
+        :type uid: int
+        """
+        # TODO: split in smaller methods
+        logger.debug('adding message')
+        if flags is None:
+            flags = tuple()
+        leap_assert_type(flags, tuple)
+
+        content_doc = self._get_empty_msg()
+        flags_doc = self._get_empty_flags_doc()
+
+        content_doc[self.MBOX_KEY] = self.mbox
+        flags_doc[self.MBOX_KEY] = self.mbox
+        # ...should get a sanity check here.
+        content_doc[self.UID_KEY] = uid
+        flags_doc[self.UID_KEY] = uid
+
+        if flags:
+            flags_doc[self.FLAGS_KEY] = map(self._stringify, flags)
+            flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags
+
+        msg = self._get_parsed_msg(raw)
+        headers = dict(msg)
+
+        logger.debug("adding. is multipart:%s" % msg.is_multipart())
+        flags_doc[self.MULTIPART_KEY] = msg.is_multipart()
+        # XXX get lower case for keys?
+        # XXX get headers doc
+        content_doc[self.HEADERS_KEY] = headers
+        # set subject based on message headers and eventually replace by
+        # subject given as param
+        if self.SUBJECT_FIELD in headers:
+            content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
+        if subject is not None:
+            content_doc[self.SUBJECT_KEY] = subject
+
+        # XXX could separate body into its own doc
+        # but should also separate multiparts
+        # that should be wrapped in MessagePart
+        content_doc[self.RAW_KEY] = self._stringify(raw)
+        content_doc[self.SIZE_KEY] = len(raw)
+
+        if not date and self.DATE_FIELD in headers:
+            content_doc[self.DATE_KEY] = headers[self.DATE_FIELD]
+        else:
+            content_doc[self.DATE_KEY] = date
+
+        logger.debug('enqueuing message for write')
+
+        ptuple = SoledadWriterPayload
+        self.soledad_writer.put(ptuple(
+            mode=ptuple.CREATE, payload=content_doc))
+        self.soledad_writer.put(ptuple(
+            mode=ptuple.CREATE, payload=flags_doc))
+
+    def remove(self, msg):
+        """
+        Removes a message.
+
+        :param msg: a  Leapmessage instance
+        :type msg: LeapMessage
+        """
+        # XXX remove
+        #self._soledad.delete_doc(msg)
+        msg.remove()
+
+    # getters
+
+    def get_msg_by_uid(self, uid):
+        """
+        Retrieves a LeapMessage by UID.
+
+        :param uid: the message uid to query by
+        :type uid: int
+
+        :return: A LeapMessage instance matching the query,
+                 or None if not found.
+        :rtype: LeapMessage
+        """
+        msg = LeapMessage(self._soledad, uid, self.mbox)
+        if not msg.does_exist():
+            return None
+        return msg
+
+    def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
+        """
+        Get all documents for the selected mailbox of the
+        passed type. By default, it returns the flag docs.
+
+        If you want acess to the content, use __iter__ instead
+
+        :return: a list of u1db documents
+        :rtype: list of SoledadDocument
+        """
+        if _type not in fields.__dict__.values():
+            raise TypeError("Wrong type passed to get_all")
+
+        if sameProxiedObjects(self._soledad, None):
+            logger.warning('Tried to get messages but soledad is None!')
+            return []
+
+        all_docs = [doc for doc in self._soledad.get_from_index(
+            SoledadBackedAccount.TYPE_MBOX_IDX,
+            _type, self.mbox)]
+
+        # inneficient, but first let's grok it and then
+        # let's worry about efficiency.
+        # XXX FIXINDEX -- should implement order by in soledad
+        return sorted(all_docs, key=lambda item: item.content['uid'])
+
+    def all_msg_iter(self):
+        """
+        Return an iterator trhough the UIDs of all messages, sorted in
+        ascending order.
+        """
+        all_uids = (doc.content[self.UID_KEY] for doc in
+                    self._soledad.get_from_index(
+                        SoledadBackedAccount.TYPE_MBOX_IDX,
+                        self.TYPE_FLAGS_VAL, self.mbox))
+        return (u for u in sorted(all_uids))
+
+    def count(self):
+        """
+        Return the count of messages for this mailbox.
+
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            SoledadBackedAccount.TYPE_MBOX_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox)
+        return count
+
+    # unseen messages
+
+    def unseen_iter(self):
+        """
+        Get an iterator for the message UIDs with no `seen` flag
+        for this mailbox.
+
+        :return: iterator through unseen message doc UIDs
+        :rtype: iterable
+        """
+        return (doc.content[self.UID_KEY] for doc in
+                self._soledad.get_from_index(
+                    SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
+                    self.TYPE_FLAGS_VAL, self.mbox, '0'))
+
+    def count_unseen(self):
+        """
+        Count all messages with the `Unseen` flag.
+
+        :returns: count
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
+            self.TYPE_FLAGS_VAL, self.mbox, '0')
+        return count
+
+    def get_unseen(self):
+        """
+        Get all messages with the `Unseen` flag
+
+        :returns: a list of LeapMessages
+        :rtype: list
+        """
+        return [LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.unseen_iter()]
+
+    # recent messages
+
+    def recent_iter(self):
+        """
+        Get an iterator for the message docs with `recent` flag.
+
+        :return: iterator through recent message docs
+        :rtype: iterable
+        """
+        return (doc.content[self.UID_KEY] for doc in
+                self._soledad.get_from_index(
+                    SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
+                    self.TYPE_FLAGS_VAL, self.mbox, '1'))
+
+    def get_recent(self):
+        """
+        Get all messages with the `Recent` flag.
+
+        :returns: a list of LeapMessages
+        :rtype: list
+        """
+        return [LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.recent_iter()]
+
+    def count_recent(self):
+        """
+        Count all messages with the `Recent` flag.
+
+        :returns: count
+        :rtype: int
+        """
+        count = self._soledad.get_count_from_index(
+            SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
+            self.TYPE_FLAGS_VAL, self.mbox, '1')
+        return count
+
+    def __len__(self):
+        """
+        Returns the number of messages on this mailbox.
+
+        :rtype: int
+        """
+        return self.count()
+
+    def __iter__(self):
+        """
+        Returns an iterator over all messages.
+
+        :returns: iterator of dicts with content for all messages.
+        :rtype: iterable
+        """
+        return (LeapMessage(self._soledad, docuid, self.mbox)
+                for docuid in self.all_msg_iter())
+
+    def __repr__(self):
+        """
+        Representation string for this object.
+        """
+        return u"<MessageCollection: mbox '%s' (%s)>" % (
+            self.mbox, self.count())
+
+    # XXX should implement __eq__ also !!! --- use a hash
+    # of content for that, will be used for dedup.
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
new file mode 100644
index 0000000..1ae19c0
--- /dev/null
+++ b/src/leap/mail/imap/parser.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# parser.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Mail parser mixins.
+"""
+import cStringIO
+import StringIO
+import re
+
+from email.parser import Parser
+
+
+class MailParser(object):
+    """
+    Mixin with utility methods to parse raw messages.
+    """
+    def __init__(self):
+        """
+        Initializes the mail parser.
+        """
+        self._parser = Parser()
+
+    def _get_parsed_msg(self, raw):
+        """
+        Return a parsed Message.
+
+        :param raw: the raw string to parse
+        :type raw: basestring, or StringIO object
+        """
+        msg = self._get_parser_fun(raw)(raw, True)
+        return msg
+
+    def _get_parser_fun(self, o):
+        """
+        Retunn the proper parser function for an object.
+
+        :param o: object
+        :type o: object
+        :param parser: an instance of email.parser.Parser
+        :type parser: email.parser.Parser
+        """
+        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+            return self._parser.parse
+        if isinstance(o, basestring):
+            return self._parser.parsestr
+        # fallback
+        return self._parser.parsestr
+
+    def _stringify(self, o):
+        """
+        Return a string object.
+
+        :param o: object
+        :type o: object
+        """
+        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+            return o.getvalue()
+        else:
+            return o
+
+
+class MBoxParser(object):
+    """
+    Utility function to parse mailbox names.
+    """
+    INBOX_NAME = "INBOX"
+    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
+    def _parse_mailbox_name(self, name):
+        """
+        :param name: the name of the mailbox
+        :type name: unicode
+
+        :rtype: unicode
+        """
+        if self.INBOX_RE.match(name):
+            # ensure inital INBOX is uppercase
+            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
+        return name
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
deleted file mode 100644
index 57587a5..0000000
--- a/src/leap/mail/imap/server.py
+++ /dev/null
@@ -1,1897 +0,0 @@
-# -*- coding: utf-8 -*-
-# server.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-"""
-Soledad-backed IMAP Server.
-"""
-import copy
-import logging
-import StringIO
-import cStringIO
-import time
-import re
-
-from collections import defaultdict, namedtuple
-from email.parser import Parser
-
-from zope.interface import implements
-from zope.proxy import sameProxiedObjects
-
-from twisted.mail import imap4
-from twisted.internet import defer
-from twisted.python import log
-
-from u1db import errors as u1db_errors
-
-from leap.common import events as leap_events
-from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.mail import get_email_charset
-from leap.mail.messageflow import IMessageConsumer, MessageProducer
-from leap.mail.decorators import deferred
-from leap.soledad.client import Soledad
-
-logger = logging.getLogger(__name__)
-
-
-class MissingIndexError(Exception):
-    """
-    Raises when tried to access a non existent index document.
-    """
-
-
-class BadIndexError(Exception):
-    """
-    Raises when index is malformed or has the wrong cardinality.
-    """
-
-
-class WithMsgFields(object):
-    """
-    Container class for class-attributes to be shared by
-    several message-related classes.
-    """
-    # Internal representation of Message
-    DATE_KEY = "date"
-    HEADERS_KEY = "headers"
-    FLAGS_KEY = "flags"
-    MBOX_KEY = "mbox"
-    RAW_KEY = "raw"
-    SUBJECT_KEY = "subject"
-    UID_KEY = "uid"
-    MULTIPART_KEY = "multi"
-    SIZE_KEY = "size"
-
-    # Mailbox specific keys
-    CLOSED_KEY = "closed"
-    CREATED_KEY = "created"
-    SUBSCRIBED_KEY = "subscribed"
-    RW_KEY = "rw"
-    LAST_UID_KEY = "lastuid"
-
-    # Document Type, for indexing
-    TYPE_KEY = "type"
-    TYPE_MESSAGE_VAL = "msg"
-    TYPE_MBOX_VAL = "mbox"
-    TYPE_FLAGS_VAL = "flags"
-    # should add also a headers val
-
-    INBOX_VAL = "inbox"
-
-    # Flags for SoledadDocument for indexing.
-    SEEN_KEY = "seen"
-    RECENT_KEY = "recent"
-
-    # Flags in Mailbox and Message
-    SEEN_FLAG = "\\Seen"
-    RECENT_FLAG = "\\Recent"
-    ANSWERED_FLAG = "\\Answered"
-    FLAGGED_FLAG = "\\Flagged"  # yo dawg
-    DELETED_FLAG = "\\Deleted"
-    DRAFT_FLAG = "\\Draft"
-    NOSELECT_FLAG = "\\Noselect"
-    LIST_FLAG = "List"  # is this OK? (no \. ie, no system flag)
-
-    # Fields in mail object
-    SUBJECT_FIELD = "Subject"
-    DATE_FIELD = "Date"
-
-fields = WithMsgFields  # alias for convenience
-
-
-class IndexedDB(object):
-    """
-    Methods dealing with the index.
-
-    This is a MixIn that needs access to the soledad instance,
-    and also assumes that a INDEXES attribute is accessible to the instance.
-
-    INDEXES must be a dictionary of type:
-    {'index-name': ['field1', 'field2']}
-    """
-    # TODO we might want to move this to soledad itself, check
-
-    def initialize_db(self):
-        """
-        Initialize the database.
-        """
-        leap_assert(self._soledad,
-                    "Need a soledad attribute accesible in the instance")
-        leap_assert_type(self.INDEXES, dict)
-
-        # Ask the database for currently existing indexes.
-        if not self._soledad:
-            logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
-            return
-        db_indexes = dict()
-        if self._soledad is not None:
-            db_indexes = dict(self._soledad.list_indexes())
-        for name, expression in SoledadBackedAccount.INDEXES.items():
-            if name not in db_indexes:
-                # The index does not yet exist.
-                self._soledad.create_index(name, *expression)
-                continue
-
-            if expression == db_indexes[name]:
-                # The index exists and is up to date.
-                continue
-            # The index exists but the definition is not what expected, so we
-            # delete it and add the proper index expression.
-            self._soledad.delete_index(name)
-            self._soledad.create_index(name, *expression)
-
-
-class MailParser(object):
-    """
-    Mixin with utility methods to parse raw messages.
-    """
-    def __init__(self):
-        """
-        Initializes the mail parser.
-        """
-        self._parser = Parser()
-
-    def _get_parsed_msg(self, raw):
-        """
-        Return a parsed Message.
-
-        :param raw: the raw string to parse
-        :type raw: basestring, or StringIO object
-        """
-        msg = self._get_parser_fun(raw)(raw, True)
-        return msg
-
-    def _get_parser_fun(self, o):
-        """
-        Retunn the proper parser function for an object.
-
-        :param o: object
-        :type o: object
-        :param parser: an instance of email.parser.Parser
-        :type parser: email.parser.Parser
-        """
-        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-            return self._parser.parse
-        if isinstance(o, basestring):
-            return self._parser.parsestr
-        # fallback
-        return self._parser.parsestr
-
-    def _stringify(self, o):
-        """
-        Return a string object.
-
-        :param o: object
-        :type o: object
-        """
-        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-            return o.getvalue()
-        else:
-            return o
-
-
-class MBoxParser(object):
-    """
-    Utility function to parse mailbox names.
-    """
-    INBOX_NAME = "INBOX"
-    INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
-    def _parse_mailbox_name(self, name):
-        """
-        :param name: the name of the mailbox
-        :type name: unicode
-
-        :rtype: unicode
-        """
-        if self.INBOX_RE.match(name):
-            # ensure inital INBOX is uppercase
-            return self.INBOX_NAME + name[len(self.INBOX_NAME):]
-        return name
-
-
-#######################################
-# Soledad Account
-#######################################
-
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
-    """
-    An implementation of IAccount and INamespacePresenteer
-    that is backed by Soledad Encrypted Documents.
-    """
-
-    implements(imap4.IAccount, imap4.INamespacePresenter)
-
-    _soledad = None
-    selected = None
-
-    TYPE_IDX = 'by-type'
-    TYPE_MBOX_IDX = 'by-type-and-mbox'
-    TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
-    TYPE_SUBS_IDX = 'by-type-and-subscribed'
-    TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
-    TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
-    # Tomas created the `recent and seen index`, but the semantic is not too
-    # correct since the recent flag is volatile.
-    TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
-    KTYPE = WithMsgFields.TYPE_KEY
-    MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL
-
-    INDEXES = {
-        # generic
-        TYPE_IDX: [KTYPE],
-        TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
-        TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY],
-
-        # mailboxes
-        TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
-        # messages
-        TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
-        TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
-        TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
-                                  'bool(recent)', 'bool(seen)'],
-    }
-
-    MBOX_KEY = MBOX_VAL
-
-    EMPTY_MBOX = {
-        WithMsgFields.TYPE_KEY: MBOX_KEY,
-        WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
-        WithMsgFields.SUBJECT_KEY: "",
-        WithMsgFields.FLAGS_KEY: [],
-        WithMsgFields.CLOSED_KEY: False,
-        WithMsgFields.SUBSCRIBED_KEY: False,
-        WithMsgFields.RW_KEY: 1,
-        WithMsgFields.LAST_UID_KEY: 0
-    }
-
-    def __init__(self, account_name, soledad=None):
-        """
-        Creates a SoledadAccountIndex that keeps track of the mailboxes
-        and subscriptions handled by this account.
-
-        :param acct_name: The name of the account (user id).
-        :type acct_name: str
-
-        :param soledad: a Soledad instance.
-        :param soledad: Soledad
-        """
-        leap_assert(soledad, "Need a soledad instance to initialize")
-        leap_assert_type(soledad, Soledad)
-
-        # XXX SHOULD assert too that the name matches the user/uuid with which
-        # soledad has been initialized.
-
-        self._account_name = self._parse_mailbox_name(account_name)
-        self._soledad = soledad
-
-        self.initialize_db()
-
-        # every user should have the right to an inbox folder
-        # at least, so let's make one!
-
-        if not self.mailboxes:
-            self.addMailbox(self.INBOX_NAME)
-
-    def _get_empty_mailbox(self):
-        """
-        Returns an empty mailbox.
-
-        :rtype: dict
-        """
-        return copy.deepcopy(self.EMPTY_MBOX)
-
-    def _get_mailbox_by_name(self, name):
-        """
-        Return an mbox document by name.
-
-        :param name: the name of the mailbox
-        :type name: str
-
-        :rtype: SoledadDocument
-        """
-        doc = self._soledad.get_from_index(
-            self.TYPE_MBOX_IDX, self.MBOX_KEY,
-            self._parse_mailbox_name(name))
-        return doc[0] if doc else None
-
-    @property
-    def mailboxes(self):
-        """
-        A list of the current mailboxes for this account.
-        """
-        return [doc.content[self.MBOX_KEY]
-                for doc in self._soledad.get_from_index(
-                    self.TYPE_IDX, self.MBOX_KEY)]
-
-    @property
-    def subscriptions(self):
-        """
-        A list of the current subscriptions for this account.
-        """
-        return [doc.content[self.MBOX_KEY]
-                for doc in self._soledad.get_from_index(
-                    self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
-
-    def getMailbox(self, name):
-        """
-        Returns a Mailbox with that name, without selecting it.
-
-        :param name: name of the mailbox
-        :type name: str
-
-        :returns: a a SoledadMailbox instance
-        :rtype: SoledadMailbox
-        """
-        name = self._parse_mailbox_name(name)
-
-        if name not in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox")
-
-        return SoledadMailbox(name, soledad=self._soledad)
-
-    ##
-    ## IAccount
-    ##
-
-    def addMailbox(self, name, creation_ts=None):
-        """
-        Add a mailbox to the account.
-
-        :param name: the name of the mailbox
-        :type name: str
-
-        :param creation_ts: an optional creation timestamp to be used as
-                            mailbox id. A timestamp will be used if no
-                            one is provided.
-        :type creation_ts: int
-
-        :returns: True if successful
-        :rtype: bool
-        """
-        name = self._parse_mailbox_name(name)
-
-        if name in self.mailboxes:
-            raise imap4.MailboxCollision, name
-
-        if not creation_ts:
-            # by default, we pass an int value
-            # taken from the current time
-            # we make sure to take enough decimals to get a unique
-            # mailbox-uidvalidity.
-            creation_ts = int(time.time() * 10E2)
-
-        mbox = self._get_empty_mailbox()
-        mbox[self.MBOX_KEY] = name
-        mbox[self.CREATED_KEY] = creation_ts
-
-        doc = self._soledad.create_doc(mbox)
-        return bool(doc)
-
-    def create(self, pathspec):
-        """
-        Create a new mailbox from the given hierarchical name.
-
-        :param pathspec: The full hierarchical name of a new mailbox to create.
-                         If any of the inferior hierarchical names to this one
-                         do not exist, they are created as well.
-        :type pathspec: str
-
-        :return: A true value if the creation succeeds.
-        :rtype: bool
-
-        :raise MailboxException: Raised if this mailbox cannot be added.
-        """
-        # TODO raise MailboxException
-        paths = filter(
-            None,
-            self._parse_mailbox_name(pathspec).split('/'))
-        for accum in range(1, len(paths)):
-            try:
-                self.addMailbox('/'.join(paths[:accum]))
-            except imap4.MailboxCollision:
-                pass
-        try:
-            self.addMailbox('/'.join(paths))
-        except imap4.MailboxCollision:
-            if not pathspec.endswith('/'):
-                return False
-        return True
-
-    def select(self, name, readwrite=1):
-        """
-        Selects a mailbox.
-
-        :param name: the mailbox to select
-        :type name: str
-
-        :param readwrite: 1 for readwrite permissions.
-        :type readwrite: int
-
-        :rtype: bool
-        """
-        name = self._parse_mailbox_name(name)
-
-        if name not in self.mailboxes:
-            return None
-
-        self.selected = name
-
-        return SoledadMailbox(
-            name, rw=readwrite,
-            soledad=self._soledad)
-
-    def delete(self, name, force=False):
-        """
-        Deletes a mailbox.
-
-        Right now it does not purge the messages, but just removes the mailbox
-        name from the mailboxes list!!!
-
-        :param name: the mailbox to be deleted
-        :type name: str
-
-        :param force: if True, it will not check for noselect flag or inferior
-                      names. use with care.
-        :type force: bool
-        """
-        name = self._parse_mailbox_name(name)
-
-        if not name in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox")
-
-        mbox = self.getMailbox(name)
-
-        if force is False:
-            # See if this box is flagged \Noselect
-            # XXX use mbox.flags instead?
-            if self.NOSELECT_FLAG in mbox.getFlags():
-                # Check for hierarchically inferior mailboxes with this one
-                # as part of their root.
-                for others in self.mailboxes:
-                    if others != name and others.startswith(name):
-                        raise imap4.MailboxException, (
-                            "Hierarchically inferior mailboxes "
-                            "exist and \\Noselect is set")
-        mbox.destroy()
-
-        # XXX FIXME --- not honoring the inferior names...
-
-        # if there are no hierarchically inferior names, we will
-        # delete it from our ken.
-        #if self._inferiorNames(name) > 1:
-            # ??! -- can this be rite?
-            #self._index.removeMailbox(name)
-
-    def rename(self, oldname, newname):
-        """
-        Renames a mailbox.
-
-        :param oldname: old name of the mailbox
-        :type oldname: str
-
-        :param newname: new name of the mailbox
-        :type newname: str
-        """
-        oldname = self._parse_mailbox_name(oldname)
-        newname = self._parse_mailbox_name(newname)
-
-        if oldname not in self.mailboxes:
-            raise imap4.NoSuchMailbox, oldname
-
-        inferiors = self._inferiorNames(oldname)
-        inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
-
-        for (old, new) in inferiors:
-            if new in self.mailboxes:
-                raise imap4.MailboxCollision, new
-
-        for (old, new) in inferiors:
-            mbox = self._get_mailbox_by_name(old)
-            mbox.content[self.MBOX_KEY] = new
-            self._soledad.put_doc(mbox)
-
-        # XXX ---- FIXME!!!! ------------------------------------
-        # until here we just renamed the index...
-        # We have to rename also the occurrence of this
-        # mailbox on ALL the messages that are contained in it!!!
-        # ... we maybe could use a reference to the doc_id
-        # in each msg, instead of the "mbox" field in msgs
-        # -------------------------------------------------------
-
-    def _inferiorNames(self, name):
-        """
-        Return hierarchically inferior mailboxes.
-
-        :param name: name of the mailbox
-        :rtype: list
-        """
-        # XXX use wildcard query instead
-        inferiors = []
-        for infname in self.mailboxes:
-            if infname.startswith(name):
-                inferiors.append(infname)
-        return inferiors
-
-    def isSubscribed(self, name):
-        """
-        Returns True if user is subscribed to this mailbox.
-
-        :param name: the mailbox to be checked.
-        :type name: str
-
-        :rtype: bool
-        """
-        mbox = self._get_mailbox_by_name(name)
-        return mbox.content.get('subscribed', False)
-
-    def _set_subscription(self, name, value):
-        """
-        Sets the subscription value for a given mailbox
-
-        :param name: the mailbox
-        :type name: str
-
-        :param value: the boolean value
-        :type value: bool
-        """
-        # maybe we should store subscriptions in another
-        # document...
-        if not name in self.mailboxes:
-            self.addMailbox(name)
-        mbox = self._get_mailbox_by_name(name)
-
-        if mbox:
-            mbox.content[self.SUBSCRIBED_KEY] = value
-            self._soledad.put_doc(mbox)
-
-    def subscribe(self, name):
-        """
-        Subscribe to this mailbox
-
-        :param name: name of the mailbox
-        :type name: str
-        """
-        name = self._parse_mailbox_name(name)
-        if name not in self.subscriptions:
-            self._set_subscription(name, True)
-
-    def unsubscribe(self, name):
-        """
-        Unsubscribe from this mailbox
-
-        :param name: name of the mailbox
-        :type name: str
-        """
-        name = self._parse_mailbox_name(name)
-        if name not in self.subscriptions:
-            raise imap4.MailboxException, "Not currently subscribed to " + name
-        self._set_subscription(name, False)
-
-    def listMailboxes(self, ref, wildcard):
-        """
-        List the mailboxes.
-
-        from rfc 3501:
-        returns a subset of names from the complete set
-        of all names available to the client.  Zero or more untagged LIST
-        replies are returned, containing the name attributes, hierarchy
-        delimiter, and name.
-
-        :param ref: reference name
-        :type ref: str
-
-        :param wildcard: mailbox name with possible wildcards
-        :type wildcard: str
-        """
-        # XXX use wildcard in index query
-        ref = self._inferiorNames(
-            self._parse_mailbox_name(ref))
-        wildcard = imap4.wildcardToRegexp(wildcard, '/')
-        return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
-
-    ##
-    ## INamespacePresenter
-    ##
-
-    def getPersonalNamespaces(self):
-        return [["", "/"]]
-
-    def getSharedNamespaces(self):
-        return None
-
-    def getOtherNamespaces(self):
-        return None
-
-    # extra, for convenience
-
-    def deleteAllMessages(self, iknowhatiamdoing=False):
-        """
-        Deletes all messages from all mailboxes.
-        Danger! high voltage!
-
-        :param iknowhatiamdoing: confirmation parameter, needs to be True
-                                 to proceed.
-        """
-        if iknowhatiamdoing is True:
-            for mbox in self.mailboxes:
-                self.delete(mbox, force=True)
-
-    def __repr__(self):
-        """
-        Representation string for this object.
-        """
-        return "<SoledadBackedAccount (%s)>" % self._account_name
-
-#######################################
-# LeapMessage, MessageCollection
-# and Mailbox
-#######################################
-
-
-class LeapMessage(fields, MailParser, MBoxParser):
-
-    implements(imap4.IMessage)
-
-    def __init__(self, soledad, uid, mbox):
-        """
-        Initializes a LeapMessage.
-
-        :param soledad: a Soledad instance
-        :type soledad: Soledad
-        :param uid: the UID for the message.
-        :type uid: int or basestring
-        :param mbox: the mbox this message belongs to
-        :type mbox: basestring
-        """
-        MailParser.__init__(self)
-        self._soledad = soledad
-        self._uid = int(uid)
-        self._mbox = self._parse_mailbox_name(mbox)
-
-        self.__cdoc = None
-
-    @property
-    def _fdoc(self):
-        """
-        An accessor to the flags docuemnt
-        """
-        return self._get_flags_doc()
-
-    @property
-    def _cdoc(self):
-        """
-        An accessor to the content docuemnt
-        """
-        if not self.__cdoc:
-            self.__cdoc = self._get_content_doc()
-        return self.__cdoc
-
-    def getUID(self):
-        """
-        Retrieve the unique identifier associated with this message
-
-        :return: uid for this message
-        :rtype: int
-        """
-        return self._uid
-
-    def getFlags(self):
-        """
-        Retrieve the flags associated with this message
-
-        :return: The flags, represented as strings
-        :rtype: tuple
-        """
-        if self._uid is None:
-            return []
-
-        flags = []
-        flag_doc = self._fdoc
-        if flag_doc:
-            flags = flag_doc.content.get(self.FLAGS_KEY, None)
-        if flags:
-            flags = map(str, flags)
-        return tuple(flags)
-
-    # setFlags, addFlags, removeFlags are not in the interface spec
-    # but we use them with store command.
-
-    def setFlags(self, flags):
-        """
-        Sets the flags for this message
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to update in the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        log.msg('setting flags: %s' % (self._uid))
-
-        doc = self._fdoc
-        doc.content[self.FLAGS_KEY] = flags
-        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-        doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
-        self._soledad.put_doc(doc)
-
-    def addFlags(self, flags):
-        """
-        Adds flags to this message.
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to add to the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        oldflags = self.getFlags()
-        self.setFlags(tuple(set(flags + oldflags)))
-
-    def removeFlags(self, flags):
-        """
-        Remove flags from this message.
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to be removed from the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        oldflags = self.getFlags()
-        self.setFlags(tuple(set(oldflags) - set(flags)))
-
-    def getInternalDate(self):
-        """
-        Retrieve the date internally associated with this message
-
-        :rtype: C{str}
-        :return: An RFC822-formatted date string.
-        """
-        return str(self._cdoc.content.get(self.DATE_KEY, ''))
-
-    #
-    # IMessagePart
-    #
-
-    # XXX we should implement this interface too for the subparts
-    # so we allow nested parts...
-
-    def getBodyFile(self):
-        """
-        Retrieve a file object containing only the body of this message.
-
-        :return: file-like object opened for reading
-        :rtype: StringIO
-        """
-        fd = StringIO.StringIO()
-
-        cdoc = self._cdoc
-        content = cdoc.content.get(self.RAW_KEY, '')
-        charset = get_email_charset(
-            unicode(cdoc.content.get(self.RAW_KEY, '')))
-        try:
-            content = content.encode(charset)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            content = content.encode(charset, 'replace')
-
-        raw = self._get_raw_msg()
-        msg = self._get_parsed_msg(raw)
-        body = msg.get_payload()
-        fd.write(body)
-        # XXX SHOULD use a separate BODY FIELD ...
-        fd.seek(0)
-        return fd
-
-    def getSize(self):
-        """
-        Return the total size, in octets, of this message.
-
-        :return: size of the message, in octets
-        :rtype: int
-        """
-        size = self._cdoc.content.get(self.SIZE_KEY, False)
-        if not size:
-            # XXX fallback, should remove when all migrated.
-            size = self.getBodyFile().len
-        return size
-
-    def _get_headers(self):
-        """
-        Return the headers dict stored in this message document.
-        """
-        # XXX get from the headers doc
-        return self._cdoc.content.get(self.HEADERS_KEY, {})
-
-    def getHeaders(self, negate, *names):
-        """
-        Retrieve a group of message headers.
-
-        :param names: The names of the headers to retrieve or omit.
-        :type names: tuple of str
-
-        :param negate: If True, indicates that the headers listed in names
-                       should be omitted from the return value, rather
-                       than included.
-        :type negate: bool
-
-        :return: A mapping of header field names to header field values
-        :rtype: dict
-        """
-        headers = self._get_headers()
-        names = map(lambda s: s.upper(), names)
-        if negate:
-            cond = lambda key: key.upper() not in names
-        else:
-            cond = lambda key: key.upper() in names
-
-        # unpack and filter original dict by negate-condition
-        filter_by_cond = [
-            map(str, (key, val)) for
-            key, val in headers.items()
-            if cond(key)]
-        return dict(filter_by_cond)
-
-    def isMultipart(self):
-        """
-        Return True if this message is multipart.
-        """
-        if self._cdoc:
-            retval = self._fdoc.content.get(self.MULTIPART_KEY, False)
-            return retval
-
-    def getSubPart(self, part):
-        """
-        Retrieve a MIME submessage
-
-        :type part: C{int}
-        :param part: The number of the part to retrieve, indexed from 0.
-        :raise IndexError: Raised if the specified part does not exist.
-        :raise TypeError: Raised if this message is not multipart.
-        :rtype: Any object implementing C{IMessagePart}.
-        :return: The specified sub-part.
-        """
-        if not self.isMultipart():
-            raise TypeError
-
-        msg = self._get_parsed_msg()
-        # XXX should wrap IMessagePart
-        return msg.get_payload()[part]
-
-    #
-    # accessors
-    #
-
-    def _get_flags_doc(self):
-        """
-        Return the document that keeps the flags for this
-        message.
-        """
-        flag_docs = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
-            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
-        flag_doc = flag_docs[0] if flag_docs else None
-        return flag_doc
-
-    def _get_content_doc(self):
-        """
-        Return the document that keeps the flags for this
-        message.
-        """
-        cont_docs = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
-            fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid))
-        cont_doc = cont_docs[0] if cont_docs else None
-        return cont_doc
-
-    def _get_raw_msg(self):
-        """
-        Return the raw msg.
-        :rtype: basestring
-        """
-        return self._cdoc.content.get(self.RAW_KEY, '')
-
-    def __getitem__(self, key):
-        """
-        Return the content of the message document.
-
-        :param key: The key
-        :type key: str
-
-        :return: The content value indexed by C{key} or None
-        :rtype: str
-        """
-        return self._cdoc.content.get(key, None)
-
-    def does_exist(self):
-        """
-        Return True if there is actually a message for this
-        UID and mbox.
-        """
-        return bool(self._fdoc)
-
-
-SoledadWriterPayload = namedtuple(
-    'SoledadWriterPayload', ['mode', 'payload'])
-
-SoledadWriterPayload.CREATE = 1
-SoledadWriterPayload.PUT = 2
-
-
-class SoledadDocWriter(object):
-    """
-    This writer will create docs serially in the local soledad database.
-    """
-
-    implements(IMessageConsumer)
-
-    def __init__(self, soledad):
-        """
-        Initialize the writer.
-
-        :param soledad: the soledad instance
-        :type soledad: Soledad
-        """
-        self._soledad = soledad
-
-    def consume(self, queue):
-        """
-        Creates a new document in soledad db.
-
-        :param queue: queue to get item from, with content of the document
-                      to be inserted.
-        :type queue: Queue
-        """
-        empty = queue.empty()
-        while not empty:
-            item = queue.get()
-            if item.mode == SoledadWriterPayload.CREATE:
-                call = self._soledad.create_doc
-            elif item.mode == SoledadWriterPayload.PUT:
-                call = self._soledad.put_doc
-
-            # should handle errors
-            try:
-                call(item.payload)
-            except u1db_errors.RevisionConflict as exc:
-                logger.error("Error: %r" % (exc,))
-                raise exc
-
-            empty = queue.empty()
-
-
-class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
-    """
-    A collection of messages, surprisingly.
-
-    It is tied to a selected mailbox name that is passed to constructor.
-    Implements a filter query over the messages contained in a soledad
-    database.
-    """
-    # XXX this should be able to produce a MessageSet methinks
-
-    EMPTY_MSG = {
-        fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL,
-        fields.UID_KEY: 1,
-        fields.MBOX_KEY: fields.INBOX_VAL,
-
-        fields.SUBJECT_KEY: "",
-        fields.DATE_KEY: "",
-        fields.RAW_KEY: "",
-
-        # XXX should separate headers into another doc
-        fields.HEADERS_KEY: {},
-    }
-
-    EMPTY_FLAGS = {
-        fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
-        fields.UID_KEY: 1,
-        fields.MBOX_KEY: fields.INBOX_VAL,
-
-        fields.FLAGS_KEY: [],
-        fields.SEEN_KEY: False,
-        fields.RECENT_KEY: True,
-        fields.MULTIPART_KEY: False,
-    }
-
-    # get from SoledadBackedAccount the needed index-related constants
-    INDEXES = SoledadBackedAccount.INDEXES
-    TYPE_IDX = SoledadBackedAccount.TYPE_IDX
-
-    def __init__(self, mbox=None, soledad=None):
-        """
-        Constructor for MessageCollection.
-
-        :param mbox: the name of the mailbox. It is the name
-                     with which we filter the query over the
-                     messages database
-        :type mbox: str
-
-        :param soledad: Soledad database
-        :type soledad: Soledad instance
-        """
-        MailParser.__init__(self)
-        leap_assert(mbox, "Need a mailbox name to initialize")
-        leap_assert(mbox.strip() != "", "mbox cannot be blank space")
-        leap_assert(isinstance(mbox, (str, unicode)),
-                    "mbox needs to be a string")
-        leap_assert(soledad, "Need a soledad instance to initialize")
-
-        # okay, all in order, keep going...
-        self.mbox = self._parse_mailbox_name(mbox)
-        self._soledad = soledad
-        self.initialize_db()
-
-        # I think of someone like nietzsche when reading this
-
-        # this will be the producer that will enqueue the content
-        # to be processed serially by the consumer (the writer). We just
-        # need to `put` the new material on its plate.
-
-        self.soledad_writer = MessageProducer(
-            SoledadDocWriter(soledad),
-            period=0.05)
-
-    def _get_empty_msg(self):
-        """
-        Returns an empty message.
-
-        :return: a dict containing a default empty message
-        :rtype: dict
-        """
-        return copy.deepcopy(self.EMPTY_MSG)
-
-    def _get_empty_flags_doc(self):
-        """
-        Returns an empty doc for storing flags.
-
-        :return:
-        :rtype:
-        """
-        return copy.deepcopy(self.EMPTY_FLAGS)
-
-    @deferred
-    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
-        """
-        Creates a new message document.
-
-        :param raw: the raw message
-        :type raw: str
-
-        :param subject: subject of the message.
-        :type subject: str
-
-        :param flags: flags
-        :type flags: list
-
-        :param date: the received date for the message
-        :type date: str
-
-        :param uid: the message uid for this mailbox
-        :type uid: int
-        """
-        # TODO: split in smaller methods
-        logger.debug('adding message')
-        if flags is None:
-            flags = tuple()
-        leap_assert_type(flags, tuple)
-
-        content_doc = self._get_empty_msg()
-        flags_doc = self._get_empty_flags_doc()
-
-        content_doc[self.MBOX_KEY] = self.mbox
-        flags_doc[self.MBOX_KEY] = self.mbox
-        # ...should get a sanity check here.
-        content_doc[self.UID_KEY] = uid
-        flags_doc[self.UID_KEY] = uid
-
-        if flags:
-            flags_doc[self.FLAGS_KEY] = map(self._stringify, flags)
-            flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags
-
-        msg = self._get_parsed_msg(raw)
-        headers = dict(msg)
-
-        logger.debug("adding. is multipart:%s" % msg.is_multipart())
-        flags_doc[self.MULTIPART_KEY] = msg.is_multipart()
-        # XXX get lower case for keys?
-        # XXX get headers doc
-        content_doc[self.HEADERS_KEY] = headers
-        # set subject based on message headers and eventually replace by
-        # subject given as param
-        if self.SUBJECT_FIELD in headers:
-            content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
-        if subject is not None:
-            content_doc[self.SUBJECT_KEY] = subject
-
-        # XXX could separate body into its own doc
-        # but should also separate multiparts
-        # that should be wrapped in MessagePart
-        content_doc[self.RAW_KEY] = self._stringify(raw)
-        content_doc[self.SIZE_KEY] = len(raw)
-
-        if not date and self.DATE_FIELD in headers:
-            content_doc[self.DATE_KEY] = headers[self.DATE_FIELD]
-        else:
-            content_doc[self.DATE_KEY] = date
-
-        logger.debug('enqueuing message for write')
-
-        ptuple = SoledadWriterPayload
-        self.soledad_writer.put(ptuple(
-            mode=ptuple.CREATE, payload=content_doc))
-        self.soledad_writer.put(ptuple(
-            mode=ptuple.CREATE, payload=flags_doc))
-
-    def remove(self, msg):
-        """
-        Removes a message.
-
-        :param msg: a u1db doc containing the message
-        :type msg: SoledadDocument
-        """
-        self._soledad.delete_doc(msg)
-
-    # getters
-
-    def get_msg_by_uid(self, uid):
-        """
-        Retrieves a LeapMessage by UID.
-
-        :param uid: the message uid to query by
-        :type uid: int
-
-        :return: A LeapMessage instance matching the query,
-                 or None if not found.
-        :rtype: LeapMessage
-        """
-        msg = LeapMessage(self._soledad, uid, self.mbox)
-        if not msg.does_exist():
-            return None
-        return msg
-
-    def get_all(self):
-        """
-        Get all message documents for the selected mailbox.
-        If you want acess to the content, use __iter__ instead
-
-        :return: a list of u1db documents
-        :rtype: list of SoledadDocument
-        """
-        # TODO change to get_all_docs and turn this
-        # into returning messages
-        if sameProxiedObjects(self._soledad, None):
-            logger.warning('Tried to get messages but soledad is None!')
-            return []
-
-        all_docs = [doc for doc in self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox)]
-
-        # inneficient, but first let's grok it and then
-        # let's worry about efficiency.
-        # XXX FIXINDEX -- should implement order by in soledad
-        return sorted(all_docs, key=lambda item: item.content['uid'])
-
-    def count(self):
-        """
-        Return the count of messages for this mailbox.
-
-        :rtype: int
-        """
-        count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox)
-        return count
-
-    # unseen messages
-
-    def unseen_iter(self):
-        """
-        Get an iterator for the message docs with no `seen` flag
-
-        :return: iterator through unseen message doc UIDs
-        :rtype: iterable
-        """
-        return (doc.content[self.UID_KEY] for doc in
-                self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-                    self.TYPE_FLAGS_VAL, self.mbox, '0'))
-
-    def count_unseen(self):
-        """
-        Count all messages with the `Unseen` flag.
-
-        :returns: count
-        :rtype: int
-        """
-        count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-            self.TYPE_FLAGS_VAL, self.mbox, '0')
-        return count
-
-    def get_unseen(self):
-        """
-        Get all messages with the `Unseen` flag
-
-        :returns: a list of LeapMessages
-        :rtype: list
-        """
-        return [LeapMessage(self._soledad, docid, self.mbox)
-                for docid in self.unseen_iter()]
-
-    # recent messages
-
-    def recent_iter(self):
-        """
-        Get an iterator for the message docs with `recent` flag.
-
-        :return: iterator through recent message docs
-        :rtype: iterable
-        """
-        return (doc.content[self.UID_KEY] for doc in
-                self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-                    self.TYPE_FLAGS_VAL, self.mbox, '1'))
-
-    def get_recent(self):
-        """
-        Get all messages with the `Recent` flag.
-
-        :returns: a list of LeapMessages
-        :rtype: list
-        """
-        return [LeapMessage(self._soledad, docid, self.mbox)
-                for docid in self.recent_iter()]
-
-    def count_recent(self):
-        """
-        Count all messages with the `Recent` flag.
-
-        :returns: count
-        :rtype: int
-        """
-        count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-            self.TYPE_FLAGS_VAL, self.mbox, '1')
-        return count
-
-    def __len__(self):
-        """
-        Returns the number of messages on this mailbox.
-
-        :rtype: int
-        """
-        return self.count()
-
-    def __iter__(self):
-        """
-        Returns an iterator over all messages.
-
-        :returns: iterator of dicts with content for all messages.
-        :rtype: iterable
-        """
-        # XXX return LeapMessage instead?! (change accordingly)
-        return (m.content for m in self.get_all())
-
-    def __repr__(self):
-        """
-        Representation string for this object.
-        """
-        return u"<MessageCollection: mbox '%s' (%s)>" % (
-            self.mbox, self.count())
-
-    # XXX should implement __eq__ also !!! --- use a hash
-    # of content for that, will be used for dedup.
-
-
-class SoledadMailbox(WithMsgFields, MBoxParser):
-    """
-    A Soledad-backed IMAP mailbox.
-
-    Implements the high-level method needed for the Mailbox interfaces.
-    The low-level database methods are contained in MessageCollection class,
-    which we instantiate and make accessible in the `messages` attribute.
-    """
-    implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
-    # XXX should finish the implementation of IMailboxListener
-
-    messages = None
-    _closed = False
-
-    INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
-                  WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
-                  WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
-                  WithMsgFields.LIST_FLAG)
-    flags = None
-
-    CMD_MSG = "MESSAGES"
-    CMD_RECENT = "RECENT"
-    CMD_UIDNEXT = "UIDNEXT"
-    CMD_UIDVALIDITY = "UIDVALIDITY"
-    CMD_UNSEEN = "UNSEEN"
-
-    _listeners = defaultdict(set)
-
-    def __init__(self, mbox, soledad=None, rw=1):
-        """
-        SoledadMailbox constructor. Needs to get passed a name, plus a
-        Soledad instance.
-
-        :param mbox: the mailbox name
-        :type mbox: str
-
-        :param soledad: a Soledad instance.
-        :type soledad: Soledad
-
-        :param rw: read-and-write flags
-        :type rw: int
-        """
-        leap_assert(mbox, "Need a mailbox name to initialize")
-        leap_assert(soledad, "Need a soledad instance to initialize")
-
-        # XXX should move to wrapper
-        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    #"soledad._db must be an instance of SQLCipherDatabase")
-
-        self.mbox = self._parse_mailbox_name(mbox)
-        self.rw = rw
-
-        self._soledad = soledad
-
-        self.messages = MessageCollection(
-            mbox=mbox, soledad=self._soledad)
-
-        if not self.getFlags():
-            self.setFlags(self.INIT_FLAGS)
-
-    @property
-    def listeners(self):
-        """
-        Returns listeners for this mbox.
-
-        The server itself is a listener to the mailbox.
-        so we can notify it (and should!) after changes in flags
-        and number of messages.
-
-        :rtype: set
-        """
-        return self._listeners[self.mbox]
-
-    def addListener(self, listener):
-        """
-        Adds a listener to the listeners queue.
-        The server adds itself as a listener when there is a SELECT,
-        so it can send EXIST commands.
-
-        :param listener: listener to add
-        :type listener: an object that implements IMailboxListener
-        """
-        logger.debug('adding mailbox listener: %s' % listener)
-        self.listeners.add(listener)
-
-    def removeListener(self, listener):
-        """
-        Removes a listener from the listeners queue.
-
-        :param listener: listener to remove
-        :type listener: an object that implements IMailboxListener
-        """
-        self.listeners.remove(listener)
-
-    def _get_mbox(self):
-        """
-        Returns mailbox document.
-
-        :return: A SoledadDocument containing this mailbox, or None if
-                 the query failed.
-        :rtype: SoledadDocument or None.
-        """
-        try:
-            query = self._soledad.get_from_index(
-                SoledadBackedAccount.TYPE_MBOX_IDX,
-                self.TYPE_MBOX_VAL, self.mbox)
-            if query:
-                return query.pop()
-        except Exception as exc:
-            logger.error("Unhandled error %r" % exc)
-
-    def getFlags(self):
-        """
-        Returns the flags defined for this mailbox.
-
-        :returns: tuple of flags for this mailbox
-        :rtype: tuple of str
-        """
-        mbox = self._get_mbox()
-        if not mbox:
-            return None
-        flags = mbox.content.get(self.FLAGS_KEY, [])
-        return map(str, flags)
-
-    def setFlags(self, flags):
-        """
-        Sets flags for this mailbox.
-
-        :param flags: a tuple with the flags
-        :type flags: tuple of str
-        """
-        leap_assert(isinstance(flags, tuple),
-                    "flags expected to be a tuple")
-        mbox = self._get_mbox()
-        if not mbox:
-            return None
-        mbox.content[self.FLAGS_KEY] = map(str, flags)
-        self._soledad.put_doc(mbox)
-
-    # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
-
-    def _get_closed(self):
-        """
-        Return the closed attribute for this mailbox.
-
-        :return: True if the mailbox is closed
-        :rtype: bool
-        """
-        mbox = self._get_mbox()
-        return mbox.content.get(self.CLOSED_KEY, False)
-
-    def _set_closed(self, closed):
-        """
-        Set the closed attribute for this mailbox.
-
-        :param closed: the state to be set
-        :type closed: bool
-        """
-        leap_assert(isinstance(closed, bool), "closed needs to be boolean")
-        mbox = self._get_mbox()
-        mbox.content[self.CLOSED_KEY] = closed
-        self._soledad.put_doc(mbox)
-
-    closed = property(
-        _get_closed, _set_closed, doc="Closed attribute.")
-
-    def _get_last_uid(self):
-        """
-        Return the last uid for this mailbox.
-
-        :return: the last uid for messages in this mailbox
-        :rtype: bool
-        """
-        mbox = self._get_mbox()
-        return mbox.content.get(self.LAST_UID_KEY, 1)
-
-    def _set_last_uid(self, uid):
-        """
-        Sets the last uid for this mailbox.
-
-        :param uid: the uid to be set
-        :type uid: int
-        """
-        leap_assert(isinstance(uid, int), "uid has to be int")
-        mbox = self._get_mbox()
-        key = self.LAST_UID_KEY
-
-        count = self.getMessageCount()
-
-        # XXX safety-catch. If we do get duplicates,
-        # we want to avoid further duplication.
-
-        if uid >= count:
-            value = uid
-        else:
-            # something is wrong,
-            # just set the last uid
-            # beyond the max msg count.
-            logger.debug("WRONG uid < count. Setting last uid to %s", count)
-            value = count
-
-        mbox.content[key] = value
-        self._soledad.put_doc(mbox)
-
-    last_uid = property(
-        _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
-
-    def getUIDValidity(self):
-        """
-        Return the unique validity identifier for this mailbox.
-
-        :return: unique validity identifier
-        :rtype: int
-        """
-        mbox = self._get_mbox()
-        return mbox.content.get(self.CREATED_KEY, 1)
-
-    def getUID(self, message):
-        """
-        Return the UID of a message in the mailbox
-
-        .. note:: this implementation does not make much sense RIGHT NOW,
-        but in the future will be useful to get absolute UIDs from
-        message sequence numbers.
-
-        :param message: the message uid
-        :type message: int
-
-        :rtype: int
-        """
-        msg = self.messages.get_msg_by_uid(message)
-        return msg.getUID()
-
-    def getUIDNext(self):
-        """
-        Return the likely UID for the next message added to this
-        mailbox. Currently it returns the higher UID incremented by
-        one.
-
-        We increment the next uid *each* time this function gets called.
-        In this way, there will be gaps if the message with the allocated
-        uid cannot be saved. But that is preferable to having race conditions
-        if we get to parallel message adding.
-
-        :rtype: int
-        """
-        self.last_uid += 1
-        return self.last_uid
-
-    def getMessageCount(self):
-        """
-        Returns the total count of messages in this mailbox.
-
-        :rtype: int
-        """
-        return self.messages.count()
-
-    def getUnseenCount(self):
-        """
-        Returns the number of messages with the 'Unseen' flag.
-
-        :return: count of messages flagged `unseen`
-        :rtype: int
-        """
-        return self.messages.count_unseen()
-
-    def getRecentCount(self):
-        """
-        Returns the number of messages with the 'Recent' flag.
-
-        :return: count of messages flagged `recent`
-        :rtype: int
-        """
-        return self.messages.count_recent()
-
-    def isWriteable(self):
-        """
-        Get the read/write status of the mailbox.
-
-        :return: 1 if mailbox is read-writeable, 0 otherwise.
-        :rtype: int
-        """
-        return self.rw
-
-    def getHierarchicalDelimiter(self):
-        """
-        Returns the character used to delimite hierarchies in mailboxes.
-
-        :rtype: str
-        """
-        return '/'
-
-    def requestStatus(self, names):
-        """
-        Handles a status request by gathering the output of the different
-        status commands.
-
-        :param names: a list of strings containing the status commands
-        :type names: iter
-        """
-        r = {}
-        if self.CMD_MSG in names:
-            r[self.CMD_MSG] = self.getMessageCount()
-        if self.CMD_RECENT in names:
-            r[self.CMD_RECENT] = self.getRecentCount()
-        if self.CMD_UIDNEXT in names:
-            r[self.CMD_UIDNEXT] = self.last_uid + 1
-        if self.CMD_UIDVALIDITY in names:
-            r[self.CMD_UIDVALIDITY] = self.getUID()
-        if self.CMD_UNSEEN in names:
-            r[self.CMD_UNSEEN] = self.getUnseenCount()
-        return defer.succeed(r)
-
-    def addMessage(self, message, flags, date=None):
-        """
-        Adds a message to this mailbox.
-
-        :param message: the raw message
-        :type message: str
-
-        :param flags: flag list
-        :type flags: list of str
-
-        :param date: timestamp
-        :type date: str
-
-        :return: a deferred that evals to None
-        """
-        # XXX we should treat the message as an IMessage from here
-        leap_assert_type(message, basestring)
-        uid_next = self.getUIDNext()
-        logger.debug('Adding msg with UID :%s' % uid_next)
-        if flags is None:
-            flags = tuple()
-        else:
-            flags = tuple(str(flag) for flag in flags)
-
-        d = self._do_add_messages(message, flags, date, uid_next)
-        d.addCallback(self._notify_new)
-
-    @deferred
-    def _do_add_messages(self, message, flags, date, uid_next):
-        """
-        Calls to the messageCollection add_msg method (deferred to thread).
-        Invoked from addMessage.
-        """
-        self.messages.add_msg(message, flags=flags, date=date,
-                              uid=uid_next)
-
-    def _notify_new(self, *args):
-        """
-        Notify of new messages to all the listeners.
-
-        :param args: ignored.
-        """
-        exists = self.getMessageCount()
-        recent = self.getRecentCount()
-        logger.debug("NOTIFY: there are %s messages, %s recent" % (
-            exists,
-            recent))
-
-        logger.debug("listeners: %s", str(self.listeners))
-        for l in self.listeners:
-            logger.debug('notifying...')
-            l.newMessages(exists, recent)
-
-    # commands, do not rename methods
-
-    def destroy(self):
-        """
-        Called before this mailbox is permanently deleted.
-
-        Should cleanup resources, and set the \\Noselect flag
-        on the mailbox.
-        """
-        self.setFlags((self.NOSELECT_FLAG,))
-        self.deleteAllDocs()
-
-        # XXX removing the mailbox in situ for now,
-        # we should postpone the removal
-        self._soledad.delete_doc(self._get_mbox())
-
-    def expunge(self):
-        """
-        Remove all messages flagged \\Deleted
-        """
-        if not self.isWriteable():
-            raise imap4.ReadOnlyMailbox
-        delete = []
-        deleted = []
-
-        for m in self.messages.get_all():
-            if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
-                delete.append(m)
-        for m in delete:
-            deleted.append(m.content)
-            self.messages.remove(m)
-
-        # XXX should return the UIDs of the deleted messages
-        # more generically
-        return [x for x in range(len(deleted))]
-
-    @deferred
-    def fetch(self, messages, uid):
-        """
-        Retrieve one or more messages in this mailbox.
-
-        from rfc 3501: The data items to be fetched can be either a single atom
-        or a parenthesized list.
-
-        :param messages: IDs of the messages to retrieve information about
-        :type messages: MessageSet
-
-        :param uid: If true, the IDs are UIDs. They are message sequence IDs
-                    otherwise.
-        :type uid: bool
-
-        :rtype: A tuple of two-tuples of message sequence numbers and
-                LeapMessage
-        """
-        result = []
-        sequence = True if uid == 0 else False
-
-        if not messages.last:
-            try:
-                iter(messages)
-            except TypeError:
-                # looks like we cannot iterate
-                messages.last = self.last_uid
-
-        # for sequence numbers (uid = 0)
-        if sequence:
-            logger.debug("Getting msg by index: INEFFICIENT call!")
-            raise NotImplementedError
-
-        else:
-            for msg_id in messages:
-                msg = self.messages.get_msg_by_uid(msg_id)
-                if msg:
-                    result.append((msg_id, msg))
-                else:
-                    logger.debug("fetch %s, no msg found!!!" % msg_id)
-
-        if self.isWriteable():
-            self._unset_recent_flag()
-        self._signal_unread_to_ui()
-
-        # XXX workaround for hangs in thunderbird
-        #return tuple(result[:100])  # --- doesn't show all!!
-        return tuple(result)
-
-    @deferred
-    def _unset_recent_flag(self):
-        """
-        Unsets `Recent` flag from a tuple of messages.
-        Called from fetch.
-
-        From RFC, about `Recent`:
-
-        Message is "recently" arrived in this mailbox.  This session
-        is the first session to have been notified about this
-        message; if the session is read-write, subsequent sessions
-        will not see \Recent set for this message.  This flag can not
-        be altered by the client.
-
-        If it is not possible to determine whether or not this
-        session is the first session to be notified about a message,
-        then that message SHOULD be considered recent.
-        """
-        log.msg('unsetting recent flags...')
-        for msg in self.messages.get_recent():
-            msg.removeFlags((fields.RECENT_FLAG,))
-        self._signal_unread_to_ui()
-
-    @deferred
-    def _signal_unread_to_ui(self):
-        """
-        Sends unread event to ui.
-        """
-        unseen = self.getUnseenCount()
-        leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
-
-    @deferred
-    def store(self, messages, flags, mode, uid):
-        """
-        Sets the flags of one or more messages.
-
-        :param messages: The identifiers of the messages to set the flags
-        :type messages: A MessageSet object with the list of messages requested
-
-        :param flags: The flags to set, unset, or add.
-        :type flags: sequence of str
-
-        :param mode: If mode is -1, these flags should be removed from the
-                     specified messages.  If mode is 1, these flags should be
-                     added to the specified messages.  If mode is 0, all
-                     existing flags should be cleared and these flags should be
-                     added.
-        :type mode: -1, 0, or 1
-
-        :param uid: If true, the IDs specified in the query are UIDs;
-                    otherwise they are message sequence IDs.
-        :type uid: bool
-
-        :return: A dict mapping message sequence numbers to sequences of
-                 str representing the flags set on the message after this
-                 operation has been performed.
-        :rtype: dict
-
-        :raise ReadOnlyMailbox: Raised if this mailbox is not open for
-                                read-write.
-        """
-        # XXX implement also sequence (uid = 0)
-        # XXX we should prevent cclient from setting Recent flag.
-        leap_assert(not isinstance(flags, basestring),
-                    "flags cannot be a string")
-        flags = tuple(flags)
-
-        if not self.isWriteable():
-            log.msg('read only mailbox!')
-            raise imap4.ReadOnlyMailbox
-
-        if not messages.last:
-            messages.last = self.messages.count()
-
-        result = {}
-        for msg_id in messages:
-            log.msg("MSG ID = %s" % msg_id)
-            msg = self.messages.get_msg_by_uid(msg_id)
-            if mode == 1:
-                msg.addFlags(flags)
-            elif mode == -1:
-                msg.removeFlags(flags)
-            elif mode == 0:
-                msg.setFlags(flags)
-            result[msg_id] = msg.getFlags()
-
-        self._signal_unread_to_ui()
-        return result
-
-    @deferred
-    def close(self):
-        """
-        Expunge and mark as closed
-        """
-        self.expunge()
-        self.closed = True
-
-    # convenience fun
-
-    def deleteAllDocs(self):
-        """
-        Deletes all docs in this mailbox
-        """
-        docs = self.messages.get_all()
-        for doc in docs:
-            self.messages._soledad.delete_doc(doc)
-
-    def __repr__(self):
-        """
-        Representation string for this mailbox.
-        """
-        return u"<SoledadMailbox: mbox '%s' (%s)>" % (
-            self.mbox, self.messages.count())
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 8756ddc..26e14c3 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
 from leap.common import events as leap_events
 from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import KeyManager
-from leap.mail.imap.server import SoledadBackedAccount
+from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
 from leap.soledad.client import Soledad
 
-- 
cgit v1.2.3


From 25a0aea875fd0d67238beed1237f7239474673ec Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 27 Dec 2013 02:06:44 -0400
Subject: First stage of the storage schema rewrite.

* Separates between flags, docs, body and attachment docs.

* Implement IMessageCopier interface: move and have fun!
  This little change is known to push forward our beloved architect
  emotional rollercoster.

* Message deduplication.

* It also fixes a hidden bug that was rendering the multipart mime
  interface useless
  (yes, the "True" parameter in the parsestr method).

* Does not handle well nested attachs, includes dirty workaround that
  flattens them.

* Includes chiiph's patch for rc2:

 * return deferred from addMessage
 * convert StringIO types to string
 * remove unneeded yields from the chain of deferreds in fetcher
---
 changes/feature_split_message_docs |   6 +
 src/leap/mail/imap/fetch.py        |   7 +-
 src/leap/mail/imap/fields.py       |  49 ++-
 src/leap/mail/imap/index.py        |   4 +-
 src/leap/mail/imap/mailbox.py      | 103 +++--
 src/leap/mail/imap/messages.py     | 831 +++++++++++++++++++++++++++++--------
 src/leap/mail/imap/parser.py       |  24 +-
 7 files changed, 808 insertions(+), 216 deletions(-)
 create mode 100644 changes/feature_split_message_docs

diff --git a/changes/feature_split_message_docs b/changes/feature_split_message_docs
new file mode 100644
index 0000000..231c36e
--- /dev/null
+++ b/changes/feature_split_message_docs
@@ -0,0 +1,6 @@
+  o Defer costly operations to a pool of threads.
+  o Split the internal representation of messages into four distinct documents:
+    1) Flags 2) Headers 3) Body 4) Attachments.
+  o Add deduplication ability to the save operation, for body and attachments.
+  o Add IMessageCopier interface to mailbox implementation, so bulk moves
+    are costless. Closes: #4654
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 0b31c3b..fdf1412 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -412,13 +412,13 @@ class LeapIncomingMail(object):
 
         # decrypt or fail gracefully
         try:
-            decrdata, valid_sig = yield self._decrypt_and_verify_data(
+            decrdata, valid_sig = self._decrypt_and_verify_data(
                 encdata, senderPubkey)
         except keymanager_errors.DecryptError as e:
             logger.warning('Failed to decrypt encrypted message (%s). '
                            'Storing message without modifications.' % str(e))
             # Bailing out!
-            yield (msg, False)
+            return (msg, False)
 
         # decrypted successully, now fix encoding and parse
         try:
@@ -441,7 +441,7 @@ class LeapIncomingMail(object):
 
         # all ok, replace payload by unencrypted payload
         msg.set_payload(decrmsg.get_payload())
-        yield (msg, valid_sig)
+        return (msg, valid_sig)
 
     def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
                                             senderPubkey):
@@ -527,6 +527,7 @@ class LeapIncomingMail(object):
         """
         log.msg('adding message to local db')
         doc, data = msgtuple
+
         if isinstance(data, list):
             data = data[0]
 
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 96b937e..40817cd 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -25,18 +25,35 @@ class WithMsgFields(object):
     Container class for class-attributes to be shared by
     several message-related classes.
     """
-    # Internal representation of Message
-    DATE_KEY = "date"
-    HEADERS_KEY = "headers"
-    FLAGS_KEY = "flags"
-    MBOX_KEY = "mbox"
+    # indexing
     CONTENT_HASH_KEY = "chash"
-    RAW_KEY = "raw"
-    SUBJECT_KEY = "subject"
+    PAYLOAD_HASH_KEY = "phash"
+
+    # Internal representation of Message
+
+    # flags doc
     UID_KEY = "uid"
+    MBOX_KEY = "mbox"
+    SEEN_KEY = "seen"
+    RECENT_KEY = "recent"
+    FLAGS_KEY = "flags"
     MULTIPART_KEY = "multi"
     SIZE_KEY = "size"
 
+    # headers
+    HEADERS_KEY = "headers"
+    NUM_PARTS_KEY = "numparts"
+    PARTS_MAP_KEY = "partmap"
+    DATE_KEY = "date"
+    SUBJECT_KEY = "subject"
+
+    # attachment
+    PART_NUMBER_KEY = "part"
+    RAW_KEY = "raw"
+
+    # content
+    BODY_KEY = "body"
+
     # Mailbox specific keys
     CLOSED_KEY = "closed"
     CREATED_KEY = "created"
@@ -55,10 +72,6 @@ class WithMsgFields(object):
 
     INBOX_VAL = "inbox"
 
-    # Flags for SoledadDocument for indexing.
-    SEEN_KEY = "seen"
-    RECENT_KEY = "recent"
-
     # Flags in Mailbox and Message
     SEEN_FLAG = "\\Seen"
     RECENT_FLAG = "\\Recent"
@@ -82,7 +95,9 @@ class WithMsgFields(object):
     TYPE_SUBS_IDX = 'by-type-and-subscribed'
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
-    TYPE_HASH_IDX = 'by-type-and-hash'
+    TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+    TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+    TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
 
     # Tomas created the `recent and seen index`, but the semantic is not too
     # correct since the recent flag is volatile.
@@ -90,7 +105,9 @@ class WithMsgFields(object):
 
     KTYPE = TYPE_KEY
     MBOX_VAL = TYPE_MBOX_VAL
-    HASH_VAL = CONTENT_HASH_KEY
+    CHASH_VAL = CONTENT_HASH_KEY
+    PHASH_VAL = PAYLOAD_HASH_KEY
+    PART_VAL = PART_NUMBER_KEY
 
     INDEXES = {
         # generic
@@ -102,7 +119,11 @@ class WithMsgFields(object):
         TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
 
         # content, headers doc
-        TYPE_HASH_IDX: [KTYPE, HASH_VAL],
+        TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
+        # attachment docs
+        TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL],
+        # attachment payload dedup
+        TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
 
         # messages
         TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
index 2280d86..5f0919a 100644
--- a/src/leap/mail/imap/index.py
+++ b/src/leap/mail/imap/index.py
@@ -21,7 +21,7 @@ import logging
 
 from leap.common.check import leap_assert, leap_assert_type
 
-from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.fields import fields
 
 
 logger = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ class IndexedDB(object):
         db_indexes = dict()
         if self._soledad is not None:
             db_indexes = dict(self._soledad.list_indexes())
-        for name, expression in SoledadBackedAccount.INDEXES.items():
+        for name, expression in fields.INDEXES.items():
             if name not in db_indexes:
                 # The index does not yet exist.
                 self._soledad.create_index(name, *expression)
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 09c06a2..5ea6f55 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -17,7 +17,13 @@
 """
 Soledad Mailbox.
 """
+import copy
+import threading
 import logging
+import time
+import StringIO
+import cStringIO
+
 from collections import defaultdict
 
 from twisted.internet import defer
@@ -45,9 +51,14 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     The low-level database methods are contained in MessageCollection class,
     which we instantiate and make accessible in the `messages` attribute.
     """
-    implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
+    implements(
+        imap4.IMailbox,
+        imap4.IMailboxInfo,
+        imap4.ICloseableMailbox,
+        imap4.IMessageCopier)
+
     # XXX should finish the implementation of IMailboxListener
-    # XXX should implement IMessageCopier too
+    # XXX should implement ISearchableMailbox too
 
     messages = None
     _closed = False
@@ -65,6 +76,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     CMD_UNSEEN = "UNSEEN"
 
     _listeners = defaultdict(set)
+    next_uid_lock = threading.Lock()
 
     def __init__(self, mbox, soledad=None, rw=1):
         """
@@ -284,8 +296,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         :rtype: int
         """
-        self.last_uid += 1
-        return self.last_uid
+        with self.next_uid_lock:
+            self.last_uid += 1
+            return self.last_uid
 
     def getMessageCount(self):
         """
@@ -366,6 +379,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         :return: a deferred that evals to None
         """
+        if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)):
+            message = message.getvalue()
         # XXX we should treat the message as an IMessage from here
         leap_assert_type(message, basestring)
         uid_next = self.getUIDNext()
@@ -375,11 +390,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         else:
             flags = tuple(str(flag) for flag in flags)
 
-        d = self._do_add_messages(message, flags, date, uid_next)
+        d = self._do_add_message(message, flags, date, uid_next)
         d.addCallback(self._notify_new)
+        return d
 
     @deferred
-    def _do_add_messages(self, message, flags, date, uid_next):
+    def _do_add_message(self, message, flags, date, uid_next):
         """
         Calls to the messageCollection add_msg method (deferred to thread).
         Invoked from addMessage.
@@ -420,28 +436,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # we should postpone the removal
         self._soledad.delete_doc(self._get_mbox())
 
+    @deferred
     def expunge(self):
         """
         Remove all messages flagged \\Deleted
         """
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
-        delete = []
         deleted = []
-
-        for m in self.messages.get_all_docs():
-            # XXX should operate with LeapMessages instead,
-            # so we don't expose the implementation.
-            # (so, iterate for m in self.messages)
-            if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
-                delete.append(m)
-        for m in delete:
-            deleted.append(m.content)
-            self.messages.remove(m)
-
-        # XXX should return the UIDs of the deleted messages
-        # more generically
-        return [x for x in range(len(deleted))]
+        for m in self.messages:
+            if self.DELETED_FLAG in m.getFlags():
+                self.messages.remove(m)
+                # XXX this would ve more efficient if we can just pass
+                # a sequence of uids.
+                deleted.append(m.getUID())
+        return deleted
 
     @deferred
     def fetch(self, messages, uid):
@@ -510,6 +519,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         session is the first session to be notified about a message,
         then that message SHOULD be considered recent.
         """
+        # TODO this fucker, for the sake of correctness, is messing with
+        # the whole collection of flag docs.
+
+        # Possible ways of action:
+        # 1. Ignore it, we want fun.
+        # 2. Trigger it with a delay
+        # 3. Route it through a queue with lesser priority than the
+        #    regularar writer.
+
+        # hmm let's try 2. in a quickndirty way...
+        time.sleep(1)
         log.msg('unsetting recent flags...')
         for msg in self.messages.get_recent():
             msg.removeFlags((fields.RECENT_FLAG,))
@@ -570,6 +590,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         for msg_id in messages:
             log.msg("MSG ID = %s" % msg_id)
             msg = self.messages.get_msg_by_uid(msg_id)
+            if not msg:
+                return result
             if mode == 1:
                 msg.addFlags(flags)
             elif mode == -1:
@@ -589,15 +611,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self.expunge()
         self.closed = True
 
-    #@deferred
-    #def copy(self, messageObject):
-        #"""
-        #Copy the given message object into this mailbox.
-        #"""
-        # XXX should just:
-        # 1. Get the message._fdoc
-        # 2. Change the UID to UIDNext for this mailbox
-        # 3. Add implements IMessageCopier
+    # IMessageCopier
+
+    @deferred
+    def copy(self, messageObject):
+        """
+        Copy the given message object into this mailbox.
+        """
+        uid_next = self.getUIDNext()
+        msg = messageObject
+
+        # XXX should use a public api instead
+        fdoc = msg._fdoc
+        if not fdoc:
+            logger.debug("Tried to copy a MSG with no fdoc")
+            return
+
+        new_fdoc = copy.deepcopy(fdoc.content)
+        new_fdoc[self.UID_KEY] = uid_next
+        new_fdoc[self.MBOX_KEY] = self.mbox
+
+        d = self._do_add_doc(new_fdoc)
+        d.addCallback(self._notify_new)
+
+    @deferred
+    def _do_add_doc(self, doc):
+        """
+        Defers the adding of a new doc.
+        :param doc: document to be created in soledad.
+        """
+        self._soledad.create_doc(doc)
 
     # convenience fun
 
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index b0d5da2..c69c023 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -29,9 +29,9 @@ from zope.interface import implements
 from zope.proxy import sameProxiedObjects
 
 from leap.common.check import leap_assert, leap_assert_type
+from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail.decorators import deferred
-from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.parser import MailParser, MBoxParser
@@ -40,6 +40,181 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer
 logger = logging.getLogger(__name__)
 
 
+def first(things):
+    """
+    Return the head of a collection.
+    """
+    try:
+        return things[0]
+    except (IndexError, TypeError):
+        return None
+
+
+class MessageBody(object):
+    """
+    IMessagePart implementor for the main
+    body of a multipart message.
+
+    Excusatio non petita: see the interface documentation.
+    """
+
+    implements(imap4.IMessagePart)
+
+    def __init__(self, fdoc, bdoc):
+        self._fdoc = fdoc
+        self._bdoc = bdoc
+
+    def getSize(self):
+        return len(self._bdoc.content[fields.BODY_KEY])
+
+    def getBodyFile(self):
+        fd = StringIO.StringIO()
+
+        if self._bdoc:
+            body = self._bdoc.content[fields.BODY_KEY]
+        else:
+            body = ""
+        charset = self._get_charset(body)
+        try:
+            body = body.encode(charset)
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            logger.error("Unicode error {0}".format(e))
+            body = body.encode(charset, 'replace')
+        fd.write(body)
+        fd.seek(0)
+        return fd
+
+    @memoized_method
+    def _get_charset(self, stuff):
+        return get_email_charset(unicode(stuff))
+
+    def getHeaders(self, negate, *names):
+        return {}
+
+    def isMultipart(self):
+        return False
+
+    def getSubPart(self, part):
+        return None
+
+
+class MessageAttachment(object):
+
+    implements(imap4.IMessagePart)
+
+    def __init__(self, msg):
+        """
+        Initializes the messagepart with a Message instance.
+        :param msg: a message instance
+        :type msg: Message
+        """
+        self._msg = msg
+
+    def getSize(self):
+        """
+        Return the total size, in octets, of this message part.
+
+        :return: size of the message, in octets
+        :rtype: int
+        """
+        if not self._msg:
+            return 0
+        return len(self._msg.as_string())
+
+    def getBodyFile(self):
+        """
+        Retrieve a file object containing only the body of this message.
+
+        :return: file-like object opened for reading
+        :rtype: StringIO
+        """
+        fd = StringIO.StringIO()
+        if self._msg:
+            body = self._msg.get_payload()
+        else:
+            logger.debug("Empty message!")
+            body = ""
+
+        # XXX should only do the dance if we're sure it's
+        # content/text-plain!!!
+        #charset = self._get_charset(body)
+        #try:
+            #body = body.encode(charset)
+        #except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            #logger.error("Unicode error {0}".format(e))
+            #body = body.encode(charset, 'replace')
+        fd.write(body)
+        fd.seek(0)
+        return fd
+
+    @memoized_method
+    def _get_charset(self, stuff):
+        # TODO put in a common class with LeapMessage
+        """
+        Gets (guesses?) the charset of a payload.
+
+        :param stuff: the stuff to guess about.
+        :type stuff: basestring
+        :returns: charset
+        """
+        # XXX existential doubt 1. wouldn't be smarter to
+        # peek into the mail headers?
+        # XXX existential doubt 2. shouldn't we make the scope
+        # of the decorator somewhat more persistent?
+        # ah! yes! and put memory bounds.
+        return get_email_charset(unicode(stuff))
+
+    def getHeaders(self, negate, *names):
+        """
+        Retrieve a group of message headers.
+
+        :param names: The names of the headers to retrieve or omit.
+        :type names: tuple of str
+
+        :param negate: If True, indicates that the headers listed in names
+                       should be omitted from the return value, rather
+                       than included.
+        :type negate: bool
+
+        :return: A mapping of header field names to header field values
+        :rtype: dict
+        """
+        if not self._msg:
+            return {}
+        headers = dict(self._msg.items())
+        names = map(lambda s: s.upper(), names)
+        if negate:
+            cond = lambda key: key.upper() not in names
+        else:
+            cond = lambda key: key.upper() in names
+
+        # unpack and filter original dict by negate-condition
+        filter_by_cond = [
+            map(str, (key, val)) for
+            key, val in headers.items()
+            if cond(key)]
+        return dict(filter_by_cond)
+
+    def isMultipart(self):
+        """
+        Return True if this message is multipart.
+        """
+        return self._msg.is_multipart()
+
+    def getSubPart(self, part):
+        """
+        Retrieve a MIME submessage
+
+        :type part: C{int}
+        :param part: The number of the part to retrieve, indexed from 0.
+        :raise IndexError: Raised if the specified part does not exist.
+        :raise TypeError: Raised if this message is not multipart.
+        :rtype: Any object implementing C{IMessagePart}.
+        :return: The specified sub-part.
+        """
+        return self._msg.get_payload()
+
+
 class LeapMessage(fields, MailParser, MBoxParser):
 
     implements(imap4.IMessage)
@@ -59,25 +234,21 @@ class LeapMessage(fields, MailParser, MBoxParser):
         self._soledad = soledad
         self._uid = int(uid)
         self._mbox = self._parse_mailbox_name(mbox)
-        self._chash = None
 
-        self.__cdoc = None
+        self.__chash = None
+        self.__bdoc = None
 
     @property
     def _fdoc(self):
         """
         An accessor to the flags document.
         """
-        return self._get_flags_doc()
-
-    @property
-    def _cdoc(self):
-        """
-        An accessor to the content document.
-        """
-        if not self.__cdoc:
-            self.__cdoc = self._get_content_doc()
-        return self.__cdoc
+        if all(map(bool, (self._uid, self._mbox))):
+            fdoc = self._get_flags_doc()
+            if fdoc:
+                self.__chash = fdoc.content.get(
+                    fields.CONTENT_HASH_KEY, None)
+            return fdoc
 
     @property
     def _chash(self):
@@ -86,7 +257,26 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         if not self._fdoc:
             return None
-        return self._fdoc.content.get(fields.CONTENT_HASH_KEY, None)
+        if not self.__chash and self._fdoc:
+            self.__chash = self._fdoc.content.get(
+                fields.CONTENT_HASH_KEY, None)
+        return self.__chash
+
+    @property
+    def _hdoc(self):
+        """
+        An accessor to the headers document.
+        """
+        return self._get_headers_doc()
+
+    @property
+    def _bdoc(self):
+        """
+        An accessor to the body document.
+        """
+        if not self.__bdoc:
+            self.__bdoc = self._get_body_doc()
+        return self.__bdoc
 
     # IMessage implementation
 
@@ -110,9 +300,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
             return []
 
         flags = []
-        flag_doc = self._fdoc
-        if flag_doc:
-            flags = flag_doc.content.get(self.FLAGS_KEY, None)
+        fdoc = self._fdoc
+        if fdoc:
+            flags = fdoc.content.get(self.FLAGS_KEY, None)
         if flags:
             flags = map(str, flags)
         return tuple(flags)
@@ -180,7 +370,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: C{str}
         :return: An RFC822-formatted date string.
         """
-        return str(self._cdoc.content.get(self.DATE_KEY, ''))
+        return str(self._hdoc.content.get(self.DATE_KEY, ''))
 
     #
     # IMessagePart
@@ -197,25 +387,38 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
+        bdoc = self._bdoc
+        if bdoc:
+            body = self._bdoc.content.get(self.BODY_KEY, "")
+        else:
+            body = ""
 
-        cdoc = self._cdoc
-        content = cdoc.content.get(self.RAW_KEY, '')
-        charset = get_email_charset(
-            unicode(cdoc.content.get(self.RAW_KEY, '')))
+        charset = self._get_charset(body)
         try:
-            content = content.encode(charset)
+            body = body.encode(charset)
         except (UnicodeEncodeError, UnicodeDecodeError) as e:
             logger.error("Unicode error {0}".format(e))
-            content = content.encode(charset, 'replace')
-
-        raw = self._get_raw_msg()
-        msg = self._get_parsed_msg(raw)
-        body = msg.get_payload()
+            body = body.encode(charset, 'replace')
         fd.write(body)
-        # XXX SHOULD use a separate BODY FIELD ...
         fd.seek(0)
         return fd
 
+    @memoized_method
+    def _get_charset(self, stuff):
+        """
+        Gets (guesses?) the charset of a payload.
+
+        :param stuff: the stuff to guess about.
+        :type stuff: basestring
+        :returns: charset
+        """
+        # XXX existential doubt 1. wouldn't be smarter to
+        # peek into the mail headers?
+        # XXX existential doubt 2. shouldn't we make the scope
+        # of the decorator somewhat more persistent?
+        # ah! yes! and put memory bounds.
+        return get_email_charset(unicode(stuff))
+
     def getSize(self):
         """
         Return the total size, in octets, of this message.
@@ -223,19 +426,17 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: size of the message, in octets
         :rtype: int
         """
-        size = self._cdoc.content.get(self.SIZE_KEY, False)
+        size = None
+        if self._fdoc:
+            size = self._fdoc.content.get(self.SIZE_KEY, False)
+        else:
+            logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
+                                                       self._uid))
         if not size:
             # XXX fallback, should remove when all migrated.
             size = self.getBodyFile().len
         return size
 
-    def _get_headers(self):
-        """
-        Return the headers dict stored in this message document.
-        """
-        # XXX get from the headers doc
-        return self._cdoc.content.get(self.HEADERS_KEY, {})
-
     def getHeaders(self, negate, *names):
         """
         Retrieve a group of message headers.
@@ -252,26 +453,49 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: dict
         """
         headers = self._get_headers()
+        if not headers:
+            return {'content-type': ''}
         names = map(lambda s: s.upper(), names)
         if negate:
             cond = lambda key: key.upper() not in names
         else:
             cond = lambda key: key.upper() in names
 
+        head = copy.deepcopy(dict(headers.items()))
+
+            # twisted imap server expects headers to be lowercase
+        head = dict(
+            map(str, (key, value)) if key.lower() != "content-type"
+            else map(str, (key.lower(), value))
+            for (key, value) in head.items())
+
         # unpack and filter original dict by negate-condition
-        filter_by_cond = [
-            map(str, (key, val)) for
-            key, val in headers.items()
-            if cond(key)]
+        filter_by_cond = [(key, val) for key, val in head.items() if cond(key)]
         return dict(filter_by_cond)
 
+    def _get_headers(self):
+        """
+        Return the headers dict for this message.
+        """
+        if self._hdoc is not None:
+            return self._hdoc.content.get(self.HEADERS_KEY, {})
+        else:
+            logger.warning(
+                "No HEADERS doc for msg %s:%s" % (
+                    self._mbox,
+                    self._uid))
+
     def isMultipart(self):
         """
         Return True if this message is multipart.
         """
-        if self._cdoc:
-            retval = self._fdoc.content.get(self.MULTIPART_KEY, False)
-            return retval
+        if self._fdoc:
+            return self._fdoc.content.get(self.MULTIPART_KEY, False)
+        else:
+            logger.warning(
+                "No FLAGS doc for msg %s:%s" % (
+                    self.mbox,
+                    self.uid))
 
     def getSubPart(self, part):
         """
@@ -284,12 +508,22 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: Any object implementing C{IMessagePart}.
         :return: The specified sub-part.
         """
+        logger.debug("Getting subpart: %s" % part)
         if not self.isMultipart():
             raise TypeError
 
-        msg = self._get_parsed_msg()
-        # XXX should wrap IMessagePart
-        return msg.get_payload()[part]
+        if part == 0:
+            # Let's get the first part, which
+            # is really the body.
+            return MessageBody(self._fdoc, self._bdoc)
+
+        attach_doc = self._get_attachment_doc(part)
+        if not attach_doc:
+            # so long and thanks for all the fish
+            logger.debug("...not today")
+            raise IndexError
+        msg_part = self._get_parsed_msg(attach_doc.content[self.RAW_KEY])
+        return MessageAttachment(msg_part)
 
     #
     # accessors
@@ -301,32 +535,87 @@ class LeapMessage(fields, MailParser, MBoxParser):
         message.
         """
         flag_docs = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_UID_IDX,
+            fields.TYPE_MBOX_UID_IDX,
             fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
-        flag_doc = flag_docs[0] if flag_docs else None
-        return flag_doc
+        return first(flag_docs)
 
-    def _get_content_doc(self):
+    def _get_headers_doc(self):
         """
-        Return the document that keeps the flags for this
+        Return the document that keeps the headers for this
+        message.
+        """
+        head_docs = self._soledad.get_from_index(
+            fields.TYPE_C_HASH_IDX,
+            fields.TYPE_HEADERS_VAL, str(self._chash))
+        return first(head_docs)
+
+    def _get_body_doc(self):
+        """
+        Return the document that keeps the body for this
         message.
         """
-        cont_docs = self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_HASH_IDX,
-            fields.TYPE_MESSAGE_VAL, self._content_hash, str(self._uid))
-        cont_doc = cont_docs[0] if cont_docs else None
-        return cont_doc
+        body_docs = self._soledad.get_from_index(
+            fields.TYPE_C_HASH_IDX,
+            fields.TYPE_MESSAGE_VAL, str(self._chash))
+        return first(body_docs)
+
+    def _get_num_parts(self):
+        """
+        Return the number of parts for a multipart message.
+        """
+        if not self.isMultipart():
+            raise TypeError(
+                "Tried to get num parts in a non-multipart message")
+        if not self._hdoc:
+            return None
+        return self._hdoc.content.get(fields.NUM_PARTS_KEY, 2)
+
+    def _get_attachment_doc(self, part):
+        """
+        Return the document that keeps the headers for this
+        message.
+
+        :param part: the part number for the multipart message.
+        :type part: int
+        """
+        if not self._hdoc:
+            return None
+        try:
+            phash = self._hdoc.content[self.PARTS_MAP_KEY][str(part)]
+        except KeyError:
+            # this is the remnant of a debug session until
+            # I found that the index is actually a string...
+            # It should be safe to just raise the KeyError now,
+            # but leaving it here while the blood is fresh...
+            logger.warning("We expected a phash in the "
+                           "index %s, but noone found" % (part, ))
+            logger.debug(self._hdoc.content[self.PARTS_MAP_KEY])
+            return None
+        attach_docs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_ATTACHMENT_VAL, str(phash))
+
+        # The following is true for the fist owner.
+        # We could use this relationship to flag the "owner"
+        # and orphan when we delete it.
+
+        #attach_docs = self._soledad.get_from_index(
+            #fields.TYPE_C_HASH_PART_IDX,
+            #fields.TYPE_ATTACHMENT_VAL, str(self._chash), str(part))
+        return first(attach_docs)
 
     def _get_raw_msg(self):
         """
         Return the raw msg.
         :rtype: basestring
         """
-        return self._cdoc.content.get(self.RAW_KEY, '')
+        # TODO deprecate this.
+        return self._bdoc.content.get(self.RAW_KEY, '')
 
     def __getitem__(self, key):
         """
-        Return the content of the message document.
+        Return an item from the content of the flags document,
+        for convenience.
 
         :param key: The key
         :type key: str
@@ -334,14 +623,73 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: The content value indexed by C{key} or None
         :rtype: str
         """
-        return self._cdoc.content.get(key, None)
+        return self._fdoc.content.get(key, None)
+
+    # setters
+
+    # XXX to be used in the messagecopier interface?!
+
+    def set_uid(self, uid):
+        """
+        Set new uid for this message.
+
+        :param uid: the new uid
+        :type uid: basestring
+        """
+        # XXX dangerous! lock?
+        self._uid = uid
+        d = self._fdoc
+        d.content[self.UID_KEY] = uid
+        self._soledad.put_doc(d)
+
+    def set_mbox(self, mbox):
+        """
+        Set new mbox for this message.
+
+        :param mbox: the new mbox
+        :type mbox: basestring
+        """
+        # XXX dangerous! lock?
+        self._mbox = mbox
+        d = self._fdoc
+        d.content[self.MBOX_KEY] = mbox
+        self._soledad.put_doc(d)
+
+    # destructor
+
+    @deferred
+    def remove(self):
+        """
+        Remove all docs associated with this message.
+        """
+        # XXX this would ve more efficient if we can just pass
+        # a sequence of uids.
+
+        # XXX For the moment we are only removing the flags and headers
+        # docs. The rest we leave there polluting your hard disk,
+        # until we think about a good way of deorphaning.
+        # Maybe a crawler of unreferenced docs.
+
+        fd = self._get_flags_doc()
+        hd = self._get_headers_doc()
+        #bd = self._get_body_doc()
+        #docs = [fd, hd, bd]
+
+        docs = [fd, hd]
+
+        #for pn in range(self._get_num_parts()[1:]):
+            #ad = self._get_attachment_doc(pn)
+            #docs.append(ad)
+
+        for d in filter(None, docs):
+            self._soledad.delete_doc(d)
 
     def does_exist(self):
         """
-        Return True if there is actually a message for this
+        Return True if there is actually a flags message for this
         UID and mbox.
         """
-        return bool(self._fdoc)
+        return self._fdoc is not None
 
 
 SoledadWriterPayload = namedtuple(
@@ -349,6 +697,8 @@ SoledadWriterPayload = namedtuple(
 
 SoledadWriterPayload.CREATE = 1
 SoledadWriterPayload.PUT = 2
+SoledadWriterPayload.BODY_CREATE = 3
+SoledadWriterPayload.ATTACHMENT_CREATE = 4
 
 
 class SoledadDocWriter(object):
@@ -378,20 +728,98 @@ class SoledadDocWriter(object):
         empty = queue.empty()
         while not empty:
             item = queue.get()
+            call = None
+            payload = item.payload
+
             if item.mode == SoledadWriterPayload.CREATE:
                 call = self._soledad.create_doc
+            elif item.mode == SoledadWriterPayload.BODY_CREATE:
+                if not self._body_does_exist(payload):
+                    call = self._soledad.create_doc
+            elif item.mode == SoledadWriterPayload.ATTACHMENT_CREATE:
+                if not self._attachment_does_exist(payload):
+                    call = self._soledad.create_doc
             elif item.mode == SoledadWriterPayload.PUT:
                 call = self._soledad.put_doc
 
-            # should handle errors
-            try:
-                call(item.payload)
-            except u1db_errors.RevisionConflict as exc:
-                logger.error("Error: %r" % (exc,))
-                raise exc
+            # XXX delete?
+
+            if call:
+                # should handle errors
+                try:
+                    call(item.payload)
+                except u1db_errors.RevisionConflict as exc:
+                    logger.error("Error: %r" % (exc,))
+                    raise exc
 
             empty = queue.empty()
 
+    """
+    Message deduplication.
+
+    We do a query for the content hashes before writing to our beloved
+    slcipher backend of Soledad. This means, by now, that:
+
+    1. We will not store the same attachment twice, only the hash of it.
+    2. We will not store the same message body twice, only the hash of it.
+
+    The first case is useful if you are always receiving the same old memes
+    from unwary friends that still have not discovered that 4chan is the
+    generator of the internet. The second will save your day if you have
+    initiated session with the same account in two different machines. I also
+    wonder why would you do that, but let's respect each other choices, like
+    with the religious celebrations, and assume that one day we'll be able
+    to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
+    Stack.
+    """
+
+    def _body_does_exist(self, doc):
+        """
+        Check whether we already have a body payload with this hash in our
+        database.
+
+        :param doc: tentative body document
+        :type doc: dict
+        :returns: True if that happens, False otherwise.
+        """
+        if not doc:
+            return False
+        chash = doc[fields.CONTENT_HASH_KEY]
+        body_docs = self._soledad.get_from_index(
+            fields.TYPE_C_HASH_IDX,
+            fields.TYPE_MESSAGE_VAL, str(chash))
+        if not body_docs:
+            return False
+        if len(body_docs) != 1:
+            logger.warning("Found more than one copy of chash %s!"
+                           % (chash,))
+        logger.debug("Found body doc with that hash! Skipping save!")
+        return True
+
+    def _attachment_does_exist(self, doc):
+        """
+        Check whether we already have an attachment payload with this hash
+        in our database.
+
+        :param doc: tentative body document
+        :type doc: dict
+        :returns: True if that happens, False otherwise.
+        """
+        if not doc:
+            return False
+        phash = doc[fields.PAYLOAD_HASH_KEY]
+        attach_docs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_ATTACHMENT_VAL, str(phash))
+        if not attach_docs:
+            return False
+
+        if len(attach_docs) != 1:
+            logger.warning("Found more than one copy of phash %s!"
+                           % (phash,))
+        logger.debug("Found attachment doc with that hash! Skipping save!")
+        return True
+
 
 class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     """
@@ -402,35 +830,62 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     database.
     """
     # XXX this should be able to produce a MessageSet methinks
-
-    EMPTY_MSG = {
-        fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL,
-        fields.UID_KEY: 1,
-        fields.MBOX_KEY: fields.INBOX_VAL,
-
-        fields.SUBJECT_KEY: "",
-        fields.DATE_KEY: "",
-        fields.RAW_KEY: "",
-
-        # XXX should separate headers into another doc
-        fields.HEADERS_KEY: {},
+    # could validate these kinds of objects turning them
+    # into a template for the class.
+    FLAGS_DOC = "FLAGS"
+    HEADERS_DOC = "HEADERS"
+    ATTACHMENT_DOC = "ATTACHMENT"
+    BODY_DOC = "BODY"
+
+    templates = {
+
+        FLAGS_DOC: {
+            fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
+            fields.UID_KEY: 1,
+            fields.MBOX_KEY: fields.INBOX_VAL,
+
+            fields.SEEN_KEY: False,
+            fields.RECENT_KEY: True,
+            fields.FLAGS_KEY: [],
+            fields.MULTIPART_KEY: False,
+            fields.SIZE_KEY: 0
+        },
+
+        HEADERS_DOC: {
+            fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
+            fields.CONTENT_HASH_KEY: "",
+
+            fields.HEADERS_KEY: {},
+            fields.NUM_PARTS_KEY: 0,
+            fields.PARTS_MAP_KEY: {},
+            fields.DATE_KEY: "",
+            fields.SUBJECT_KEY: ""
+        },
+
+        ATTACHMENT_DOC: {
+            fields.TYPE_KEY: fields.TYPE_ATTACHMENT_VAL,
+            fields.PART_NUMBER_KEY: 0,
+            fields.CONTENT_HASH_KEY:  "",
+            fields.PAYLOAD_HASH_KEY: "",
+
+            fields.RAW_KEY: ""
+        },
+
+        BODY_DOC: {
+            fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL,
+            fields.CONTENT_HASH_KEY: "",
+
+            fields.BODY_KEY: "",
+
+            # this should not be needed,
+            # but let's keep the raw msg for some time
+            # until we are sure we can reconstruct
+            # the original msg from our disection.
+            fields.RAW_KEY: "",
+
+        }
     }
 
-    EMPTY_FLAGS = {
-        fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
-        fields.UID_KEY: 1,
-        fields.MBOX_KEY: fields.INBOX_VAL,
-
-        fields.FLAGS_KEY: [],
-        fields.SEEN_KEY: False,
-        fields.RECENT_KEY: True,
-        fields.MULTIPART_KEY: False,
-    }
-
-    # get from SoledadBackedAccount the needed index-related constants
-    INDEXES = SoledadBackedAccount.INDEXES
-    TYPE_IDX = SoledadBackedAccount.TYPE_IDX
-
     def __init__(self, mbox=None, soledad=None):
         """
         Constructor for MessageCollection.
@@ -465,23 +920,16 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             SoledadDocWriter(soledad),
             period=0.05)
 
-    def _get_empty_msg(self):
+    def _get_empty_doc(self, _type=FLAGS_DOC):
         """
-        Returns an empty message.
-
-        :return: a dict containing a default empty message
+        Returns an empty doc for storing different message parts.
+        Defaults to returning a template for a flags document.
+        :return: a dict with the template
         :rtype: dict
         """
-        return copy.deepcopy(self.EMPTY_MSG)
-
-    def _get_empty_flags_doc(self):
-        """
-        Returns an empty doc for storing flags.
-
-        :return:
-        :rtype:
-        """
-        return copy.deepcopy(self.EMPTY_FLAGS)
+        if not _type in self.templates.keys():
+            raise TypeError("Improper type passed to _get_empty_doc")
+        return copy.deepcopy(self.templates[_type])
 
     @deferred
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
@@ -509,52 +957,107 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             flags = tuple()
         leap_assert_type(flags, tuple)
 
-        content_doc = self._get_empty_msg()
-        flags_doc = self._get_empty_flags_doc()
-
-        content_doc[self.MBOX_KEY] = self.mbox
-        flags_doc[self.MBOX_KEY] = self.mbox
-        # ...should get a sanity check here.
-        content_doc[self.UID_KEY] = uid
-        flags_doc[self.UID_KEY] = uid
-
-        if flags:
-            flags_doc[self.FLAGS_KEY] = map(self._stringify, flags)
-            flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags
+        # docs for flags, headers, and body
+        fd, hd, bd = map(
+            lambda t: self._get_empty_doc(t),
+            (self.FLAGS_DOC, self.HEADERS_DOC, self.BODY_DOC))
 
         msg = self._get_parsed_msg(raw)
         headers = dict(msg)
-
-        logger.debug("adding. is multipart:%s" % msg.is_multipart())
-        flags_doc[self.MULTIPART_KEY] = msg.is_multipart()
-        # XXX get lower case for keys?
-        # XXX get headers doc
-        content_doc[self.HEADERS_KEY] = headers
-        # set subject based on message headers and eventually replace by
-        # subject given as param
-        if self.SUBJECT_FIELD in headers:
-            content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
-        if subject is not None:
-            content_doc[self.SUBJECT_KEY] = subject
-
-        # XXX could separate body into its own doc
-        # but should also separate multiparts
-        # that should be wrapped in MessagePart
-        content_doc[self.RAW_KEY] = self._stringify(raw)
-        content_doc[self.SIZE_KEY] = len(raw)
-
+        raw_str = msg.as_string()
+        chash = self._get_hash(msg)
+        multi = msg.is_multipart()
+
+        attaches = []
+        inner_parts = []
+
+        if multi:
+            # XXX should walk down recursively
+            # in a better way.  but fixing this quick
+            # to have an rc.
+            # XXX should pick the content-type in txt
+            body = first(msg.get_payload()).get_payload()
+            if isinstance(body, list):
+                # allowing one nesting level for now...
+                body, rest = body[0].get_payload(), body[1:]
+                for p in rest:
+                    inner_parts.append(p)
+        else:
+            body = msg.get_payload()
+        logger.debug("adding msg (multipart:%s)" % multi)
+
+        # flags doc ---------------------------------------
+        fd[self.MBOX_KEY] = self.mbox
+        fd[self.UID_KEY] = uid
+        fd[self.CONTENT_HASH_KEY] = chash
+        fd[self.MULTIPART_KEY] = multi
+        fd[self.SIZE_KEY] = len(raw_str)
+        if flags:
+            fd[self.FLAGS_KEY] = map(self._stringify, flags)
+            fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
+            fd[self.RECENT_KEY] = self.RECENT_FLAG in flags
+
+        # headers doc ----------------------------------------
+        hd[self.CONTENT_HASH_KEY] = chash
+        hd[self.HEADERS_KEY] = headers
+        if not subject and self.SUBJECT_FIELD in headers:
+            hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
+        else:
+            hd[self.SUBJECT_KEY] = subject
         if not date and self.DATE_FIELD in headers:
-            content_doc[self.DATE_KEY] = headers[self.DATE_FIELD]
+            hd[self.DATE_KEY] = headers[self.DATE_FIELD]
         else:
-            content_doc[self.DATE_KEY] = date
-
-        logger.debug('enqueuing message for write')
-
+            hd[self.DATE_KEY] = date
+        if multi:
+            hd[self.NUM_PARTS_KEY] = len(msg.get_payload())
+
+        # body doc
+        bd[self.CONTENT_HASH_KEY] = chash
+        bd[self.BODY_KEY] = body
+        # in an ideal world, we would not need to save a copy of the
+        # raw message. But we'll keep it until we can be sure that
+        # we can rebuild the original message from the parts.
+        bd[self.RAW_KEY] = raw_str
+
+        docs = [fd, hd]
+
+        # attachment docs
+        if multi:
+            outer_parts = msg.get_payload()
+            parts = outer_parts + inner_parts
+
+            # skip first part, we already got it in body
+            to_attach = ((i, m) for i, m in enumerate(parts) if i > 0)
+            for index, part_msg in to_attach:
+                att_doc = self._get_empty_doc(self.ATTACHMENT_DOC)
+                att_doc[self.PART_NUMBER_KEY] = index
+                att_doc[self.CONTENT_HASH_KEY] = chash
+                phash = self._get_hash(part_msg)
+                att_doc[self.PAYLOAD_HASH_KEY] = phash
+                att_doc[self.RAW_KEY] = part_msg.as_string()
+
+                # keep a pointer to the payload hash in the
+                # headers doc, under the parts_map
+                hd[self.PARTS_MAP_KEY][str(index)] = phash
+                attaches.append(att_doc)
+
+        # Saving ... -------------------------------
+        # ok, there we go...
+        logger.debug('enqueuing message docs for write')
         ptuple = SoledadWriterPayload
+
+        # first, regular docs: flags and headers
+        for doc in docs:
+            self.soledad_writer.put(ptuple(
+                mode=ptuple.CREATE, payload=doc))
+        # second, try to create body doc.
         self.soledad_writer.put(ptuple(
-            mode=ptuple.CREATE, payload=content_doc))
-        self.soledad_writer.put(ptuple(
-            mode=ptuple.CREATE, payload=flags_doc))
+            mode=ptuple.BODY_CREATE, payload=bd))
+        # and last, but not least, try to create
+        # attachment docs if not already there.
+        for at in attaches:
+            self.soledad_writer.put(ptuple(
+                mode=ptuple.ATTACHMENT_CREATE, payload=at))
 
     def remove(self, msg):
         """
@@ -563,8 +1066,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param msg: a  Leapmessage instance
         :type msg: LeapMessage
         """
-        # XXX remove
-        #self._soledad.delete_doc(msg)
         msg.remove()
 
     # getters
@@ -596,14 +1097,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :rtype: list of SoledadDocument
         """
         if _type not in fields.__dict__.values():
-            raise TypeError("Wrong type passed to get_all")
+            raise TypeError("Wrong type passed to get_all_docs")
 
         if sameProxiedObjects(self._soledad, None):
             logger.warning('Tried to get messages but soledad is None!')
             return []
 
         all_docs = [doc for doc in self._soledad.get_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
+            fields.TYPE_MBOX_IDX,
             _type, self.mbox)]
 
         # inneficient, but first let's grok it and then
@@ -618,8 +1119,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         all_uids = (doc.content[self.UID_KEY] for doc in
                     self._soledad.get_from_index(
-                        SoledadBackedAccount.TYPE_MBOX_IDX,
-                        self.TYPE_FLAGS_VAL, self.mbox))
+                        fields.TYPE_MBOX_IDX,
+                        fields.TYPE_FLAGS_VAL, self.mbox))
         return (u for u in sorted(all_uids))
 
     def count(self):
@@ -629,7 +1130,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :rtype: int
         """
         count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_IDX,
+            fields.TYPE_MBOX_IDX,
             fields.TYPE_FLAGS_VAL, self.mbox)
         return count
 
@@ -645,8 +1146,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         return (doc.content[self.UID_KEY] for doc in
                 self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-                    self.TYPE_FLAGS_VAL, self.mbox, '0'))
+                    fields.TYPE_MBOX_SEEN_IDX,
+                    fields.TYPE_FLAGS_VAL, self.mbox, '0'))
 
     def count_unseen(self):
         """
@@ -656,8 +1157,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :rtype: int
         """
         count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
-            self.TYPE_FLAGS_VAL, self.mbox, '0')
+            fields.TYPE_MBOX_SEEN_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox, '0')
         return count
 
     def get_unseen(self):
@@ -681,8 +1182,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         return (doc.content[self.UID_KEY] for doc in
                 self._soledad.get_from_index(
-                    SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-                    self.TYPE_FLAGS_VAL, self.mbox, '1'))
+                    fields.TYPE_MBOX_RECT_IDX,
+                    fields.TYPE_FLAGS_VAL, self.mbox, '1'))
 
     def get_recent(self):
         """
@@ -702,8 +1203,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :rtype: int
         """
         count = self._soledad.get_count_from_index(
-            SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
-            self.TYPE_FLAGS_VAL, self.mbox, '1')
+            fields.TYPE_MBOX_RECT_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox, '1')
         return count
 
     def __len__(self):
@@ -731,5 +1232,5 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         return u"<MessageCollection: mbox '%s' (%s)>" % (
             self.mbox, self.count())
 
-    # XXX should implement __eq__ also !!! --- use a hash
-    # of content for that, will be used for dedup.
+    # XXX should implement __eq__ also !!!
+    # --- use the content hash for that, will be used for dedup.
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
index 1ae19c0..306dcf0 100644
--- a/src/leap/mail/imap/parser.py
+++ b/src/leap/mail/imap/parser.py
@@ -19,10 +19,14 @@ Mail parser mixins.
 """
 import cStringIO
 import StringIO
+import hashlib
 import re
 
+from email.message import Message
 from email.parser import Parser
 
+from leap.common.check import leap_assert_type
+
 
 class MailParser(object):
     """
@@ -34,16 +38,30 @@ class MailParser(object):
         """
         self._parser = Parser()
 
-    def _get_parsed_msg(self, raw):
+    def _get_parsed_msg(self, raw, headersonly=False):
         """
         Return a parsed Message.
 
         :param raw: the raw string to parse
         :type raw: basestring, or StringIO object
+
+        :param headersonly: True for parsing only the headers.
+        :type headersonly: bool
         """
-        msg = self._get_parser_fun(raw)(raw, True)
+        msg = self._get_parser_fun(raw)(raw, headersonly=headersonly)
         return msg
 
+    def _get_hash(self, msg):
+        """
+        Returns a hash of the string representation of the raw message,
+        suitable for indexing the inmutable pieces.
+
+        :param msg: a Message object
+        :type msg: Message
+        """
+        leap_assert_type(msg, Message)
+        return hashlib.sha256(msg.as_string()).hexdigest()
+
     def _get_parser_fun(self, o):
         """
         Retunn the proper parser function for an object.
@@ -67,6 +85,8 @@ class MailParser(object):
         :param o: object
         :type o: object
         """
+        # XXX Maybe we don't need no more, we're using
+        # msg.as_string()
         if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
             return o.getvalue()
         else:
-- 
cgit v1.2.3


From a912729c4788d46d648a72126226741b63e0a37c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 2 Jan 2014 17:14:03 -0400
Subject: add documentation to the decorator, fix errorback.

* it also fixes the traceback in the errorback, thanks to
  chiiph, who reads documentation instead of whinning :D

* other minor documentation corrections
---
 src/leap/mail/decorators.py    | 68 ++++++++++++++++++++++++++++++++++++------
 src/leap/mail/imap/fetch.py    |  4 +--
 src/leap/mail/imap/messages.py |  5 +++-
 3 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py
index 9e49605..024a139 100644
--- a/src/leap/mail/decorators.py
+++ b/src/leap/mail/decorators.py
@@ -19,13 +19,10 @@ Useful decorators for mail package.
 """
 import logging
 import os
-import sys
-import traceback
 
 from functools import wraps
 
 from twisted.internet.threads import deferToThread
-from twisted.python import log
 
 logger = logging.getLogger(__name__)
 
@@ -41,27 +38,68 @@ def deferred(f):
     method wrapper.
     """
     class descript(object):
+        """
+        The class to be used as decorator.
+
+        It takes any method as the passed object.
+        """
+
         def __init__(self, f):
+            """
+            Initializes the decorator object.
+
+            :param f: the decorated function
+            :type f: callable
+            """
             self.f = f
 
         def __get__(self, instance, klass):
+            """
+            Descriptor implementation.
+
+            At creation time, the decorated `method` is unbound.
+
+            It will dispatch the make_unbound method if we still do not
+            have an instance available, and the make_bound method when the
+            method has already been bound to the instance.
+
+            :param instance: the instance of the class, or None if not exist.
+            :type instance: instantiated class or None.
+            """
             if instance is None:
                 # Class method was requested
                 return self.make_unbound(klass)
             return self.make_bound(instance)
 
         def _errback(self, failure):
-            err = failure.value
-            logger.warning('error in method: %s' % (self.f.__name__))
-            logger.exception(err)
-            log.err(err)
+            """
+            Errorback that logs the exception catched.
+
+            :param failure: a twisted failure
+            :type failure: Failure
+            """
+            logger.warning('Error in method: %s' % (self.f.__name__))
+            logger.exception(failure.getTraceback())
 
         def make_unbound(self, klass):
+            """
+            Return a wrapped function with the unbound call, during the
+            early access to the decortad method. This gets passed
+            only the class (not the instance since it does not yet exist).
+
+            :param klass: the class to which the still unbound method belongs
+            :type klass: type
+            """
 
             @wraps(self.f)
             def wrapper(*args, **kwargs):
                 """
-                this doc will vanish
+                We're temporarily wrapping the decorated method, but this
+                should not be called, since our application should use
+                the bound-wrapped method after this decorator class has been
+                used.
+
+                This documentation will vanish at runtime.
                 """
                 raise TypeError(
                     'unbound method {}() must be called with {} instance '
@@ -72,11 +110,23 @@ def deferred(f):
             return wrapper
 
         def make_bound(self, instance):
+            """
+            Return a function that wraps the bound method call,
+            after we are able to access the instance object.
+
+            :param instance: an instance of the class the decorated method,
+                             now bound, belongs to.
+            :type instance: object
+            """
 
             @wraps(self.f)
             def wrapper(*args, **kwargs):
                 """
-                This documentation will disapear
+                Do a proper function wrapper that defers the decorated method
+                call to a separated thread if the LEAPMAIL_DEBUG
+                environment variable is set.
+
+                This documentation will vanish at runtime.
                 """
                 if not os.environ.get('LEAPMAIL_DEBUG'):
                     d = deferToThread(self.f, instance, *args, **kwargs)
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index fdf1412..cb200be 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -455,8 +455,8 @@ class LeapIncomingMail(object):
         :param senderPubkey: The key of the sender of the message.
         :type senderPubkey: OpenPGPKey
 
-        :return: A unitary tuple containing a decrypted message and
-                 a bool indicating wether the signature is valid.
+        :return: A tuple containing a decrypted message and
+                 a bool indicating whether the signature is valid.
         :rtype: (Message, bool)
         """
         log.msg('maybe decrypting inline encrypted msg')
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index c69c023..47c40d5 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -695,6 +695,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
 SoledadWriterPayload = namedtuple(
     'SoledadWriterPayload', ['mode', 'payload'])
 
+# TODO we could consider using enum here:
+# https://pypi.python.org/pypi/enum
+
 SoledadWriterPayload.CREATE = 1
 SoledadWriterPayload.PUT = 2
 SoledadWriterPayload.BODY_CREATE = 3
@@ -758,7 +761,7 @@ class SoledadDocWriter(object):
     Message deduplication.
 
     We do a query for the content hashes before writing to our beloved
-    slcipher backend of Soledad. This means, by now, that:
+    sqlcipher backend of Soledad. This means, by now, that:
 
     1. We will not store the same attachment twice, only the hash of it.
     2. We will not store the same message body twice, only the hash of it.
-- 
cgit v1.2.3


From 5585ff784940dee267576d097076de66797f9188 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 2 Jan 2014 16:08:09 -0400
Subject: fix tests after rewrite

---
 src/leap/mail/imap/fields.py          |   3 +
 src/leap/mail/imap/mailbox.py         |  41 ++++---
 src/leap/mail/imap/messages.py        |  94 +++++++++++++---
 src/leap/mail/imap/tests/test_imap.py | 196 ++++++++++++++++++++++------------
 4 files changed, 227 insertions(+), 107 deletions(-)

diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 40817cd..bc536fe 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -35,6 +35,7 @@ class WithMsgFields(object):
     UID_KEY = "uid"
     MBOX_KEY = "mbox"
     SEEN_KEY = "seen"
+    DEL_KEY = "deleted"
     RECENT_KEY = "recent"
     FLAGS_KEY = "flags"
     MULTIPART_KEY = "multi"
@@ -95,6 +96,7 @@ class WithMsgFields(object):
     TYPE_SUBS_IDX = 'by-type-and-subscribed'
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+    TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
     TYPE_C_HASH_IDX = 'by-type-and-contenthash'
     TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
     TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
@@ -128,6 +130,7 @@ class WithMsgFields(object):
         # messages
         TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
         TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
+        TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
         TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
                                   'bool(recent)', 'bool(seen)'],
     }
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 5ea6f55..10087f6 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -390,18 +390,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         else:
             flags = tuple(str(flag) for flag in flags)
 
-        d = self._do_add_message(message, flags, date, uid_next)
+        d = self._do_add_message(message, flags=flags, date=date, uid=uid_next)
         d.addCallback(self._notify_new)
         return d
 
     @deferred
-    def _do_add_message(self, message, flags, date, uid_next):
+    def _do_add_message(self, message, flags, date, uid):
         """
         Calls to the messageCollection add_msg method (deferred to thread).
         Invoked from addMessage.
         """
-        self.messages.add_msg(message, flags=flags, date=date,
-                              uid=uid_next)
+        self.messages.add_msg(message, flags=flags, date=date, uid=uid)
 
     def _notify_new(self, *args):
         """
@@ -436,21 +435,29 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # we should postpone the removal
         self._soledad.delete_doc(self._get_mbox())
 
-    @deferred
+    def _close_cb(self, result):
+        self.closed = True
+
+    def close(self):
+        """
+        Expunge and mark as closed
+        """
+        d = self.expunge()
+        d.addCallback(self._close_cb)
+        return d
+
+    def _expunge_cb(self, result):
+        return result
+
     def expunge(self):
         """
         Remove all messages flagged \\Deleted
         """
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
-        deleted = []
-        for m in self.messages:
-            if self.DELETED_FLAG in m.getFlags():
-                self.messages.remove(m)
-                # XXX this would ve more efficient if we can just pass
-                # a sequence of uids.
-                deleted.append(m.getUID())
-        return deleted
+        d = self.messages.remove_all_deleted()
+        d.addCallback(self._expunge_cb)
+        return d
 
     @deferred
     def fetch(self, messages, uid):
@@ -603,14 +610,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self._signal_unread_to_ui()
         return result
 
-    @deferred
-    def close(self):
-        """
-        Expunge and mark as closed
-        """
-        self.expunge()
-        self.closed = True
-
     # IMessageCopier
 
     @deferred
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 47c40d5..80411f9 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -20,9 +20,11 @@ LeapMessage and MessageCollection.
 import copy
 import logging
 import StringIO
-from collections import namedtuple
+
+from collections import defaultdict, namedtuple
 
 from twisted.mail import imap4
+from twisted.internet import defer
 from twisted.python import log
 from u1db import errors as u1db_errors
 from zope.interface import implements
@@ -182,6 +184,7 @@ class MessageAttachment(object):
         if not self._msg:
             return {}
         headers = dict(self._msg.items())
+
         names = map(lambda s: s.upper(), names)
         if negate:
             cond = lambda key: key.upper() not in names
@@ -329,6 +332,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         doc.content[self.FLAGS_KEY] = flags
         doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
         doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
+        doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
         self._soledad.put_doc(doc)
 
     def addFlags(self, flags):
@@ -455,6 +459,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         headers = self._get_headers()
         if not headers:
             return {'content-type': ''}
+
         names = map(lambda s: s.upper(), names)
         if negate:
             cond = lambda key: key.upper() not in names
@@ -465,8 +470,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
             # twisted imap server expects headers to be lowercase
         head = dict(
-            map(str, (key, value)) if key.lower() != "content-type"
-            else map(str, (key.lower(), value))
+            (str(key), map(str, value)) if key.lower() != "content-type"
+            else (str(key.lower(), map(str, value)))
             for (key, value) in head.items())
 
         # unpack and filter original dict by negate-condition
@@ -670,6 +675,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
         # until we think about a good way of deorphaning.
         # Maybe a crawler of unreferenced docs.
 
+        uid = self._uid
+        print "removing...", uid
+
         fd = self._get_flags_doc()
         hd = self._get_headers_doc()
         #bd = self._get_body_doc()
@@ -682,7 +690,11 @@ class LeapMessage(fields, MailParser, MBoxParser):
             #docs.append(ad)
 
         for d in filter(None, docs):
-            self._soledad.delete_doc(d)
+            try:
+                self._soledad.delete_doc(d)
+            except Exception as exc:
+                logger.error(exc)
+        return uid
 
     def does_exist(self):
         """
@@ -849,6 +861,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
             fields.SEEN_KEY: False,
             fields.RECENT_KEY: True,
+            fields.DEL_KEY: False,
             fields.FLAGS_KEY: [],
             fields.MULTIPART_KEY: False,
             fields.SIZE_KEY: 0
@@ -921,7 +934,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         self.soledad_writer = MessageProducer(
             SoledadDocWriter(soledad),
-            period=0.05)
+            period=0.02)
 
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
@@ -966,7 +979,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             (self.FLAGS_DOC, self.HEADERS_DOC, self.BODY_DOC))
 
         msg = self._get_parsed_msg(raw)
-        headers = dict(msg)
+        headers = defaultdict(list)
+        for k, v in msg.items():
+            headers[k].append(v)
         raw_str = msg.as_string()
         chash = self._get_hash(msg)
         multi = msg.is_multipart()
@@ -987,7 +1002,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                     inner_parts.append(p)
         else:
             body = msg.get_payload()
-        logger.debug("adding msg (multipart:%s)" % multi)
+        logger.debug("adding msg with uid %s (multipart:%s)" % (
+            uid, multi))
 
         # flags doc ---------------------------------------
         fd[self.MBOX_KEY] = self.mbox
@@ -998,26 +1014,33 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         if flags:
             fd[self.FLAGS_KEY] = map(self._stringify, flags)
             fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
-            fd[self.RECENT_KEY] = self.RECENT_FLAG in flags
+            fd[self.DEL_KEY] = self.DELETED_FLAG in flags
+            fd[self.RECENT_KEY] = True  # set always by default
 
         # headers doc ----------------------------------------
         hd[self.CONTENT_HASH_KEY] = chash
         hd[self.HEADERS_KEY] = headers
+
+        print "headers"
+        import pprint
+        pprint.pprint(headers)
+
         if not subject and self.SUBJECT_FIELD in headers:
-            hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
+            hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
         else:
             hd[self.SUBJECT_KEY] = subject
         if not date and self.DATE_FIELD in headers:
-            hd[self.DATE_KEY] = headers[self.DATE_FIELD]
+            hd[self.DATE_KEY] = first(headers[self.DATE_FIELD])
         else:
             hd[self.DATE_KEY] = date
         if multi:
+            # XXX fix for multipart nested case
             hd[self.NUM_PARTS_KEY] = len(msg.get_payload())
 
         # body doc
         bd[self.CONTENT_HASH_KEY] = chash
         bd[self.BODY_KEY] = body
-        # in an ideal world, we would not need to save a copy of the
+        # XXX in an ideal world, we would not need to save a copy of the
         # raw message. But we'll keep it until we can be sure that
         # we can rebuild the original message from the parts.
         bd[self.RAW_KEY] = raw_str
@@ -1062,14 +1085,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             self.soledad_writer.put(ptuple(
                 mode=ptuple.ATTACHMENT_CREATE, payload=at))
 
-    def remove(self, msg):
+    def _remove_cb(self, result):
+        return result
+
+    def remove_all_deleted(self):
+        """
+        Removes all messages flagged as deleted.
         """
-        Removes a message.
+        delete_deferl = []
+        for msg in self.get_deleted():
+            delete_deferl.append(msg.remove())
+        d1 = defer.gatherResults(delete_deferl, consumeErrors=True)
+        d1.addCallback(self._remove_cb)
+        return d1
 
-        :param msg: a  Leapmessage instance
+    def remove(self, msg):
+        """
+        Remove a given msg.
+        :param msg: the message to be removed
         :type msg: LeapMessage
         """
-        msg.remove()
+        d = msg.remove()
+        d.addCallback(self._remove_cb)
+        return d
 
     # getters
 
@@ -1178,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     def recent_iter(self):
         """
-        Get an iterator for the message docs with `recent` flag.
+        Get an iterator for the message UIDs with `recent` flag.
 
         :return: iterator through recent message docs
         :rtype: iterable
@@ -1210,6 +1248,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             fields.TYPE_FLAGS_VAL, self.mbox, '1')
         return count
 
+    # deleted messages
+
+    def deleted_iter(self):
+        """
+        Get an iterator for the message UIDs with `deleted` flag.
+
+        :return: iterator through deleted message docs
+        :rtype: iterable
+        """
+        return (doc.content[self.UID_KEY] for doc in
+                self._soledad.get_from_index(
+                    fields.TYPE_MBOX_DEL_IDX,
+                    fields.TYPE_FLAGS_VAL, self.mbox, '1'))
+
+    def get_deleted(self):
+        """
+        Get all messages with the `Deleted` flag.
+
+        :returns: a generator of LeapMessages
+        :rtype: generator
+        """
+        return (LeapMessage(self._soledad, docid, self.mbox)
+                for docid in self.deleted_iter())
+
     def __len__(self):
         """
         Returns the number of messages on this mailbox.
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index ea75854..e1bed8c 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -25,7 +25,7 @@ XXX add authors from the original twisted tests.
 @license: GPLv3, see included LICENSE file
 """
 # XXX review license of the original tests!!!
-from nose.twistedtools import deferred
+from email import parser
 
 try:
     from cStringIO import StringIO
@@ -36,9 +36,13 @@ import os
 import types
 import tempfile
 import shutil
+import time
+
+from itertools import chain
 
 
 from mock import Mock
+from nose.twistedtools import deferred, stop_reactor
 
 
 from twisted.mail import imap4
@@ -58,9 +62,9 @@ import twisted.cred.portal
 # import u1db
 
 from leap.common.testing.basetest import BaseLeapTest
-from leap.mail.imap.server import SoledadMailbox
-from leap.mail.imap.server import SoledadBackedAccount
-from leap.mail.imap.server import MessageCollection
+from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.messages import MessageCollection
 
 from leap.soledad.client import Soledad
 from leap.soledad.client import SoledadCrypto
@@ -321,6 +325,9 @@ class IMAP4HelperMixin(BaseLeapTest):
         for mb in self.server.theAccount.mailboxes:
             self.server.theAccount.delete(mb)
 
+        # email parser
+        self.parser = parser.Parser()
+
     def tearDown(self):
         """
         tearDown method called after each test.
@@ -389,6 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
     """
     Tests for the MessageCollection class
     """
+    count = 0
 
     def setUp(self):
         """
@@ -396,34 +404,35 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         We override mixin method since we are only testing
         MessageCollection interface in this particular TestCase
         """
-        self.messages = MessageCollection("testmbox", self._soledad)
-        for m in self.messages.get_all():
-            self.messages.remove(m)
+        self.messages = MessageCollection("testmbox%s" % (self.count,),
+                                          self._soledad)
+        MessageCollectionTestCase.count += 1
 
     def tearDown(self):
         """
         tearDown method for each test
-        Delete the message collection
         """
         del self.messages
 
+    def wait(self):
+        time.sleep(2)
+
     def testEmptyMessage(self):
         """
         Test empty message and collection
         """
-        em = self.messages._get_empty_msg()
+        em = self.messages._get_empty_doc()
         self.assertEqual(
             em,
             {
-                "date": '',
                 "flags": [],
-                "headers": {},
                 "mbox": "inbox",
-                "raw": "",
                 "recent": True,
                 "seen": False,
-                "subject": "",
-                "type": "msg",
+                "deleted": False,
+                "multi": False,
+                "size": 0,
+                "type": "flags",
                 "uid": 1,
             })
         self.assertEqual(self.messages.count(), 0)
@@ -432,23 +441,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Add multiple messages
         """
+        # TODO really profile addition
         mc = self.messages
+        print "messages", self.messages
         self.assertEqual(self.messages.count(), 0)
-        mc.add_msg('Stuff', subject="test1")
-        self.assertEqual(self.messages.count(), 1)
-        mc.add_msg('Stuff', subject="test2")
-        self.assertEqual(self.messages.count(), 2)
-        mc.add_msg('Stuff', subject="test3")
-        self.assertEqual(self.messages.count(), 3)
-        mc.add_msg('Stuff', subject="test4")
+        mc.add_msg('Stuff', uid=1, subject="test1")
+        mc.add_msg('Stuff', uid=2, subject="test2")
+        mc.add_msg('Stuff', uid=3, subject="test3")
+        mc.add_msg('Stuff', uid=4, subject="test4")
+        self.wait()
         self.assertEqual(self.messages.count(), 4)
-        mc.add_msg('Stuff', subject="test5")
-        mc.add_msg('Stuff', subject="test6")
-        mc.add_msg('Stuff', subject="test7")
-        mc.add_msg('Stuff', subject="test8")
-        mc.add_msg('Stuff', subject="test9")
-        mc.add_msg('Stuff', subject="test10")
-        self.assertEqual(self.messages.count(), 10)
+        mc.add_msg('Stuff', uid=5, subject="test5")
+        mc.add_msg('Stuff', uid=6, subject="test6")
+        mc.add_msg('Stuff', uid=7, subject="test7")
+        self.wait()
+        self.assertEqual(self.messages.count(), 7)
+        self.wait()
 
     def testRecentCount(self):
         """
@@ -456,45 +464,48 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         mc = self.messages
         self.assertEqual(self.messages.count_recent(), 0)
-        mc.add_msg('Stuff', subject="test1", uid=1)
+        mc.add_msg('Stuff', uid=1, subject="test1")
         # For the semantics defined in the RFC, we auto-add the
         # recent flag by default.
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 1)
-        mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',))
+        mc.add_msg('Stuff', subject="test2", uid=2,
+                   flags=('\\Deleted',))
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 2)
-        mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',))
+        mc.add_msg('Stuff', subject="test3", uid=3,
+                   flags=('\\Recent',))
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 3)
         mc.add_msg('Stuff', subject="test4", uid=4,
                    flags=('\\Deleted', '\\Recent'))
+        self.wait()
         self.assertEqual(self.messages.count_recent(), 4)
 
-        for m in mc:
-            msg = self.messages.get_msg_by_uid(m.get('uid'))
-            msg_newflags = msg.removeFlags(('\\Recent',))
-            self._soledad.put_doc(msg_newflags)
-
+        for msg in mc:
+            msg.removeFlags(('\\Recent',))
         self.assertEqual(mc.count_recent(), 0)
 
     def testFilterByMailbox(self):
         """
         Test that queries filter by selected mailbox
         """
+        def wait():
+            time.sleep(1)
+
         mc = self.messages
         self.assertEqual(self.messages.count(), 0)
-        mc.add_msg('', subject="test1")
-        self.assertEqual(self.messages.count(), 1)
-        mc.add_msg('', subject="test2")
-        self.assertEqual(self.messages.count(), 2)
-        mc.add_msg('', subject="test3")
+        mc.add_msg('', uid=1, subject="test1")
+        mc.add_msg('', uid=2, subject="test2")
+        mc.add_msg('', uid=3, subject="test3")
+        wait()
         self.assertEqual(self.messages.count(), 3)
-
-        newmsg = mc._get_empty_msg()
+        newmsg = mc._get_empty_doc()
         newmsg['mailbox'] = "mailbox/foo"
-        newmsg['subject'] = "test another mailbox"
         mc._soledad.create_doc(newmsg)
         self.assertEqual(mc.count(), 3)
         self.assertEqual(
-            len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4)
+            len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
 
 
 class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
@@ -1174,16 +1185,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(0.5)
+
         def append():
             return self.client.append(
                 'root/subthing',
                 message,
-                ['\\SEEN', '\\DELETED'],
+                ('\\SEEN', '\\DELETED'),
                 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
             )
 
         d1 = self.connected.addCallback(strip(login))
         d1.addCallbacks(strip(append), self._ebGeneral)
+        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
@@ -1191,17 +1206,31 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestFullAppend(self, ignored, infile):
         mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing')
+        time.sleep(0.5)
         self.assertEqual(1, len(mb.messages))
 
+        msg = mb.messages.get_msg_by_uid(1)
         self.assertEqual(
-            ['\\SEEN', '\\DELETED'],
-            mb.messages[1].content['flags'])
+            ('\\SEEN', '\\DELETED'),
+            msg.getFlags())
 
         self.assertEqual(
             'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
-            mb.messages[1].content['date'])
+            msg.getInternalDate())
+
+        parsed = self.parser.parse(open(infile))
+        body = parsed.get_payload()
+        headers = parsed.items()
+        self.assertEqual(
+            body,
+            msg.getBodyFile().read())
+
+        msg_headers = msg.getHeaders(True, "",)
+        gotheaders = list(chain(
+            *[[(k, item) for item in v] for (k, v) in msg_headers.items()]))
 
-        self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+        self.assertItemsEqual(
+            headers, gotheaders)
 
     @deferred(timeout=None)
     def testPartialAppend(self):
@@ -1209,12 +1238,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test partially appending a message to the mailbox
         """
         infile = util.sibpath(__file__, 'rfc822.message')
-        message = open(infile)
         SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
 
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(1)
+
         def append():
             message = file(infile)
             return self.client.sendCommand(
@@ -1226,6 +1257,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                 )
             )
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(strip(append), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -1235,15 +1267,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestPartialAppend(self, ignored, infile):
         mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
-
+        time.sleep(1)
         self.assertEqual(1, len(mb.messages))
+        msg = mb.messages.get_msg_by_uid(1)
         self.assertEqual(
-            ['\\SEEN', ],
-            mb.messages[1].content['flags']
+            ('\\SEEN', ),
+            msg.getFlags()
         )
+        #self.assertEqual(
+            #'Right now', msg.getInternalDate())
+        parsed = self.parser.parse(open(infile))
+        body = parsed.get_payload()
         self.assertEqual(
-            'Right now', mb.messages[1].content['date'])
-        self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+            body,
+            msg.getBodyFile().read())
 
     @deferred(timeout=None)
     def testCheck(self):
@@ -1279,14 +1316,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.server.theAccount.addMailbox(name)
 
         m = SimpleLEAPServer.theAccount.getMailbox(name)
-        m.messages.add_msg('', subject="Message 1",
+        m.messages.add_msg('test 1', uid=1, subject="Message 1",
                            flags=('\\Deleted', 'AnotherFlag'))
-        m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
-        m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
+        m.messages.add_msg('test 2', uid=2, subject="Message 2",
+                           flags=('AnotherFlag',))
+        m.messages.add_msg('test 3', uid=3, subject="Message 3",
+                           flags=('\\Deleted',))
 
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(1)
+
         def select():
             return self.client.select(name)
 
@@ -1294,6 +1336,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             return self.client.close()
 
         d = self.connected.addCallback(strip(login))
+        d.addCallbacks(strip(wait), self._ebGeneral)
         d.addCallbacks(strip(select), self._ebGeneral)
         d.addCallbacks(strip(close), self._ebGeneral)
         d.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -1302,8 +1345,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestClose(self, ignored, m):
         self.assertEqual(len(m.messages), 1)
+        messages = [msg for msg in m.messages]
+        self.assertFalse(messages[0] is None)
         self.assertEqual(
-            m.messages[1].content['subject'],
+            messages[0]._hdoc.content['subject'],
             'Message 2')
         self.failUnless(m.closed)
 
@@ -1315,17 +1360,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         name = 'mailbox-expunge'
         SimpleLEAPServer.theAccount.addMailbox(name)
         m = SimpleLEAPServer.theAccount.getMailbox(name)
-        m.messages.add_msg('', subject="Message 1",
+        m.messages.add_msg('test 1', uid=1, subject="Message 1",
                            flags=('\\Deleted', 'AnotherFlag'))
-        self.failUnless(m.messages.count() == 1)
-        m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
-        self.failUnless(m.messages.count() == 2)
-        m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
-        self.failUnless(m.messages.count() == 3)
+        m.messages.add_msg('test 2', uid=2, subject="Message 2",
+                           flags=('AnotherFlag',))
+        m.messages.add_msg('test 3', uid=3, subject="Message 3",
+                           flags=('\\Deleted',))
 
         def login():
             return self.client.login('testuser', 'password-test')
 
+        def wait():
+            time.sleep(2)
+
         def select():
             return self.client.select('mailbox-expunge')
 
@@ -1338,6 +1385,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.results = None
         d1 = self.connected.addCallback(strip(login))
+        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(strip(select), self._ebGeneral)
         d1.addCallbacks(strip(expunge), self._ebGeneral)
         d1.addCallbacks(expunged, self._ebGeneral)
@@ -1348,12 +1396,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestExpunge(self, ignored, m):
         # we only left 1 mssage with no deleted flag
-        self.assertEqual(m.messages.count(), 1)
+        self.assertEqual(len(m.messages), 1)
+        messages = [msg for msg in m.messages]
         self.assertEqual(
-            m.messages[1].content['subject'],
+            messages[0]._hdoc.content['subject'],
             'Message 2')
-        self.assertEqual(self.results, [0, 1])
-        # XXX fix this thing with the indexes...
+        # the uids of the deleted messages
+        self.assertItemsEqual(self.results, [1, 3])
 
 
 class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
@@ -1363,3 +1412,10 @@ class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
     """
     # XXX coming soon to your screens!
     pass
+
+
+def tearDownModule():
+    """
+    Tear down functions for module level
+    """
+    stop_reactor()
-- 
cgit v1.2.3


From e1946f1653dbbb6fcf61569bc873ab061965664e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 6 Jan 2014 04:44:05 -0400
Subject: tests infrastructure for multipart

---
 src/leap/mail/imap/mailbox.py                      |   5 +
 .../mail/imap/tests/rfc822.multi-signed.message    | 238 +++++++++++++++++++++
 src/leap/mail/imap/tests/rfc822.multi.message      |  96 +++++++++
 src/leap/mail/imap/tests/test_imap.py              |  78 ++++++-
 4 files changed, 412 insertions(+), 5 deletions(-)
 create mode 100644 src/leap/mail/imap/tests/rfc822.multi-signed.message
 create mode 100644 src/leap/mail/imap/tests/rfc822.multi.message

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 10087f6..1d76d4d 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -478,6 +478,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 LeapMessage
         """
         result = []
+
+        # XXX DEBUG -------------
+        print "getting uid", uid
+        print "in mbox", self.mbox
+
         sequence = True if uid == 0 else False
 
         if not messages.last:
diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message
new file mode 100644
index 0000000..9907c2d
--- /dev/null
+++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message
@@ -0,0 +1,238 @@
+Date: Mon, 6 Jan 2014 04:40:47 -0400
+From: Kali Kaneko <kali@leap.se>
+To: penguin@example.com
+Subject: signed message
+Message-ID: <20140106084047.GA21317@samsara.lan>
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha1;
+	protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.21 (2012-12-30)
+
+
+--z9ECzHErBrwFF8sy
+Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l"
+Content-Disposition: inline
+
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+
+This is an example of a signed message,
+with attachments.
+
+
+--=20
+Nihil sine chao! =E2=88=B4
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: attachment; filename="attach.txt"
+
+this is attachment in plain text.
+
+--z0eOaCaDLjvTGF2l
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="hack.ico"
+Content-Transfer-Encoding: base64
+
+AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA
+KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG
+RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA
+PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl
+5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA
+/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ
+yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A
+Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK
+ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK
+LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP
+QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy
+AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs
+AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA
+AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA
+gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d
+HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA
+x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7
++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA
+AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5
++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA
+OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK
+igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA
+JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra
+2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA
+xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj
+owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB
+AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA
+AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d
+XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d
+XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA
+AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB
+AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm
+X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC
+AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B
+bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ
+S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu
+J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y
+AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N
+KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB
+XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A
+AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA
+AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d
+XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA
+AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr
+RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA
+AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A
+Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI
+yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA
+CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys
+rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA
+vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d
+HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA
+urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx
+cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA
+CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo
+6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA
+2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7
+OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA
+UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp
+qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA
+lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa
+WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB
+AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA
+ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA
+AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB
+AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB
+AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA
+tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA
+AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB
+AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB
+AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA
+AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd
+AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB
+AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB
+AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8
+ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2
+NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF
+RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB
+lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA
+AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa
+WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA
+AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX
+AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB
+AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA
+AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA
+AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA
+AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA
+AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB
+AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB
+AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA
+ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA
+AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7
+LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA
+AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA
+ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4
+RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi
+JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2
+NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK
+T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN
+UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA
+AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA
+W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA
+AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB
+l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB
+AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+
+WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA
+AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv
+RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA
+AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj
+AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB
+AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA
+AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA
+AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA
+dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A
+AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB
+AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW
+pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+
+--z0eOaCaDLjvTGF2l--
+
+--z9ECzHErBrwFF8sy
+Content-Type: application/pgp-signature
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv
+kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl
+vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK
+PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC
+w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw
+sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr
+BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN
+QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt
+mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ
+jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8
+gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X
+sSdfcAhT7Tno7PB/Acoh
+=+okv
+-----END PGP SIGNATURE-----
+
+--z9ECzHErBrwFF8sy--
diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message
new file mode 100644
index 0000000..30f74e5
--- /dev/null
+++ b/src/leap/mail/imap/tests/rfc822.multi.message
@@ -0,0 +1,96 @@
+Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
+From: Doug Sauder <doug@penguin.example.com>
+To: Joe Blow <blow@example.com>
+Subject: Test message from PINE
+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
+MIME-Version: 1.0
+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
+
+  This message is in MIME format.  The first part should be readable text,
+  while the remaining parts are likely unreadable without MIME-aware tools.
+  Send mail to mime@docserver.cac.washington.edu for more info.
+
+---1463757054-952513540-958744548=:8452
+Content-Type: TEXT/PLAIN; charset=US-ASCII
+
+This is a test message from PINE MUA.
+
+
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="redball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="redball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
+vAAAAABJRU5ErkJggg==
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="blueball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="blueball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
+Z3VldDZzO7wAAAAASUVORK5CYII=
+---1463757054-952513540-958744548=:8452--
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index e1bed8c..8c1cf20 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -357,11 +357,11 @@ class IMAP4HelperMixin(BaseLeapTest):
 
         # XXX we also should put this in a mailbox!
 
-        self._soledad.messages.add_msg('', subject="test1")
-        self._soledad.messages.add_msg('', subject="test2")
-        self._soledad.messages.add_msg('', subject="test3")
+        self._soledad.messages.add_msg('', uid=1, subject="test1")
+        self._soledad.messages.add_msg('', uid=2, subject="test2")
+        self._soledad.messages.add_msg('', uid=3, subject="test3")
         # XXX should change Flags too
-        self._soledad.messages.add_msg('', subject="test4")
+        self._soledad.messages.add_msg('', uid=4, subject="test4")
 
     def delete_all_docs(self):
         """
@@ -1405,10 +1405,78 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertItemsEqual(self.results, [1, 3])
 
 
+class StoreAndFetchTestCase(unittest.TestCase, IMAP4HelperMixin):
+    """
+    Several tests to check that the internal storage representation
+    is able to render the message structures as we expect them.
+    """
+    # TODO get rid of the fucking sleeps with a proper defer
+    # management.
+
+    def setUp(self):
+        IMAP4HelperMixin.setUp(self)
+        MBOX_NAME = "multipart/SIGNED"
+        self.received_messages = self.received_uid = None
+        self.result = None
+
+        self.server.state = 'select'
+
+        infile = util.sibpath(__file__, 'rfc822.multi-signed.message')
+        raw = open(infile).read()
+
+        self.server.theAccount.addMailbox(MBOX_NAME)
+        mbox = self.server.theAccount.getMailbox(MBOX_NAME)
+        time.sleep(1)
+        self.server.mbox = mbox
+        self.server.mbox.messages.add_msg(raw, uid=1)
+        time.sleep(1)
+
+    def addListener(self, x):
+        pass
+
+    def removeListener(self, x):
+        pass
+
+    def _fetchWork(self, uids):
+
+        def result(R):
+            self.result = R
+
+        self.connected.addCallback(
+            lambda _: self.function(
+                uids, uid=1)  # do NOT use seq numbers!
+            ).addCallback(result).addCallback(
+            self._cbStopClient).addErrback(self._ebGeneral)
+
+        d = loopback.loopbackTCP(self.server, self.client, noisy=False)
+        d.addCallback(lambda x: self.assertEqual(self.result, self.expected))
+        return d
+
+    @deferred(timeout=None)
+    def testMultiBody(self):
+        """
+        Test that a multipart signed message is retrieved the same
+        as we stored it.
+        """
+        time.sleep(1)
+        self.function = self.client.fetchBody
+        messages = '1'
+
+        # XXX review. This probably should give everything?
+
+        self.expected = {1: {
+            'RFC822.TEXT': 'This is an example of a signed message,\n'
+                           'with attachments.\n\n\n--=20\n'
+                           'Nihil sine chao! =E2=88=B4\n',
+            'UID': '1'}}
+        print "test multi: fetch uid", messages
+        return self._fetchWork(messages)
+
+
 class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     """
-    Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}.
+    Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}.
     """
     # XXX coming soon to your screens!
     pass
-- 
cgit v1.2.3


From a203337d155a6e7186980ef175642adc91d472fe Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 7 Jan 2014 14:23:25 -0400
Subject: move utility to its own

---
 src/leap/mail/imap/messages.py | 11 +----------
 src/leap/mail/utils.py         | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 10 deletions(-)
 create mode 100644 src/leap/mail/utils.py

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 80411f9..bfe913c 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -33,6 +33,7 @@ from zope.proxy import sameProxiedObjects
 from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
+from leap.mail.utils import first
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
@@ -42,16 +43,6 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer
 logger = logging.getLogger(__name__)
 
 
-def first(things):
-    """
-    Return the head of a collection.
-    """
-    try:
-        return things[0]
-    except (IndexError, TypeError):
-        return None
-
-
 class MessageBody(object):
     """
     IMessagePart implementor for the main
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
new file mode 100644
index 0000000..2480efc
--- /dev/null
+++ b/src/leap/mail/utils.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Small utilities.
+"""
+
+
+def first(things):
+    """
+    Return the head of a collection.
+    """
+    try:
+        return things[0]
+    except (IndexError, TypeError):
+        return None
-- 
cgit v1.2.3


From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 7 Jan 2014 11:34:08 -0400
Subject: Second stage of the new year's storage rewrite.

* documents of only three types:
  * flags
  * headers
  * content
* add algorithm for walking the parsed message tree.
* treat special cases like a multipart with a single part.
* modify add_msg to use the walk routine
* modify twisted interfaces to use the new storage schema.
* tests for different multipart cases
* fix multipart detection typo in the fetch

This is a merge proposal for the 0.5.0-rc3.

known bugs
----------
Some things are still know not to work well at this point
(some cases of multipart messages do not display the bodies).

IMAP server also is left in a bad internal state after a logout/login.
---
 src/leap/mail/decorators.py                        |   5 +
 src/leap/mail/imap/fetch.py                        |   2 +-
 src/leap/mail/imap/fields.py                       |  26 +-
 src/leap/mail/imap/messages.py                     | 722 +++++++++++----------
 src/leap/mail/imap/service/imap.py                 |   2 +
 .../mail/imap/tests/rfc822.multi-minimal.message   |  16 +
 src/leap/mail/imap/tests/rfc822.plain.message      |  66 ++
 src/leap/mail/imap/tests/walktree.py               | 117 ++++
 src/leap/mail/walk.py                              | 160 +++++
 9 files changed, 768 insertions(+), 348 deletions(-)
 create mode 100644 src/leap/mail/imap/tests/rfc822.multi-minimal.message
 create mode 100644 src/leap/mail/imap/tests/rfc822.plain.message
 create mode 100644 src/leap/mail/imap/tests/walktree.py
 create mode 100644 src/leap/mail/walk.py

diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py
index 024a139..d5eac97 100644
--- a/src/leap/mail/decorators.py
+++ b/src/leap/mail/decorators.py
@@ -27,6 +27,11 @@ from twisted.internet.threads import deferToThread
 logger = logging.getLogger(__name__)
 
 
+# TODO
+# Should write a helper to be able to pass a timeout argument.
+# See this answer: http://stackoverflow.com/a/19019648/1157664
+# And the notes by glyph and jpcalderone
+
 def deferred(f):
     """
     Decorator, for deferring methods to Threads.
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index cb200be..604a2ea 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -404,7 +404,7 @@ class LeapIncomingMail(object):
         """
         log.msg('decrypting multipart encrypted msg')
         msg = copy.deepcopy(msg)
-        self._multipart_sanity_check(msg)
+        self._msg_multipart_sanity_check(msg)
 
         # parse message and get encrypted content
         pgpencmsg = msg.get_payload()[1]
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index bc536fe..2545adf 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -43,17 +43,17 @@ class WithMsgFields(object):
 
     # headers
     HEADERS_KEY = "headers"
-    NUM_PARTS_KEY = "numparts"
-    PARTS_MAP_KEY = "partmap"
     DATE_KEY = "date"
     SUBJECT_KEY = "subject"
-
-    # attachment
-    PART_NUMBER_KEY = "part"
-    RAW_KEY = "raw"
+    # XXX DELETE-ME
+    #NUM_PARTS_KEY = "numparts"  # not needed?!
+    PARTS_MAP_KEY = "part_map"
+    BODY_KEY = "body"  # link to phash of body
 
     # content
-    BODY_KEY = "body"
+    LINKED_FROM_KEY = "lkf"
+    RAW_KEY = "raw"
+    CTYPE_KEY = "ctype"
 
     # Mailbox specific keys
     CLOSED_KEY = "closed"
@@ -65,11 +65,13 @@ class WithMsgFields(object):
     # Document Type, for indexing
     TYPE_KEY = "type"
     TYPE_MBOX_VAL = "mbox"
-    TYPE_MESSAGE_VAL = "msg"
     TYPE_FLAGS_VAL = "flags"
     TYPE_HEADERS_VAL = "head"
-    TYPE_ATTACHMENT_VAL = "attach"
-    # should add also a headers val
+    TYPE_CONTENT_VAL = "cnt"
+
+    # XXX DEPRECATE
+    #TYPE_MESSAGE_VAL = "msg"
+    #TYPE_ATTACHMENT_VAL = "attach"
 
     INBOX_VAL = "inbox"
 
@@ -109,7 +111,6 @@ class WithMsgFields(object):
     MBOX_VAL = TYPE_MBOX_VAL
     CHASH_VAL = CONTENT_HASH_KEY
     PHASH_VAL = PAYLOAD_HASH_KEY
-    PART_VAL = PART_NUMBER_KEY
 
     INDEXES = {
         # generic
@@ -122,8 +123,7 @@ class WithMsgFields(object):
 
         # content, headers doc
         TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
-        # attachment docs
-        TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL],
+
         # attachment payload dedup
         TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
 
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index bfe913c..37e4311 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -33,6 +33,7 @@ from zope.proxy import sameProxiedObjects
 from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
+from leap.mail import walk
 from leap.mail.utils import first
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
@@ -43,65 +44,58 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer
 logger = logging.getLogger(__name__)
 
 
-class MessageBody(object):
-    """
-    IMessagePart implementor for the main
-    body of a multipart message.
-
-    Excusatio non petita: see the interface documentation.
-    """
+# TODO ------------------------------------------------------------
 
-    implements(imap4.IMessagePart)
-
-    def __init__(self, fdoc, bdoc):
-        self._fdoc = fdoc
-        self._bdoc = bdoc
-
-    def getSize(self):
-        return len(self._bdoc.content[fields.BODY_KEY])
+# [ ] Add linked-from info.
+# [ ] Delete incoming mail only after successful write!
+# [ ] Remove UID from syncable db. Store only those indexes locally.
+# [ ] Send patch to twisted for bug in imap4.py:5717  (content-type can be
+#     none? lower-case?)
 
-    def getBodyFile(self):
-        fd = StringIO.StringIO()
-
-        if self._bdoc:
-            body = self._bdoc.content[fields.BODY_KEY]
-        else:
-            body = ""
-        charset = self._get_charset(body)
-        try:
-            body = body.encode(charset)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            body = body.encode(charset, 'replace')
-        fd.write(body)
-        fd.seek(0)
-        return fd
-
-    @memoized_method
-    def _get_charset(self, stuff):
-        return get_email_charset(unicode(stuff))
-
-    def getHeaders(self, negate, *names):
-        return {}
+def lowerdict(_dict):
+    """
+    Return a dict with the keys in lowercase.
 
-    def isMultipart(self):
-        return False
+    :param _dict: the dict to convert
+    :rtype: dict
+    """
+    return dict((key.lower(), value)
+                for key, value in _dict.items())
 
-    def getSubPart(self, part):
-        return None
 
+class MessagePart(object):
+    """
+    IMessagePart implementor.
+    It takes a subpart message and is able to find
+    the inner parts.
 
-class MessageAttachment(object):
+    Excusatio non petita: see the interface documentation.
+    """
 
     implements(imap4.IMessagePart)
 
-    def __init__(self, msg):
+    def __init__(self, soledad, part_map):
         """
-        Initializes the messagepart with a Message instance.
-        :param msg: a message instance
-        :type msg: Message
+        Initializes the MessagePart.
+
+        :param part_map: a dictionary containing the parts map for this
+                         message
+        :type part_map: dict
         """
-        self._msg = msg
+        # TODO
+        # It would be good to pass the uid/mailbox also
+        # for references while debugging.
+
+        # We have a problem on bulk moves, and is
+        # that when the fetch on the new mailbox is done
+        # the parts maybe are not complete.
+        # So we should be able to fail with empty
+        # docs until we solve that. The ideal would be
+        # to gather the results of the deferred operations
+        # to signal the operation is complete.
+        #leap_assert(part_map, "part map dict cannot be null")
+        self._soledad = soledad
+        self._pmap = part_map
 
     def getSize(self):
         """
@@ -110,9 +104,12 @@ class MessageAttachment(object):
         :return: size of the message, in octets
         :rtype: int
         """
-        if not self._msg:
+        if not self._pmap:
             return 0
-        return len(self._msg.as_string())
+        size = self._pmap.get('size', None)
+        if not size:
+            logger.error("Message part cannot find size in the partmap")
+        return size
 
     def getBodyFile(self):
         """
@@ -122,24 +119,91 @@ class MessageAttachment(object):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        if self._msg:
-            body = self._msg.get_payload()
+        if self._pmap:
+            multi = self._pmap.get('multi')
+            if not multi:
+                phash = self._pmap.get("phash", None)
+            else:
+                pmap = self._pmap.get('part_map')
+                first_part = pmap.get('1', None)
+                if first_part:
+                    phash = first_part['phash']
+
+            if not phash:
+                logger.warning("Could not find phash for this subpart!")
+                payload = str("")
+            else:
+                payload = self._get_payload_from_document(phash)
+
         else:
-            logger.debug("Empty message!")
-            body = ""
-
-        # XXX should only do the dance if we're sure it's
-        # content/text-plain!!!
-        #charset = self._get_charset(body)
-        #try:
-            #body = body.encode(charset)
-        #except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            #logger.error("Unicode error {0}".format(e))
-            #body = body.encode(charset, 'replace')
-        fd.write(body)
+            logger.warning("Message with no part_map!")
+            payload = str("")
+
+        if payload:
+            #headers = self.getHeaders(True)
+            #headers = lowerdict(headers)
+            #content_type = headers.get('content-type', "")
+            content_type = self._get_ctype_from_document(phash)
+            charset_split = content_type.split('charset=')
+            # XXX fuck all this, use a regex!
+            if len(charset_split) > 1:
+                charset = charset_split[1]
+                if charset:
+                    charset = charset.strip()
+            else:
+                charset = None
+            if not charset:
+                charset = self._get_charset(payload)
+            try:
+                payload = payload.encode(charset)
+            except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                logger.error("Unicode error {0}".format(e))
+                payload = payload.encode(charset, 'replace')
+
+        fd.write(payload)
         fd.seek(0)
         return fd
 
+    # TODO cache the phash retrieval
+    def _get_payload_from_document(self, phash):
+        """
+        Gets the message payload from the content document.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: basestring
+        """
+        cdocs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+
+        cdoc = first(cdocs)
+        if not cdoc:
+            logger.warning(
+                "Could not find the content doc "
+                "for phash %s" % (phash,))
+        payload = cdoc.content.get(fields.RAW_KEY, "")
+        return payload
+
+    # TODO cache the pahash retrieval
+    def _get_ctype_from_document(self, phash):
+        """
+        Gets the content-type from the content document.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: basestring
+        """
+        cdocs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+
+        cdoc = first(cdocs)
+        if not cdoc:
+            logger.warning(
+                "Could not find the content doc "
+                "for phash %s" % (phash,))
+        ctype = cdoc.content.get('ctype', "")
+        return ctype
+
     @memoized_method
     def _get_charset(self, stuff):
         # TODO put in a common class with LeapMessage
@@ -150,8 +214,6 @@ class MessageAttachment(object):
         :type stuff: basestring
         :returns: charset
         """
-        # XXX existential doubt 1. wouldn't be smarter to
-        # peek into the mail headers?
         # XXX existential doubt 2. shouldn't we make the scope
         # of the decorator somewhat more persistent?
         # ah! yes! and put memory bounds.
@@ -172,9 +234,17 @@ class MessageAttachment(object):
         :return: A mapping of header field names to header field values
         :rtype: dict
         """
-        if not self._msg:
+        if not self._pmap:
+            logger.warning("No pmap in Subpart!")
             return {}
-        headers = dict(self._msg.items())
+        headers = dict(self._pmap.get("headers", []))
+
+        # twisted imap server expects *some* headers to be lowercase
+        # We could use a CaseInsensitiveDict here...
+        headers = dict(
+            (str(key), str(value)) if key.lower() != "content-type"
+            else (str(key.lower()), str(value))
+            for (key, value) in headers.items())
 
         names = map(lambda s: s.upper(), names)
         if negate:
@@ -187,13 +257,18 @@ class MessageAttachment(object):
             map(str, (key, val)) for
             key, val in headers.items()
             if cond(key)]
-        return dict(filter_by_cond)
+        filtered = dict(filter_by_cond)
+        return filtered
 
     def isMultipart(self):
         """
         Return True if this message is multipart.
         """
-        return self._msg.is_multipart()
+        if not self._pmap:
+            logger.warning("Could not get part map!")
+            return False
+        multi = self._pmap.get("multi", False)
+        return multi
 
     def getSubPart(self, part):
         """
@@ -206,10 +281,30 @@ class MessageAttachment(object):
         :rtype: Any object implementing C{IMessagePart}.
         :return: The specified sub-part.
         """
-        return self._msg.get_payload()
+        if not self.isMultipart():
+            raise TypeError
+        sub_pmap = self._pmap.get("part_map", {})
+        try:
+            part_map = sub_pmap[str(part + 1)]
+        except KeyError:
+            logger.debug("getSubpart for %s: KeyError" % (part,))
+            raise IndexError
+
+        # XXX check for validity
+        return MessagePart(self._soledad, part_map)
 
 
 class LeapMessage(fields, MailParser, MBoxParser):
+    """
+    The main representation of a message.
+
+    It indexes the messages in one mailbox by a combination
+    of uid+mailbox name.
+    """
+
+    # TODO this has to change.
+    # Should index primarily by chash, and keep a local-lonly
+    # UID table.
 
     implements(imap4.IMessage)
 
@@ -268,6 +363,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         An accessor to the body document.
         """
+        if not self._hdoc:
+            return None
         if not self.__bdoc:
             self.__bdoc = self._get_body_doc()
         return self.__bdoc
@@ -320,6 +417,11 @@ class LeapMessage(fields, MailParser, MBoxParser):
         log.msg('setting flags: %s' % (self._uid))
 
         doc = self._fdoc
+        if not doc:
+            logger.warning(
+                "Could not find FDOC for %s:%s while setting flags!" %
+                (self._mbox, self._uid))
+            return
         doc.content[self.FLAGS_KEY] = flags
         doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
         doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
@@ -384,16 +486,25 @@ class LeapMessage(fields, MailParser, MBoxParser):
         fd = StringIO.StringIO()
         bdoc = self._bdoc
         if bdoc:
-            body = self._bdoc.content.get(self.BODY_KEY, "")
+            body = str(self._bdoc.content.get(self.RAW_KEY, ""))
         else:
-            body = ""
+            logger.warning("No BDOC found for message.")
+            body = str("")
+
+        # XXX not needed, isn't it? ---- ivan?
+        #if bdoc:
+            #content_type = bdoc.content.get('content-type', "")
+            #charset = content_type.split('charset=')[1]
+            #if charset:
+                #charset = charset.strip()
+            #if not charset:
+                #charset = self._get_charset(body)
+            #try:
+                #body = str(body.encode(charset))
+            #except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                #logger.error("Unicode error {0}".format(e))
+                #body = str(body.encode(charset, 'replace'))
 
-        charset = self._get_charset(body)
-        try:
-            body = body.encode(charset)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            body = body.encode(charset, 'replace')
         fd.write(body)
         fd.seek(0)
         return fd
@@ -407,8 +518,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :type stuff: basestring
         :returns: charset
         """
-        # XXX existential doubt 1. wouldn't be smarter to
-        # peek into the mail headers?
+        # TODO get from subpart headers
         # XXX existential doubt 2. shouldn't we make the scope
         # of the decorator somewhat more persistent?
         # ah! yes! and put memory bounds.
@@ -447,9 +557,11 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: A mapping of header field names to header field values
         :rtype: dict
         """
+        # TODO split in smaller methods
         headers = self._get_headers()
         if not headers:
-            return {'content-type': ''}
+            logger.warning("No headers found")
+            return {str('content-type'): str('')}
 
         names = map(lambda s: s.upper(), names)
         if negate:
@@ -457,16 +569,20 @@ class LeapMessage(fields, MailParser, MBoxParser):
         else:
             cond = lambda key: key.upper() in names
 
-        head = copy.deepcopy(dict(headers.items()))
+        if isinstance(headers, list):
+            headers = dict(headers)
 
-            # twisted imap server expects headers to be lowercase
-        head = dict(
-            (str(key), map(str, value)) if key.lower() != "content-type"
-            else (str(key.lower(), map(str, value)))
-            for (key, value) in head.items())
+        # twisted imap server expects *some* headers to be lowercase
+        # XXX refactor together with MessagePart method
+        headers = dict(
+            (str(key), str(value)) if key.lower() != "content-type"
+            else (str(key.lower()), str(value))
+            for (key, value) in headers.items())
 
         # unpack and filter original dict by negate-condition
-        filter_by_cond = [(key, val) for key, val in head.items() if cond(key)]
+        filter_by_cond = [(key, val) for key, val
+                          in headers.items() if cond(key)]
+
         return dict(filter_by_cond)
 
     def _get_headers(self):
@@ -474,7 +590,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the headers dict for this message.
         """
         if self._hdoc is not None:
-            return self._hdoc.content.get(self.HEADERS_KEY, {})
+            headers = self._hdoc.content.get(self.HEADERS_KEY, {})
+            return headers
+
         else:
             logger.warning(
                 "No HEADERS doc for msg %s:%s" % (
@@ -486,12 +604,13 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return True if this message is multipart.
         """
         if self._fdoc:
-            return self._fdoc.content.get(self.MULTIPART_KEY, False)
+            is_multipart = self._fdoc.content.get(self.MULTIPART_KEY, False)
+            return is_multipart
         else:
             logger.warning(
                 "No FLAGS doc for msg %s:%s" % (
-                    self.mbox,
-                    self.uid))
+                    self._mbox,
+                    self._uid))
 
     def getSubPart(self, part):
         """
@@ -504,27 +623,33 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: Any object implementing C{IMessagePart}.
         :return: The specified sub-part.
         """
-        logger.debug("Getting subpart: %s" % part)
         if not self.isMultipart():
             raise TypeError
-
-        if part == 0:
-            # Let's get the first part, which
-            # is really the body.
-            return MessageBody(self._fdoc, self._bdoc)
-
-        attach_doc = self._get_attachment_doc(part)
-        if not attach_doc:
-            # so long and thanks for all the fish
-            logger.debug("...not today")
+        try:
+            pmap_dict = self._get_part_from_parts_map(part + 1)
+        except KeyError:
+            logger.debug("getSubpart for %s: KeyError" % (part,))
             raise IndexError
-        msg_part = self._get_parsed_msg(attach_doc.content[self.RAW_KEY])
-        return MessageAttachment(msg_part)
+        return MessagePart(self._soledad, pmap_dict)
 
     #
     # accessors
     #
 
+    def _get_part_from_parts_map(self, part):
+        """
+        Get a part map from the headers doc
+
+        :raises: KeyError if key does not exist
+        :rtype: dict
+        """
+        if not self._hdoc:
+            logger.warning("Tried to get part but no HDOC found!")
+            return None
+
+        pmap = self._hdoc.content.get(fields.PARTS_MAP_KEY, {})
+        return pmap[str(part)]
+
     def _get_flags_doc(self):
         """
         Return the document that keeps the flags for this
@@ -550,63 +675,16 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the document that keeps the body for this
         message.
         """
-        body_docs = self._soledad.get_from_index(
-            fields.TYPE_C_HASH_IDX,
-            fields.TYPE_MESSAGE_VAL, str(self._chash))
-        return first(body_docs)
-
-    def _get_num_parts(self):
-        """
-        Return the number of parts for a multipart message.
-        """
-        if not self.isMultipart():
-            raise TypeError(
-                "Tried to get num parts in a non-multipart message")
-        if not self._hdoc:
-            return None
-        return self._hdoc.content.get(fields.NUM_PARTS_KEY, 2)
-
-    def _get_attachment_doc(self, part):
-        """
-        Return the document that keeps the headers for this
-        message.
-
-        :param part: the part number for the multipart message.
-        :type part: int
-        """
-        if not self._hdoc:
-            return None
-        try:
-            phash = self._hdoc.content[self.PARTS_MAP_KEY][str(part)]
-        except KeyError:
-            # this is the remnant of a debug session until
-            # I found that the index is actually a string...
-            # It should be safe to just raise the KeyError now,
-            # but leaving it here while the blood is fresh...
-            logger.warning("We expected a phash in the "
-                           "index %s, but noone found" % (part, ))
-            logger.debug(self._hdoc.content[self.PARTS_MAP_KEY])
+        body_phash = self._hdoc.content.get(
+            fields.BODY_KEY, None)
+        if not body_phash:
+            logger.warning("No body phash for this document!")
             return None
-        attach_docs = self._soledad.get_from_index(
+        body_docs = self._soledad.get_from_index(
             fields.TYPE_P_HASH_IDX,
-            fields.TYPE_ATTACHMENT_VAL, str(phash))
-
-        # The following is true for the fist owner.
-        # We could use this relationship to flag the "owner"
-        # and orphan when we delete it.
+            fields.TYPE_CONTENT_VAL, str(body_phash))
 
-        #attach_docs = self._soledad.get_from_index(
-            #fields.TYPE_C_HASH_PART_IDX,
-            #fields.TYPE_ATTACHMENT_VAL, str(self._chash), str(part))
-        return first(attach_docs)
-
-    def _get_raw_msg(self):
-        """
-        Return the raw msg.
-        :rtype: basestring
-        """
-        # TODO deprecate this.
-        return self._bdoc.content.get(self.RAW_KEY, '')
+        return first(body_docs)
 
     def __getitem__(self, key):
         """
@@ -658,27 +736,22 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         Remove all docs associated with this message.
         """
-        # XXX this would ve more efficient if we can just pass
-        # a sequence of uids.
-
         # XXX For the moment we are only removing the flags and headers
         # docs. The rest we leave there polluting your hard disk,
         # until we think about a good way of deorphaning.
         # Maybe a crawler of unreferenced docs.
 
+        # XXX implement elijah's idea of using a PUT document as a
+        # token to ensure consistency in the removal.
+
         uid = self._uid
-        print "removing...", uid
 
         fd = self._get_flags_doc()
-        hd = self._get_headers_doc()
+        #hd = self._get_headers_doc()
         #bd = self._get_body_doc()
         #docs = [fd, hd, bd]
 
-        docs = [fd, hd]
-
-        #for pn in range(self._get_num_parts()[1:]):
-            #ad = self._get_attachment_doc(pn)
-            #docs.append(ad)
+        docs = [fd]
 
         for d in filter(None, docs):
             try:
@@ -703,8 +776,7 @@ SoledadWriterPayload = namedtuple(
 
 SoledadWriterPayload.CREATE = 1
 SoledadWriterPayload.PUT = 2
-SoledadWriterPayload.BODY_CREATE = 3
-SoledadWriterPayload.ATTACHMENT_CREATE = 4
+SoledadWriterPayload.CONTENT_CREATE = 3
 
 
 class SoledadDocWriter(object):
@@ -723,6 +795,38 @@ class SoledadDocWriter(object):
         """
         self._soledad = soledad
 
+    def _get_call_for_item(self, item):
+        """
+        Return the proper call type for a given item.
+
+        :param item: one of the types defined under the
+                     attributes of SoledadWriterPayload
+        :type item: int
+        """
+        call = None
+        payload = item.payload
+
+        if item.mode == SoledadWriterPayload.CREATE:
+            call = self._soledad.create_doc
+        elif (item.mode == SoledadWriterPayload.CONTENT_CREATE
+              and not self._content_does_exist(payload)):
+                call = self._soledad.create_doc
+        elif item.mode == SoledadWriterPayload.PUT:
+            call = self._soledad.put_doc
+        return call
+
+    def _process(self, queue):
+        """
+        Return the item and the proper call type for the next
+        item in the queue if any.
+
+        :param queue: the queue from where we'll pick item.
+        :type queue: Queue
+        """
+        item = queue.get()
+        call = self._get_call_for_item(item)
+        return item, call
+
     def consume(self, queue):
         """
         Creates a new document in soledad db.
@@ -733,24 +837,10 @@ class SoledadDocWriter(object):
         """
         empty = queue.empty()
         while not empty:
-            item = queue.get()
-            call = None
-            payload = item.payload
-
-            if item.mode == SoledadWriterPayload.CREATE:
-                call = self._soledad.create_doc
-            elif item.mode == SoledadWriterPayload.BODY_CREATE:
-                if not self._body_does_exist(payload):
-                    call = self._soledad.create_doc
-            elif item.mode == SoledadWriterPayload.ATTACHMENT_CREATE:
-                if not self._attachment_does_exist(payload):
-                    call = self._soledad.create_doc
-            elif item.mode == SoledadWriterPayload.PUT:
-                call = self._soledad.put_doc
-
-            # XXX delete?
+            item, call = self._process(queue)
 
             if call:
+                # XXX should handle the delete case
                 # should handle errors
                 try:
                     call(item.payload)
@@ -779,33 +869,10 @@ class SoledadDocWriter(object):
     Stack.
     """
 
-    def _body_does_exist(self, doc):
+    def _content_does_exist(self, doc):
         """
-        Check whether we already have a body payload with this hash in our
-        database.
-
-        :param doc: tentative body document
-        :type doc: dict
-        :returns: True if that happens, False otherwise.
-        """
-        if not doc:
-            return False
-        chash = doc[fields.CONTENT_HASH_KEY]
-        body_docs = self._soledad.get_from_index(
-            fields.TYPE_C_HASH_IDX,
-            fields.TYPE_MESSAGE_VAL, str(chash))
-        if not body_docs:
-            return False
-        if len(body_docs) != 1:
-            logger.warning("Found more than one copy of chash %s!"
-                           % (chash,))
-        logger.debug("Found body doc with that hash! Skipping save!")
-        return True
-
-    def _attachment_does_exist(self, doc):
-        """
-        Check whether we already have an attachment payload with this hash
-        in our database.
+        Check whether we already have a content document for a payload
+        with this hash in our database.
 
         :param doc: tentative body document
         :type doc: dict
@@ -816,7 +883,7 @@ class SoledadDocWriter(object):
         phash = doc[fields.PAYLOAD_HASH_KEY]
         attach_docs = self._soledad.get_from_index(
             fields.TYPE_P_HASH_IDX,
-            fields.TYPE_ATTACHMENT_VAL, str(phash))
+            fields.TYPE_CONTENT_VAL, str(phash))
         if not attach_docs:
             return False
 
@@ -840,15 +907,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     # into a template for the class.
     FLAGS_DOC = "FLAGS"
     HEADERS_DOC = "HEADERS"
-    ATTACHMENT_DOC = "ATTACHMENT"
-    BODY_DOC = "BODY"
+    CONTENT_DOC = "CONTENT"
 
     templates = {
 
         FLAGS_DOC: {
             fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
-            fields.UID_KEY: 1,
+            fields.UID_KEY: 1,  # XXX moe to a local table
             fields.MBOX_KEY: fields.INBOX_VAL,
+            fields.CONTENT_HASH_KEY: "",
 
             fields.SEEN_KEY: False,
             fields.RECENT_KEY: True,
@@ -862,35 +929,28 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
             fields.CONTENT_HASH_KEY: "",
 
+            fields.DATE_KEY: "",
+            fields.SUBJECT_KEY: "",
+
             fields.HEADERS_KEY: {},
-            fields.NUM_PARTS_KEY: 0,
             fields.PARTS_MAP_KEY: {},
-            fields.DATE_KEY: "",
-            fields.SUBJECT_KEY: ""
         },
 
-        ATTACHMENT_DOC: {
-            fields.TYPE_KEY: fields.TYPE_ATTACHMENT_VAL,
-            fields.PART_NUMBER_KEY: 0,
-            fields.CONTENT_HASH_KEY:  "",
+        CONTENT_DOC: {
+            fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
             fields.PAYLOAD_HASH_KEY: "",
+            fields.LINKED_FROM_KEY: [],
+            fields.CTYPE_KEY: "",  # should index by this too
 
-            fields.RAW_KEY: ""
-        },
-
-        BODY_DOC: {
-            fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL,
-            fields.CONTENT_HASH_KEY: "",
-
-            fields.BODY_KEY: "",
-
-            # this should not be needed,
-            # but let's keep the raw msg for some time
-            # until we are sure we can reconstruct
-            # the original msg from our disection.
+            # should only get inmutable headers parts
+            # (for indexing)
+            fields.HEADERS_KEY: {},
             fields.RAW_KEY: "",
+            fields.PARTS_MAP_KEY: {},
+            fields.HEADERS_KEY: {},
+            fields.MULTIPART_KEY: False,
+        },
 
-        }
     }
 
     def __init__(self, mbox=None, soledad=None):
@@ -938,128 +998,124 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             raise TypeError("Improper type passed to _get_empty_doc")
         return copy.deepcopy(self.templates[_type])
 
-    @deferred
-    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
+    def _do_parse(self, raw):
         """
-        Creates a new message document.
+        Parse raw message and return it along with
+        relevant information about its outer level.
 
         :param raw: the raw message
-        :type raw: str
-
-        :param subject: subject of the message.
-        :type subject: str
-
-        :param flags: flags
-        :type flags: list
-
-        :param date: the received date for the message
-        :type date: str
-
-        :param uid: the message uid for this mailbox
-        :type uid: int
+        :type raw: StringIO or basestring
+        :return: msg, chash, size, multi
+        :rtype: tuple
         """
-        # TODO: split in smaller methods
-        logger.debug('adding message')
-        if flags is None:
-            flags = tuple()
-        leap_assert_type(flags, tuple)
-
-        # docs for flags, headers, and body
-        fd, hd, bd = map(
-            lambda t: self._get_empty_doc(t),
-            (self.FLAGS_DOC, self.HEADERS_DOC, self.BODY_DOC))
-
         msg = self._get_parsed_msg(raw)
-        headers = defaultdict(list)
-        for k, v in msg.items():
-            headers[k].append(v)
-        raw_str = msg.as_string()
         chash = self._get_hash(msg)
+        size = len(msg.as_string())
         multi = msg.is_multipart()
+        return msg, chash, size, multi
 
-        attaches = []
-        inner_parts = []
-
-        if multi:
-            # XXX should walk down recursively
-            # in a better way.  but fixing this quick
-            # to have an rc.
-            # XXX should pick the content-type in txt
-            body = first(msg.get_payload()).get_payload()
-            if isinstance(body, list):
-                # allowing one nesting level for now...
-                body, rest = body[0].get_payload(), body[1:]
-                for p in rest:
-                    inner_parts.append(p)
-        else:
-            body = msg.get_payload()
-        logger.debug("adding msg with uid %s (multipart:%s)" % (
-            uid, multi))
+    def _populate_flags(self, flags, uid, chash, size, multi):
+        """
+        Return a flags doc.
+
+        XXX Missing DOC -----------
+        """
+        fd = self._get_empty_doc(self.FLAGS_DOC)
 
-        # flags doc ---------------------------------------
         fd[self.MBOX_KEY] = self.mbox
         fd[self.UID_KEY] = uid
         fd[self.CONTENT_HASH_KEY] = chash
+        fd[self.SIZE_KEY] = size
         fd[self.MULTIPART_KEY] = multi
-        fd[self.SIZE_KEY] = len(raw_str)
         if flags:
             fd[self.FLAGS_KEY] = map(self._stringify, flags)
             fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
             fd[self.DEL_KEY] = self.DELETED_FLAG in flags
             fd[self.RECENT_KEY] = True  # set always by default
+        return fd
 
-        # headers doc ----------------------------------------
+    def _populate_headr(self, msg, chash, subject, date):
+        """
+        Return a headers doc.
+
+        XXX Missing DOC -----------
+        """
+        headers = defaultdict(list)
+        for k, v in msg.items():
+            headers[k].append(v)
+
+        # "fix" for repeated headers.
+        for k, v in headers.items():
+            newline = "\n%s: " % (k,)
+            headers[k] = newline.join(v)
+
+        hd = self._get_empty_doc(self.HEADERS_DOC)
         hd[self.CONTENT_HASH_KEY] = chash
         hd[self.HEADERS_KEY] = headers
 
-        print "headers"
-        import pprint
-        pprint.pprint(headers)
-
         if not subject and self.SUBJECT_FIELD in headers:
             hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
         else:
             hd[self.SUBJECT_KEY] = subject
+
         if not date and self.DATE_FIELD in headers:
             hd[self.DATE_KEY] = first(headers[self.DATE_FIELD])
         else:
             hd[self.DATE_KEY] = date
-        if multi:
-            # XXX fix for multipart nested case
-            hd[self.NUM_PARTS_KEY] = len(msg.get_payload())
-
-        # body doc
-        bd[self.CONTENT_HASH_KEY] = chash
-        bd[self.BODY_KEY] = body
-        # XXX in an ideal world, we would not need to save a copy of the
-        # raw message. But we'll keep it until we can be sure that
-        # we can rebuild the original message from the parts.
-        bd[self.RAW_KEY] = raw_str
+        return hd
+
+    @deferred
+    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
+        """
+        Creates a new message document.
+
+        :param raw: the raw message
+        :type raw: str
+
+        :param subject: subject of the message.
+        :type subject: str
+
+        :param flags: flags
+        :type flags: list
+
+        :param date: the received date for the message
+        :type date: str
+
+        :param uid: the message uid for this mailbox
+        :type uid: int
+        """
+        # TODO signal that we can delete the original message!-----
+        # when all the processing is done.
+
+        # TODO add the linked-from info !
+
+        logger.debug('adding message')
+        if flags is None:
+            flags = tuple()
+        leap_assert_type(flags, tuple)
+
+        # parse
+        msg, chash, size, multi = self._do_parse(raw)
+
+        fd = self._populate_flags(flags, uid, chash, size, multi)
+        hd = self._populate_headr(msg, chash, subject, date)
+
+        parts = walk.get_parts(msg)
+        body_phash_fun = [walk.get_body_phash_simple,
+                          walk.get_body_phash_multi][int(multi)]
+        body_phash = body_phash_fun(walk.get_payloads(msg))
+        parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+
+        # add parts map to header doc
+        # (body, multi, part_map)
+        for key in parts_map:
+            hd[key] = parts_map[key]
+        del parts_map
 
         docs = [fd, hd]
+        cdocs = walk.get_raw_docs(msg, parts)
 
-        # attachment docs
-        if multi:
-            outer_parts = msg.get_payload()
-            parts = outer_parts + inner_parts
-
-            # skip first part, we already got it in body
-            to_attach = ((i, m) for i, m in enumerate(parts) if i > 0)
-            for index, part_msg in to_attach:
-                att_doc = self._get_empty_doc(self.ATTACHMENT_DOC)
-                att_doc[self.PART_NUMBER_KEY] = index
-                att_doc[self.CONTENT_HASH_KEY] = chash
-                phash = self._get_hash(part_msg)
-                att_doc[self.PAYLOAD_HASH_KEY] = phash
-                att_doc[self.RAW_KEY] = part_msg.as_string()
-
-                # keep a pointer to the payload hash in the
-                # headers doc, under the parts_map
-                hd[self.PARTS_MAP_KEY][str(index)] = phash
-                attaches.append(att_doc)
-
-        # Saving ... -------------------------------
-        # ok, there we go...
+        # Saving
         logger.debug('enqueuing message docs for write')
         ptuple = SoledadWriterPayload
 
@@ -1067,14 +1123,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         for doc in docs:
             self.soledad_writer.put(ptuple(
                 mode=ptuple.CREATE, payload=doc))
-        # second, try to create body doc.
-        self.soledad_writer.put(ptuple(
-            mode=ptuple.BODY_CREATE, payload=bd))
+
         # and last, but not least, try to create
-        # attachment docs if not already there.
-        for at in attaches:
+        # content docs if not already there.
+        for cd in cdocs:
             self.soledad_writer.put(ptuple(
-                mode=ptuple.ATTACHMENT_CREATE, payload=at))
+                mode=ptuple.CONTENT_CREATE, payload=cd))
 
     def _remove_cb(self, result):
         return result
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 26e14c3..234996d 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -87,6 +87,8 @@ class LeapIMAPServer(imap4.IMAP4Server):
         :param line: the line from the server, without the line delimiter.
         :type line: str
         """
+        print "RECV: STATE (%s)" % self.state
+
         if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
             # by now.
diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message
new file mode 100644
index 0000000..582297c
--- /dev/null
+++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message
@@ -0,0 +1,16 @@
+Content-Type: multipart/mixed; boundary="===============6203542367371144092=="
+MIME-Version: 1.0
+Subject: [TEST] 010 - Inceptos cum lorem risus congue
+From: testmailbitmaskspam@gmail.com
+To: test_c5@dev.bitmask.net
+
+--===============6203542367371144092==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+Howdy from python!
+The subject: [TEST] 010 - Inceptos cum lorem risus congue
+Current date & time: Wed Jan  8 16:36:21 2014
+Trying to attach: []
+--===============6203542367371144092==--
diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message
new file mode 100644
index 0000000..fc627c3
--- /dev/null
+++ b/src/leap/mail/imap/tests/rfc822.plain.message
@@ -0,0 +1,66 @@
+From pyar-bounces@python.org.ar Wed Jan  8 14:46:02 2014
+Return-Path: <pyar-bounces@python.org.ar>
+X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net
+X-Spam-Level: **
+X-Spam-Pyzor: Reported 0 times.
+X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE,
+	CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP,
+	NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled
+	version=3.3.2
+Delivered-To: kali@leap.se
+Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33])
+	(using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))
+	(Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified))
+	by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F
+	for <kali@leap.se>; Wed,  8 Jan 2014 18:46:02 +0000 (UTC)
+Received: from pyar.usla.org.ar (unknown [190.228.30.157])
+	by mx1.riseup.net (Postfix) with ESMTP id F244C533F4
+	for <kali@leap.se>; Wed,  8 Jan 2014 10:46:01 -0800 (PST)
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+	by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F
+	for <kali@leap.se>; Wed,  8 Jan 2014 15:46:00 -0300 (ART)
+MIME-Version: 1.0
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+From: pyar-request@python.org.ar
+To: kali@leap.se
+Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+Reply-To: pyar-request@python.org.ar
+Auto-Submitted: auto-replied
+Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar>
+Date: Wed, 08 Jan 2014 15:45:59 -0300
+Precedence: bulk
+X-BeenThere: pyar@python.org.ar
+X-Mailman-Version: 2.1.15
+List-Id: Python Argentina <pyar.python.org.ar>
+X-List-Administrivia: yes
+Errors-To: pyar-bounces@python.org.ar
+Sender: "pyar" <pyar-bounces@python.org.ar>
+X-Virus-Scanned: clamav-milter 0.97.8 at mx1
+X-Virus-Status: Clean
+
+Mailing list subscription confirmation notice for mailing list pyar
+
+We have received a request de kaliyuga@riseup.net for subscription of
+your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar
+mailing list.  To confirm that you want to be added to this mailing
+list, simply reply to this message, keeping the Subject: header
+intact.  Or visit this web page:
+
+    http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b=
+3377148ac2
+
+
+Or include the following line -- and only the following line -- in a
+message to pyar-request@python.org.ar:
+
+    confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+
+Note that simply sending a `reply' to this message should work from
+most mail readers, since that usually leaves the Subject: line in the
+right form (additional "Re:" text in the Subject: is okay).
+
+If you do not wish to be subscribed to this list, please simply
+disregard this message.  If you think you are being maliciously
+subscribed to the list, or have any other questions, send them to
+pyar-owner@python.org.ar.
diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py
new file mode 100644
index 0000000..1626f65
--- /dev/null
+++ b/src/leap/mail/imap/tests/walktree.py
@@ -0,0 +1,117 @@
+#t -*- coding: utf-8 -*-
+# walktree.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the walktree module.
+"""
+import os
+from email import parser
+
+from leap.mail import walk as W
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+p = parser.Parser()
+
+# TODO pass an argument of the type of message
+
+##################################################
+# Input from hell
+
+#msg = p.parse(open('rfc822.multi-signed.message'))
+#msg = p.parse(open('rfc822.plain.message'))
+msg = p.parse(open('rfc822.multi-minimal.message'))
+DO_CHECK = False
+#################################################
+
+parts = W.get_parts(msg)
+
+if DEBUG:
+    def trim(item):
+        item = item[:10]
+    [trim(part["phash"]) for part in parts if part.get('phash', None)]
+
+raw_docs = list(W.get_raw_docs(msg, parts))
+
+body_phash_fun = [W.get_body_phash_simple,
+                  W.get_body_phash_multi][int(msg.is_multipart())]
+body_phash = body_phash_fun(W.get_payloads(msg))
+parts_map = W.walk_msg_tree(parts, body_phash=body_phash)
+
+
+# TODO add missing headers!
+expected = {
+    'body': '1ddfa80485',
+    'multi': True,
+    'part_map': {
+        1: {
+            'headers': {'Content-Disposition': 'inline',
+                        'Content-Type': 'multipart/mixed; '
+                        'boundary="z0eOaCaDLjvTGF2l"'},
+            'multi': True,
+            'part_map': {1: {'ctype': 'text/plain',
+                             'headers': [
+                                 ('Content-Type',
+                                  'text/plain; charset=utf-8'),
+                                 ('Content-Disposition',
+                                  'inline'),
+                                 ('Content-Transfer-Encoding',
+                                  'quoted-printable')],
+                             'multi': False,
+                             'parts': 1,
+                             'phash': '1ddfa80485',
+                             'size': 206},
+                         2: {'ctype': 'text/plain',
+                             'headers': [('Content-Type',
+                                          'text/plain; charset=us-ascii'),
+                                         ('Content-Disposition',
+                                          'attachment; '
+                                          'filename="attach.txt"')],
+                             'multi': False,
+                             'parts': 1,
+                             'phash': '7a94e4d769',
+                             'size': 133},
+                         3: {'ctype': 'application/octet-stream',
+                             'headers': [('Content-Type',
+                                          'application/octet-stream'),
+                                         ('Content-Disposition',
+                                          'attachment; filename="hack.ico"'),
+                                         ('Content-Transfer-Encoding',
+                                          'base64')],
+                             'multi': False,
+                             'parts': 1,
+                             'phash': 'c42cccebbd',
+                             'size': 12736}}},
+        2: {'ctype': 'application/pgp-signature',
+            'headers': [('Content-Type', 'application/pgp-signature')],
+            'multi': False,
+            'parts': 1,
+            'phash': '8f49fbf749',
+            'size': 877}}}
+
+if DEBUG and DO_CHECK:
+    # TODO turn this into a proper unittest
+    assert(parts_map == expected)
+    print "Structure: OK"
+
+
+import pprint
+print
+print "RAW DOCS"
+pprint.pprint(raw_docs)
+print
+print "PARTS MAP"
+pprint.pprint(parts_map)
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
new file mode 100644
index 0000000..820b8c7
--- /dev/null
+++ b/src/leap/mail/walk.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# walk.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for walking along a message tree.
+"""
+import hashlib
+import os
+
+from leap.mail.utils import first
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+if DEBUG:
+    get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10]
+else:
+    get_hash = lambda s: hashlib.sha256(s).hexdigest()
+
+
+"""
+Get interesting message parts
+"""
+get_parts = lambda msg: [
+    {'multi': part.is_multipart(),
+     'ctype': part.get_content_type(),
+     'size': len(part.as_string()),
+     'parts': len(part.get_payload())
+        if isinstance(part.get_payload(), list)
+        else 1,
+     'headers': part.items(),
+     'phash': get_hash(part.get_payload())
+        if not part.is_multipart() else None}
+    for part in msg.walk()]
+
+"""
+Utility lambda functions for getting the parts vector and the
+payloads from the original message.
+"""
+
+get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts)
+get_payloads = lambda msg: ((x.get_payload(),
+                             dict(((str.lower(k), v) for k, v in (x.items()))))
+                            for x in msg.walk())
+
+get_body_phash_simple = lambda payloads: first(
+    [get_hash(payload) for payload, headers in payloads
+     if "text/plain" in headers.get('content-type')])
+
+get_body_phash_multi = lambda payloads: (first(
+    [get_hash(payload) for payload, headers in payloads
+     if "text/plain" in headers.get('content-type')])
+    or get_body_phash_simple(payloads))
+
+"""
+On getting the raw docs, we get also some of the headers to be able to
+index the content. Here we remove any mutable part, as the the filename
+in the content disposition.
+"""
+
+get_raw_docs = lambda msg, parts: (
+    {"type": "cnt",  # type content they'll be
+     "raw": payload if not DEBUG else payload[:100],
+     "phash": get_hash(payload),
+     "content-disposition": first(headers.get(
+         'content-disposition', '').split(';')),
+     "content-type": headers.get(
+         'content-type', ''),
+     "content-transfer-encoding": headers.get(
+         'content-transfer-type', '')}
+    for payload, headers in get_payloads(msg)
+    if not isinstance(payload, list))
+
+
+def walk_msg_tree(parts, body_phash=None):
+    """
+    Take a list of interesting items of a message subparts structure,
+    and return a dict of dicts almost ready to be written to the content
+    documents that will be stored in Soledad.
+
+    It walks down the subparts in the parsed message tree, and collapses
+    the leaf docuents into a wrapper document until no multipart submessages
+    are left. To achieve this, it iteratively calculates a wrapper vector of
+    all documents in the sequence that have more than one part and have unitary
+    documents to their right. To collapse a multipart, take as many
+    unitary documents as parts the submessage contains, and replace the object
+    in the sequence with the new wrapper document.
+
+    :param parts: A list of dicts containing the interesting properties for
+                  the message structure. Normally this has been generated by
+                  doing a message walk.
+    :type parts: list of dicts.
+    :param body_phash: the payload hash of the body part, to be included
+                       in the outer content doc for convenience.
+    :type body_phash: basestring or None
+    """
+    # parts vector
+    pv = list(get_parts_vector(parts))
+
+    if len(parts) == 2:
+        inner_headers = parts[1].get("headers", None)
+
+    if DEBUG:
+        print "parts vector: ", pv
+        print
+
+    # wrappers vector
+    getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False
+                        for i in range(len(pv) - 1)]
+    wv = getwv(pv)
+
+    # do until no wrapper document is left
+    while any(wv):
+        wind = wv.index(True)  # wrapper index
+        nsub = pv[wind]  # number of subparts to pick
+        slic = parts[wind + 1:wind + 1 + nsub]  # slice with subparts
+
+        cwra = {
+            "multi": True,
+            "part_map": dict((index + 1, part)  # content wrapper
+                             for index, part in enumerate(slic)),
+            "headers": dict(parts[wind]['headers'])
+        }
+
+        # remove subparts and substitue wrapper
+        map(lambda i: parts.remove(i), slic)
+        parts[wind] = cwra
+
+        # refresh vectors for this iteration
+        pv = list(get_parts_vector(parts))
+        wv = getwv(pv)
+
+    outer = parts[0]
+    outer.pop('headers')
+    if not "part_map" in outer:
+        # we have a multipart with 1 part only, so kind of fix it
+        # although it would be prettier if I take this special case at
+        # the beginning of the walk.
+        pdoc = {"multi": True,
+                "part_map": {1: outer}}
+        pdoc["part_map"][1]["multi"] = False
+        if not pdoc["part_map"][1].get("phash", None):
+            pdoc["part_map"][1]["phash"] = body_phash
+        pdoc["part_map"][1]["headers"] = inner_headers
+    else:
+        pdoc = outer
+    pdoc["body"] = body_phash
+    return pdoc
-- 
cgit v1.2.3


From e4f2914517ea11aeef60aa74be50116e1979f34d Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 8 Jan 2014 21:39:27 -0400
Subject: handle all fetches as sequential

* this allows quick testing using telnet, and the use of other
  less sofisticated MUAs.
---
 src/leap/mail/imap/mailbox.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 1d76d4d..7c01490 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -479,11 +479,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         result = []
 
-        # XXX DEBUG -------------
-        print "getting uid", uid
-        print "in mbox", self.mbox
+        # For the moment our UID is sequential, so we
+        # can treat them all the same.
+        # Change this to the flag that twisted expects when we
+        # switch to content-hash based index + local UID table.
 
-        sequence = True if uid == 0 else False
+        sequence = False
+        #sequence = True if uid == 0 else False
 
         if not messages.last:
             try:
-- 
cgit v1.2.3


From 28236ecdf46360acd09a53130646e2a082583de6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 8 Jan 2014 22:24:46 -0400
Subject: add a quick message fetching utility for tests

---
 src/leap/mail/imap/tests/getmail | 282 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 282 insertions(+)
 create mode 100755 src/leap/mail/imap/tests/getmail

diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail
new file mode 100755
index 0000000..17e195c
--- /dev/null
+++ b/src/leap/mail/imap/tests/getmail
@@ -0,0 +1,282 @@
+#!/usr/bin/env python
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE in twisted for details.
+
+# Modifications by LEAP Developers 2014 to fit
+# Bitmask configuration settings.
+
+
+"""
+Simple IMAP4 client which displays the subjects of all messages in a
+particular mailbox.
+"""
+
+import sys
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+
+class TrivialPrompter(basic.LineReceiver):
+    from os import linesep as delimiter
+
+    promptDeferred = None
+
+    def prompt(self, msg):
+        assert self.promptDeferred is None
+        self.display(msg)
+        self.promptDeferred = defer.Deferred()
+        return self.promptDeferred
+
+    def display(self, msg):
+        self.transport.write(msg)
+
+    def lineReceived(self, line):
+        if self.promptDeferred is None:
+            return
+        d, self.promptDeferred = self.promptDeferred, None
+        d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+    """
+    A client with callbacks for greeting messages from an IMAP server.
+    """
+    greetDeferred = None
+
+    def serverGreeting(self, caps):
+        self.serverCapabilities = caps
+        if self.greetDeferred is not None:
+            d, self.greetDeferred = self.greetDeferred, None
+            d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+    usedUp = False
+
+    protocol = SimpleIMAP4Client
+
+    def __init__(self, username, onConn):
+        self.ctx = ssl.ClientContextFactory()
+
+        self.username = username
+        self.onConn = onConn
+
+    def buildProtocol(self, addr):
+        """
+        Initiate the protocol instance. Since we are building a simple IMAP
+        client, we don't bother checking what capabilities the server has. We
+        just add all the authenticators twisted.mail has.  Note: Gmail no
+        longer uses any of the methods below, it's been using XOAUTH since
+        2010.
+        """
+        assert not self.usedUp
+        self.usedUp = True
+
+        p = self.protocol(self.ctx)
+        p.factory = self
+        p.greetDeferred = self.onConn
+
+        p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+        p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+        p.registerAuthenticator(
+            imap4.CramMD5ClientAuthenticator(self.username))
+
+        return p
+
+    def clientConnectionFailed(self, connector, reason):
+        d, self.onConn = self.onConn, None
+        d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+    """
+    Initial callback - invoked after the server sends us its greet message.
+    """
+    # Hook up stdio
+    tp = TrivialPrompter()
+    stdio.StandardIO(tp)
+
+    # And make it easily accessible
+    proto.prompt = tp.prompt
+    proto.display = tp.display
+
+    # Try to authenticate securely
+    return proto.authenticate(
+        password).addCallback(
+        cbAuthentication,
+        proto).addErrback(
+        ebAuthentication, proto, username, password
+    )
+
+
+def ebConnection(reason):
+    """
+    Fallback error-handler. If anything goes wrong, log it and quit.
+    """
+    log.startLogging(sys.stdout)
+    log.err(reason)
+    return reason
+
+
+def cbAuthentication(result, proto):
+    """
+    Callback after authentication has succeeded.
+
+    Lists a bunch of mailboxes.
+    """
+    return proto.list("", "*"
+        ).addCallback(cbMailboxList, proto
+        )
+
+
+def ebAuthentication(failure, proto, username, password):
+    """
+    Errback invoked when authentication fails.
+
+    If it failed because no SASL mechanisms match, offer the user the choice
+    of logging in insecurely.
+
+    If you are trying to connect to your Gmail account, you will be here!
+    """
+    failure.trap(imap4.NoSupportedAuthentication)
+    return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+    """
+    insecure-login.
+    """
+    return proto.login(username, password
+        ).addCallback(cbAuthentication, proto
+        )
+
+
+def cbMailboxList(result, proto):
+    """
+    Callback invoked when a list of mailboxes has been retrieved.
+    """
+    result = [e[2] for e in result]
+    s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)])
+    if not s:
+        return defer.fail(Exception("No mailboxes exist on server!"))
+    return proto.prompt(s + "\nWhich mailbox? [1] "
+        ).addCallback(cbPickMailbox, proto, result
+        )
+
+
+def cbPickMailbox(result, proto, mboxes):
+    """
+    When the user selects a mailbox, "examine" it.
+    """
+    mbox = mboxes[int(result or '1') - 1]
+    return proto.examine(mbox
+        ).addCallback(cbExamineMbox, proto
+        )
+
+
+def cbExamineMbox(result, proto):
+    """
+    Callback invoked when examine command completes.
+
+    Retrieve the subject header of every message in the mailbox.
+    """
+    return proto.fetchSpecific('1:*',
+                               headerType='HEADER.FIELDS',
+                               headerArgs=['SUBJECT'],
+        ).addCallback(cbFetch, proto,
+        )
+
+
+def cbFetch(result, proto):
+    """
+    Display headers.
+    """
+    if result:
+        keys = result.keys()
+        keys.sort()
+        for k in keys:
+            proto.display('%s %s' % (k, result[k][0][2]))
+    else:
+        print "Hey, an empty mailbox!"
+
+    return proto.prompt("\nWhich message? [1] (Q quits) "
+                        ).addCallback(cbPickMessage, proto)
+
+
+def cbPickMessage(result, proto):
+    """
+    Pick a message.
+    """
+    if result == "Q":
+        print "Bye!"
+        return proto.logout()
+
+    return proto.fetchSpecific(
+        '%s' % result,
+        headerType='',
+        headerArgs=['BODY.PEEK[]'],
+        ).addCallback(cbShowmessage, proto)
+
+
+def cbShowmessage(result, proto):
+    """
+    Display message.
+    """
+    if result:
+        keys = result.keys()
+        keys.sort()
+        for k in keys:
+            proto.display('%s %s' % (k, result[k][0][2]))
+    else:
+        print "Hey, an empty message!"
+
+    return proto.logout()
+
+
+def cbClose(result):
+    """
+    Close the connection when we finish everything.
+    """
+    from twisted.internet import reactor
+    reactor.stop()
+
+
+def main():
+    import sys
+
+    if len(sys.argv) != 3:
+        print "Usage: getmail <user> <pass>"
+        sys.exit()
+
+    hostname = "localhost"
+    port = "1984"
+    username = sys.argv[1]
+    password = sys.argv[2]
+
+    onConn = defer.Deferred(
+        ).addCallback(cbServerGreeting, username, password
+        ).addErrback(ebConnection
+        ).addBoth(cbClose)
+
+    factory = SimpleIMAP4ClientFactory(username, onConn)
+
+    from twisted.internet import reactor
+    if port == '993':
+        reactor.connectSSL(
+            hostname, int(port), factory, ssl.ClientContextFactory())
+    else:
+        if not port:
+            port = 143
+        reactor.connectTCP(hostname, int(port), factory)
+    reactor.run()
+
+
+if __name__ == '__main__':
+    main()
-- 
cgit v1.2.3


From 3bd8a7b669ac490b8f8e6c33550963c37069e383 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 8 Jan 2014 23:17:47 -0400
Subject: changes file updated

---
 changes/feature_split_message_docs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/changes/feature_split_message_docs b/changes/feature_split_message_docs
index 231c36e..0109501 100644
--- a/changes/feature_split_message_docs
+++ b/changes/feature_split_message_docs
@@ -1,6 +1,7 @@
   o Defer costly operations to a pool of threads.
-  o Split the internal representation of messages into four distinct documents:
-    1) Flags 2) Headers 3) Body 4) Attachments.
+  o Split the internal representation of messages into three distinct documents:
+    1) Flags 2) Headers 3) Content.
+  o Make use of the Twisted MIME interface.
   o Add deduplication ability to the save operation, for body and attachments.
   o Add IMessageCopier interface to mailbox implementation, so bulk moves
     are costless. Closes: #4654
-- 
cgit v1.2.3


From f9a9a695526afea8eef4ccba8904fb215acc43dd Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 8 Jan 2014 23:29:18 -0400
Subject: add a flag to be able to close the session

---
 changes/bug_4925_close_session     |  1 +
 src/leap/mail/imap/account.py      |  1 +
 src/leap/mail/imap/service/imap.py | 12 +++---------
 3 files changed, 5 insertions(+), 9 deletions(-)
 create mode 100644 changes/bug_4925_close_session

diff --git a/changes/bug_4925_close_session b/changes/bug_4925_close_session
new file mode 100644
index 0000000..93dab55
--- /dev/null
+++ b/changes/bug_4925_close_session
@@ -0,0 +1 @@
+  o Add a flag to be able to reset the session. Closes: #4925
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index fd861e7..8caafef 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -46,6 +46,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
     _soledad = None
     selected = None
+    closed = False
 
     def __init__(self, account_name, soledad=None):
         """
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 234996d..dfd4862 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -71,15 +71,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
         # populate the test account properly (and only once
         # per session)
 
-        # theAccount = SoledadBackedAccount(
-        #     user, soledad=soledad)
-
-        # ---------------------------------
-        # XXX pre-populate acct for tests!!
-        # populate_test_account(theAccount)
-        # ---------------------------------
-        #self.theAccount = theAccount
-
     def lineReceived(self, line):
         """
         Attempt to parse a single line from the server.
@@ -88,6 +79,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
         :type line: str
         """
         print "RECV: STATE (%s)" % self.state
+        if self.theAccount.closed is True and self.state != "unauth":
+            log.msg("Closing the session. State: unauth")
+            self.state = "unauth"
 
         if "login" in line.lower():
             # avoid to log the pass, even though we are using a dummy auth
-- 
cgit v1.2.3


From 4381131163161c00f00a3eb300041374aa06d370 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Sat, 28 Dec 2013 20:09:03 -0200
Subject: Convert unicode to str when raising in IMAP server (#4830).

---
 .../bug_4830_convert-unicode-to-str-when-raising   |  1 +
 src/leap/mail/imap/account.py                      | 32 ++++++++++++++++++----
 src/leap/mail/imap/parser.py                       |  5 ++++
 3 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 changes/bug_4830_convert-unicode-to-str-when-raising

diff --git a/changes/bug_4830_convert-unicode-to-str-when-raising b/changes/bug_4830_convert-unicode-to-str-when-raising
new file mode 100644
index 0000000..86d9b1c
--- /dev/null
+++ b/changes/bug_4830_convert-unicode-to-str-when-raising
@@ -0,0 +1 @@
+  o Convert unicode to str when raising exceptions in IMAP server (#4830).
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index fd861e7..8f5b57b 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -36,6 +36,23 @@ from leap.soledad.client import Soledad
 #######################################
 
 
+def _unicode_as_str(text):
+    """
+    Return some representation of C{text} as a str.
+
+    This is here mainly because Twisted's exception methods are not able to
+    print unicode text.
+
+    :param text: The text to convert.
+    :type text: unicode
+
+    :return: A representation of C{text} as str.
+    :rtype: str
+    """
+    # XXX is there a better str representation for unicode?
+    return repr(text)
+
+
 class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     """
     An implementation of IAccount and INamespacePresenteer
@@ -128,7 +145,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         name = self._parse_mailbox_name(name)
 
         if name not in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox")
+            raise imap4.MailboxException("No such mailbox: %s" %
+                                         _unicode_as_str(name))
 
         return SoledadMailbox(name, soledad=self._soledad)
 
@@ -154,7 +172,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         name = self._parse_mailbox_name(name)
 
         if name in self.mailboxes:
-            raise imap4.MailboxCollision, name
+            raise imap4.MailboxCollision, _unicode_as_str(name)
 
         if not creation_ts:
             # by default, we pass an int value
@@ -240,7 +258,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         name = self._parse_mailbox_name(name)
 
         if not name in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox")
+            raise imap4.MailboxException("No such mailbox: %s" %
+                                         _unicode_as_str(name))
 
         mbox = self.getMailbox(name)
 
@@ -279,14 +298,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         newname = self._parse_mailbox_name(newname)
 
         if oldname not in self.mailboxes:
-            raise imap4.NoSuchMailbox, oldname
+            raise imap4.NoSuchMailbox, _unicode_as_str(oldname)
 
         inferiors = self._inferiorNames(oldname)
         inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
 
         for (old, new) in inferiors:
             if new in self.mailboxes:
-                raise imap4.MailboxCollision, new
+                raise imap4.MailboxCollision, _unicode_as_str(new)
 
         for (old, new) in inferiors:
             mbox = self._get_mailbox_by_name(old)
@@ -367,7 +386,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         """
         name = self._parse_mailbox_name(name)
         if name not in self.subscriptions:
-            raise imap4.MailboxException, "Not currently subscribed to " + name
+            raise imap4.MailboxException, \
+                "Not currently subscribed to %s" % _unicode_as_str(name)
         self._set_subscription(name, False)
 
     def listMailboxes(self, ref, wildcard):
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
index 306dcf0..6a9ace9 100644
--- a/src/leap/mail/imap/parser.py
+++ b/src/leap/mail/imap/parser.py
@@ -102,6 +102,11 @@ class MBoxParser(object):
 
     def _parse_mailbox_name(self, name):
         """
+        Return a normalized representation of the mailbox C{name}.
+
+        This method ensures that an eventual initial 'inbox' part of a
+        mailbox name is made uppercase.
+
         :param name: the name of the mailbox
         :type name: unicode
 
-- 
cgit v1.2.3


From 4de544034406e3331280bdc86d10b3304d90c1d9 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Thu, 9 Jan 2014 16:44:04 -0300
Subject: Remove unneded repr wrapper.

Also use pep8 recommended raise format:
    raise Exception("message")
    # instead of:
    raise Exception, "message"
---
 src/leap/mail/imap/account.py | 33 +++++++--------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 8f5b57b..6b10583 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -36,23 +36,6 @@ from leap.soledad.client import Soledad
 #######################################
 
 
-def _unicode_as_str(text):
-    """
-    Return some representation of C{text} as a str.
-
-    This is here mainly because Twisted's exception methods are not able to
-    print unicode text.
-
-    :param text: The text to convert.
-    :type text: unicode
-
-    :return: A representation of C{text} as str.
-    :rtype: str
-    """
-    # XXX is there a better str representation for unicode?
-    return repr(text)
-
-
 class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     """
     An implementation of IAccount and INamespacePresenteer
@@ -145,8 +128,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         name = self._parse_mailbox_name(name)
 
         if name not in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox: %s" %
-                                         _unicode_as_str(name))
+            raise imap4.MailboxException("No such mailbox: %r" % name)
 
         return SoledadMailbox(name, soledad=self._soledad)
 
@@ -172,7 +154,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         name = self._parse_mailbox_name(name)
 
         if name in self.mailboxes:
-            raise imap4.MailboxCollision, _unicode_as_str(name)
+            raise imap4.MailboxCollision(repr(name))
 
         if not creation_ts:
             # by default, we pass an int value
@@ -258,8 +240,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         name = self._parse_mailbox_name(name)
 
         if not name in self.mailboxes:
-            raise imap4.MailboxException("No such mailbox: %s" %
-                                         _unicode_as_str(name))
+            raise imap4.MailboxException("No such mailbox: %r" % name)
 
         mbox = self.getMailbox(name)
 
@@ -298,14 +279,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         newname = self._parse_mailbox_name(newname)
 
         if oldname not in self.mailboxes:
-            raise imap4.NoSuchMailbox, _unicode_as_str(oldname)
+            raise imap4.NoSuchMailbox(repr(oldname))
 
         inferiors = self._inferiorNames(oldname)
         inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
 
         for (old, new) in inferiors:
             if new in self.mailboxes:
-                raise imap4.MailboxCollision, _unicode_as_str(new)
+                raise imap4.MailboxCollision(repr(new))
 
         for (old, new) in inferiors:
             mbox = self._get_mailbox_by_name(old)
@@ -386,8 +367,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         """
         name = self._parse_mailbox_name(name)
         if name not in self.subscriptions:
-            raise imap4.MailboxException, \
-                "Not currently subscribed to %s" % _unicode_as_str(name)
+            raise imap4.MailboxException(
+                "Not currently subscribed to %r" % name)
         self._set_subscription(name, False)
 
     def listMailboxes(self, ref, wildcard):
-- 
cgit v1.2.3


From bffdcddee55d1045be5d5c8378f712283863b6bf Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 9 Jan 2014 18:11:58 -0400
Subject: check for none

---
 changes/bug_4933_check_for_none | 1 +
 src/leap/mail/walk.py           | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)
 create mode 100644 changes/bug_4933_check_for_none

diff --git a/changes/bug_4933_check_for_none b/changes/bug_4933_check_for_none
new file mode 100644
index 0000000..33f3bd5
--- /dev/null
+++ b/changes/bug_4933_check_for_none
@@ -0,0 +1 @@
+  o Check for none in payload detection. Closes: #4933
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 820b8c7..dc13345 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -57,11 +57,13 @@ get_payloads = lambda msg: ((x.get_payload(),
 
 get_body_phash_simple = lambda payloads: first(
     [get_hash(payload) for payload, headers in payloads
-     if "text/plain" in headers.get('content-type')])
+     if payloads
+     and "text/plain" in headers.get('content-type')])
 
 get_body_phash_multi = lambda payloads: (first(
     [get_hash(payload) for payload, headers in payloads
-     if "text/plain" in headers.get('content-type')])
+     if payloads
+     and "text/plain" in headers.get('content-type')])
     or get_body_phash_simple(payloads))
 
 """
-- 
cgit v1.2.3


From ddb50ed05ae7141c2f9c2aece9e24681e0d5d696 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 9 Jan 2014 20:03:02 -0400
Subject: Check for none in innerheaders

This was causing a bug, among other things, when saving to the Sent
folder for some messages. Closes #4914
---
 src/leap/mail/walk.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index dc13345..1871752 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -111,8 +111,8 @@ def walk_msg_tree(parts, body_phash=None):
     # parts vector
     pv = list(get_parts_vector(parts))
 
-    if len(parts) == 2:
-        inner_headers = parts[1].get("headers", None)
+    inner_headers = parts[1].get("headers", None) if (
+        len(parts) == 2) else None
 
     if DEBUG:
         print "parts vector: ", pv
@@ -155,7 +155,8 @@ def walk_msg_tree(parts, body_phash=None):
         pdoc["part_map"][1]["multi"] = False
         if not pdoc["part_map"][1].get("phash", None):
             pdoc["part_map"][1]["phash"] = body_phash
-        pdoc["part_map"][1]["headers"] = inner_headers
+        if inner_headers:
+            pdoc["part_map"][1]["headers"] = inner_headers
     else:
         pdoc = outer
     pdoc["body"] = body_phash
-- 
cgit v1.2.3


From 241cde270f1ef37aa33332934869a58b143885d1 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Sat, 11 Jan 2014 20:31:08 -0400
Subject: add offline flag

---
 changes/feature_4943-offline-flag  |  1 +
 src/leap/mail/imap/service/imap.py | 18 +++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)
 create mode 100644 changes/feature_4943-offline-flag

diff --git a/changes/feature_4943-offline-flag b/changes/feature_4943-offline-flag
new file mode 100644
index 0000000..6edfd4d
--- /dev/null
+++ b/changes/feature_4943-offline-flag
@@ -0,0 +1 @@
+  o Add a flag for offline mode in imap. Related to #4943
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index dfd4862..c48e5c5 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -178,6 +178,7 @@ def run_service(*args, **kwargs):
     check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD)
     userid = kwargs.get('userid', None)
     leap_check(userid is not None, "need an user id")
+    offline = kwargs.get('offline', False)
 
     uuid = soledad._get_uuid()
     factory = LeapIMAPFactory(uuid, userid, soledad)
@@ -187,12 +188,15 @@ def run_service(*args, **kwargs):
     try:
         tport = reactor.listenTCP(port, factory,
                                   interface="localhost")
-        fetcher = LeapIncomingMail(
-            keymanager,
-            soledad,
-            factory.theAccount,
-            check_period,
-            userid)
+        if not offline:
+            fetcher = LeapIncomingMail(
+                keymanager,
+                soledad,
+                factory.theAccount,
+                check_period,
+                userid)
+        else:
+            fetcher = None
     except CannotListenError:
         logger.error("IMAP Service failed to start: "
                      "cannot listen in port %s" % (port,))
@@ -200,7 +204,7 @@ def run_service(*args, **kwargs):
         logger.error("Error launching IMAP service: %r" % (exc,))
     else:
         # all good.
-        fetcher.start_loop()
+        # (the caller has still to call fetcher.start_loop)
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
         leap_events.signal(IMAP_SERVICE_STARTED, str(port))
         return fetcher, tport, factory
-- 
cgit v1.2.3


From cf231b4536652fadfe03169c97688a0c76606dca Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 13 Jan 2014 10:24:51 -0400
Subject: avoid failure if no content-type

---
 src/leap/mail/walk.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 1871752..dd3b745 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -57,13 +57,12 @@ get_payloads = lambda msg: ((x.get_payload(),
 
 get_body_phash_simple = lambda payloads: first(
     [get_hash(payload) for payload, headers in payloads
-     if payloads
-     and "text/plain" in headers.get('content-type')])
+     if payloads])
 
 get_body_phash_multi = lambda payloads: (first(
     [get_hash(payload) for payload, headers in payloads
      if payloads
-     and "text/plain" in headers.get('content-type')])
+     and "text/plain" in headers.get('content-type', '')])
     or get_body_phash_simple(payloads))
 
 """
-- 
cgit v1.2.3


From 51eaab77deedf0c923fe40cf3d346fa879bf2ae3 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 13 Jan 2014 13:20:00 -0400
Subject: Add check for uniqueness when adding mails.

Check by mbox + content-hash
---
 changes/bug_4949-check-fdoc-uniqueness |  2 ++
 src/leap/mail/imap/fields.py           |  4 +++
 src/leap/mail/imap/mailbox.py          |  6 ++--
 src/leap/mail/imap/messages.py         | 50 ++++++++++++++++++++++++++++++++++
 4 files changed, 59 insertions(+), 3 deletions(-)
 create mode 100644 changes/bug_4949-check-fdoc-uniqueness

diff --git a/changes/bug_4949-check-fdoc-uniqueness b/changes/bug_4949-check-fdoc-uniqueness
new file mode 100644
index 0000000..bf49d1f
--- /dev/null
+++ b/changes/bug_4949-check-fdoc-uniqueness
@@ -0,0 +1,2 @@
+  o Check for flags doc uniqueness before adding a message. Avoids duplicates of
+    a single message in the same mailbox while copying or moving. Closes: #4949
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 2545adf..70af61f 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -99,6 +99,7 @@ class WithMsgFields(object):
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
     TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+    TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
     TYPE_C_HASH_IDX = 'by-type-and-contenthash'
     TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
     TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
@@ -121,6 +122,9 @@ class WithMsgFields(object):
         # mailboxes
         TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
 
+        # fdocs uniqueness
+        TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
+
         # content, headers doc
         TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
 
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 7c01490..c9e8684 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -125,7 +125,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def addListener(self, listener):
         """
-        Adds a listener to the listeners queue.
+        Add a listener to the listeners queue.
         The server adds itself as a listener when there is a SELECT,
         so it can send EXIST commands.
 
@@ -137,7 +137,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def removeListener(self, listener):
         """
-        Removes a listener from the listeners queue.
+        Remove a listener from the listeners queue.
 
         :param listener: listener to remove
         :type listener: an object that implements IMailboxListener
@@ -146,7 +146,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def _get_mbox(self):
         """
-        Returns mailbox document.
+        Return mailbox document.
 
         :return: A SoledadDocument containing this mailbox, or None if
                  the query failed.
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 37e4311..a3fcd87 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1064,10 +1064,27 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             hd[self.DATE_KEY] = date
         return hd
 
+    def _fdoc_already_exists(self, chash):
+        """
+        Check whether we can find a flags doc for this mailbox with the
+        given content-hash. It enforces that we can only have the same maessage
+        listed once for a a given mailbox.
+
+        :param chash: the content-hash to check about.
+        :type chash: basestring
+        :return: False, if it does not exist, or UID.
+        """
+        exist = self._get_fdoc_from_chash(chash)
+        if exist:
+            return exist.content.get(fields.UID_KEY, "unknown-uid")
+        else:
+            return False
+
     @deferred
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
         """
         Creates a new message document.
+        Here lives the magic of the leap mail. Well, in soledad, really.
 
         :param raw: the raw message
         :type raw: str
@@ -1097,6 +1114,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # parse
         msg, chash, size, multi = self._do_parse(raw)
 
+        # check for uniqueness.
+        if self._fdoc_already_exists(chash):
+            logger.warning("We already have that message in this mailbox.")
+            # note that this operation will leave holes in the UID sequence,
+            # but we're gonna change that all the same for a local-only table.
+            # so not touch it by the moment.
+            return False
+
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
 
@@ -1156,6 +1181,31 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # getters
 
+    def _get_fdoc_from_chash(self, chash):
+        """
+        Return a flags document for this mailbox with a given chash.
+
+        :return: A SoledadDocument containing the Flags Document, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
+        """
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MBOX_C_HASH_IDX,
+                fields.TYPE_FLAGS_VAL, self.mbox, chash)
+            if query:
+                if len(query) > 1:
+                    logger.warning(
+                        "More than one fdoc found for this chash, "
+                        "we got a duplicate!!")
+                    # XXX we could take action, like trigger a background
+                    # process to kill dupes.
+                return query.pop()
+            else:
+                return None
+        except Exception as exc:
+            logger.exception("Unhandled error %r" % exc)
+
     def get_msg_by_uid(self, uid):
         """
         Retrieves a LeapMessage by UID.
-- 
cgit v1.2.3


From 5adc6b66839b15c23980355774d8d24aba4918bd Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 13 Jan 2014 14:51:13 -0400
Subject: Restore the encoding of the messages. Fixes: #4956

We still are getting wrong output with unicode chars, but this
at least avoids breaking the fetch command.
---
 src/leap/mail/imap/messages.py | 47 +++++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 26 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index a3fcd87..7b49c80 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -19,6 +19,7 @@ LeapMessage and MessageCollection.
 """
 import copy
 import logging
+import re
 import StringIO
 
 from collections import defaultdict, namedtuple
@@ -63,6 +64,10 @@ def lowerdict(_dict):
                 for key, value in _dict.items())
 
 
+CHARSET_PATTERN = r"""charset=([\w-]+)"""
+CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
+
+
 class MessagePart(object):
     """
     IMessagePart implementor.
@@ -140,18 +145,9 @@ class MessagePart(object):
             payload = str("")
 
         if payload:
-            #headers = self.getHeaders(True)
-            #headers = lowerdict(headers)
-            #content_type = headers.get('content-type', "")
             content_type = self._get_ctype_from_document(phash)
-            charset_split = content_type.split('charset=')
-            # XXX fuck all this, use a regex!
-            if len(charset_split) > 1:
-                charset = charset_split[1]
-                if charset:
-                    charset = charset.strip()
-            else:
-                charset = None
+            charset = first(CHARSET_RE.findall(content_type))
+            logger.debug("Got charset from header: %s" % (charset,))
             if not charset:
                 charset = self._get_charset(payload)
             try:
@@ -483,28 +479,27 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: file-like object opened for reading
         :rtype: StringIO
         """
+        # TODO refactor with getBodyFile in MessagePart
         fd = StringIO.StringIO()
         bdoc = self._bdoc
         if bdoc:
-            body = str(self._bdoc.content.get(self.RAW_KEY, ""))
+            body = self._bdoc.content.get(self.RAW_KEY, "")
+            content_type = bdoc.content.get('content-type', "")
+            charset = first(CHARSET_RE.findall(content_type))
+            logger.debug("Got charset from header: %s" % (charset,))
+            if not charset:
+                charset = self._get_charset(body)
+            try:
+                body = body.decode(charset).encode(charset)
+            except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                logger.error("Unicode error {0}".format(e))
+                body = body.encode(charset, 'replace')
+
+        # We are still returning funky characters from here.
         else:
             logger.warning("No BDOC found for message.")
             body = str("")
 
-        # XXX not needed, isn't it? ---- ivan?
-        #if bdoc:
-            #content_type = bdoc.content.get('content-type', "")
-            #charset = content_type.split('charset=')[1]
-            #if charset:
-                #charset = charset.strip()
-            #if not charset:
-                #charset = self._get_charset(body)
-            #try:
-                #body = str(body.encode(charset))
-            #except (UnicodeEncodeError, UnicodeDecodeError) as e:
-                #logger.error("Unicode error {0}".format(e))
-                #body = str(body.encode(charset, 'replace'))
-
         fd.write(body)
         fd.seek(0)
         return fd
-- 
cgit v1.2.3


From 4856f32ec75cda000fc794d0ac93990e0d1e42f6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 13 Jan 2014 17:58:02 -0400
Subject: Very limited support for SEARCH Commands. Closes: #4209

limited to HEADER Message-ID.

This is a quick workaround for avoiding duplicate saves in Drafts Folder.
but we'll get there!
---
 changes/feature_enable-search-by-msg-id |  3 ++
 src/leap/mail/imap/fields.py            | 13 +++--
 src/leap/mail/imap/mailbox.py           | 45 ++++++++++++++++
 src/leap/mail/imap/messages.py          | 93 +++++++++++++++++++++++++++++----
 4 files changed, 137 insertions(+), 17 deletions(-)
 create mode 100644 changes/feature_enable-search-by-msg-id

diff --git a/changes/feature_enable-search-by-msg-id b/changes/feature_enable-search-by-msg-id
new file mode 100644
index 0000000..accc12f
--- /dev/null
+++ b/changes/feature_enable-search-by-msg-id
@@ -0,0 +1,3 @@
+  o Ability to support SEARCH Commands, limited to HEADER Message-ID.
+    This is a quick workaround for avoiding duplicate saves in Drafts Folder.
+    Closes: #4209
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 70af61f..3d2ac92 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -45,13 +45,12 @@ class WithMsgFields(object):
     HEADERS_KEY = "headers"
     DATE_KEY = "date"
     SUBJECT_KEY = "subject"
-    # XXX DELETE-ME
-    #NUM_PARTS_KEY = "numparts"  # not needed?!
     PARTS_MAP_KEY = "part_map"
     BODY_KEY = "body"  # link to phash of body
+    MSGID_KEY = "msgid"
 
     # content
-    LINKED_FROM_KEY = "lkf"
+    LINKED_FROM_KEY = "lkf"  # XXX not implemented yet!
     RAW_KEY = "raw"
     CTYPE_KEY = "ctype"
 
@@ -69,10 +68,6 @@ class WithMsgFields(object):
     TYPE_HEADERS_VAL = "head"
     TYPE_CONTENT_VAL = "cnt"
 
-    # XXX DEPRECATE
-    #TYPE_MESSAGE_VAL = "msg"
-    #TYPE_ATTACHMENT_VAL = "attach"
-
     INBOX_VAL = "inbox"
 
     # Flags in Mailbox and Message
@@ -96,6 +91,7 @@ class WithMsgFields(object):
     TYPE_MBOX_IDX = 'by-type-and-mbox'
     TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
     TYPE_SUBS_IDX = 'by-type-and-subscribed'
+    TYPE_MSGID_IDX = 'by-type-and-message-id'
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
     TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
@@ -125,6 +121,9 @@ class WithMsgFields(object):
         # fdocs uniqueness
         TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
 
+        # headers doc - search by msgid.
+        TYPE_MSGID_IDX: [KTYPE, MSGID_KEY],
+
         # content, headers doc
         TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
 
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index c9e8684..ccbf5c2 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -39,6 +39,7 @@ from leap.mail.decorators import deferred
 from leap.mail.imap.fields import WithMsgFields, fields
 from leap.mail.imap.messages import MessageCollection
 from leap.mail.imap.parser import MBoxParser
+from leap.mail.utils import first
 
 logger = logging.getLogger(__name__)
 
@@ -55,6 +56,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         imap4.IMailbox,
         imap4.IMailboxInfo,
         imap4.ICloseableMailbox,
+        imap4.ISearchableMailbox,
         imap4.IMessageCopier)
 
     # XXX should finish the implementation of IMailboxListener
@@ -617,6 +619,49 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self._signal_unread_to_ui()
         return result
 
+    # ISearchableMailbox
+
+    def search(self, query, uid):
+        """
+        Search for messages that meet the given query criteria.
+
+        Warning: this is half-baked, and it might give problems since
+        it offers the SearchableInterface.
+        We'll be implementing it asap.
+
+        :param query: The search criteria
+        :type query: list
+
+        :param uid: If true, the IDs specified in the query are UIDs;
+                    otherwise they are message sequence IDs.
+        :type uid: bool
+
+        :return: A list of message sequence numbers or message UIDs which
+                 match the search criteria or a C{Deferred} whose callback
+                 will be invoked with such a list.
+        :rtype: C{list} or C{Deferred}
+        """
+        # TODO see if we can raise w/o interrupting flow
+        #:raise IllegalQueryError: Raised when query is not valid.
+        # example query:
+        #  ['UNDELETED', 'HEADER', 'Message-ID',
+        #   '52D44F11.9060107@dev.bitmask.net']
+
+        # TODO hardcoding for now! -- we'll support generic queries later on
+        # but doing a quickfix for avoiding duplicat saves in the draft folder.
+        # See issue #4209
+
+        if query[1] == 'HEADER' and query[2].lower() == "message-id":
+            msgid = str(query[3]).strip()
+            d = self.messages._get_uid_from_msgid(str(msgid))
+            d1 = defer.gatherResults([d])
+            # we want a list, so return it all the same
+            return d1
+
+        # nothing implemented for any other query
+        logger.warning("Cannot process query: %s" % (query,))
+        return []
+
     # IMessageCopier
 
     @deferred
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 7b49c80..a3d29d6 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -20,6 +20,8 @@ LeapMessage and MessageCollection.
 import copy
 import logging
 import re
+import threading
+import time
 import StringIO
 
 from collections import defaultdict, namedtuple
@@ -44,6 +46,7 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer
 
 logger = logging.getLogger(__name__)
 
+read_write_lock = threading.Lock()
 
 # TODO ------------------------------------------------------------
 
@@ -53,6 +56,7 @@ logger = logging.getLogger(__name__)
 # [ ] Send patch to twisted for bug in imap4.py:5717  (content-type can be
 #     none? lower-case?)
 
+
 def lowerdict(_dict):
     """
     Return a dict with the keys in lowercase.
@@ -60,12 +64,17 @@ def lowerdict(_dict):
     :param _dict: the dict to convert
     :rtype: dict
     """
+    # TODO should properly implement a CaseInsensitive dict.
+    # Look into requests code.
     return dict((key.lower(), value)
                 for key, value in _dict.items())
 
 
 CHARSET_PATTERN = r"""charset=([\w-]+)"""
+MSGID_PATTERN = r"""<([\w@.]+)>"""
+
 CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
+MSGID_RE = re.compile(MSGID_PATTERN)
 
 
 class MessagePart(object):
@@ -897,6 +906,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     Implements a filter query over the messages contained in a soledad
     database.
     """
+
     # XXX this should be able to produce a MessageSet methinks
     # could validate these kinds of objects turning them
     # into a template for the class.
@@ -1044,9 +1054,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             newline = "\n%s: " % (k,)
             headers[k] = newline.join(v)
 
+        lower_headers = lowerdict(headers)
+        msgid = first(MSGID_RE.findall(
+            lower_headers.get('message-id', '')))
+
         hd = self._get_empty_doc(self.HEADERS_DOC)
         hd[self.CONTENT_HASH_KEY] = chash
         hd[self.HEADERS_KEY] = headers
+        hd[self.MSGID_KEY] = msgid
 
         if not subject and self.SUBJECT_FIELD in headers:
             hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
@@ -1139,16 +1154,17 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         logger.debug('enqueuing message docs for write')
         ptuple = SoledadWriterPayload
 
-        # first, regular docs: flags and headers
-        for doc in docs:
-            self.soledad_writer.put(ptuple(
-                mode=ptuple.CREATE, payload=doc))
+        with read_write_lock:
+            # first, regular docs: flags and headers
+            for doc in docs:
+                self.soledad_writer.put(ptuple(
+                    mode=ptuple.CREATE, payload=doc))
 
-        # and last, but not least, try to create
-        # content docs if not already there.
-        for cd in cdocs:
-            self.soledad_writer.put(ptuple(
-                mode=ptuple.CONTENT_CREATE, payload=cd))
+            # and last, but not least, try to create
+            # content docs if not already there.
+            for cd in cdocs:
+                self.soledad_writer.put(ptuple(
+                    mode=ptuple.CONTENT_CREATE, payload=cd))
 
     def _remove_cb(self, result):
         return result
@@ -1174,7 +1190,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         d.addCallback(self._remove_cb)
         return d
 
-    # getters
+    # getters: specific queries
 
     def _get_fdoc_from_chash(self, chash):
         """
@@ -1201,6 +1217,63 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         except Exception as exc:
             logger.exception("Unhandled error %r" % exc)
 
+    def _get_uid_from_msgidCb(self, msgid):
+        hdoc = None
+        with read_write_lock:
+            try:
+                query = self._soledad.get_from_index(
+                    fields.TYPE_MSGID_IDX,
+                    fields.TYPE_HEADERS_VAL, msgid)
+                if query:
+                    if len(query) > 1:
+                        logger.warning(
+                            "More than one hdoc found for this msgid, "
+                            "we got a duplicate!!")
+                        # XXX we could take action, like trigger a background
+                        # process to kill dupes.
+                    hdoc = query.pop()
+            except Exception as exc:
+                logger.exception("Unhandled error %r" % exc)
+
+        if hdoc is None:
+            logger.warning("Could not find hdoc for msgid %s"
+                           % (msgid,))
+            return None
+        msg_chash = hdoc.content.get(fields.CONTENT_HASH_KEY)
+        fdoc = self._get_fdoc_from_chash(msg_chash)
+        if not fdoc:
+            logger.warning("Could not find fdoc for msgid %s"
+                           % (msgid,))
+            return None
+        return fdoc.content.get(fields.UID_KEY, None)
+
+    @deferred
+    def _get_uid_from_msgid(self, msgid):
+        """
+        Return a UID for a given message-id.
+
+        It first gets the headers-doc for that msg-id, and
+        it found it queries the flags doc for the current mailbox
+        for the matching content-hash.
+
+        :return: A UID, or None
+        """
+        # We need to wait a little bit, cause in some of the cases
+        # the query is received right after we've saved the document,
+        # and we cannot find it otherwise. This seems to be enough.
+
+        # Doing a sleep since we'll be calling this in a secondary thread,
+        # but we'll should be able to collect the results after a
+        # reactor.callLater.
+        # Maybe we can implement something like NOT_DONE_YET in the web
+        # framework, and return from the callback?
+        # See: http://jcalderone.livejournal.com/50226.html
+        # reactor.callLater(0.3, self._get_uid_from_msgidCb, msgid)
+        time.sleep(0.3)
+        return self._get_uid_from_msgidCb(msgid)
+
+    # getters: generic for a mailbox
+
     def get_msg_by_uid(self, uid):
         """
         Retrieves a LeapMessage by UID.
-- 
cgit v1.2.3


From 2b53238ce5211bc23da8d1e8903335daa12ca02e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 14 Jan 2014 16:28:07 -0400
Subject: remove locks (moved to soledad client)

---
 src/leap/mail/imap/mailbox.py  | 11 +++++++---
 src/leap/mail/imap/messages.py | 50 +++++++++++++++++++-----------------------
 src/leap/mail/messageflow.py   |  2 --
 3 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index ccbf5c2..cd782b2 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -39,7 +39,6 @@ from leap.mail.decorators import deferred
 from leap.mail.imap.fields import WithMsgFields, fields
 from leap.mail.imap.messages import MessageCollection
 from leap.mail.imap.parser import MBoxParser
-from leap.mail.utils import first
 
 logger = logging.getLogger(__name__)
 
@@ -60,7 +59,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         imap4.IMessageCopier)
 
     # XXX should finish the implementation of IMailboxListener
-    # XXX should implement ISearchableMailbox too
+    # XXX should complately implement ISearchableMailbox too
 
     messages = None
     _closed = False
@@ -78,6 +77,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     CMD_UNSEEN = "UNSEEN"
 
     _listeners = defaultdict(set)
+
     next_uid_lock = threading.Lock()
 
     def __init__(self, mbox, soledad=None, rw=1):
@@ -161,7 +161,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             if query:
                 return query.pop()
         except Exception as exc:
-            logger.error("Unhandled error %r" % exc)
+            logger.exception("Unhandled error %r" % exc)
 
     def getFlags(self):
         """
@@ -226,6 +226,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: bool
         """
         mbox = self._get_mbox()
+        if not mbox:
+            logger.error("We could not get a mbox!")
+            # XXX It looks like it has been corrupted.
+            # We need to be able to survive this.
+            return None
         return mbox.content.get(self.LAST_UID_KEY, 1)
 
     def _set_last_uid(self, uid):
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index a3d29d6..7c17dbe 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -46,8 +46,6 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer
 
 logger = logging.getLogger(__name__)
 
-read_write_lock = threading.Lock()
-
 # TODO ------------------------------------------------------------
 
 # [ ] Add linked-from info.
@@ -1154,17 +1152,16 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         logger.debug('enqueuing message docs for write')
         ptuple = SoledadWriterPayload
 
-        with read_write_lock:
-            # first, regular docs: flags and headers
-            for doc in docs:
-                self.soledad_writer.put(ptuple(
-                    mode=ptuple.CREATE, payload=doc))
+        # first, regular docs: flags and headers
+        for doc in docs:
+            self.soledad_writer.put(ptuple(
+                mode=ptuple.CREATE, payload=doc))
 
-            # and last, but not least, try to create
-            # content docs if not already there.
-            for cd in cdocs:
-                self.soledad_writer.put(ptuple(
-                    mode=ptuple.CONTENT_CREATE, payload=cd))
+        # and last, but not least, try to create
+        # content docs if not already there.
+        for cd in cdocs:
+            self.soledad_writer.put(ptuple(
+                mode=ptuple.CONTENT_CREATE, payload=cd))
 
     def _remove_cb(self, result):
         return result
@@ -1219,21 +1216,20 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     def _get_uid_from_msgidCb(self, msgid):
         hdoc = None
-        with read_write_lock:
-            try:
-                query = self._soledad.get_from_index(
-                    fields.TYPE_MSGID_IDX,
-                    fields.TYPE_HEADERS_VAL, msgid)
-                if query:
-                    if len(query) > 1:
-                        logger.warning(
-                            "More than one hdoc found for this msgid, "
-                            "we got a duplicate!!")
-                        # XXX we could take action, like trigger a background
-                        # process to kill dupes.
-                    hdoc = query.pop()
-            except Exception as exc:
-                logger.exception("Unhandled error %r" % exc)
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MSGID_IDX,
+                fields.TYPE_HEADERS_VAL, msgid)
+            if query:
+                if len(query) > 1:
+                    logger.warning(
+                        "More than one hdoc found for this msgid, "
+                        "we got a duplicate!!")
+                    # XXX we could take action, like trigger a background
+                    # process to kill dupes.
+                hdoc = query.pop()
+        except Exception as exc:
+            logger.exception("Unhandled error %r" % exc)
 
         if hdoc is None:
             logger.warning("Could not find hdoc for msgid %s"
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index a0a571d..ac26e45 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -121,8 +121,6 @@ class MessageProducer(object):
         """
         if not self._loop.running:
             self._loop.start(self._period, now=True)
-        else:
-            print "was running..., not starting"
 
     def stop(self):
         """
-- 
cgit v1.2.3


From fc7ef201ea169e76123e15db346ac8d882d93c02 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 15 Jan 2014 16:57:18 -0400
Subject: remove use of soledad_writer

Since the soledad client lock gets us covered with writes now, it makes
no sense to enqueue using the messageconsumer.

The SoledadWriter is left orphaned by now. We might want to reuse it
to enqueue low priority tasks that need a strategy of retries in case
of revisionconflicts.

the MessageConsumer abstraction should also be useful for the case of
the smtp queue.
---
 src/leap/mail/imap/messages.py | 163 +++++++++++++++++++++++++----------------
 1 file changed, 99 insertions(+), 64 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 7c17dbe..b35b808 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -20,7 +20,6 @@ LeapMessage and MessageCollection.
 import copy
 import logging
 import re
-import threading
 import time
 import StringIO
 
@@ -51,8 +50,6 @@ logger = logging.getLogger(__name__)
 # [ ] Add linked-from info.
 # [ ] Delete incoming mail only after successful write!
 # [ ] Remove UID from syncable db. Store only those indexes locally.
-# [ ] Send patch to twisted for bug in imap4.py:5717  (content-type can be
-#     none? lower-case?)
 
 
 def lowerdict(_dict):
@@ -657,10 +654,27 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the document that keeps the flags for this
         message.
         """
-        flag_docs = self._soledad.get_from_index(
-            fields.TYPE_MBOX_UID_IDX,
-            fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
-        return first(flag_docs)
+        result = {}
+        try:
+            flag_docs = self._soledad.get_from_index(
+                fields.TYPE_MBOX_UID_IDX,
+                fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+            result = first(flag_docs)
+        except Exception as exc:
+            # ugh! Something's broken down there!
+            logger.warning("FUCKING ERROR ----- getting for UID:", self._uid)
+            logger.exception(exc)
+            try:
+                flag_docs = self._soledad.get_from_index(
+                    fields.TYPE_MBOX_UID_IDX,
+                    fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+                result = first(flag_docs)
+            except Exception as exc:
+                # ugh! Something's broken down there!
+                logger.warning("FUCKING ERROR, 2nd time -----")
+                logger.exception(exc)
+        finally:
+            return result
 
     def _get_headers_doc(self):
         """
@@ -770,6 +784,51 @@ class LeapMessage(fields, MailParser, MBoxParser):
         return self._fdoc is not None
 
 
+class ContentDedup(object):
+    """
+    Message deduplication.
+
+    We do a query for the content hashes before writing to our beloved
+    sqlcipher backend of Soledad. This means, by now, that:
+
+    1. We will not store the same attachment twice, only the hash of it.
+    2. We will not store the same message body twice, only the hash of it.
+
+    The first case is useful if you are always receiving the same old memes
+    from unwary friends that still have not discovered that 4chan is the
+    generator of the internet. The second will save your day if you have
+    initiated session with the same account in two different machines. I also
+    wonder why would you do that, but let's respect each other choices, like
+    with the religious celebrations, and assume that one day we'll be able
+    to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
+    Stack.
+    """
+
+    def _content_does_exist(self, doc):
+        """
+        Check whether we already have a content document for a payload
+        with this hash in our database.
+
+        :param doc: tentative body document
+        :type doc: dict
+        :returns: True if that happens, False otherwise.
+        """
+        if not doc:
+            return False
+        phash = doc[fields.PAYLOAD_HASH_KEY]
+        attach_docs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+        if not attach_docs:
+            return False
+
+        if len(attach_docs) != 1:
+            logger.warning("Found more than one copy of phash %s!"
+                           % (phash,))
+        logger.debug("Found attachment doc with that hash! Skipping save!")
+        return True
+
+
 SoledadWriterPayload = namedtuple(
     'SoledadWriterPayload', ['mode', 'payload'])
 
@@ -781,6 +840,13 @@ SoledadWriterPayload.PUT = 2
 SoledadWriterPayload.CONTENT_CREATE = 3
 
 
+"""
+SoledadDocWriter was used to avoid writing to the db from multiple threads.
+Its use here has been deprecated in favor of a local rw_lock in the client.
+But we might want to reuse in in the near future to implement priority queues.
+"""
+
+
 class SoledadDocWriter(object):
     """
     This writer will create docs serially in the local soledad database.
@@ -852,51 +918,9 @@ class SoledadDocWriter(object):
 
             empty = queue.empty()
 
-    """
-    Message deduplication.
 
-    We do a query for the content hashes before writing to our beloved
-    sqlcipher backend of Soledad. This means, by now, that:
-
-    1. We will not store the same attachment twice, only the hash of it.
-    2. We will not store the same message body twice, only the hash of it.
-
-    The first case is useful if you are always receiving the same old memes
-    from unwary friends that still have not discovered that 4chan is the
-    generator of the internet. The second will save your day if you have
-    initiated session with the same account in two different machines. I also
-    wonder why would you do that, but let's respect each other choices, like
-    with the religious celebrations, and assume that one day we'll be able
-    to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
-    Stack.
-    """
-
-    def _content_does_exist(self, doc):
-        """
-        Check whether we already have a content document for a payload
-        with this hash in our database.
-
-        :param doc: tentative body document
-        :type doc: dict
-        :returns: True if that happens, False otherwise.
-        """
-        if not doc:
-            return False
-        phash = doc[fields.PAYLOAD_HASH_KEY]
-        attach_docs = self._soledad.get_from_index(
-            fields.TYPE_P_HASH_IDX,
-            fields.TYPE_CONTENT_VAL, str(phash))
-        if not attach_docs:
-            return False
-
-        if len(attach_docs) != 1:
-            logger.warning("Found more than one copy of phash %s!"
-                           % (phash,))
-        logger.debug("Found attachment doc with that hash! Skipping save!")
-        return True
-
-
-class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
+class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
+                        ContentDedup):
     """
     A collection of messages, surprisingly.
 
@@ -1145,23 +1169,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             hd[key] = parts_map[key]
         del parts_map
 
-        docs = [fd, hd]
-        cdocs = walk.get_raw_docs(msg, parts)
-
         # Saving
-        logger.debug('enqueuing message docs for write')
-        ptuple = SoledadWriterPayload
 
         # first, regular docs: flags and headers
-        for doc in docs:
-            self.soledad_writer.put(ptuple(
-                mode=ptuple.CREATE, payload=doc))
+        self._soledad.create_doc(fd)
+
+        # XXX should check for content duplication on headers too
+        # but with chash. !!!
+        self._soledad.create_doc(hd)
 
         # and last, but not least, try to create
         # content docs if not already there.
-        for cd in cdocs:
-            self.soledad_writer.put(ptuple(
-                mode=ptuple.CONTENT_CREATE, payload=cd))
+        cdocs = walk.get_raw_docs(msg, parts)
+        for cdoc in cdocs:
+            if not self._content_does_exist(cdoc):
+                self._soledad.create_doc(cdoc)
 
     def _remove_cb(self, result):
         return result
@@ -1312,17 +1334,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX FIXINDEX -- should implement order by in soledad
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
-    def all_msg_iter(self):
+    def all_uid_iter(self):
         """
         Return an iterator trhough the UIDs of all messages, sorted in
         ascending order.
         """
+        # XXX we should get this from the uid table, local-only
         all_uids = (doc.content[self.UID_KEY] for doc in
                     self._soledad.get_from_index(
                         fields.TYPE_MBOX_IDX,
                         fields.TYPE_FLAGS_VAL, self.mbox))
         return (u for u in sorted(all_uids))
 
+    def all_flags(self):
+        """
+        Return a dict with all flags documents for this mailbox.
+        """
+        all_flags = dict(((
+            doc.content[self.UID_KEY],
+            doc.content[self.FLAGS_KEY]) for doc in
+            self._soledad.get_from_index(
+                fields.TYPE_MBOX_IDX,
+                fields.TYPE_FLAGS_VAL, self.mbox)))
+        return all_flags
+
     def count(self):
         """
         Return the count of messages for this mailbox.
@@ -1447,7 +1482,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :rtype: iterable
         """
         return (LeapMessage(self._soledad, docuid, self.mbox)
-                for docuid in self.all_msg_iter())
+                for docuid in self.all_uid_iter())
 
     def __repr__(self):
         """
-- 
cgit v1.2.3


From 90f4338da088394ade1663871a23b8fb0a4c0d66 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 15 Jan 2014 17:05:24 -0400
Subject: Performance improvement on FLAGS-only FETCH

* Compute the intersection set of the uids on a FETCH, so we avoid
  iterating through the non-existant UIDs.
* Dispatch FLAGS query to our specialized method, that fetches all the
  flags documents and return objects that only specify one subset of the
  MessagePart interface, apt to render flags quickly with less queries
  overhead.
* Overwrite the do_FETCH command in the imap Server to use fetch_flags.
* Use deferLater for a better dispatch of tasks in the reactor.
---
 src/leap/mail/imap/mailbox.py      | 94 +++++++++++++++++++++++++++++---------
 src/leap/mail/imap/messages.py     | 11 +----
 src/leap/mail/imap/service/imap.py | 37 ++++++++++++++-
 3 files changed, 109 insertions(+), 33 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index cd782b2..94070ac 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -20,13 +20,13 @@ Soledad Mailbox.
 import copy
 import threading
 import logging
-import time
 import StringIO
 import cStringIO
 
 from collections import defaultdict
 
 from twisted.internet import defer
+from twisted.internet.task import deferLater
 from twisted.python import log
 
 from twisted.mail import imap4
@@ -59,7 +59,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         imap4.IMessageCopier)
 
     # XXX should finish the implementation of IMailboxListener
-    # XXX should complately implement ISearchableMailbox too
+    # XXX should completely implement ISearchableMailbox too
 
     messages = None
     _closed = False
@@ -467,15 +467,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         return d
 
     @deferred
-    def fetch(self, messages, uid):
+    def fetch(self, messages_asked, uid):
         """
         Retrieve one or more messages in this mailbox.
 
         from rfc 3501: The data items to be fetched can be either a single atom
         or a parenthesized list.
 
-        :param messages: IDs of the messages to retrieve information about
-        :type messages: MessageSet
+        :param messages_asked: IDs of the messages to retrieve information
+                               about
+        :type messages_asked: MessageSet
 
         :param uid: If true, the IDs are UIDs. They are message sequence IDs
                     otherwise.
@@ -484,7 +485,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: A tuple of two-tuples of message sequence numbers and
                 LeapMessage
         """
-        result = []
+        from twisted.internet import reactor
 
         # For the moment our UID is sequential, so we
         # can treat them all the same.
@@ -494,12 +495,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         sequence = False
         #sequence = True if uid == 0 else False
 
-        if not messages.last:
+        if not messages_asked.last:
             try:
-                iter(messages)
+                iter(messages_asked)
             except TypeError:
                 # looks like we cannot iterate
-                messages.last = self.last_uid
+                messages_asked.last = self.last_uid
+
+        set_asked = set(messages_asked)
+        set_exist = set(self.messages.all_uid_iter())
+        seq_messg = set_asked.intersection(set_exist)
+        getmsg = lambda msgid: self.messages.get_msg_by_uid(msgid)
 
         # for sequence numbers (uid = 0)
         if sequence:
@@ -507,20 +513,68 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             raise NotImplementedError
 
         else:
-            for msg_id in messages:
-                msg = self.messages.get_msg_by_uid(msg_id)
-                if msg:
-                    result.append((msg_id, msg))
-                else:
-                    logger.debug("fetch %s, no msg found!!!" % msg_id)
+            result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
 
         if self.isWriteable():
+            deferLater(reactor, 30, self._unset_recent_flag)
+            # XXX I should rewrite the scheduler so it handles a
+            # set of queues with different priority.
             self._unset_recent_flag()
-        self._signal_unread_to_ui()
 
-        # XXX workaround for hangs in thunderbird
-        #return tuple(result[:100])  # --- doesn't show all!!
-        return tuple(result)
+        # this should really be called as a final callback of
+        # the do_FETCH method...
+        deferLater(reactor, 1, self._signal_unread_to_ui)
+        return result
+
+    @deferred
+    def fetch_flags(self, messages_asked, uid):
+        """
+        A fast method to fetch all flags, tricking just the
+        needed subset of the MIME interface that's needed to satisfy
+        a generic FLAGS query.
+        Given how LEAP Mail is supposed to work without local cache,
+        this query is going to be quite common, and also we expect
+        it to be in the form 1:* at the beginning of a session, so
+        it's not bad to fetch all the flags doc at once.
+
+        :param messages_asked: IDs of the messages to retrieve information
+                               about
+        :type messages_asked: MessageSet
+
+        :param uid: If true, the IDs are UIDs. They are message sequence IDs
+                    otherwise.
+        :type uid: bool
+
+        :return: A tuple of two-tuples of message sequence numbers and
+                flagsPart, which is a only a partial implementation of
+                MessagePart.
+        :rtype: tuple
+        """
+        class flagsPart(object):
+            def __init__(self, uid, flags):
+                self.uid = uid
+                self.flags = flags
+
+            def getUID(self):
+                return self.uid
+
+            def getFlags(self):
+                return map(str, self.flags)
+
+        if not messages_asked.last:
+            try:
+                iter(messages_asked)
+            except TypeError:
+                # looks like we cannot iterate
+                messages_asked.last = self.last_uid
+
+        set_asked = set(messages_asked)
+        set_exist = set(self.messages.all_uid_iter())
+        seq_messg = set_asked.intersection(set_exist)
+        all_flags = self.messages.all_flags()
+        result = ((msgid, flagsPart(
+            msgid, all_flags[msgid])) for msgid in seq_messg)
+        return result
 
     @deferred
     def _unset_recent_flag(self):
@@ -549,8 +603,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # 3. Route it through a queue with lesser priority than the
         #    regularar writer.
 
-        # hmm let's try 2. in a quickndirty way...
-        time.sleep(1)
         log.msg('unsetting recent flags...')
         for msg in self.messages.get_recent():
             msg.removeFlags((fields.RECENT_FLAG,))
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index b35b808..22de356 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -662,17 +662,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
             result = first(flag_docs)
         except Exception as exc:
             # ugh! Something's broken down there!
-            logger.warning("FUCKING ERROR ----- getting for UID:", self._uid)
+            logger.warning("ERROR while getting flags for UID: %s" % self._uid)
             logger.exception(exc)
-            try:
-                flag_docs = self._soledad.get_from_index(
-                    fields.TYPE_MBOX_UID_IDX,
-                    fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
-                result = first(flag_docs)
-            except Exception as exc:
-                # ugh! Something's broken down there!
-                logger.warning("FUCKING ERROR, 2nd time -----")
-                logger.exception(exc)
         finally:
             return result
 
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index c48e5c5..e877869 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -22,6 +22,7 @@ from copy import copy
 import logging
 
 from twisted.internet.protocol import ServerFactory
+from twisted.internet.defer import maybeDeferred
 from twisted.internet.error import CannotListenError
 from twisted.mail import imap4
 from twisted.python import log
@@ -78,7 +79,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
         :param line: the line from the server, without the line delimiter.
         :type line: str
         """
-        print "RECV: STATE (%s)" % self.state
         if self.theAccount.closed is True and self.state != "unauth":
             log.msg("Closing the session. State: unauth")
             self.state = "unauth"
@@ -89,7 +89,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
             msg = line[:7] + " [...]"
         else:
             msg = copy(line)
-        log.msg('rcv: %s' % msg)
+        log.msg('rcv (%s): %s' % (self.state, msg))
         imap4.IMAP4Server.lineReceived(self, line)
 
     def authenticateLogin(self, username, password):
@@ -111,6 +111,39 @@ class LeapIMAPServer(imap4.IMAP4Server):
         leap_events.signal(IMAP_CLIENT_LOGIN, "1")
         return imap4.IAccount, self.theAccount, lambda: None
 
+    def do_FETCH(self, tag, messages, query, uid=0):
+        """
+        Overwritten fetch dispatcher to use the fast fetch_flags
+        method
+        """
+        log.msg("LEAP Overwritten fetch...")
+        if not query:
+            self.sendPositiveResponse(tag, 'FETCH complete')
+            return  # XXX ???
+
+        cbFetch = self._IMAP4Server__cbFetch
+        ebFetch = self._IMAP4Server__ebFetch
+
+        if str(query[0]) == "flags":
+            self._oldTimeout = self.setTimeout(None)
+            # no need to call iter, we get a generator
+            maybeDeferred(
+                self.mbox.fetch_flags, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(ebFetch, tag)
+        else:
+            self._oldTimeout = self.setTimeout(None)
+            # no need to call iter, we get a generator
+            maybeDeferred(
+                self.mbox.fetch, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(ebFetch, tag)
+
+    select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
+                    imap4.IMAP4Server.arg_fetchatt)
+
 
 class IMAPAuthRealm(object):
     """
-- 
cgit v1.2.3


From 1069e7b9470fb63f70a43fa72407fcd4276d550d Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 15 Jan 2014 16:43:18 -0400
Subject: Update service initialization file

This will need to place a configuration file with:

* userid
* uuid
* password (optional)

Use it for even faster startup times, and running under the native
twisted reactor.
---
 .gitignore                                 |   1 +
 src/leap/mail/imap/service/imap-server.tac | 182 +++++++++++++++++++++--------
 2 files changed, 136 insertions(+), 47 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0512b87..3a80621 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ lib/
 local/
 share/
 MANIFEST
+twistd.pid
diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index da72cae..b65bb17 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -1,69 +1,157 @@
+# -*- coding: utf-8 -*-
+# imap-server.tac 
+# Copyright (C) 2013,2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+TAC file for initialization of the imap service using twistd.
+
+Use this for debugging and testing the imap server using a native reactor.
+
+For now, and for debugging/testing purposes, you need
+to pass a config file with the following structure:
+
+[leap_mail]
+userid = "user@provider"
+uuid = "deadbeefdeadabad"
+passwd = "supersecret" # optional, will get prompted if not found.
+"""
 import ConfigParser
+import getpass
 import os
+import sys
 
-from leap.soledad.client import Soledad
+from leap.keymanager import KeyManager
 from leap.mail.imap.service import imap
-from leap.common.config import get_path_prefix
-
-
-config = ConfigParser.ConfigParser()
-config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')])
-
-userID = config.get('mail', 'address')
-privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read()
-nickserver_url = ""
+from leap.soledad.client import Soledad
 
-d = {}
+from twisted.application import service, internet
 
-for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'):
-    d[key] = config.get('mail', key)
 
+# TODO should get this initializers from some authoritative mocked source
+# We might want to put them the soledad itself.
 
-def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url,
-                               server_pemfile, token):
+def initialize_soledad(uuid, email, passwd,
+                       secrets, localdb,
+                       gnupg_home, tempdir):
     """
     Initializes soledad by hand
 
-    :param user_uuid:
-    :param soledad_pass:
-    :param server_url:
-    :param server_pemfile:
-    :param token:
-
+    :param email: ID for the user
+    :param gnupg_home: path to home used by gnupg
+    :param tempdir: path to temporal dir
     :rtype: Soledad instance
     """
+    # XXX TODO unify with an authoritative source of mocks
+    # for soledad (or partial initializations).
+    # This is copied from the imap tests.
+
+    server_url = "http://provider"
+    cert_file = ""
 
-    base_config = get_path_prefix()
+    class Mock(object):
+        def __init__(self, return_value=None):
+            self._return = return_value
 
-    secret_path = os.path.join(
-        base_config, "leap", "soledad", "%s.secret" % user_uuid)
-    soledad_path = os.path.join(
-        base_config, "leap", "soledad", "%s-mailbox.db" % user_uuid)
+        def __call__(self, *args, **kwargs):
+            return self._return
 
-    _soledad = Soledad(
-        user_uuid,
-        soledad_pass,
-        secret_path,
-        soledad_path,
+    class MockSharedDB(object):
+
+        get_doc = Mock()
+        put_doc = Mock()
+        lock = Mock(return_value=('atoken', 300))
+        unlock = Mock(return_value=True)
+
+        def __call__(self):
+            return self
+
+    Soledad._shared_db = MockSharedDB()
+    soledad = Soledad(
+        uuid,
+        passwd,
+        secrets,
+        localdb,
         server_url,
-        server_pemfile,
-        token)
+        cert_file)
+
+    return soledad
+
+######################################################################
+# Remember to set your config files, see module documentation above!
+######################################################################
+
+print "[+] Running LEAP IMAP Service"
+
+
+bmconf = os.environ.get("LEAP_MAIL_CONF", "")
+if not bmconf:
+	print "[-] Please set LEAP_MAIL_CONF environment variable pointing to your config."
+	sys.exit(1)
+SECTION = "leap_mail"
+cp = ConfigParser.ConfigParser()
+cp.read(bmconf)
+
+userid = cp.get(SECTION, "userid")
+uuid = cp.get(SECTION, "uuid")
+passwd = unicode(cp.get(SECTION, "passwd"))
+
+# XXX get this right from the environment variable !!!
+port = 1984
+
+if not userid or not uuid:
+	print "[-] Config file missing userid or uuid field"
+	sys.exit(1)
+
+if not passwd:
+	passwd = unicode(getpass.getpass("Soledad passphrase: "))
+
+
+secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,))
+localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,))
+
+# XXX Is this really used? Should point it to user var dirs defined in xdg?
+gnupg_home = "/tmp/"
+tempdir = "/tmp/"
+
+###################################################
+
+# Ad-hoc soledad/keymanager initialization.
+
+soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir)
+km_args = (userid, "https://localhost", soledad)
+km_kwargs = { 
+	"session_id":  "", 
+	"ca_cert_path":  "", 
+	"api_uri":  "", 
+	"api_version":  "", 
+	"uid": uuid,
+	"gpgbinary": "/usr/bin/gpg"
+}   
+keymanager = KeyManager(*km_args, **km_kwargs)
+
+##################################################
 
-    return _soledad
+# Ok, let's expose the application object for the twistd application
+# framework to pick up from here...
 
-soledad = initialize_soledad_mailbox(
-    d['uid'],
-    d['passphrase'],
-    d['server'],
-    d['pemfile'],
-    d['token'])
 
-# import the private key ---- should sync it from remote!
-from leap.common.keymanager.openpgp import OpenPGPScheme
-opgp = OpenPGPScheme(soledad)
-opgp.put_ascii_key(privkey)
+def getIMAPService():
+    factory = imap.LeapIMAPFactory(uuid, userid, soledad)
+    return internet.TCPServer(port, factory, interface="localhost")
 
-from leap.common.keymanager import KeyManager
-keymanager = KeyManager(userID, nickserver_url, soledad, d['token'])
 
-imap.run_service(soledad, keymanager)
+application = service.Application("LEAP IMAP Application")
+service = getIMAPService()
+service.setServiceParent(application)
-- 
cgit v1.2.3


From 77def3fb8d8b8106ebb341e6cb7ee9987ce6e2e9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:13:05 -0400
Subject: Dispatch the flags query if it's the only one.

ie, we got something like FETCH 1:* (FLAGS)
but not for FETCH 1:* (FLAGS INTERNALDATE)
---
 src/leap/mail/imap/service/imap.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index e877869..8c5b488 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -124,7 +124,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
         cbFetch = self._IMAP4Server__cbFetch
         ebFetch = self._IMAP4Server__ebFetch
 
-        if str(query[0]) == "flags":
+        print "QUERY: ", query
+
+        if len(query) == 1 and str(query[0]) == "flags":
             self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
             maybeDeferred(
-- 
cgit v1.2.3


From 61454c82de35778d27bcd1a2c89fe20ee3d5b142 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:15:27 -0400
Subject: patch UIDVALIDITY response for conformance to the spec

testimap was choking on this.
---
 src/leap/mail/imap/service/imap.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 8c5b488..6e03456 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -146,6 +146,36 @@ class LeapIMAPServer(imap4.IMAP4Server):
     select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
                     imap4.IMAP4Server.arg_fetchatt)
 
+    def _cbSelectWork(self, mbox, cmdName, tag):
+        """
+        Callback for selectWork, patched to avoid conformance errors due to
+        incomplete UIDVALIDITY line.
+        """
+        if mbox is None:
+            self.sendNegativeResponse(tag, 'No such mailbox')
+            return
+        if '\\noselect' in [s.lower() for s in mbox.getFlags()]:
+            self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
+            return
+
+        flags = mbox.getFlags()
+        self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS')
+        self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT')
+        self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
+
+        # Patched -------------------------------------------------------
+        # imaptest was complaining about the incomplete line, we're adding
+        # "UIDs valid" here.
+        self.sendPositiveResponse(
+            None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity())
+        # ----------------------------------------------------------------
+
+        s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY'
+        mbox.addListener(self)
+        self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName))
+        self.state = 'select'
+        self.mbox = mbox
+
 
 class IMAPAuthRealm(object):
     """
-- 
cgit v1.2.3


From ae56191d2d6f2953bd49f43b9dedb322a7f0db8c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:18:11 -0400
Subject: reset last uid on expunge

---
 src/leap/mail/imap/mailbox.py  |  1 +
 src/leap/mail/imap/messages.py | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 94070ac..86dac77 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -463,6 +463,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
         d = self.messages.remove_all_deleted()
+        d.addCallback(self.messages.reset_last_uid)
         d.addCallback(self._expunge_cb)
         return d
 
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 22de356..02df38e 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1337,6 +1337,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                         fields.TYPE_FLAGS_VAL, self.mbox))
         return (u for u in sorted(all_uids))
 
+    def reset_last_uid(self, param):
+        """
+        Set the last uid to the highest uid found.
+        Used while expunging, passed as a callback.
+        """
+        try:
+            self.last_uid = max(self.all_uid_iter()) + 1
+        except ValueError:
+            # empty sequence
+            pass
+        return param
+
     def all_flags(self):
         """
         Return a dict with all flags documents for this mailbox.
-- 
cgit v1.2.3


From 759a3fff83252c6ef67434a860574da49b066df4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:19:31 -0400
Subject: fix internaldate storage

---
 src/leap/mail/imap/messages.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 02df38e..1b996b6 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -467,7 +467,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: C{str}
         :return: An RFC822-formatted date string.
         """
-        return str(self._hdoc.content.get(self.DATE_KEY, ''))
+        date = self._hdoc.content.get(self.DATE_KEY, '')
+        return str(date)
 
     #
     # IMessagePart
@@ -1077,12 +1078,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         hd[self.MSGID_KEY] = msgid
 
         if not subject and self.SUBJECT_FIELD in headers:
-            hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
+            hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
         else:
             hd[self.SUBJECT_KEY] = subject
 
         if not date and self.DATE_FIELD in headers:
-            hd[self.DATE_KEY] = first(headers[self.DATE_FIELD])
+            hd[self.DATE_KEY] = headers[self.DATE_FIELD]
         else:
             hd[self.DATE_KEY] = date
         return hd
-- 
cgit v1.2.3


From 557fac26982aa1360ed51d158869312d6438eb84 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:24:27 -0400
Subject: factor out bound and filter for msg seqs

---
 src/leap/mail/imap/mailbox.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 86dac77..84eb528 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -467,6 +467,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d.addCallback(self._expunge_cb)
         return d
 
+    def _bound_seq(self, messages_asked):
+        """
+        Put an upper bound to a messages sequence if this is open.
+
+        :param messages_asked: IDs of the messages.
+        :type messages_asked: MessageSet
+        :rtype: MessageSet
+        """
+        if not messages_asked.last:
+            try:
+                iter(messages_asked)
+            except TypeError:
+                # looks like we cannot iterate
+                messages_asked.last = self.last_uid
+        return messages_asked
+
+    def _filter_msg_seq(self, messages_asked):
+        """
+        Filter a message sequence returning only the ones that do exist in the
+        collection.
+
+        :param messages_asked: IDs of the messages.
+        :type messages_asked: MessageSet
+        :rtype: set
+        """
+        set_asked = set(messages_asked)
+        set_exist = set(self.messages.all_uid_iter())
+        seq_messg = set_asked.intersection(set_exist)
+        return seq_messg
+
     @deferred
     def fetch(self, messages_asked, uid):
         """
-- 
cgit v1.2.3


From a660231e918df6698b6dcfad9d1845bd77ee6f8f Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:26:01 -0400
Subject: Fix grave bug with iteration in STORE

This was in the root for problems with Trash behavior. Closes: #4958
Make use of the refactored utilities for bounding and filtering
sequences.
---
 src/leap/mail/imap/mailbox.py | 37 ++++++++++++-------------------------
 1 file changed, 12 insertions(+), 25 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 84eb528..137f9f5 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -526,17 +526,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         sequence = False
         #sequence = True if uid == 0 else False
 
-        if not messages_asked.last:
-            try:
-                iter(messages_asked)
-            except TypeError:
-                # looks like we cannot iterate
-                messages_asked.last = self.last_uid
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
 
-        set_asked = set(messages_asked)
-        set_exist = set(self.messages.all_uid_iter())
-        seq_messg = set_asked.intersection(set_exist)
-        getmsg = lambda msgid: self.messages.get_msg_by_uid(msgid)
 
         # for sequence numbers (uid = 0)
         if sequence:
@@ -563,6 +555,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         A fast method to fetch all flags, tricking just the
         needed subset of the MIME interface that's needed to satisfy
         a generic FLAGS query.
+
         Given how LEAP Mail is supposed to work without local cache,
         this query is going to be quite common, and also we expect
         it to be in the form 1:* at the beginning of a session, so
@@ -592,16 +585,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             def getFlags(self):
                 return map(str, self.flags)
 
-        if not messages_asked.last:
-            try:
-                iter(messages_asked)
-            except TypeError:
-                # looks like we cannot iterate
-                messages_asked.last = self.last_uid
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
 
-        set_asked = set(messages_asked)
-        set_exist = set(self.messages.all_uid_iter())
-        seq_messg = set_asked.intersection(set_exist)
         all_flags = self.messages.all_flags()
         result = ((msgid, flagsPart(
             msgid, all_flags[msgid])) for msgid in seq_messg)
@@ -648,7 +634,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
 
     @deferred
-    def store(self, messages, flags, mode, uid):
+    def store(self, messages_asked, flags, mode, uid):
         """
         Sets the flags of one or more messages.
 
@@ -677,25 +663,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :raise ReadOnlyMailbox: Raised if this mailbox is not open for
                                 read-write.
         """
+        from twisted.internet import reactor
         # XXX implement also sequence (uid = 0)
         # XXX we should prevent cclient from setting Recent flag.
         leap_assert(not isinstance(flags, basestring),
                     "flags cannot be a string")
         flags = tuple(flags)
 
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
+
         if not self.isWriteable():
             log.msg('read only mailbox!')
             raise imap4.ReadOnlyMailbox
 
-        if not messages.last:
-            messages.last = self.messages.count()
-
         result = {}
-        for msg_id in messages:
+        for msg_id in seq_messg:
             log.msg("MSG ID = %s" % msg_id)
             msg = self.messages.get_msg_by_uid(msg_id)
             if not msg:
-                return result
+                continue
             if mode == 1:
                 msg.addFlags(flags)
             elif mode == -1:
-- 
cgit v1.2.3


From 6c7207a5667d8158572b2a900a3506e3c3ecc6e5 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 17:33:20 -0400
Subject: Temporal refactor setting of recent flag.

This flag is set way too often, and is damaging performance.
Will move it to a single doc per mailbox in subsequente commits.
---
 src/leap/mail/imap/mailbox.py | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 137f9f5..cf09bc4 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -529,6 +529,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         messages_asked = self._bound_seq(messages_asked)
         seq_messg = self._filter_msg_seq(messages_asked)
 
+        def getmsg(msgid):
+            if self.isWriteable():
+                deferLater(reactor, 2, self._unset_recent_flag, messages_asked)
+            return self.messages.get_msg_by_uid(msgid)
 
         # for sequence numbers (uid = 0)
         if sequence:
@@ -538,12 +542,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         else:
             result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
 
-        if self.isWriteable():
-            deferLater(reactor, 30, self._unset_recent_flag)
-            # XXX I should rewrite the scheduler so it handles a
-            # set of queues with different priority.
-            self._unset_recent_flag()
-
         # this should really be called as a final callback of
         # the do_FETCH method...
         deferLater(reactor, 1, self._signal_unread_to_ui)
@@ -594,7 +592,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         return result
 
     @deferred
-    def _unset_recent_flag(self):
+    def _unset_recent_flag(self, message_uid):
         """
         Unsets `Recent` flag from a tuple of messages.
         Called from fetch.
@@ -610,19 +608,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         If it is not possible to determine whether or not this
         session is the first session to be notified about a message,
         then that message SHOULD be considered recent.
-        """
-        # TODO this fucker, for the sake of correctness, is messing with
-        # the whole collection of flag docs.
 
-        # Possible ways of action:
-        # 1. Ignore it, we want fun.
-        # 2. Trigger it with a delay
-        # 3. Route it through a queue with lesser priority than the
-        #    regularar writer.
+        :param message_uids: the sequence of msg ids to update.
+        :type message_uids: sequence
+        """
+        # XXX deprecate this!
+        # move to a mailbox-level call, and do it in batches!
 
-        log.msg('unsetting recent flags...')
-        for msg in self.messages.get_recent():
-            msg.removeFlags((fields.RECENT_FLAG,))
+        log.msg('unsetting recent flag: %s' % message_uid)
+        msg = self.messages.get_msg_by_uid(message_uid)
+        msg.removeFlags((fields.RECENT_FLAG,))
         self._signal_unread_to_ui()
 
     @deferred
@@ -691,7 +686,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 msg.setFlags(flags)
             result[msg_id] = msg.getFlags()
 
-        self._signal_unread_to_ui()
+        # this should really be called as a final callback of
+        # the do_FETCH method...
+        deferLater(reactor, 1, self._signal_unread_to_ui)
         return result
 
     # ISearchableMailbox
@@ -758,6 +755,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         new_fdoc[self.MBOX_KEY] = self.mbox
 
         d = self._do_add_doc(new_fdoc)
+        # XXX notify should be done when all the
+        # copies in the batch are finished.
         d.addCallback(self._notify_new)
 
     @deferred
-- 
cgit v1.2.3


From 9f9701d42be385aa9a6d7e72fd10104b0025971b Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 22:01:20 -0400
Subject: Separate RECENT Flag to a mailbox document.

this way we avoid a bunch of writes.
---
 src/leap/mail/imap/fields.py       |   2 +
 src/leap/mail/imap/mailbox.py      |  82 +++++++------------
 src/leap/mail/imap/messages.py     | 163 +++++++++++++++++++++++++++----------
 src/leap/mail/imap/service/imap.py |  28 ++++++-
 4 files changed, 177 insertions(+), 98 deletions(-)

diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 3d2ac92..bc928a1 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -60,6 +60,7 @@ class WithMsgFields(object):
     SUBSCRIBED_KEY = "subscribed"
     RW_KEY = "rw"
     LAST_UID_KEY = "lastuid"
+    RECENTFLAGS_KEY = "rct"
 
     # Document Type, for indexing
     TYPE_KEY = "type"
@@ -67,6 +68,7 @@ class WithMsgFields(object):
     TYPE_FLAGS_VAL = "flags"
     TYPE_HEADERS_VAL = "head"
     TYPE_CONTENT_VAL = "cnt"
+    TYPE_RECENT_VAL = "rct"
 
     INBOX_VAL = "inbox"
 
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index cf09bc4..bd69d12 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -398,18 +398,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             flags = tuple(str(flag) for flag in flags)
 
         d = self._do_add_message(message, flags=flags, date=date, uid=uid_next)
-        d.addCallback(self._notify_new)
         return d
 
-    @deferred
     def _do_add_message(self, message, flags, date, uid):
         """
         Calls to the messageCollection add_msg method (deferred to thread).
         Invoked from addMessage.
         """
-        self.messages.add_msg(message, flags=flags, date=date, uid=uid)
+        d = self.messages.add_msg(message, flags=flags, date=date, uid=uid)
+        # XXX notify after batch APPEND?
+        d.addCallback(self.notify_new)
+        return d
 
-    def _notify_new(self, *args):
+    def notify_new(self, *args):
         """
         Notify of new messages to all the listeners.
 
@@ -463,8 +464,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
         d = self.messages.remove_all_deleted()
-        d.addCallback(self.messages.reset_last_uid)
         d.addCallback(self._expunge_cb)
+        d.addCallback(self.messages.reset_last_uid)
         return d
 
     def _bound_seq(self, messages_asked):
@@ -480,7 +481,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 iter(messages_asked)
             except TypeError:
                 # looks like we cannot iterate
-                messages_asked.last = self.last_uid
+                try:
+                    messages_asked.last = self.last_uid
+                except ValueError:
+                    pass
         return messages_asked
 
     def _filter_msg_seq(self, messages_asked):
@@ -529,10 +533,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         messages_asked = self._bound_seq(messages_asked)
         seq_messg = self._filter_msg_seq(messages_asked)
 
-        def getmsg(msgid):
-            if self.isWriteable():
-                deferLater(reactor, 2, self._unset_recent_flag, messages_asked)
-            return self.messages.get_msg_by_uid(msgid)
+        getmsg = lambda uid: self.messages.get_msg_by_uid(uid)
 
         # for sequence numbers (uid = 0)
         if sequence:
@@ -544,7 +545,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         # this should really be called as a final callback of
         # the do_FETCH method...
-        deferLater(reactor, 1, self._signal_unread_to_ui)
+
         return result
 
     @deferred
@@ -591,37 +592,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             msgid, all_flags[msgid])) for msgid in seq_messg)
         return result
 
-    @deferred
-    def _unset_recent_flag(self, message_uid):
-        """
-        Unsets `Recent` flag from a tuple of messages.
-        Called from fetch.
-
-        From RFC, about `Recent`:
-
-        Message is "recently" arrived in this mailbox.  This session
-        is the first session to have been notified about this
-        message; if the session is read-write, subsequent sessions
-        will not see \Recent set for this message.  This flag can not
-        be altered by the client.
-
-        If it is not possible to determine whether or not this
-        session is the first session to be notified about a message,
-        then that message SHOULD be considered recent.
-
-        :param message_uids: the sequence of msg ids to update.
-        :type message_uids: sequence
-        """
-        # XXX deprecate this!
-        # move to a mailbox-level call, and do it in batches!
-
-        log.msg('unsetting recent flag: %s' % message_uid)
-        msg = self.messages.get_msg_by_uid(message_uid)
-        msg.removeFlags((fields.RECENT_FLAG,))
-        self._signal_unread_to_ui()
-
-    @deferred
-    def _signal_unread_to_ui(self):
+    def signal_unread_to_ui(self):
         """
         Sends unread event to ui.
         """
@@ -687,8 +658,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             result[msg_id] = msg.getFlags()
 
         # this should really be called as a final callback of
-        # the do_FETCH method...
-        deferLater(reactor, 1, self._signal_unread_to_ui)
+        # the do_STORE method...
+        # XXX ---
+        #deferLater(reactor, 1, self._signal_unread_to_ui)
         return result
 
     # ISearchableMailbox
@@ -741,6 +713,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Copy the given message object into this mailbox.
         """
+        from twisted.internet import reactor
         uid_next = self.getUIDNext()
         msg = messageObject
 
@@ -753,17 +726,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         new_fdoc = copy.deepcopy(fdoc.content)
         new_fdoc[self.UID_KEY] = uid_next
         new_fdoc[self.MBOX_KEY] = self.mbox
+        self._do_add_doc(new_fdoc)
+        deferLater(reactor, 1, self.notify_new)
 
-        d = self._do_add_doc(new_fdoc)
-        # XXX notify should be done when all the
-        # copies in the batch are finished.
-        d.addCallback(self._notify_new)
-
-    @deferred
     def _do_add_doc(self, doc):
         """
-        Defers the adding of a new doc.
+        Defer the adding of a new doc.
+
         :param doc: document to be created in soledad.
+        :type doc: dict
         """
         self._soledad.create_doc(doc)
 
@@ -771,12 +742,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def deleteAllDocs(self):
         """
-        Deletes all docs in this mailbox
+        Delete all docs in this mailbox
         """
         docs = self.messages.get_all_docs()
         for doc in docs:
             self.messages._soledad.delete_doc(doc)
 
+    def unset_recent_flags(self, uids):
+        """
+        Unset Recent flag for a sequence of UIDs.
+        """
+        seq_messg = self._bound_seq(uids)
+        self.messages.unset_recent_flags(seq_messg)
+
     def __repr__(self):
         """
         Representation string for this mailbox.
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 1b996b6..6556b12 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -21,6 +21,7 @@ import copy
 import logging
 import re
 import time
+import threading
 import StringIO
 
 from collections import defaultdict, namedtuple
@@ -308,7 +309,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     implements(imap4.IMessage)
 
-    def __init__(self, soledad, uid, mbox):
+    def __init__(self, soledad, uid, mbox, collection=None):
         """
         Initializes a LeapMessage.
 
@@ -318,11 +319,14 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :type uid: int or basestring
         :param mbox: the mbox this message belongs to
         :type mbox: basestring
+        :param collection: a reference to the parent collection object
+        :type collection: MessageCollection
         """
         MailParser.__init__(self)
         self._soledad = soledad
         self._uid = int(uid)
         self._mbox = self._parse_mailbox_name(mbox)
+        self._collection = collection
 
         self.__chash = None
         self.__bdoc = None
@@ -373,7 +377,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     def getUID(self):
         """
-        Retrieve the unique identifier associated with this message
+        Retrieve the unique identifier associated with this Message.
 
         :return: uid for this message
         :rtype: int
@@ -382,18 +386,26 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     def getFlags(self):
         """
-        Retrieve the flags associated with this message
+        Retrieve the flags associated with this Message.
 
         :return: The flags, represented as strings
         :rtype: tuple
         """
         if self._uid is None:
             return []
+        uid = self._uid
 
         flags = []
         fdoc = self._fdoc
         if fdoc:
             flags = fdoc.content.get(self.FLAGS_KEY, None)
+
+        msgcol = self._collection
+
+        # We treat the recent flag specially: gotten from
+        # a mailbox-level document.
+        if msgcol and uid in msgcol.recent_flags:
+            flags.append(fields.RECENT_FLAG)
         if flags:
             flags = map(str, flags)
         return tuple(flags)
@@ -414,7 +426,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: SoledadDocument
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        log.msg('setting flags: %s' % (self._uid))
+        log.msg('setting flags: %s (%s)' % (self._uid, flags))
 
         doc = self._fdoc
         if not doc:
@@ -424,7 +436,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
             return
         doc.content[self.FLAGS_KEY] = flags
         doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-        doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
         doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
         self._soledad.put_doc(doc)
 
@@ -927,6 +938,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     FLAGS_DOC = "FLAGS"
     HEADERS_DOC = "HEADERS"
     CONTENT_DOC = "CONTENT"
+    RECENT_DOC = "RECENT"
 
     templates = {
 
@@ -937,7 +949,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             fields.CONTENT_HASH_KEY: "",
 
             fields.SEEN_KEY: False,
-            fields.RECENT_KEY: True,
             fields.DEL_KEY: False,
             fields.FLAGS_KEY: [],
             fields.MULTIPART_KEY: False,
@@ -970,12 +981,25 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             fields.MULTIPART_KEY: False,
         },
 
+        RECENT_DOC: {
+            fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
+            fields.MBOX_KEY: fields.INBOX_VAL,
+            fields.RECENTFLAGS_KEY: [],
+        }
     }
 
+    _rdoc_lock = threading.Lock()
+
     def __init__(self, mbox=None, soledad=None):
         """
         Constructor for MessageCollection.
 
+        On initialization, we ensure that we have a document for
+        storing the recent flags. The nature of this flag make us wanting
+        to store the set of the UIDs with this flag at the level of the
+        MessageCollection for each mailbox, instead of treating them
+        as a property of each message.
+
         :param mbox: the name of the mailbox. It is the name
                      with which we filter the query over the
                      messages database
@@ -994,17 +1018,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         # okay, all in order, keep going...
         self.mbox = self._parse_mailbox_name(mbox)
         self._soledad = soledad
+        self.__rflags = None
         self.initialize_db()
 
-        # I think of someone like nietzsche when reading this
-
-        # this will be the producer that will enqueue the content
-        # to be processed serially by the consumer (the writer). We just
-        # need to `put` the new material on its plate.
-
-        self.soledad_writer = MessageProducer(
-            SoledadDocWriter(soledad),
-            period=0.02)
+        # ensure that we have a recent-flags doc
+        self._get_or_create_rdoc()
 
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
@@ -1017,6 +1035,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             raise TypeError("Improper type passed to _get_empty_doc")
         return copy.deepcopy(self.templates[_type])
 
+    def _get_or_create_rdoc(self):
+        """
+        Try to retrieve the recent-flags doc for this MessageCollection,
+        and create one if not found.
+        """
+        rdoc = self._get_recent_doc()
+        if not rdoc:
+            rdoc = self._get_empty_doc(self.RECENT_DOC)
+            if self.mbox != fields.INBOX_VAL:
+                rdoc[fields.MBOX_KEY] = self.mbox
+            self._soledad.create_doc(rdoc)
+
     def _do_parse(self, raw):
         """
         Parse raw message and return it along with
@@ -1161,7 +1191,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             hd[key] = parts_map[key]
         del parts_map
 
-        # Saving
+        # Saving ----------------------------------------
+        self.set_recent_flag(uid)
 
         # first, regular docs: flags and headers
         self._soledad.create_doc(fd)
@@ -1203,6 +1234,76 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
     # getters: specific queries
 
+    def _get_recent_flags(self):
+        """
+        An accessor for the recent-flags set for this mailbox.
+        """
+        if not self.__rflags:
+            rdoc = self._get_recent_doc()
+            self.__rflags = set(rdoc.content.get(
+                fields.RECENTFLAGS_KEY, []))
+        return self.__rflags
+
+    def _set_recent_flags(self, value):
+        """
+        Setter for the recent-flags set for this mailbox.
+        """
+        rdoc = self._get_recent_doc()
+        newv = set(value)
+        self.__rflags = newv
+
+        with self._rdoc_lock:
+            rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
+            # XXX should deferLater 0 it?
+            self._soledad.put_doc(rdoc)
+
+    recent_flags = property(
+        _get_recent_flags, _set_recent_flags,
+        doc="Set of UIDs with the recent flag for this mailbox.")
+
+    def unset_recent_flags(self, uids):
+        """
+        Unset Recent flag for a sequence of uids.
+        """
+        self.recent_flags = self.recent_flags.difference(
+            set(uids))
+
+    def unset_recent_flag(self, uid):
+        """
+        Unset Recent flag for a given uid.
+        """
+        self.recent_flags = self.recent_flags.difference(
+            set([uid]))
+
+    def set_recent_flag(self, uid):
+        """
+        Set Recent flag for a given uid.
+        """
+        self.recent_flags = self.recent_flags.union(
+            set([uid]))
+
+    def _get_recent_doc(self):
+        """
+        Get recent-flags document for this inbox.
+        """
+        # TODO refactor this try-catch structure into a utility
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MBOX_IDX,
+                fields.TYPE_RECENT_VAL, self.mbox)
+            if query:
+                if len(query) > 1:
+                    logger.warning(
+                        "More than one rdoc found for this mbox, "
+                        "we got a duplicate!!")
+                    # XXX we could take action, like trigger a background
+                    # process to kill dupes.
+                return query.pop()
+            else:
+                return None
+        except Exception as exc:
+            logger.exception("Unhandled error %r" % exc)
+
     def _get_fdoc_from_chash(self, chash):
         """
         Return a flags document for this mailbox with a given chash.
@@ -1287,6 +1388,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     def get_msg_by_uid(self, uid):
         """
         Retrieves a LeapMessage by UID.
+        This is used primarity in the Mailbox fetch and store methods.
 
         :param uid: the message uid to query by
         :type uid: int
@@ -1295,7 +1397,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                  or None if not found.
         :rtype: LeapMessage
         """
-        msg = LeapMessage(self._soledad, uid, self.mbox)
+        msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
         if not msg.does_exist():
             return None
         return msg
@@ -1412,28 +1514,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
     # recent messages
 
-    def recent_iter(self):
-        """
-        Get an iterator for the message UIDs with `recent` flag.
-
-        :return: iterator through recent message docs
-        :rtype: iterable
-        """
-        return (doc.content[self.UID_KEY] for doc in
-                self._soledad.get_from_index(
-                    fields.TYPE_MBOX_RECT_IDX,
-                    fields.TYPE_FLAGS_VAL, self.mbox, '1'))
-
-    def get_recent(self):
-        """
-        Get all messages with the `Recent` flag.
-
-        :returns: a list of LeapMessages
-        :rtype: list
-        """
-        return [LeapMessage(self._soledad, docid, self.mbox)
-                for docid in self.recent_iter()]
-
     def count_recent(self):
         """
         Count all messages with the `Recent` flag.
@@ -1441,10 +1521,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         :returns: count
         :rtype: int
         """
-        count = self._soledad.get_count_from_index(
-            fields.TYPE_MBOX_RECT_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox, '1')
-        return count
+        return len(self.recent_flags)
 
     # deleted messages
 
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 6e03456..a3ef098 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -24,6 +24,7 @@ import logging
 from twisted.internet.protocol import ServerFactory
 from twisted.internet.defer import maybeDeferred
 from twisted.internet.error import CannotListenError
+from twisted.internet.task import deferLater
 from twisted.mail import imap4
 from twisted.python import log
 from twisted import cred
@@ -116,6 +117,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         Overwritten fetch dispatcher to use the fast fetch_flags
         method
         """
+        from twisted.internet import reactor
         log.msg("LEAP Overwritten fetch...")
         if not query:
             self.sendPositiveResponse(tag, 'FETCH complete')
@@ -124,8 +126,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
         cbFetch = self._IMAP4Server__cbFetch
         ebFetch = self._IMAP4Server__ebFetch
 
-        print "QUERY: ", query
-
         if len(query) == 1 and str(query[0]) == "flags":
             self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
@@ -141,11 +141,32 @@ class LeapIMAPServer(imap4.IMAP4Server):
                 self.mbox.fetch, messages, uid=uid
             ).addCallback(
                 cbFetch, tag, query, uid
-            ).addErrback(ebFetch, tag)
+            ).addErrback(
+                ebFetch, tag)
+
+        deferLater(reactor,
+                   2, self.mbox.unset_recent_flags, messages)
+        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
 
     select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
                     imap4.IMAP4Server.arg_fetchatt)
 
+    def do_COPY(self, tag, messages, mailbox, uid=0):
+        from twisted.internet import reactor
+        imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
+        deferLater(reactor,
+                   2, self.mbox.unset_recent_flags, messages)
+        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
+
+    select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
+                   imap4.IMAP4Server.arg_astring)
+
+    def notifyNew(self, ignored):
+        """
+        Notify new messages to listeners.
+        """
+        self.mbox.notify_new()
+
     def _cbSelectWork(self, mbox, cmdName, tag):
         """
         Callback for selectWork, patched to avoid conformance errors due to
@@ -177,6 +198,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         self.mbox = mbox
 
 
+
 class IMAPAuthRealm(object):
     """
     Dummy authentication realm. Do not use in production!
-- 
cgit v1.2.3


From 8ebd48d923466db51b9ea5698f51d1f12867a7cb Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 16 Jan 2014 23:46:08 -0400
Subject: refactor common pattern to utility function

---
 src/leap/mail/imap/messages.py | 104 +++++++++++++++++++----------------------
 1 file changed, 48 insertions(+), 56 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 6556b12..f968c47 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -25,6 +25,7 @@ import threading
 import StringIO
 
 from collections import defaultdict, namedtuple
+from functools import partial
 
 from twisted.mail import imap4
 from twisted.internet import defer
@@ -42,7 +43,7 @@ from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.parser import MailParser, MBoxParser
-from leap.mail.messageflow import IMessageConsumer, MessageProducer
+from leap.mail.messageflow import IMessageConsumer
 
 logger = logging.getLogger(__name__)
 
@@ -66,6 +67,31 @@ def lowerdict(_dict):
                 for key, value in _dict.items())
 
 
+def try_unique_query(curried):
+    """
+    Try to execute a query that is expected to have a
+    single outcome, and log a warning if more than one document found.
+
+    :param curried: a curried function
+    :type curried: callable
+    """
+    leap_assert(callable(curried), "A callable is expected")
+    try:
+        query = curried()
+        if query:
+            if len(query) > 1:
+                # TODO we could take action, like trigger a background
+                # process to kill dupes.
+                name = getattr(curried, 'expected', 'doc')
+                logger.warning(
+                    "More than one %s found for this mbox, "
+                    "we got a duplicate!!" % (name,))
+            return query.pop()
+        else:
+            return None
+    except Exception as exc:
+        logger.exception("Unhandled error %r" % exc)
+
 CHARSET_PATTERN = r"""charset=([\w-]+)"""
 MSGID_PATTERN = r"""<([\w@.]+)>"""
 
@@ -1286,23 +1312,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         """
         Get recent-flags document for this inbox.
         """
-        # TODO refactor this try-catch structure into a utility
-        try:
-            query = self._soledad.get_from_index(
-                fields.TYPE_MBOX_IDX,
-                fields.TYPE_RECENT_VAL, self.mbox)
-            if query:
-                if len(query) > 1:
-                    logger.warning(
-                        "More than one rdoc found for this mbox, "
-                        "we got a duplicate!!")
-                    # XXX we could take action, like trigger a background
-                    # process to kill dupes.
-                return query.pop()
-            else:
-                return None
-        except Exception as exc:
-            logger.exception("Unhandled error %r" % exc)
+        curried = partial(
+            self._soledad.get_from_index,
+            fields.TYPE_MBOX_IDX,
+            fields.TYPE_RECENT_VAL, self.mbox)
+        curried.expected = "rdoc"
+        return try_unique_query(curried)
 
     def _get_fdoc_from_chash(self, chash):
         """
@@ -1312,39 +1327,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                  the query failed.
         :rtype: SoledadDocument or None.
         """
-        try:
-            query = self._soledad.get_from_index(
-                fields.TYPE_MBOX_C_HASH_IDX,
-                fields.TYPE_FLAGS_VAL, self.mbox, chash)
-            if query:
-                if len(query) > 1:
-                    logger.warning(
-                        "More than one fdoc found for this chash, "
-                        "we got a duplicate!!")
-                    # XXX we could take action, like trigger a background
-                    # process to kill dupes.
-                return query.pop()
-            else:
-                return None
-        except Exception as exc:
-            logger.exception("Unhandled error %r" % exc)
+        curried = partial(
+            self._soledad.get_from_index,
+            fields.TYPE_MBOX_C_HASH_IDX,
+            fields.TYPE_FLAGS_VAL, self.mbox, chash)
+        curried.expected = "fdoc"
+        return try_unique_query(curried)
 
     def _get_uid_from_msgidCb(self, msgid):
         hdoc = None
-        try:
-            query = self._soledad.get_from_index(
-                fields.TYPE_MSGID_IDX,
-                fields.TYPE_HEADERS_VAL, msgid)
-            if query:
-                if len(query) > 1:
-                    logger.warning(
-                        "More than one hdoc found for this msgid, "
-                        "we got a duplicate!!")
-                    # XXX we could take action, like trigger a background
-                    # process to kill dupes.
-                hdoc = query.pop()
-        except Exception as exc:
-            logger.exception("Unhandled error %r" % exc)
+        curried = partial(
+            self._soledad.get_from_index,
+            fields.TYPE_MSGID_IDX,
+            fields.TYPE_HEADERS_VAL, msgid)
+        curried.expected = "hdoc"
+        hdoc = try_unique_query(curried)
 
         if hdoc is None:
             logger.warning("Could not find hdoc for msgid %s"
@@ -1373,13 +1370,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         # the query is received right after we've saved the document,
         # and we cannot find it otherwise. This seems to be enough.
 
-        # Doing a sleep since we'll be calling this in a secondary thread,
-        # but we'll should be able to collect the results after a
-        # reactor.callLater.
-        # Maybe we can implement something like NOT_DONE_YET in the web
-        # framework, and return from the callback?
-        # See: http://jcalderone.livejournal.com/50226.html
-        # reactor.callLater(0.3, self._get_uid_from_msgidCb, msgid)
+        # XXX do a deferLater instead ??
         time.sleep(0.3)
         return self._get_uid_from_msgidCb(msgid)
 
@@ -1426,6 +1417,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         # inneficient, but first let's grok it and then
         # let's worry about efficiency.
         # XXX FIXINDEX -- should implement order by in soledad
+        # FIXME ----------------------------------------------
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
     def all_uid_iter(self):
@@ -1573,4 +1565,4 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             self.mbox, self.count())
 
     # XXX should implement __eq__ also !!!
-    # --- use the content hash for that, will be used for dedup.
+    # use chash...
-- 
cgit v1.2.3


From 9ef1cd79397d811575826025b924c615e6ce2aa4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 17 Jan 2014 02:51:31 -0400
Subject: Add a fetch_headers for mass-header fetch queries

---
 src/leap/mail/imap/fields.py       |   2 +
 src/leap/mail/imap/mailbox.py      |  76 +++++++++++++++---
 src/leap/mail/imap/messages.py     | 159 +++++++++++++++++++++++++++++++++++--
 src/leap/mail/imap/service/imap.py |  12 ++-
 4 files changed, 231 insertions(+), 18 deletions(-)

diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index bc928a1..886ee63 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -61,6 +61,7 @@ class WithMsgFields(object):
     RW_KEY = "rw"
     LAST_UID_KEY = "lastuid"
     RECENTFLAGS_KEY = "rct"
+    HDOCS_SET_KEY = "hdocset"
 
     # Document Type, for indexing
     TYPE_KEY = "type"
@@ -69,6 +70,7 @@ class WithMsgFields(object):
     TYPE_HEADERS_VAL = "head"
     TYPE_CONTENT_VAL = "cnt"
     TYPE_RECENT_VAL = "rct"
+    TYPE_HDOCS_SET_VAL = "hdocset"
 
     INBOX_VAL = "inbox"
 
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index bd69d12..b186e75 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -466,6 +466,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = self.messages.remove_all_deleted()
         d.addCallback(self._expunge_cb)
         d.addCallback(self.messages.reset_last_uid)
+
+        # XXX DEBUG -------------------
+        # FIXME !!!
+        # XXX should remove the hdocset too!!!
         return d
 
     def _bound_seq(self, messages_asked):
@@ -520,8 +524,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: A tuple of two-tuples of message sequence numbers and
                 LeapMessage
         """
-        from twisted.internet import reactor
-
         # For the moment our UID is sequential, so we
         # can treat them all the same.
         # Change this to the flag that twisted expects when we
@@ -532,20 +534,14 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         messages_asked = self._bound_seq(messages_asked)
         seq_messg = self._filter_msg_seq(messages_asked)
-
         getmsg = lambda uid: self.messages.get_msg_by_uid(uid)
 
         # for sequence numbers (uid = 0)
         if sequence:
             logger.debug("Getting msg by index: INEFFICIENT call!")
             raise NotImplementedError
-
         else:
             result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
-
-        # this should really be called as a final callback of
-        # the do_FETCH method...
-
         return result
 
     @deferred
@@ -558,7 +554,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Given how LEAP Mail is supposed to work without local cache,
         this query is going to be quite common, and also we expect
         it to be in the form 1:* at the beginning of a session, so
-        it's not bad to fetch all the flags doc at once.
+        it's not bad to fetch all the FLAGS docs at once.
 
         :param messages_asked: IDs of the messages to retrieve information
                                about
@@ -592,6 +588,55 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             msgid, all_flags[msgid])) for msgid in seq_messg)
         return result
 
+    @deferred
+    def fetch_headers(self, messages_asked, uid):
+        """
+        A fast method to fetch all headers, tricking just the
+        needed subset of the MIME interface that's needed to satisfy
+        a generic HEADERS query.
+
+        Given how LEAP Mail is supposed to work without local cache,
+        this query is going to be quite common, and also we expect
+        it to be in the form 1:* at the beginning of a session, so
+        **MAYBE** it's not too bad to fetch all the HEADERS docs at once.
+
+        :param messages_asked: IDs of the messages to retrieve information
+                               about
+        :type messages_asked: MessageSet
+
+        :param uid: If true, the IDs are UIDs. They are message sequence IDs
+                    otherwise.
+        :type uid: bool
+
+        :return: A tuple of two-tuples of message sequence numbers and
+                headersPart, which is a only a partial implementation of
+                MessagePart.
+        :rtype: tuple
+        """
+        class headersPart(object):
+            def __init__(self, uid, headers):
+                self.uid = uid
+                self.headers = headers
+
+            def getUID(self):
+                return self.uid
+
+            def getHeaders(self, _):
+                return dict(
+                    (str(key), str(value))
+                    for key, value in
+                    self.headers.items())
+
+        messages_asked = self._bound_seq(messages_asked)
+        seq_messg = self._filter_msg_seq(messages_asked)
+
+        all_chash = self.messages.all_flags_chash()
+        all_headers = self.messages.all_headers()
+        result = ((msgid, headersPart(
+            msgid, all_headers.get(all_chash.get(msgid, 'nil'), {})))
+            for msgid in seq_messg)
+        return result
+
     def signal_unread_to_ui(self):
         """
         Sends unread event to ui.
@@ -629,7 +674,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :raise ReadOnlyMailbox: Raised if this mailbox is not open for
                                 read-write.
         """
-        from twisted.internet import reactor
         # XXX implement also sequence (uid = 0)
         # XXX we should prevent cclient from setting Recent flag.
         leap_assert(not isinstance(flags, basestring),
@@ -657,10 +701,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 msg.setFlags(flags)
             result[msg_id] = msg.getFlags()
 
+        # After changing flags, we want to signal again to the
+        # UI because the number of unread might have changed.
+        # Hoever, we should probably limit this to INBOX only?
         # this should really be called as a final callback of
         # the do_STORE method...
-        # XXX ---
-        #deferLater(reactor, 1, self._signal_unread_to_ui)
+        from twisted.internet import reactor
+        deferLater(reactor, 1, self._signal_unread_to_ui)
         return result
 
     # ISearchableMailbox
@@ -727,6 +774,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         new_fdoc[self.UID_KEY] = uid_next
         new_fdoc[self.MBOX_KEY] = self.mbox
         self._do_add_doc(new_fdoc)
+
+        # XXX should use a public api instead
+        hdoc = msg._hdoc
+        self.messages.add_hdocset_docid(hdoc.doc_id)
+
         deferLater(reactor, 1, self.notify_new)
 
     def _do_add_doc(self, doc):
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index f968c47..7a21009 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -964,10 +964,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     FLAGS_DOC = "FLAGS"
     HEADERS_DOC = "HEADERS"
     CONTENT_DOC = "CONTENT"
+    """
+    RECENT_DOC is a document that stores a list of the UIDs
+    with the recent flag for this mailbox. It deserves a special treatment
+    because:
+    (1) it cannot be set by the user
+    (2) it's a flag that we set inmediately after a fetch, which is quite
+        often.
+    (3) we need to be able to set/unset it in batches without doing a single
+        write for each element in the sequence.
+    """
     RECENT_DOC = "RECENT"
+    """
+    HDOCS_SET_DOC is a document that stores a set of the Document-IDs
+    (the u1db index) for all the headers documents for a given mailbox.
+    We use it to prefetch massively all the headers for a mailbox.
+    This is the second massive query, after fetching all the FLAGS,  that
+    a MUA will do in a case where we do not have local disk cache.
+    """
+    HDOCS_SET_DOC = "HDOCS_SET"
 
     templates = {
 
+        # Message Level
+
         FLAGS_DOC: {
             fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
             fields.UID_KEY: 1,  # XXX moe to a local table
@@ -1007,14 +1027,25 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             fields.MULTIPART_KEY: False,
         },
 
+        # Mailbox Level
+
         RECENT_DOC: {
             fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
             fields.MBOX_KEY: fields.INBOX_VAL,
             fields.RECENTFLAGS_KEY: [],
+        },
+
+        HDOCS_SET_DOC: {
+            fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL,
+            fields.MBOX_KEY: fields.INBOX_VAL,
+            fields.HDOCS_SET_KEY: [],
         }
+
+
     }
 
     _rdoc_lock = threading.Lock()
+    _hdocset_lock = threading.Lock()
 
     def __init__(self, mbox=None, soledad=None):
         """
@@ -1045,10 +1076,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         self.mbox = self._parse_mailbox_name(mbox)
         self._soledad = soledad
         self.__rflags = None
+        self.__hdocset = None
         self.initialize_db()
 
-        # ensure that we have a recent-flags doc
+        # ensure that we have a recent-flags and a hdocs-sec doc
         self._get_or_create_rdoc()
+        self._get_or_create_hdocset()
 
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
@@ -1073,6 +1106,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                 rdoc[fields.MBOX_KEY] = self.mbox
             self._soledad.create_doc(rdoc)
 
+    def _get_or_create_hdocset(self):
+        """
+        Try to retrieve the hdocs-set doc for this MessageCollection,
+        and create one if not found.
+        """
+        hdocset = self._get_hdocset_doc()
+        if not hdocset:
+            hdocset = self._get_empty_doc(self.HDOCS_SET_DOC)
+            if self.mbox != fields.INBOX_VAL:
+                hdocset[fields.MBOX_KEY] = self.mbox
+            self._soledad.create_doc(hdocset)
+
     def _do_parse(self, raw):
         """
         Parse raw message and return it along with
@@ -1222,10 +1267,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
         # first, regular docs: flags and headers
         self._soledad.create_doc(fd)
-
         # XXX should check for content duplication on headers too
         # but with chash. !!!
-        self._soledad.create_doc(hd)
+        hdoc = self._soledad.create_doc(hd)
+        # We add the newly created hdoc to the fast-access set of
+        # headers documents associated with the mailbox.
+        self.add_hdocset_docid(hdoc.doc_id)
 
         # and last, but not least, try to create
         # content docs if not already there.
@@ -1258,7 +1305,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         d.addCallback(self._remove_cb)
         return d
 
+    #
     # getters: specific queries
+    #
+
+    # recent flags
 
     def _get_recent_flags(self):
         """
@@ -1310,14 +1361,85 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
     def _get_recent_doc(self):
         """
-        Get recent-flags document for this inbox.
+        Get recent-flags document for this mailbox.
         """
         curried = partial(
             self._soledad.get_from_index,
             fields.TYPE_MBOX_IDX,
             fields.TYPE_RECENT_VAL, self.mbox)
         curried.expected = "rdoc"
-        return try_unique_query(curried)
+        with self._rdoc_lock:
+            return try_unique_query(curried)
+
+    # headers-docs-set
+
+    def _get_hdocset(self):
+        """
+        An accessor for the hdocs-set for this mailbox.
+        """
+        if not self.__hdocset:
+            hdocset_doc = self._get_hdocset_doc()
+            value = set(hdocset_doc.content.get(
+                fields.HDOCS_SET_KEY, []))
+            self.__hdocset = value
+        return self.__hdocset
+
+    def _set_hdocset(self, value):
+        """
+        Setter for the hdocs-set for this mailbox.
+        """
+        hdocset_doc = self._get_hdocset_doc()
+        newv = set(value)
+        self.__hdocset = newv
+
+        with self._hdocset_lock:
+            hdocset_doc.content[fields.HDOCS_SET_KEY] = list(newv)
+            # XXX should deferLater 0 it?
+            self._soledad.put_doc(hdocset_doc)
+
+    _hdocset = property(
+        _get_hdocset, _set_hdocset,
+        doc="Set of Document-IDs for the headers docs associated "
+            "with this mailbox.")
+
+    def _get_hdocset_doc(self):
+        """
+        Get hdocs-set document for this mailbox.
+        """
+        curried = partial(
+            self._soledad.get_from_index,
+            fields.TYPE_MBOX_IDX,
+            fields.TYPE_HDOCS_SET_VAL, self.mbox)
+        curried.expected = "hdocset"
+        with self._hdocset_lock:
+            hdocset_doc = try_unique_query(curried)
+        return hdocset_doc
+
+    def remove_hdocset_docids(self, docids):
+        """
+        Remove the given document IDs from the set of
+        header-documents associated with this mailbox.
+        """
+        self._hdocset = self._hdocset.difference(
+            set(docids))
+
+    def remove_hdocset_docid(self, docid):
+        """
+        Remove the given document ID from the set of
+        header-documents associated with this mailbox.
+        """
+        self._hdocset = self._hdocset.difference(
+            set([docid]))
+
+    def add_hdocset_docid(self, docid):
+        """
+        Add the given document ID to the set of
+        header-documents associated with this mailbox.
+        """
+        hdocset = self._hdocset
+        self._hdocset = hdocset.union(set([docid]))
+
+    # individual doc getters, message layer.
 
     def _get_fdoc_from_chash(self, chash):
         """
@@ -1456,6 +1578,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                 fields.TYPE_FLAGS_VAL, self.mbox)))
         return all_flags
 
+    def all_flags_chash(self):
+        """
+        Return a dict with the content-hash for all flag documents
+        for this mailbox.
+        """
+        all_flags_chash = dict(((
+            doc.content[self.UID_KEY],
+            doc.content[self.CONTENT_HASH_KEY]) for doc in
+            self._soledad.get_from_index(
+                fields.TYPE_MBOX_IDX,
+                fields.TYPE_FLAGS_VAL, self.mbox)))
+        return all_flags_chash
+
+    def all_headers(self):
+        """
+        Return a dict with all the headers documents for this
+        mailbox.
+        """
+        all_headers = dict(((
+            doc.content[self.CONTENT_HASH_KEY],
+            doc.content[self.HEADERS_KEY]) for doc in
+            self._soledad.get_docs(self._hdocset)))
+        return all_headers
+
     def count(self):
         """
         Return the count of messages for this mailbox.
@@ -1509,6 +1655,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     def count_recent(self):
         """
         Count all messages with the `Recent` flag.
+        It just retrieves the length of the recent_flags set,
+        which is stored in a specific type of document for
+        this collection.
 
         :returns: count
         :rtype: int
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index a3ef098..a1d3ab7 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -123,6 +123,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
             self.sendPositiveResponse(tag, 'FETCH complete')
             return  # XXX ???
 
+        print "QUERY ", query
+        print query[0]
+
         cbFetch = self._IMAP4Server__cbFetch
         ebFetch = self._IMAP4Server__ebFetch
 
@@ -134,6 +137,14 @@ class LeapIMAPServer(imap4.IMAP4Server):
             ).addCallback(
                 cbFetch, tag, query, uid
             ).addErrback(ebFetch, tag)
+        elif len(query) == 1 and str(query[0]) == "rfc822.header":
+            self._oldTimeout = self.setTimeout(None)
+            # no need to call iter, we get a generator
+            maybeDeferred(
+                self.mbox.fetch_headers, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(ebFetch, tag)
         else:
             self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
@@ -198,7 +209,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
         self.mbox = mbox
 
 
-
 class IMAPAuthRealm(object):
     """
     Dummy authentication realm. Do not use in production!
-- 
cgit v1.2.3


From 7b558eb23208d6de0b115fa453334421cc941e44 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Fri, 17 Jan 2014 14:59:09 -0300
Subject: Add custom json.loads method.

This allows us to support the use of an `str` parameter that won't be
converted to unicode.
So in the case of a string containing bytes with different encodings
this won't break.
---
 src/leap/mail/utils.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 100 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 2480efc..93388d3 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -15,8 +15,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-Small utilities.
+Mail utilities.
 """
+import json
+import traceback
 
 
 def first(things):
@@ -27,3 +29,100 @@ def first(things):
         return things[0]
     except (IndexError, TypeError):
         return None
+
+
+class CustomJsonScanner(object):
+    """
+    This class is a context manager definition used to monkey patch the default
+    json string parsing behavior.
+    The emails can have more than one encoding, so the `str` objects have more
+    than one encoding and json does not support direct work with `str`
+    (only `unicode`).
+    """
+
+    def _parse_string_str(self, s, idx, *args, **kwargs):
+        """
+        Parses the string "s" starting at the point idx and returns an `str`
+        object. Which basically means it works exactly the same as the regular
+        JSON string parsing, except that it doesn't try to decode utf8.
+        We need this because mail raw strings might have bytes in multiple
+        encodings.
+
+        :param s: the string we want to parse
+        :type s: str
+        :param idx: the starting point for parsing
+        :type idx: int
+
+        :returns: the parsed string and the index where the
+                  string ends.
+        :rtype: tuple (str, int)
+        """
+        # NOTE: we just want to use this monkey patched version if we are
+        # calling the loads from our custom method. Otherwise, we use the
+        # json's default parser.
+        monkey_patched = False
+        for i in traceback.extract_stack():
+            # look for json_loads method in the call stack
+            if i[2] == json_loads.__name__:
+                monkey_patched = True
+                break
+
+        if not monkey_patched:
+            return self._orig_scanstring(s, idx, *args, **kwargs)
+
+        found = False
+        end = s.find("\"", idx)
+        while not found:
+            try:
+                if s[end-1] != "\\":
+                    found = True
+                else:
+                    end = s.find("\"", end+1)
+            except Exception:
+                found = True
+        return s[idx:end].decode("string-escape"), end+1
+
+    def __enter__(self):
+        """
+        Replace the json methods with the needed ones.
+        Also make a backup to restore them later.
+        """
+        # backup original values
+        self._orig_make_scanner = json.scanner.make_scanner
+        self._orig_scanstring = json.decoder.scanstring
+
+        # We need the make_scanner function to be the python one so we can
+        # monkey_patch the json string parsing
+        json.scanner.make_scanner = json.scanner.py_make_scanner
+
+        # And now we monkey patch the money method
+        json.decoder.scanstring = self._parse_string_str
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Restores the backuped methods.
+        """
+        # restore original values
+        json.scanner.make_scanner = self._orig_make_scanner
+        json.decoder.scanstring = self._orig_scanstring
+
+
+def json_loads(data):
+    """
+    It works as json.loads but supporting multiple encodings in the same
+    string and accepting an `str` parameter that won't be converted to unicode.
+
+    :param data: the string to load the objects from
+    :type data: str
+
+    :returns: the corresponding python object result of parsing 'data', this
+              behaves similarly as json.loads, with the exception of that
+              returns always `str` instead of `unicode`.
+    """
+    obj = None
+    with CustomJsonScanner():
+        # We need to use the cls parameter in order to trigger the code
+        # that will let us control the string parsing method.
+        obj = json.loads(data, cls=json.JSONDecoder)
+
+    return obj
-- 
cgit v1.2.3


From 28694a321a81f4cbe5f4873cdc55e6d3f471dd48 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Fri, 17 Jan 2014 15:07:37 -0300
Subject: Fix encodings usage, use custom json.loads method.

Also remove some unused imports.
---
 src/leap/mail/imap/fetch.py    | 19 +++++--------------
 src/leap/mail/imap/messages.py |  4 ++--
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 604a2ea..817ad6a 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -18,9 +18,7 @@
 Incoming mail fetcher.
 """
 import copy
-import json
 import logging
-#import ssl
 import threading
 import time
 import sys
@@ -34,7 +32,6 @@ from StringIO import StringIO
 from twisted.python import log
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
-#from twisted.internet.threads import deferToThread
 from zope.proxy import sameProxiedObjects
 
 from leap.common import events as leap_events
@@ -49,6 +46,7 @@ from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.mail.decorators import deferred
+from leap.mail.utils import json_loads
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
@@ -321,7 +319,8 @@ class LeapIncomingMail(object):
         """
         log.msg('processing decrypted doc')
         doc, data = msgtuple
-        msg = json.loads(data)
+        msg = json_loads(data)
+
         if not isinstance(msg, dict):
             defer.returnValue(False)
         if not msg.get(self.INCOMING_KEY, False):
@@ -338,16 +337,15 @@ class LeapIncomingMail(object):
         Tries to decrypt a gpg message if data looks like one.
 
         :param data: the text to be decrypted.
-        :type data: unicode
+        :type data: str
         :return: data, possibly descrypted.
         :rtype: str
         """
+        leap_assert_type(data, str)
         log.msg('maybe decrypting doc')
-        leap_assert_type(data, unicode)
 
         # parse the original message
         encoding = get_email_charset(data)
-        data = data.encode(encoding)
         msg = self._parser.parsestr(data)
 
         # try to obtain sender public key
@@ -420,13 +418,6 @@ class LeapIncomingMail(object):
             # Bailing out!
             return (msg, False)
 
-        # decrypted successully, now fix encoding and parse
-        try:
-            decrdata = decrdata.encode(encoding)
-        except (UnicodeEncodeError, UnicodeDecodeError) as e:
-            logger.error("Unicode error {0}".format(e))
-            decrdata = decrdata.encode(encoding, 'replace')
-
         decrmsg = self._parser.parsestr(decrdata)
         # remove original message's multipart/encrypted content-type
         del(msg['content-type'])
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 22de356..28bd272 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -494,8 +494,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
             if not charset:
                 charset = self._get_charset(body)
             try:
-                body = body.decode(charset).encode(charset)
-            except (UnicodeEncodeError, UnicodeDecodeError) as e:
+                body = body.encode(charset)
+            except UnicodeError as e:
                 logger.error("Unicode error {0}".format(e))
                 body = body.encode(charset, 'replace')
 
-- 
cgit v1.2.3


From 98fa323ef8220a6ca330972e45ee56e811c03f69 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Fri, 17 Jan 2014 15:08:49 -0300
Subject: Update VERSION_COMPAT, add changes file for #4838.

---
 changes/VERSION_COMPAT            | 3 ++-
 changes/handle-unicode-characters | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 changes/handle-unicode-characters

diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
index 1d5643f..03caa3e 100644
--- a/changes/VERSION_COMPAT
+++ b/changes/VERSION_COMPAT
@@ -9,4 +9,5 @@
 # BEGIN DEPENDENCY LIST -------------------------
 # leap.foo.bar>=x.y.z
 leap.soledad.client 0.5.0 # get_count_by_index
-
+leap.common 0.3.7 # get_email_charset
+leap.keymanager 0.3.8 # openpgp.decrypt
diff --git a/changes/handle-unicode-characters b/changes/handle-unicode-characters
new file mode 100644
index 0000000..052c543
--- /dev/null
+++ b/changes/handle-unicode-characters
@@ -0,0 +1 @@
+  o Handle correctly unicode characters in emails. Closes #4838.
-- 
cgit v1.2.3


From a50c880f43d1aef00fd233318d9413e01fb3aa3f Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 20 Jan 2014 11:38:19 -0400
Subject: Fix typo in the signal_unread method. Closes: #5001

It had been made public to be called from the overwritten methods
in service.imap
---
 src/leap/mail/imap/mailbox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index b186e75..a167531 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -707,7 +707,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # this should really be called as a final callback of
         # the do_STORE method...
         from twisted.internet import reactor
-        deferLater(reactor, 1, self._signal_unread_to_ui)
+        deferLater(reactor, 1, self.signal_unread_to_ui)
         return result
 
     # ISearchableMailbox
-- 
cgit v1.2.3


From ca9ba607ec09036db387dda6704b5956fc7baae5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= <chiiph@leap.se>
Date: Mon, 20 Jan 2014 13:29:13 -0300
Subject: Fix search command filter

---
 src/leap/mail/imap/mailbox.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index a167531..174361f 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -742,12 +742,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # but doing a quickfix for avoiding duplicat saves in the draft folder.
         # See issue #4209
 
-        if query[1] == 'HEADER' and query[2].lower() == "message-id":
-            msgid = str(query[3]).strip()
-            d = self.messages._get_uid_from_msgid(str(msgid))
-            d1 = defer.gatherResults([d])
-            # we want a list, so return it all the same
-            return d1
+        if len(query) > 2:
+            if query[1] == 'HEADER' and query[2].lower() == "message-id":
+                msgid = str(query[3]).strip()
+                d = self.messages._get_uid_from_msgid(str(msgid))
+                d1 = defer.gatherResults([d])
+                # we want a list, so return it all the same
+                return d1
 
         # nothing implemented for any other query
         logger.warning("Cannot process query: %s" % (query,))
-- 
cgit v1.2.3


From 90d3062b764c405398adb22a5fbaf4f89c6e8d26 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 20 Jan 2014 12:56:30 -0400
Subject: make the read/write operations over sets atomic Fixes: #5009

---
 src/leap/mail/imap/messages.py | 100 ++++++++++++++++++++++++-----------------
 1 file changed, 59 insertions(+), 41 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index d2c0950..378738e 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1044,8 +1044,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
     }
 
+    # Different locks for wrapping both the u1db document getting/setting
+    # and the property getting/settting in an atomic operation.
+
+    # TODO we would abstract this to a SoledadProperty class
+
     _rdoc_lock = threading.Lock()
+    _rdoc_property_lock = threading.Lock()
     _hdocset_lock = threading.Lock()
+    _hdocset_property_lock = threading.Lock()
 
     def __init__(self, mbox=None, soledad=None):
         """
@@ -1316,20 +1323,20 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         An accessor for the recent-flags set for this mailbox.
         """
         if not self.__rflags:
-            rdoc = self._get_recent_doc()
-            self.__rflags = set(rdoc.content.get(
-                fields.RECENTFLAGS_KEY, []))
+            with self._rdoc_lock:
+                rdoc = self._get_recent_doc()
+                self.__rflags = set(rdoc.content.get(
+                    fields.RECENTFLAGS_KEY, []))
         return self.__rflags
 
     def _set_recent_flags(self, value):
         """
         Setter for the recent-flags set for this mailbox.
         """
-        rdoc = self._get_recent_doc()
-        newv = set(value)
-        self.__rflags = newv
-
         with self._rdoc_lock:
+            rdoc = self._get_recent_doc()
+            newv = set(value)
+            self.__rflags = newv
             rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
             # XXX should deferLater 0 it?
             self._soledad.put_doc(rdoc)
@@ -1338,38 +1345,44 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
 
+    def _get_recent_doc(self):
+        """
+        Get recent-flags document for this mailbox.
+        """
+        curried = partial(
+            self._soledad.get_from_index,
+            fields.TYPE_MBOX_IDX,
+            fields.TYPE_RECENT_VAL, self.mbox)
+        curried.expected = "rdoc"
+        rdoc = try_unique_query(curried)
+        return rdoc
+
+    # Property-set modification (protected by a different
+    # lock to give atomicity to the read/write operation)
+
     def unset_recent_flags(self, uids):
         """
         Unset Recent flag for a sequence of uids.
         """
-        self.recent_flags = self.recent_flags.difference(
-            set(uids))
+        with self._rdoc_property_lock:
+            self.recent_flags = self.recent_flags.difference(
+                set(uids))
 
     def unset_recent_flag(self, uid):
         """
         Unset Recent flag for a given uid.
         """
-        self.recent_flags = self.recent_flags.difference(
-            set([uid]))
+        with self._rdoc_property_lock:
+            self.recent_flags = self.recent_flags.difference(
+                set([uid]))
 
     def set_recent_flag(self, uid):
         """
         Set Recent flag for a given uid.
         """
-        self.recent_flags = self.recent_flags.union(
-            set([uid]))
-
-    def _get_recent_doc(self):
-        """
-        Get recent-flags document for this mailbox.
-        """
-        curried = partial(
-            self._soledad.get_from_index,
-            fields.TYPE_MBOX_IDX,
-            fields.TYPE_RECENT_VAL, self.mbox)
-        curried.expected = "rdoc"
-        with self._rdoc_lock:
-            return try_unique_query(curried)
+        with self._rdoc_property_lock:
+            self.recent_flags = self.recent_flags.union(
+                set([uid]))
 
     # headers-docs-set
 
@@ -1378,21 +1391,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         An accessor for the hdocs-set for this mailbox.
         """
         if not self.__hdocset:
-            hdocset_doc = self._get_hdocset_doc()
-            value = set(hdocset_doc.content.get(
-                fields.HDOCS_SET_KEY, []))
-            self.__hdocset = value
+            with self._hdocset_lock:
+                hdocset_doc = self._get_hdocset_doc()
+                value = set(hdocset_doc.content.get(
+                    fields.HDOCS_SET_KEY, []))
+                self.__hdocset = value
         return self.__hdocset
 
     def _set_hdocset(self, value):
         """
         Setter for the hdocs-set for this mailbox.
         """
-        hdocset_doc = self._get_hdocset_doc()
-        newv = set(value)
-        self.__hdocset = newv
-
         with self._hdocset_lock:
+            hdocset_doc = self._get_hdocset_doc()
+            newv = set(value)
+            self.__hdocset = newv
             hdocset_doc.content[fields.HDOCS_SET_KEY] = list(newv)
             # XXX should deferLater 0 it?
             self._soledad.put_doc(hdocset_doc)
@@ -1411,33 +1424,38 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             fields.TYPE_MBOX_IDX,
             fields.TYPE_HDOCS_SET_VAL, self.mbox)
         curried.expected = "hdocset"
-        with self._hdocset_lock:
-            hdocset_doc = try_unique_query(curried)
+        hdocset_doc = try_unique_query(curried)
         return hdocset_doc
 
+    # Property-set modification (protected by a different
+    # lock to give atomicity to the read/write operation)
+
     def remove_hdocset_docids(self, docids):
         """
         Remove the given document IDs from the set of
         header-documents associated with this mailbox.
         """
-        self._hdocset = self._hdocset.difference(
-            set(docids))
+        with self._hdocset_property_lock:
+            self._hdocset = self._hdocset.difference(
+                set(docids))
 
     def remove_hdocset_docid(self, docid):
         """
         Remove the given document ID from the set of
         header-documents associated with this mailbox.
         """
-        self._hdocset = self._hdocset.difference(
-            set([docid]))
+        with self._hdocset_property_lock:
+            self._hdocset = self._hdocset.difference(
+                set([docid]))
 
     def add_hdocset_docid(self, docid):
         """
         Add the given document ID to the set of
         header-documents associated with this mailbox.
         """
-        hdocset = self._hdocset
-        self._hdocset = hdocset.union(set([docid]))
+        with self._hdocset_property_lock:
+            self._hdocset = self._hdocset.union(
+                set([docid]))
 
     # individual doc getters, message layer.
 
-- 
cgit v1.2.3


From 22c106a7306446a3fa9689f5942a86a53ec884b4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 21 Jan 2014 01:04:37 -0400
Subject: workaround for recursionlimit due to qtreactor

---
 src/leap/mail/imap/mailbox.py      |  4 ++++
 src/leap/mail/imap/messages.py     |  2 --
 src/leap/mail/imap/service/imap.py | 23 +++++++++++++++++++----
 3 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 174361f..38c58cb 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -765,6 +765,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         uid_next = self.getUIDNext()
         msg = messageObject
 
+        # XXX DEBUG ----------------------------------------
+        #print "copying MESSAGE from %s (%s) to %s (%s)" % (
+            #msg._mbox, msg._uid, self.mbox, uid_next)
+
         # XXX should use a public api instead
         fdoc = msg._fdoc
         if not fdoc:
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 378738e..cd4d85f 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -528,7 +528,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
             body = self._bdoc.content.get(self.RAW_KEY, "")
             content_type = bdoc.content.get('content-type', "")
             charset = first(CHARSET_RE.findall(content_type))
-            logger.debug("Got charset from header: %s" % (charset,))
             if not charset:
                 charset = self._get_charset(body)
             try:
@@ -665,7 +664,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
         try:
             pmap_dict = self._get_part_from_parts_map(part + 1)
         except KeyError:
-            logger.debug("getSubpart for %s: KeyError" % (part,))
             raise IndexError
         return MessagePart(self._soledad, pmap_dict)
 
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index a1d3ab7..ad22da6 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -49,6 +49,25 @@ from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED
 from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START
 from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN
 
+######################################################
+# Temporary workaround for RecursionLimit when using
+# qt4reactor. Do remove when we move to poll or select
+# reactor, which do not show those problems. See #4974
+import resource
+import sys
+
+try:
+    sys.setrecursionlimit(10**6)
+except Exception:
+    print "Error setting recursion limit"
+try:
+    # Increase max stack size from 8MB to 256MB
+    resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1))
+except Exception:
+    print "Error setting stack size"
+
+######################################################
+
 
 class LeapIMAPServer(imap4.IMAP4Server):
     """
@@ -118,14 +137,10 @@ class LeapIMAPServer(imap4.IMAP4Server):
         method
         """
         from twisted.internet import reactor
-        log.msg("LEAP Overwritten fetch...")
         if not query:
             self.sendPositiveResponse(tag, 'FETCH complete')
             return  # XXX ???
 
-        print "QUERY ", query
-        print query[0]
-
         cbFetch = self._IMAP4Server__cbFetch
         ebFetch = self._IMAP4Server__ebFetch
 
-- 
cgit v1.2.3


From 379f7fd742d1e79a575f0f723bcddb01cc611067 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Tue, 21 Jan 2014 16:18:15 -0200
Subject: Prevent double base64 encoding of attachments when signing (#5014).

---
 ...bug_5014_fix-attachment-processing-when-signing |  1 +
 src/leap/mail/smtp/rfc3156.py                      | 28 +++++++++++++++-------
 2 files changed, 21 insertions(+), 8 deletions(-)
 create mode 100644 changes/bug_5014_fix-attachment-processing-when-signing

diff --git a/changes/bug_5014_fix-attachment-processing-when-signing b/changes/bug_5014_fix-attachment-processing-when-signing
new file mode 100644
index 0000000..c12e35e
--- /dev/null
+++ b/changes/bug_5014_fix-attachment-processing-when-signing
@@ -0,0 +1 @@
+  o Correctly process attachments when signing. Fixes #5014.
diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py
index 9739531..2c6d4a7 100644
--- a/src/leap/mail/smtp/rfc3156.py
+++ b/src/leap/mail/smtp/rfc3156.py
@@ -24,6 +24,7 @@ import base64
 from abc import ABCMeta, abstractmethod
 from StringIO import StringIO
 
+from twisted.python import log
 from email.mime.application import MIMEApplication
 from email.mime.multipart import MIMEMultipart
 from email import errors
@@ -145,14 +146,25 @@ def encode_base64(msg):
     :param msg: The non-multipart message to be encoded.
     :type msg: email.message.Message
     """
-    orig = msg.get_payload()
-    encdata = _bencode(orig)
-    msg.set_payload(encdata)
-    # replace or set the Content-Transfer-Encoding header.
-    try:
-        msg.replace_header('Content-Transfer-Encoding', 'base64')
-    except KeyError:
-        msg['Content-Transfer-Encoding'] = 'base64'
+    encoding = msg.get('Content-Transfer-Encoding', None)
+    # XXX Python's email module can only decode quoted-printable, base64 and
+    # uuencoded data, so we might have to implement other decoding schemes in
+    # order to support RFC 3156 properly and correctly calculate signatures
+    # for multipart attachments (eg. 7bit or 8bit encoded attachments). For
+    # now, if content is already encoded as base64 or if it is encoded with
+    # some unknown encoding, we just pass.
+    if encoding is None or encoding.lower() in ['quoted-printable',
+            'x-uuencode', 'uue', 'x-uue']:
+        orig = msg.get_payload(decode=True)
+        encdata = _bencode(orig)
+        msg.set_payload(encdata)
+        # replace or set the Content-Transfer-Encoding header.
+        try:
+            msg.replace_header('Content-Transfer-Encoding', 'base64')
+        except KeyError:
+            msg['Content-Transfer-Encoding'] = 'base64'
+    elif encoding is not 'base64':
+        log.err('Unknown content-transfer-encoding: %s' % encoding)
 
 
 def encode_base64_rec(msg):
-- 
cgit v1.2.3


From 9347da67f253a697f5a1c3bd380263f61c62abed Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Tue, 21 Jan 2014 16:32:24 -0200
Subject: Restrict adding outgoing footer to text/plain messages.

---
 ...g_restrict-adding-outgoing-footer-to-text-plain-messages |  1 +
 src/leap/mail/smtp/gateway.py                               | 13 ++++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages

diff --git a/changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages b/changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages
new file mode 100644
index 0000000..9983404
--- /dev/null
+++ b/changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages
@@ -0,0 +1 @@
+  o Restrict adding outgoing footer to text/plain messages.
diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py
index bef5c6d..ef398d1 100644
--- a/src/leap/mail/smtp/gateway.py
+++ b/src/leap/mail/smtp/gateway.py
@@ -600,13 +600,16 @@ class EncryptedMessage(object):
             self._msg = self._origmsg
             return
 
-        # add a nice footer to the outgoing message
         from_address = validate_address(self._fromAddress.addrstr)
         username, domain = from_address.split('@')
-        self.lines.append('--')
-        self.lines.append('%s - https://%s/key/%s' %
-                          (self.FOOTER_STRING, domain, username))
-        self.lines.append('')
+
+        # add a nice footer to the outgoing message
+        if self._origmsg.get_content_type() == 'text/plain':
+            self.lines.append('--')
+            self.lines.append('%s - https://%s/key/%s' %
+                              (self.FOOTER_STRING, domain, username))
+            self.lines.append('')
+
         self._origmsg = self.parseMessage()
 
         # get sender and recipient data
-- 
cgit v1.2.3


From c2e052a08789057d550a0442caa28b27ebc4b416 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Wed, 22 Jan 2014 11:01:05 -0300
Subject: Add find_charset helper and use where is needed.

---
 src/leap/mail/imap/messages.py | 13 +++++--------
 src/leap/mail/utils.py         | 25 +++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index cd4d85f..862a9f2 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -38,7 +38,7 @@ from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail import walk
-from leap.mail.utils import first
+from leap.mail.utils import first, find_charset
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
@@ -92,10 +92,7 @@ def try_unique_query(curried):
     except Exception as exc:
         logger.exception("Unhandled error %r" % exc)
 
-CHARSET_PATTERN = r"""charset=([\w-]+)"""
 MSGID_PATTERN = r"""<([\w@.]+)>"""
-
-CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
 MSGID_RE = re.compile(MSGID_PATTERN)
 
 
@@ -177,9 +174,9 @@ class MessagePart(object):
 
         if payload:
             content_type = self._get_ctype_from_document(phash)
-            charset = first(CHARSET_RE.findall(content_type))
+            charset = find_charset(content_type)
             logger.debug("Got charset from header: %s" % (charset,))
-            if not charset:
+            if charset is None:
                 charset = self._get_charset(payload)
             try:
                 payload = payload.encode(charset)
@@ -527,8 +524,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         if bdoc:
             body = self._bdoc.content.get(self.RAW_KEY, "")
             content_type = bdoc.content.get('content-type', "")
-            charset = first(CHARSET_RE.findall(content_type))
-            if not charset:
+            charset = find_charset(content_type)
+            if charset is None:
                 charset = self._get_charset(body)
             try:
                 body = body.encode(charset)
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 93388d3..6c79227 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -18,9 +18,14 @@
 Mail utilities.
 """
 import json
+import re
 import traceback
 
 
+CHARSET_PATTERN = r"""charset=([\w-]+)"""
+CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
+
+
 def first(things):
     """
     Return the head of a collection.
@@ -31,6 +36,26 @@ def first(things):
         return None
 
 
+def find_charset(thing, default=None):
+    """
+    Looks into the object 'thing' for a charset specification.
+    It searchs into the object's `repr`.
+
+    :param thing: the object to look into.
+    :type thing: object
+    :param default: the dafault charset to return if no charset is found.
+    :type default: str
+
+    :returns: the charset or 'default'
+    :rtype: str or None
+    """
+    charset = first(CHARSET_RE.findall(repr(thing)))
+    if charset is None:
+        charset = default
+
+    return charset
+
+
 class CustomJsonScanner(object):
     """
     This class is a context manager definition used to monkey patch the default
-- 
cgit v1.2.3


From 61af338b0dee8a56cd0f302502fe7cd9dc8bc5d1 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Wed, 22 Jan 2014 11:03:58 -0300
Subject: Handle non-ascii headers. Closes #5021.

---
 src/leap/mail/imap/messages.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 862a9f2..5bb5f1c 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -605,18 +605,26 @@ class LeapMessage(fields, MailParser, MBoxParser):
         if isinstance(headers, list):
             headers = dict(headers)
 
+        # default to most likely standard
+        charset = find_charset(headers, "utf-8")
+
         # twisted imap server expects *some* headers to be lowercase
         # XXX refactor together with MessagePart method
-        headers = dict(
-            (str(key), str(value)) if key.lower() != "content-type"
-            else (str(key.lower()), str(value))
-            for (key, value) in headers.items())
+        headers2 = dict()
+        for key, value in headers.items():
+            if key.lower() == "content-type":
+                key = key.lower()
 
-        # unpack and filter original dict by negate-condition
-        filter_by_cond = [(key, val) for key, val
-                          in headers.items() if cond(key)]
+            if not isinstance(key, str):
+                key = key.encode(charset, 'replace')
+            if not isinstance(value, str):
+                value = value.encode(charset, 'replace')
+
+            # filter original dict by negate-condition
+            if cond(key):
+                headers2[key] = value
 
-        return dict(filter_by_cond)
+        return headers2
 
     def _get_headers(self):
         """
-- 
cgit v1.2.3


From b5b6b72c8f97140dda78b24ffecaed06f39ea932 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Wed, 22 Jan 2014 11:05:15 -0300
Subject: Add changes file for #5021.

---
 changes/bug-5021_handle-non-ascii-headers | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/bug-5021_handle-non-ascii-headers

diff --git a/changes/bug-5021_handle-non-ascii-headers b/changes/bug-5021_handle-non-ascii-headers
new file mode 100644
index 0000000..098cfa0
--- /dev/null
+++ b/changes/bug-5021_handle-non-ascii-headers
@@ -0,0 +1 @@
+  o Handle non-ascii headers. Closes #5021.
-- 
cgit v1.2.3


From 460539c51b431b6d16c45ecd8216ab1e0471d106 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= <chiiph@leap.se>
Date: Wed, 22 Jan 2014 18:42:12 -0300
Subject: Properly parse apple mail

---
 changes/bug_properly_parse_apple_mails | 1 +
 src/leap/mail/walk.py                  | 9 +++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 changes/bug_properly_parse_apple_mails

diff --git a/changes/bug_properly_parse_apple_mails b/changes/bug_properly_parse_apple_mails
new file mode 100644
index 0000000..1bf42ae
--- /dev/null
+++ b/changes/bug_properly_parse_apple_mails
@@ -0,0 +1 @@
+  o Properly parse emails crafted by Mail.app. Fixes #5013.
\ No newline at end of file
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index dd3b745..27d672c 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -143,6 +143,15 @@ def walk_msg_tree(parts, body_phash=None):
         pv = list(get_parts_vector(parts))
         wv = getwv(pv)
 
+    if all(x == 1 for x in pv):
+        # special case in the rightmost element
+        main_pmap = parts[0]['part_map']
+        last_part = max(main_pmap.keys())
+        main_pmap[last_part]['part_map'] = {}
+        for partind in range(len(pv) - 1):
+            print partind+1, len(parts)
+            main_pmap[last_part]['part_map'][partind] = parts[partind+1]
+
     outer = parts[0]
     outer.pop('headers')
     if not "part_map" in outer:
-- 
cgit v1.2.3


From bded9833da985034a11c30b342388c397798a585 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 10:49:30 -0400
Subject: add constants to dict keys

---
 src/leap/mail/walk.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 27d672c..856daa3 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -107,10 +107,16 @@ def walk_msg_tree(parts, body_phash=None):
                        in the outer content doc for convenience.
     :type body_phash: basestring or None
     """
+    PART_MAP = "part_map"
+    MULTI = "multi"
+    HEADERS = "headers"
+    PHASH = "phash"
+    BODY = "body"
+
     # parts vector
     pv = list(get_parts_vector(parts))
 
-    inner_headers = parts[1].get("headers", None) if (
+    inner_headers = parts[1].get(HEADERS, None) if (
         len(parts) == 2) else None
 
     if DEBUG:
@@ -129,10 +135,10 @@ def walk_msg_tree(parts, body_phash=None):
         slic = parts[wind + 1:wind + 1 + nsub]  # slice with subparts
 
         cwra = {
-            "multi": True,
-            "part_map": dict((index + 1, part)  # content wrapper
-                             for index, part in enumerate(slic)),
-            "headers": dict(parts[wind]['headers'])
+            MULTI: True,
+            PART_MAP: dict((index + 1, part)  # content wrapper
+                           for index, part in enumerate(slic)),
+            HEADERS: dict(parts[wind][HEADERS])
         }
 
         # remove subparts and substitue wrapper
@@ -153,19 +159,19 @@ def walk_msg_tree(parts, body_phash=None):
             main_pmap[last_part]['part_map'][partind] = parts[partind+1]
 
     outer = parts[0]
-    outer.pop('headers')
-    if not "part_map" in outer:
+    outer.pop(HEADERS)
+    if not PART_MAP in outer:
         # we have a multipart with 1 part only, so kind of fix it
         # although it would be prettier if I take this special case at
         # the beginning of the walk.
-        pdoc = {"multi": True,
-                "part_map": {1: outer}}
-        pdoc["part_map"][1]["multi"] = False
-        if not pdoc["part_map"][1].get("phash", None):
-            pdoc["part_map"][1]["phash"] = body_phash
+        pdoc = {MULTI: True,
+                PART_MAP: {1: outer}}
+        pdoc[PART_MAP][1][MULTI] = False
+        if not pdoc[PART_MAP][1].get(PHASH, None):
+            pdoc[PART_MAP][1][PHASH] = body_phash
         if inner_headers:
-            pdoc["part_map"][1]["headers"] = inner_headers
+            pdoc[PART_MAP][1][HEADERS] = inner_headers
     else:
         pdoc = outer
-    pdoc["body"] = body_phash
+    pdoc[BODY] = body_phash
     return pdoc
-- 
cgit v1.2.3


From 90b870c48c0c27e3a366902c5b76986a5140258c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 10:49:52 -0400
Subject: add check for none in part_map special case

---
 src/leap/mail/walk.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 856daa3..30cb70a 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -151,12 +151,13 @@ def walk_msg_tree(parts, body_phash=None):
 
     if all(x == 1 for x in pv):
         # special case in the rightmost element
-        main_pmap = parts[0]['part_map']
-        last_part = max(main_pmap.keys())
-        main_pmap[last_part]['part_map'] = {}
-        for partind in range(len(pv) - 1):
-            print partind+1, len(parts)
-            main_pmap[last_part]['part_map'][partind] = parts[partind+1]
+        main_pmap = parts[0].get(PART_MAP, None)
+        if main_pmap is not None:
+            last_part = max(main_pmap.keys())
+            main_pmap[last_part][PART_MAP] = {}
+            for partind in range(len(pv) - 1):
+                print partind+1, len(parts)
+                main_pmap[last_part][PART_MAP][partind] = parts[partind + 1]
 
     outer = parts[0]
     outer.pop(HEADERS)
-- 
cgit v1.2.3


From 8b7c63dd7369eff2262b6f2a2d574f46a67657a0 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Thu, 23 Jan 2014 15:34:28 -0200
Subject: Handle upper and lowercase base64 encoded outgoing attachments.

---
 src/leap/mail/smtp/rfc3156.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py
index 2c6d4a7..62a0675 100644
--- a/src/leap/mail/smtp/rfc3156.py
+++ b/src/leap/mail/smtp/rfc3156.py
@@ -147,14 +147,15 @@ def encode_base64(msg):
     :type msg: email.message.Message
     """
     encoding = msg.get('Content-Transfer-Encoding', None)
+    if encoding is not None:
+        encoding = encoding.lower()
     # XXX Python's email module can only decode quoted-printable, base64 and
     # uuencoded data, so we might have to implement other decoding schemes in
     # order to support RFC 3156 properly and correctly calculate signatures
     # for multipart attachments (eg. 7bit or 8bit encoded attachments). For
     # now, if content is already encoded as base64 or if it is encoded with
     # some unknown encoding, we just pass.
-    if encoding is None or encoding.lower() in ['quoted-printable',
-            'x-uuencode', 'uue', 'x-uue']:
+    if encoding in [None, 'quoted-printable', 'x-uuencode', 'uue', 'x-uue']:
         orig = msg.get_payload(decode=True)
         encdata = _bencode(orig)
         msg.set_payload(encdata)
-- 
cgit v1.2.3


From 05c51ee2af6af6025b01831fd3b395cad80fe9f1 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Sun, 26 Jan 2014 18:09:51 -0400
Subject: Script for reproducible imaptest runs.

---
 src/leap/mail/imap/tests/.gitignore          |   1 +
 src/leap/mail/imap/tests/leap_tests_imap.zsh | 160 +++++++++++++++++++++++++++
 2 files changed, 161 insertions(+)
 create mode 100644 src/leap/mail/imap/tests/.gitignore
 create mode 100755 src/leap/mail/imap/tests/leap_tests_imap.zsh

diff --git a/src/leap/mail/imap/tests/.gitignore b/src/leap/mail/imap/tests/.gitignore
new file mode 100644
index 0000000..60baa9c
--- /dev/null
+++ b/src/leap/mail/imap/tests/.gitignore
@@ -0,0 +1 @@
+data/*
diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh
new file mode 100755
index 0000000..7ba408c
--- /dev/null
+++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh
@@ -0,0 +1,160 @@
+#!/bin/zsh
+# BATCH STRESS TEST FOR IMAP ----------------------
+# http://imgs.xkcd.com/comics/science.jpg
+#
+# Run imaptest against a LEAP IMAP server
+# for a fixed period of time, and collect output.
+#
+# Author: Kali Kaneko
+# Date: 2014 01 26
+#
+# To run, you need to have `imaptest` in your path.
+# See:
+# http://www.imapwiki.org/ImapTest/Installation
+#
+# For the tests, I'm using a 10MB file sample that
+# can be downloaded from:
+# http://www.dovecot.org/tmp/dovecot-crlf
+#
+# Want to contribute to benchmarking?
+#
+# 1. Create a pristine account in a bitmask provider.
+#
+# 2. Launch your bitmask client, with different flags
+#    if you desire.
+#
+#    For example to try the nosync flag in sqlite:
+#
+#    LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log
+#
+# 3. Run at several points in time (ie: just after
+#    launching the bitmask client. one minute after,
+#    ten minutes after)
+#
+#    mkdir data
+#    cd data
+#    ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log  
+#
+# 4. Submit your results to: kali at leap dot se
+#    together with the logs of the bitmask run.
+#
+# Please provide also details about your system, and
+# the type of hard disk setup you are running against.
+#
+#
+# Edit these variables --------------------------
+
+USER="test_f14@dev.bitmask.net"
+MBOX="~/leap/imaptest/data/dovecot-crlf"
+
+HOST="localhost"
+PORT="1984"
+
+IMAPTEST="imaptest"
+GREP="/bin/grep"
+
+# -----------------------------------------------
+#
+# These should be kept constant across benchmarking
+# runs across different machines, for comparability.
+
+DURATION=100
+NUM_MSG=200
+
+
+# TODO add another function, and a cli flag, to be able
+# to take several aggretates spaced in time, along a period
+# of several minutes.
+
+imaptest_cmd() {
+  stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \
+	  port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \
+	  no_pipelining 2>/dev/null
+}
+
+stress_imap() 	{
+  mknod imap_pipe p
+  cat imap_pipe | tee output &
+  imaptest_cmd >> imap_pipe
+}
+
+wait_and_kill() {
+  while : 
+  do
+    sleep $DURATION
+    pkill -2 imaptest
+    rm imap_pipe
+    break
+  done
+}
+
+print_results() {
+	sleep 1
+	echo
+	echo
+	echo "AGGREGATED RESULTS"
+	echo "----------------------"
+	echo "\tavg\tstdev"
+	$GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \
+	awk '
+function avg(data, count) {
+    sum=0;
+    for( x=0; x <= count-1; x++) {
+        sum += data[x];
+    }
+    return sum/count;
+}
+function std_dev(data, count) {
+    sum=0;
+    for( x=0; x <= count-1; x++) {
+        sum += data[x];
+    }
+    average = sum/count;
+
+    sumsq=0;
+    for( x=0; x <= count-1; x++) {
+        sumsq += (data[x] - average)^2;
+    }
+    return sqrt(sumsq/count);
+}
+BEGIN {
+  cnt = 0
+} END {
+
+printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR));
+printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR));
+printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR));
+printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR));
+printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR));
+printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR));
+printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR));
+printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR));
+printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR));
+printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR));
+printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR));
+
+print ""
+print "TOT samples", NR;
+}
+{
+  it = cnt++;
+  array[1][it] = $1;
+  array[2][it] = $2;
+  array[3][it] = $3;
+  array[4][it] = $4;
+  array[5][it] = $5;
+  array[6][it] = $6;
+  array[7][it] = $7;
+  array[8][it] = $8;
+  array[9][it] = $9;
+  array[10][it] = $10;
+  array[11][it] = $11;
+}'
+}
+
+
+echo "[+] LEAP IMAP TESTS"
+echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages"
+wait_and_kill &
+stress_imap
+print_results
-- 
cgit v1.2.3


From e9db0eb4802e528142000d7a2f7da0c9135fce44 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Sun, 26 Jan 2014 19:49:39 -0400
Subject: temporarily remove notify after adding msg

---
 src/leap/mail/imap/mailbox.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 38c58cb..0131ce0 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -406,8 +406,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Invoked from addMessage.
         """
         d = self.messages.add_msg(message, flags=flags, date=date, uid=uid)
-        # XXX notify after batch APPEND?
-        d.addCallback(self.notify_new)
+        # XXX Removing notify temporarily.
+        # This is interfering with imaptest results. I'm not clear if it's
+        # because we clutter the logging or because the set of listeners is
+        # ever-growing. We should come up with some smart way of dealing with
+        # it, or maybe just disabling it using an environmental variable since
+        # we will only have just a few listeners in the regular desktop case.
+        #d.addCallback(self.notify_new)
         return d
 
     def notify_new(self, *args):
@@ -422,7 +427,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             exists,
             recent))
 
-        logger.debug("listeners: %s", str(self.listeners))
         for l in self.listeners:
             logger.debug('notifying...')
             l.newMessages(exists, recent)
-- 
cgit v1.2.3


From 36baa3a0e93df779145515f879ed3efdab014bea Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Sun, 26 Jan 2014 20:27:53 -0400
Subject: Allow passing user and mbox as parameters

Increase default testing duration to 200 secs.
---
 src/leap/mail/imap/tests/leap_tests_imap.zsh | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh
index 7ba408c..676d1a8 100755
--- a/src/leap/mail/imap/tests/leap_tests_imap.zsh
+++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh
@@ -41,8 +41,10 @@
 # Please provide also details about your system, and
 # the type of hard disk setup you are running against.
 #
-#
-# Edit these variables --------------------------
+
+# ------------------------------------------------
+# Edit these variables if you are too lazy to pass
+# the user and mbox as parameters. Like me.
 
 USER="test_f14@dev.bitmask.net"
 MBOX="~/leap/imaptest/data/dovecot-crlf"
@@ -50,15 +52,16 @@ MBOX="~/leap/imaptest/data/dovecot-crlf"
 HOST="localhost"
 PORT="1984"
 
-IMAPTEST="imaptest"
+# in case you have it aliased
 GREP="/bin/grep"
+IMAPTEST="imaptest"
 
 # -----------------------------------------------
 #
 # These should be kept constant across benchmarking
 # runs across different machines, for comparability.
 
-DURATION=100
+DURATION=200
 NUM_MSG=200
 
 
@@ -153,6 +156,21 @@ print "TOT samples", NR;
 }
 
 
+{ test $1 = "--help" } && {
+ echo "Usage: $0 [user@provider] [/path/to/sample.mbox]"
+ exit 0
+}
+
+# If the first parameter is passed, take it as the user
+{ test $1 } && {
+ USER=$1
+}
+
+# If the second parameter is passed, take it as the mbox
+{ test $2 } && {
+ MBOX=$2
+}
+
 echo "[+] LEAP IMAP TESTS"
 echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages"
 wait_and_kill &
-- 
cgit v1.2.3


From 17ea6fd404fd606c74776dc05ce769a7df43569a Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Mon, 27 Jan 2014 14:49:16 -0300
Subject: Use repr() on exceptions, inform if using 'replace'.

---
 src/leap/mail/imap/messages.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 5bb5f1c..34304ea 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -178,10 +178,11 @@ class MessagePart(object):
             logger.debug("Got charset from header: %s" % (charset,))
             if charset is None:
                 charset = self._get_charset(payload)
+                logger.debug("Got charset: %s" % (charset,))
             try:
                 payload = payload.encode(charset)
             except (UnicodeEncodeError, UnicodeDecodeError) as e:
-                logger.error("Unicode error {0}".format(e))
+                logger.error("Unicode error, using 'replace'. {0!r}".format(e))
                 payload = payload.encode(charset, 'replace')
 
         fd.write(payload)
@@ -530,7 +531,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             try:
                 body = body.encode(charset)
             except UnicodeError as e:
-                logger.error("Unicode error {0}".format(e))
+                logger.error("Unicode error, using 'replace'. {0!r}".format(e))
                 body = body.encode(charset, 'replace')
 
         # We are still returning funky characters from here.
-- 
cgit v1.2.3


From 4ae6ad57a0f80143e3ded867c1fdd2264804a775 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 21 Jan 2014 19:22:09 -0400
Subject: memory store for append/fetch/copy

---
 src/leap/mail/imap/account.py      |  13 +-
 src/leap/mail/imap/interfaces.py   |  93 ++++++++
 src/leap/mail/imap/mailbox.py      |  62 +++--
 src/leap/mail/imap/memorystore.py  | 478 +++++++++++++++++++++++++++++++++++++
 src/leap/mail/imap/messages.py     | 206 ++++++++++++----
 src/leap/mail/imap/service/imap.py |  20 +-
 src/leap/mail/messageflow.py       |  39 ++-
 src/leap/mail/size.py              |  57 +++++
 8 files changed, 884 insertions(+), 84 deletions(-)
 create mode 100644 src/leap/mail/imap/interfaces.py
 create mode 100644 src/leap/mail/imap/memorystore.py
 create mode 100644 src/leap/mail/size.py

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index ce83079..7641ea8 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -48,7 +48,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     selected = None
     closed = False
 
-    def __init__(self, account_name, soledad=None):
+    def __init__(self, account_name, soledad=None, memstore=None):
         """
         Creates a SoledadAccountIndex that keeps track of the mailboxes
         and subscriptions handled by this account.
@@ -57,7 +57,9 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :type acct_name: str
 
         :param soledad: a Soledad instance.
-        :param soledad: Soledad
+        :type soledad: Soledad
+        :param memstore: a MemoryStore instance.
+        :type memstore: MemoryStore
         """
         leap_assert(soledad, "Need a soledad instance to initialize")
         leap_assert_type(soledad, Soledad)
@@ -67,6 +69,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         self._account_name = self._parse_mailbox_name(account_name)
         self._soledad = soledad
+        self._memstore = memstore
 
         self.initialize_db()
 
@@ -131,7 +134,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         if name not in self.mailboxes:
             raise imap4.MailboxException("No such mailbox: %r" % name)
 
-        return SoledadMailbox(name, soledad=self._soledad)
+        return SoledadMailbox(name, soledad=self._soledad,
+                              memstore=self._memstore)
 
     ##
     ## IAccount
@@ -221,8 +225,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         self.selected = name
 
         return SoledadMailbox(
-            name, rw=readwrite,
-            soledad=self._soledad)
+            name, self._soledad, self._memstore, readwrite)
 
     def delete(self, name, force=False):
         """
diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py
new file mode 100644
index 0000000..585165a
--- /dev/null
+++ b/src/leap/mail/imap/interfaces.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# interfaces.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Interfaces for the IMAP module.
+"""
+from zope.interface import Interface, Attribute
+
+
+class IMessageContainer(Interface):
+    """
+    I am a container around the different documents that a message
+    is split into.
+    """
+    fdoc = Attribute('The flags document for this message, if any.')
+    hdoc = Attribute('The headers document for this message, if any.')
+    cdocs = Attribute('The dict of content documents for this message, '
+                      'if any.')
+
+    def walk(self):
+        """
+        Return an iterator to the docs for all the parts.
+
+        :rtype: iterator
+        """
+
+
+class IMessageStore(Interface):
+    """
+    I represent a generic storage for LEAP Messages.
+    """
+
+    def create_message(self, mbox, uid, message):
+        """
+        Put the passed message into this IMessageStore.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        :param message: a IMessageContainer implementor.
+        """
+
+    def put_message(self, mbox, uid, message):
+        """
+        Put the passed message into this IMessageStore.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        :param message: a IMessageContainer implementor.
+        """
+
+    def remove_message(self, mbox, uid):
+        """
+        Remove the given message from this IMessageStore.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        """
+
+    def get_message(self, mbox, uid):
+        """
+        Get a IMessageContainer for the given mbox and uid combination.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        """
+
+
+class IMessageStoreWriter(Interface):
+    """
+    I represent a storage that is able to write its contents to another
+    different IMessageStore.
+    """
+
+    def write_messages(self, store):
+        """
+        Write the documents in this IMessageStore to a different
+        storage. Usually this will be done from a MemoryStorage to a DbStorage.
+
+        :param store: another IMessageStore implementor.
+        """
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 0131ce0..9babe6b 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -37,6 +37,7 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
 from leap.mail.decorators import deferred
 from leap.mail.imap.fields import WithMsgFields, fields
+from leap.mail.imap.memorystore import MessageDict
 from leap.mail.imap.messages import MessageCollection
 from leap.mail.imap.parser import MBoxParser
 
@@ -80,7 +81,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     next_uid_lock = threading.Lock()
 
-    def __init__(self, mbox, soledad=None, rw=1):
+    def __init__(self, mbox, soledad, memstore, rw=1):
         """
         SoledadMailbox constructor. Needs to get passed a name, plus a
         Soledad instance.
@@ -91,9 +92,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param soledad: a Soledad instance.
         :type soledad: Soledad
 
-        :param rw: read-and-write flags
+        :param memstore: a MemoryStore instance
+        :type memstore: MemoryStore
+
+        :param rw: read-and-write flag for this mailbox
         :type rw: int
         """
+        print "got memstore: ", memstore
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
@@ -105,9 +110,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self.rw = rw
 
         self._soledad = soledad
+        self._memstore = memstore
 
         self.messages = MessageCollection(
-            mbox=mbox, soledad=self._soledad)
+            mbox=mbox, soledad=self._soledad, memstore=self._memstore)
 
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
@@ -231,7 +237,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             # XXX It looks like it has been corrupted.
             # We need to be able to survive this.
             return None
-        return mbox.content.get(self.LAST_UID_KEY, 1)
+        last = mbox.content.get(self.LAST_UID_KEY, 1)
+        if self._memstore:
+            last = max(last, self._memstore.get_last_uid(mbox))
+        return last
 
     def _set_last_uid(self, uid):
         """
@@ -259,6 +268,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             value = count
 
         mbox.content[key] = value
+        # XXX this should be set in the memorystore instead!!!
         self._soledad.put_doc(mbox)
 
     last_uid = property(
@@ -532,12 +542,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # can treat them all the same.
         # Change this to the flag that twisted expects when we
         # switch to content-hash based index + local UID table.
+        print
+        print "FETCHING..."
 
         sequence = False
         #sequence = True if uid == 0 else False
 
         messages_asked = self._bound_seq(messages_asked)
+        print "asked: ", messages_asked
         seq_messg = self._filter_msg_seq(messages_asked)
+
+        print "seq: ", seq_messg
         getmsg = lambda uid: self.messages.get_msg_by_uid(uid)
 
         # for sequence numbers (uid = 0)
@@ -769,36 +784,41 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         uid_next = self.getUIDNext()
         msg = messageObject
 
-        # XXX DEBUG ----------------------------------------
-        #print "copying MESSAGE from %s (%s) to %s (%s)" % (
-            #msg._mbox, msg._uid, self.mbox, uid_next)
-
         # XXX should use a public api instead
         fdoc = msg._fdoc
+        hdoc = msg._hdoc
         if not fdoc:
             logger.debug("Tried to copy a MSG with no fdoc")
             return
 
+        #old_mbox = fdoc.content[self.MBOX_KEY]
+        #old_uid = fdoc.content[self.UID_KEY]
+        #old_key = old_mbox, old_uid
+        #print "copying from OLD MBOX ", old_mbox
+
+        # XXX bit doubt... to duplicate in memory
+        # or not to...?
+        # I think it should be ok to duplicate as long as we're
+        # careful at the hour of writes...
+        # We could use also proxies, but it will break when
+        # the original mailbox is flushed.
+
+        # XXX DEBUG ----------------------------------------
+        #print "copying MESSAGE from %s (%s) to %s (%s)" % (
+            #msg._mbox, msg._uid, self.mbox, uid_next)
+
         new_fdoc = copy.deepcopy(fdoc.content)
         new_fdoc[self.UID_KEY] = uid_next
         new_fdoc[self.MBOX_KEY] = self.mbox
-        self._do_add_doc(new_fdoc)
+        self._memstore.put(self.mbox, uid_next, MessageDict(
+            new_fdoc, hdoc.content))
 
-        # XXX should use a public api instead
-        hdoc = msg._hdoc
-        self.messages.add_hdocset_docid(hdoc.doc_id)
+        # XXX use memory store
+        if hasattr(hdoc, 'doc_id'):
+            self.messages.add_hdocset_docid(hdoc.doc_id)
 
         deferLater(reactor, 1, self.notify_new)
 
-    def _do_add_doc(self, doc):
-        """
-        Defer the adding of a new doc.
-
-        :param doc: document to be created in soledad.
-        :type doc: dict
-        """
-        self._soledad.create_doc(doc)
-
     # convenience fun
 
     def deleteAllDocs(self):
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
new file mode 100644
index 0000000..b8829e0
--- /dev/null
+++ b/src/leap/mail/imap/memorystore.py
@@ -0,0 +1,478 @@
+# -*- coding: utf-8 -*-
+# memorystore.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+In-memory transient store for a LEAPIMAPServer.
+"""
+import contextlib
+import logging
+import weakref
+
+from collections import namedtuple
+
+from twisted.internet.task import LoopingCall
+from zope.interface import implements
+
+from leap.mail import size
+from leap.mail.messageflow import MessageProducer
+from leap.mail.messageparts import MessagePartType
+from leap.mail.imap import interfaces
+from leap.mail.imap.fields import fields
+
+logger = logging.getLogger(__name__)
+
+
+"""
+A MessagePartDoc is a light wrapper around the dictionary-like
+data that we pass along for message parts. It can be used almost everywhere
+that you would expect a SoledadDocument, since it has a dict under the
+`content` attribute.
+
+We also keep some metadata on it, relative in part to the message as a whole,
+and sometimes to a part in particular only.
+
+* `new` indicates that the document has just been created. SoledadStore
+  should just create a new doc for all the related message parts.
+* `store` indicates the type of store a given MessagePartDoc lives in.
+  We currently use this to indicate that  the document comes from memeory,
+  but we should probably get rid of it as soon as we extend the use of the
+  SoledadStore interface along LeapMessage, MessageCollection and Mailbox.
+* `part` is one of the MessagePartType enums.
+
+* `dirty` indicates that, while we already have the document in Soledad,
+  we have modified its state in memory, so we need to put_doc instead while
+  dumping the MemoryStore contents.
+  `dirty` attribute would only apply to flags-docs and linkage-docs.
+
+
+  XXX this is still not implemented!
+
+"""
+
+MessagePartDoc = namedtuple(
+    'MessagePartDoc',
+    ['new', 'dirty', 'part', 'store', 'content'])
+
+
+class ReferenciableDict(dict):
+    """
+    A dict that can be weak-referenced.
+
+    Some builtin objects are not weak-referenciable unless
+    subclassed. So we do.
+
+    Used to return pointers to the items in the MemoryStore.
+    """
+
+
+class MessageWrapper(object):
+    """
+    A simple nested dictionary container around the different message subparts.
+    """
+    implements(interfaces.IMessageContainer)
+
+    FDOC = "fdoc"
+    HDOC = "hdoc"
+    CDOCS = "cdocs"
+
+    # XXX can use this to limit the memory footprint,
+    # or is it too premature to optimize?
+    # Does it work well together with the interfaces.implements?
+
+    #__slots__ = ["_dict", "_new", "_dirty", "memstore"]
+
+    def __init__(self, fdoc=None, hdoc=None, cdocs=None,
+                 from_dict=None, memstore=None,
+                 new=True, dirty=False):
+        self._dict = {}
+
+        self._new = new
+        self._dirty = dirty
+        self.memstore = memstore
+
+        if from_dict is not None:
+            self.from_dict(from_dict)
+        else:
+            if fdoc is not None:
+                self._dict[self.FDOC] = ReferenciableDict(fdoc)
+            if hdoc is not None:
+                self._dict[self.HDOC] = ReferenciableDict(hdoc)
+            if cdocs is not None:
+                self._dict[self.CDOCS] = ReferenciableDict(cdocs)
+
+    # properties
+
+    @property
+    def new(self):
+        return self._new
+
+    def set_new(self, value=True):
+        self._new = value
+
+    @property
+    def dirty(self):
+        return self._dirty
+
+    def set_dirty(self, value=True):
+        self._dirty = value
+
+    # IMessageContainer
+
+    @property
+    def fdoc(self):
+        _fdoc = self._dict.get(self.FDOC, None)
+        if _fdoc:
+            content_ref = weakref.proxy(_fdoc)
+        else:
+            logger.warning("NO FDOC!!!")
+            content_ref = {}
+        return MessagePartDoc(new=self.new, dirty=self.dirty,
+                              store=self._storetype,
+                              part=MessagePartType.fdoc,
+                              content=content_ref)
+
+    @property
+    def hdoc(self):
+        _hdoc = self._dict.get(self.HDOC, None)
+        if _hdoc:
+            content_ref = weakref.proxy(_hdoc)
+        else:
+            logger.warning("NO HDOC!!!!")
+            content_ref = {}
+        return MessagePartDoc(new=self.new, dirty=self.dirty,
+                              store=self._storetype,
+                              part=MessagePartType.hdoc,
+                              content=content_ref)
+
+    @property
+    def cdocs(self):
+        _cdocs = self._dict.get(self.CDOCS, None)
+        if _cdocs:
+            return weakref.proxy(_cdocs)
+        else:
+            return {}
+
+    def walk(self):
+        """
+        Generator that iterates through all the parts, returning
+        MessagePartDoc.
+        """
+        yield self.fdoc
+        yield self.hdoc
+        for cdoc in self.cdocs.values():
+            # XXX this will break ----
+            content_ref = weakref.proxy(cdoc)
+            yield MessagePartDoc(new=self.new, dirty=self.dirty,
+                                 store=self._storetype,
+                                 part=MessagePartType.cdoc,
+                                 content=content_ref)
+
+    # i/o
+
+    def as_dict(self):
+        """
+        Return a dict representation of the parts contained.
+        """
+        return self._dict
+
+    def from_dict(self, msg_dict):
+        """
+        Populate MessageWrapper parts from a dictionary.
+        It expects the same format that we use in a
+        MessageWrapper.
+        """
+        fdoc, hdoc, cdocs = map(
+            lambda part: msg_dict.get(part, None),
+            [self.FDOC, self.HDOC, self.CDOCS])
+        self._dict[self.FDOC] = fdoc
+        self._dict[self.HDOC] = hdoc
+        self._dict[self.CDOCS] = cdocs
+
+
+@contextlib.contextmanager
+def set_bool_flag(obj, att):
+    """
+    Set a boolean flag to True while we're doing our thing.
+    Just to let the world know.
+    """
+    setattr(obj, att, True)
+    try:
+        yield True
+    except RuntimeError as exc:
+        logger.exception(exc)
+    finally:
+        setattr(obj, att, False)
+
+
+class MemoryStore(object):
+    """
+    An in-memory store to where we can write the different parts that
+    we split the messages into and buffer them until we write them to the
+    permanent storage.
+
+    It uses MessageWrapper instances to represent the message-parts, which are
+    indexed by mailbox name and UID.
+
+    It also can be passed a permanent storage as a paremeter (any implementor
+    of IMessageStore, in this case a SoledadStore). In this case, a periodic
+    dump of the messages stored in memory will be done. The period of the
+    writes to the permanent storage is controled by the write_period parameter
+    in the constructor.
+    """
+    implements(interfaces.IMessageStore)
+    implements(interfaces.IMessageStoreWriter)
+
+    producer = None
+
+    # TODO We will want to index by chash when we transition to local-only
+    # UIDs.
+    # TODO should store RECENT-FLAGS too
+    # TODO should store HDOCSET too (use weakrefs!) -- will need to subclass
+    # TODO do use dirty flag (maybe use namedtuples for that) so we can use it
+    # also as a read-cache.
+
+    WRITING_FLAG = "_writing"
+
+    def __init__(self, permanent_store=None, write_period=60):
+        """
+        Initialize a MemoryStore.
+
+        :param permanent_store: a IMessageStore implementor to dump
+                                messages to.
+        :type permanent_store: IMessageStore
+        :param write_period: the interval to dump messages to disk, in seconds.
+        :type write_period: int
+        """
+        self._permanent_store = permanent_store
+        self._write_period = write_period
+
+        # Internal Storage
+        self._msg_store = {}
+        self._phash_store = {}
+
+        # TODO ----------------- implement mailbox-level flags store too! ----
+        self._rflags_store = {}
+        self._hdocset_store = {}
+        # TODO ----------------- implement mailbox-level flags store too! ----
+
+        # New and dirty flags, to set MessageWrapper State.
+        self._new = set([])
+        self._dirty = set([])
+
+        # Flag for signaling we're busy writing to the disk storage.
+        setattr(self, self.WRITING_FLAG, False)
+
+        if self._permanent_store is not None:
+            # this producer spits its messages to the permanent store
+            # consumer using a queue. We will use that to put
+            # our messages to be written.
+            self.producer = MessageProducer(permanent_store,
+                                            period=0.1)
+            # looping call for dumping to SoledadStore
+            self._write_loop = LoopingCall(self.write_messages,
+                                           permanent_store)
+
+            # We can start the write loop right now, why wait?
+            self._start_write_loop()
+
+    def _start_write_loop(self):
+        """
+        Start loop for writing to disk database.
+        """
+        if not self._write_loop.running:
+            self._write_loop.start(self._write_period, now=True)
+
+    def _stop_write_loop(self):
+        """
+        Stop loop for writing to disk database.
+        """
+        if self._write_loop.running:
+            self._write_loop.stop()
+
+    # IMessageStore
+
+    # XXX this would work well for whole message operations.
+    # We would have to add a put_flags operation to modify only
+    # the flags doc (and set the dirty flag accordingly)
+
+    def create_message(self, mbox, uid, message):
+        """
+        Create the passed message into this MemoryStore.
+
+        By default we consider that any message is a new message.
+        """
+        print "adding new doc to memstore %s (%s)" % (mbox, uid)
+        key = mbox, uid
+        self._new.add(key)
+
+        msg_dict = message.as_dict()
+        self._msg_store[key] = msg_dict
+
+        cdocs = message.cdocs
+
+        dirty = key in self._dirty
+        new = key in self._new
+
+        # XXX should capture this in log...
+
+        for cdoc_key in cdocs.keys():
+            print "saving cdoc"
+            cdoc = self._msg_store[key]['cdocs'][cdoc_key]
+
+            # XXX this should be done in the MessageWrapper constructor
+            # instead...
+            # first we make it weak-referenciable
+            referenciable_cdoc = ReferenciableDict(cdoc)
+            self._msg_store[key]['cdocs'][cdoc_key] = MessagePartDoc(
+                new=new, dirty=dirty, store="mem",
+                part=MessagePartType.cdoc,
+                content=referenciable_cdoc)
+            phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None)
+            if not phash:
+                continue
+            self._phash_store[phash] = weakref.proxy(referenciable_cdoc)
+
+    def put_message(self, mbox, uid, msg):
+        """
+        Put an existing message.
+        """
+        return NotImplementedError()
+
+    def get_message(self, mbox, uid):
+        """
+        Get a MessageWrapper for the given mbox and uid combination.
+
+        :return: MessageWrapper or None
+        """
+        key = mbox, uid
+        msg_dict = self._msg_store.get(key, None)
+        if msg_dict:
+            new, dirty = self._get_new_dirty_state(key)
+            return MessageWrapper(from_dict=msg_dict,
+                                  memstore=weakref.proxy(self))
+        else:
+            return None
+
+    def remove_message(self, mbox, uid):
+        """
+        Remove a Message from this MemoryStore.
+        """
+        raise NotImplementedError()
+
+    # IMessageStoreWriter
+
+    def write_messages(self, store):
+        """
+        Write the message documents in this MemoryStore to a different store.
+        """
+        # XXX pass if it's writing (ie, the queue is not empty...)
+        # See how to make the writing_flag aware of the queue state...
+        print "writing messages to producer..."
+
+        with set_bool_flag(self, self.WRITING_FLAG):
+            for msg_wrapper in self.all_msg_iter():
+                self.producer.push(msg_wrapper)
+
+    # MemoryStore specific methods.
+
+    def get_uids(self, mbox):
+        """
+        Get all uids for a given mbox.
+        """
+        all_keys = self._msg_store.keys()
+        return [uid for m, uid in all_keys if m == mbox]
+
+    def get_last_uid(self, mbox):
+        """
+        Get the highest UID for a given mbox.
+        """
+        # XXX should get from msg_store keys instead!
+        if not self._new:
+            return 0
+        return max(self.get_uids(mbox))
+
+    def count_new_mbox(self, mbox):
+        """
+        Count the new messages by inbox.
+        """
+        return len([(m, uid) for m, uid in self._new if mbox == mbox])
+
+    def count_new(self):
+        """
+        Count all the new messages in the MemoryStore.
+        """
+        return len(self._new)
+
+    def get_by_phash(self, phash):
+        """
+        Return a content-document by its payload-hash.
+        """
+        doc = self._phash_store.get(phash, None)
+
+        # XXX have to keep a mapping between phash and its linkage
+        # info, to know if this payload is been already saved or not.
+        # We will be able to get this from the linkage-docs,
+        # not yet implemented.
+        new = True
+        dirty = False
+        return MessagePartDoc(
+            new=new, dirty=dirty, store="mem",
+            part=MessagePartType.cdoc,
+            content=doc)
+
+    def all_msg_iter(self):
+        """
+        Return generator that iterates through all messages in the store.
+        """
+        return (self.get_message(*key)
+                for key in sorted(self._msg_store.keys()))
+
+    def _get_new_dirty_state(self, key):
+        """
+        Return `new` and `dirty` flags for a given message.
+        """
+        return map(lambda _set: key in _set, (self._new, self._dirty))
+
+    @property
+    def is_writing(self):
+        """
+        Property that returns whether the store is currently writing its
+        internal state to a permanent storage.
+
+        Used to evaluate whether the CHECK command can inform that the field
+        is clear to proceed, or waiting for the write operations to complete
+        is needed instead.
+
+        :rtype: bool
+        """
+        # XXX this should probably return a deferred !!!
+        return getattr(self, self.WRITING_FLAG)
+
+    def put_part(self, part_type, value):
+        """
+        Put the passed part into this IMessageStore.
+        `part` should be one of: fdoc, hdoc, cdoc
+        """
+        # XXX turn that into a enum
+
+    # Memory management.
+
+    def get_size(self):
+        """
+        Return the size of the internal storage.
+        Use for calculating the limit beyond which we should flush the store.
+        """
+        return size.get_size(self._msg_store)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 34304ea..ef0b0a1 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -42,6 +42,7 @@ from leap.mail.utils import first, find_charset
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
+from leap.mail.imap.memorystore import MessageDict
 from leap.mail.imap.parser import MailParser, MBoxParser
 from leap.mail.messageflow import IMessageConsumer
 
@@ -49,11 +50,20 @@ logger = logging.getLogger(__name__)
 
 # TODO ------------------------------------------------------------
 
+# [ ] Add ref to incoming message during add_msg
 # [ ] Add linked-from info.
 # [ ] Delete incoming mail only after successful write!
 # [ ] Remove UID from syncable db. Store only those indexes locally.
 
 
+# XXX no longer needed, since i'm using proxies instead of direct weakrefs
+def maybe_call(thing):
+    """
+    Return the same thing, or the result of its invocation if it is a callable.
+    """
+    return thing() if callable(thing) else thing
+
+
 def lowerdict(_dict):
     """
     Return a dict with the keys in lowercase.
@@ -333,7 +343,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     implements(imap4.IMessage)
 
-    def __init__(self, soledad, uid, mbox, collection=None):
+    def __init__(self, soledad, uid, mbox, collection=None, container=None):
         """
         Initializes a LeapMessage.
 
@@ -345,12 +355,15 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :type mbox: basestring
         :param collection: a reference to the parent collection object
         :type collection: MessageCollection
+        :param container: a IMessageContainer implementor instance
+        :type container: IMessageContainer
         """
         MailParser.__init__(self)
         self._soledad = soledad
         self._uid = int(uid)
         self._mbox = self._parse_mailbox_name(mbox)
         self._collection = collection
+        self._container = container
 
         self.__chash = None
         self.__bdoc = None
@@ -361,12 +374,28 @@ class LeapMessage(fields, MailParser, MBoxParser):
         An accessor to the flags document.
         """
         if all(map(bool, (self._uid, self._mbox))):
-            fdoc = self._get_flags_doc()
+            fdoc = None
+            if self._container is not None:
+                fdoc = self._container.fdoc
+            if not fdoc:
+                fdoc = self._get_flags_doc()
             if fdoc:
-                self.__chash = fdoc.content.get(
+                fdoc_content = maybe_call(fdoc.content)
+                self.__chash = fdoc_content.get(
                     fields.CONTENT_HASH_KEY, None)
             return fdoc
 
+    @property
+    def _hdoc(self):
+        """
+        An accessor to the headers document.
+        """
+        if self._container is not None:
+            hdoc = self._container.hdoc
+            if hdoc:
+                return hdoc
+        return self._get_headers_doc()
+
     @property
     def _chash(self):
         """
@@ -375,17 +404,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
         if not self._fdoc:
             return None
         if not self.__chash and self._fdoc:
-            self.__chash = self._fdoc.content.get(
+            self.__chash = maybe_call(self._fdoc.content).get(
                 fields.CONTENT_HASH_KEY, None)
         return self.__chash
 
-    @property
-    def _hdoc(self):
-        """
-        An accessor to the headers document.
-        """
-        return self._get_headers_doc()
-
     @property
     def _bdoc(self):
         """
@@ -422,7 +444,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         flags = []
         fdoc = self._fdoc
         if fdoc:
-            flags = fdoc.content.get(self.FLAGS_KEY, None)
+            flags = maybe_call(fdoc.content).get(self.FLAGS_KEY, None)
 
         msgcol = self._collection
 
@@ -449,6 +471,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: a SoledadDocument instance
         :rtype: SoledadDocument
         """
+        # XXX use memory store ...!
+
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         log.msg('setting flags: %s (%s)' % (self._uid, flags))
 
@@ -461,7 +485,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
         doc.content[self.FLAGS_KEY] = flags
         doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
         doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
-        self._soledad.put_doc(doc)
+
+        if getattr(doc, 'store', None) != "mem":
+            self._soledad.put_doc(doc)
 
     def addFlags(self, flags):
         """
@@ -521,18 +547,26 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         # TODO refactor with getBodyFile in MessagePart
         fd = StringIO.StringIO()
-        bdoc = self._bdoc
-        if bdoc:
-            body = self._bdoc.content.get(self.RAW_KEY, "")
-            content_type = bdoc.content.get('content-type', "")
+        if self._bdoc is not None:
+            bdoc_content = self._bdoc.content
+            body = bdoc_content.get(self.RAW_KEY, "")
+            content_type = bdoc_content.get('content-type', "")
             charset = find_charset(content_type)
+            logger.debug('got charset from content-type: %s' % charset)
             if charset is None:
                 charset = self._get_charset(body)
             try:
                 body = body.encode(charset)
             except UnicodeError as e:
-                logger.error("Unicode error, using 'replace'. {0!r}".format(e))
-                body = body.encode(charset, 'replace')
+                logger.error("Unicode error {0}".format(e))
+                logger.debug("Attempted to encode with: %s" % charset)
+                try:
+                    body = body.encode(charset, 'replace')
+                except UnicodeError as e:
+                    try:
+                        body = body.encode('utf-8', 'replace')
+                    except:
+                        pass
 
         # We are still returning funky characters from here.
         else:
@@ -567,7 +601,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         size = None
         if self._fdoc:
-            size = self._fdoc.content.get(self.SIZE_KEY, False)
+            fdoc_content = maybe_call(self._fdoc.content)
+            size = fdoc_content.get(self.SIZE_KEY, False)
         else:
             logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
                                                        self._uid))
@@ -632,7 +667,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the headers dict for this message.
         """
         if self._hdoc is not None:
-            headers = self._hdoc.content.get(self.HEADERS_KEY, {})
+            hdoc_content = maybe_call(self._hdoc.content)
+            headers = hdoc_content.get(self.HEADERS_KEY, {})
             return headers
 
         else:
@@ -646,7 +682,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return True if this message is multipart.
         """
         if self._fdoc:
-            is_multipart = self._fdoc.content.get(self.MULTIPART_KEY, False)
+            fdoc_content = maybe_call(self._fdoc.content)
+            is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
             return is_multipart
         else:
             logger.warning(
@@ -688,7 +725,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
             logger.warning("Tried to get part but no HDOC found!")
             return None
 
-        pmap = self._hdoc.content.get(fields.PARTS_MAP_KEY, {})
+        hdoc_content = maybe_call(self._hdoc.content)
+        pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
         return pmap[str(part)]
 
     def _get_flags_doc(self):
@@ -724,16 +762,33 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the document that keeps the body for this
         message.
         """
-        body_phash = self._hdoc.content.get(
+        hdoc_content = maybe_call(self._hdoc.content)
+        body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
         if not body_phash:
             logger.warning("No body phash for this document!")
             return None
-        body_docs = self._soledad.get_from_index(
-            fields.TYPE_P_HASH_IDX,
-            fields.TYPE_CONTENT_VAL, str(body_phash))
 
-        return first(body_docs)
+        # XXX get from memstore too...
+        # if memstore: memstore.get_phrash
+        # memstore should keep a dict with weakrefs to the
+        # phash doc...
+
+        if self._container is not None:
+            bdoc = self._container.memstore.get_by_phash(body_phash)
+            if bdoc:
+                return bdoc
+            else:
+                print "no doc for that phash found!"
+
+        # no memstore or no doc found there
+        if self._soledad:
+            body_docs = self._soledad.get_from_index(
+                fields.TYPE_P_HASH_IDX,
+                fields.TYPE_CONTENT_VAL, str(body_phash))
+            return first(body_docs)
+        else:
+            logger.error("No phash in container, and no soledad found!")
 
     def __getitem__(self, key):
         """
@@ -746,7 +801,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: The content value indexed by C{key} or None
         :rtype: str
         """
-        return self._fdoc.content.get(key, None)
+        return maybe_call(self._fdoc.content).get(key, None)
 
     # setters
 
@@ -790,6 +845,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         # until we think about a good way of deorphaning.
         # Maybe a crawler of unreferenced docs.
 
+        # XXX remove from memory store!!!
+
         # XXX implement elijah's idea of using a PUT document as a
         # token to ensure consistency in the removal.
 
@@ -957,7 +1014,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     """
     A collection of messages, surprisingly.
 
-    It is tied to a selected mailbox name that is passed to constructor.
+    It is tied to a selected mailbox name that is passed to its constructor.
     Implements a filter query over the messages contained in a soledad
     database.
     """
@@ -1058,7 +1115,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     _hdocset_lock = threading.Lock()
     _hdocset_property_lock = threading.Lock()
 
-    def __init__(self, mbox=None, soledad=None):
+    def __init__(self, mbox=None, soledad=None, memstore=None):
         """
         Constructor for MessageCollection.
 
@@ -1068,13 +1125,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         MessageCollection for each mailbox, instead of treating them
         as a property of each message.
 
+        We are passed an instance of MemoryStore, the same for the
+        SoledadBackedAccount, that we use as a read cache and a buffer
+        for writes.
+
         :param mbox: the name of the mailbox. It is the name
                      with which we filter the query over the
-                     messages database
+                     messages database.
         :type mbox: str
-
         :param soledad: Soledad database
         :type soledad: Soledad instance
+        :param memstore: a MemoryStore instance
+        :type memstore: MemoryStore
         """
         MailParser.__init__(self)
         leap_assert(mbox, "Need a mailbox name to initialize")
@@ -1086,6 +1148,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         # okay, all in order, keep going...
         self.mbox = self._parse_mailbox_name(mbox)
         self._soledad = soledad
+        self._memstore = memstore
+
         self.__rflags = None
         self.__hdocset = None
         self.initialize_db()
@@ -1241,6 +1305,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         # when all the processing is done.
 
         # TODO add the linked-from info !
+        # TODO add reference to the original message
 
         logger.debug('adding message')
         if flags is None:
@@ -1273,24 +1338,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
             hd[key] = parts_map[key]
         del parts_map
 
-        # Saving ----------------------------------------
-        self.set_recent_flag(uid)
+        # The MessageContainer expects a dict, zero-indexed
+        # XXX review-me
+        cdocs = dict((index, doc) for index, doc in
+                     enumerate(walk.get_raw_docs(msg, parts)))
+        print "cdocs is", cdocs
 
-        # first, regular docs: flags and headers
-        self._soledad.create_doc(fd)
+        # Saving ----------------------------------------
         # XXX should check for content duplication on headers too
         # but with chash. !!!
-        hdoc = self._soledad.create_doc(hd)
+
+        # XXX adapt hdocset to use memstore
+        #hdoc = self._soledad.create_doc(hd)
         # We add the newly created hdoc to the fast-access set of
         # headers documents associated with the mailbox.
-        self.add_hdocset_docid(hdoc.doc_id)
+        #self.add_hdocset_docid(hdoc.doc_id)
 
-        # and last, but not least, try to create
-        # content docs if not already there.
-        cdocs = walk.get_raw_docs(msg, parts)
-        for cdoc in cdocs:
-            if not self._content_does_exist(cdoc):
-                self._soledad.create_doc(cdoc)
+        # XXX move to memory store too
+        # self.set_recent_flag(uid)
+
+        # TODO ---- add reference to original doc, to be deleted
+        # after writes are done.
+        msg_container = MessageDict(fd, hd, cdocs)
+        self._memstore.put(self.mbox, uid, msg_container)
 
     def _remove_cb(self, result):
         return result
@@ -1321,6 +1391,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
     #
 
     # recent flags
+    # XXX FIXME -------------------------------------
+    # This should be rewritten to use memory store.
 
     def _get_recent_flags(self):
         """
@@ -1390,6 +1462,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
     # headers-docs-set
 
+    # XXX FIXME -------------------------------------
+    # This should be rewritten to use memory store.
+
     def _get_hdocset(self):
         """
         An accessor for the hdocs-set for this mailbox.
@@ -1532,7 +1607,16 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
                  or None if not found.
         :rtype: LeapMessage
         """
-        msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
+        print "getting msg by id!"
+        msg_container = self._memstore.get(self.mbox, uid)
+        print "msg container", msg_container
+        if msg_container is not None:
+            print "getting LeapMessage (from memstore)"
+            msg = LeapMessage(None, uid, self.mbox, collection=self,
+                              container=msg_container)
+            print "got msg:", msg
+        else:
+            msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
         if not msg.does_exist():
             return None
         return msg
@@ -1570,11 +1654,19 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         ascending order.
         """
         # XXX we should get this from the uid table, local-only
-        all_uids = (doc.content[self.UID_KEY] for doc in
-                    self._soledad.get_from_index(
-                        fields.TYPE_MBOX_IDX,
-                        fields.TYPE_FLAGS_VAL, self.mbox))
-        return (u for u in sorted(all_uids))
+        # XXX FIXME -------------
+        # This should be cached in the memstoretoo
+        db_uids = set([doc.content[self.UID_KEY] for doc in
+                       self._soledad.get_from_index(
+                           fields.TYPE_MBOX_IDX,
+                           fields.TYPE_FLAGS_VAL, self.mbox)])
+        if self._memstore is not None:
+            mem_uids = self._memstore.get_uids(self.mbox)
+            uids = db_uids.union(set(mem_uids))
+        else:
+            uids = db_uids
+
+        return (u for u in sorted(uids))
 
     def reset_last_uid(self, param):
         """
@@ -1592,12 +1684,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         """
         Return a dict with all flags documents for this mailbox.
         """
+        # XXX get all from memstore and cahce it there
         all_flags = dict(((
             doc.content[self.UID_KEY],
             doc.content[self.FLAGS_KEY]) for doc in
             self._soledad.get_from_index(
                 fields.TYPE_MBOX_IDX,
                 fields.TYPE_FLAGS_VAL, self.mbox)))
+        if self._memstore is not None:
+            # XXX
+            uids = self._memstore.get_uids(self.mbox)
+            fdocs = [(uid, self._memstore.get(self.mbox, uid).fdoc)
+                     for uid in uids]
+            for uid, doc in fdocs:
+                all_flags[uid] = doc.content[self.FLAGS_KEY]
+
         return all_flags
 
     def all_flags_chash(self):
@@ -1630,9 +1731,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
 
         :rtype: int
         """
+        # XXX We could cache this in memstore too until next write...
         count = self._soledad.get_count_from_index(
             fields.TYPE_MBOX_IDX,
             fields.TYPE_FLAGS_VAL, self.mbox)
+        if self._memstore is not None:
+            count += self._memstore.count_new()
         return count
 
     # unseen messages
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index ad22da6..71b9950 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -36,6 +36,7 @@ from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import KeyManager
 from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
+from leap.mail.imap.memorystore import MemoryStore
 from leap.soledad.client import Soledad
 
 # The default port in which imap service will run
@@ -69,6 +70,8 @@ except Exception:
 ######################################################
 
 
+# TODO move this to imap.server
+
 class LeapIMAPServer(imap4.IMAP4Server):
     """
     An IMAP4 Server with mailboxes backed by soledad
@@ -256,11 +259,15 @@ class LeapIMAPFactory(ServerFactory):
         self._uuid = uuid
         self._userid = userid
         self._soledad = soledad
+        self._memstore = MemoryStore()
 
         theAccount = SoledadBackedAccount(
-            uuid, soledad=soledad)
+            uuid, soledad=soledad,
+            memstore=self._memstore)
         self.theAccount = theAccount
 
+        # XXX how to pass the store along?
+
     def buildProtocol(self, addr):
         "Return a protocol suitable for the job."
         imapProtocol = LeapIMAPServer(
@@ -323,3 +330,14 @@ def run_service(*args, **kwargs):
 
     # not ok, signal error.
     leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port))
+
+    def checkpoint(self):
+        """
+        Called when the client issues a CHECK command.
+
+        This should perform any checkpoint operations required by the server.
+        It may be a long running operation, but may not block.  If it returns
+        a deferred, the client will only be informed of success (or failure)
+        when the deferred's callback (or errback) is invoked.
+        """
+        return None
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index ac26e45..ed6abcd 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -25,12 +25,15 @@ from zope.interface import Interface, implements
 
 
 class IMessageConsumer(Interface):
+    """
+    I consume messages from a queue.
+    """
 
     def consume(self, queue):
         """
         Consumes the passed item.
 
-        :param item: q queue where we put the object to be consumed.
+        :param item: a queue where we put the object to be consumed.
         :type item: object
         """
         # TODO we could add an optional type to be passed
@@ -40,6 +43,28 @@ class IMessageConsumer(Interface):
         # the queue, maybe wrapped in an object with a retries attribute.
 
 
+class IMessageProducer(Interface):
+    """
+    I produce messages and put them in a store to be consumed by other
+    entities.
+    """
+
+    def push(self, item):
+        """
+        Push a new item in the queue.
+        """
+
+    def start(self):
+        """
+        Start producing items.
+        """
+
+    def stop(self):
+        """
+        Stop producing items.
+        """
+
+
 class DummyMsgConsumer(object):
 
     implements(IMessageConsumer)
@@ -62,6 +87,8 @@ class MessageProducer(object):
     deferred chain and leave further processing detached from the calling loop,
     as in the case of smtp.
     """
+    implements(IMessageProducer)
+
     # TODO this can be seen as a first step towards properly implementing
     # components that implement IPushProducer / IConsumer  interfaces.
     # However, I need to think more about how to pause the streaming.
@@ -92,7 +119,7 @@ class MessageProducer(object):
 
     def _check_for_new(self):
         """
-        Checks for new items in the internal queue, and calls the consume
+        Check for new items in the internal queue, and calls the consume
         method in the consumer.
 
         If the queue is found empty, the loop is stopped. It will be started
@@ -102,11 +129,11 @@ class MessageProducer(object):
         if self._queue.empty():
             self.stop()
 
-    # public methods
+    # public methods: IMessageProducer
 
-    def put(self, item):
+    def push(self, item):
         """
-        Puts a new item in the queue.
+        Push a new item in the queue.
 
         If the queue was empty, we will start the loop again.
         """
@@ -117,7 +144,7 @@ class MessageProducer(object):
 
     def start(self):
         """
-        Starts polling for new items.
+        Start polling for new items.
         """
         if not self._loop.running:
             self._loop.start(self._period, now=True)
diff --git a/src/leap/mail/size.py b/src/leap/mail/size.py
new file mode 100644
index 0000000..4880d71
--- /dev/null
+++ b/src/leap/mail/size.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# size.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Recursively get size of objects.
+"""
+from gc import collect
+from itertools import chain
+from sys import getsizeof
+
+
+def _get_size(item, seen):
+    known_types = {dict: lambda d: chain.from_iterable(d.items())}
+    default_size = getsizeof(0)
+
+    def size_walk(item):
+        if id(item) in seen:
+            return 0
+        seen.add(id(item))
+        s = getsizeof(item, default_size)
+        for _type, fun in known_types.iteritems():
+            if isinstance(item, _type):
+                s += sum(map(size_walk, fun(item)))
+                break
+        return s
+
+    return size_walk(item)
+
+
+def get_size(item):
+    """
+    Return the cumulative size of a given object.
+
+    Currently it supports only dictionaries, and seemingly leaks
+    some memory, so use with care.
+
+    :param item: the item which size wants to be computed
+    """
+    seen = set()
+    size = _get_size(item, seen)
+    #print "len(seen) ", len(seen)
+    del seen
+    collect()
+    return size
-- 
cgit v1.2.3


From 23b2a2c10ddd6dbb15e3f532d526ac0a53bd788b Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 00:27:19 -0400
Subject: move server to its own file

---
 src/leap/mail/imap/server.py       | 199 +++++++++++++++++++++++++++++++++++++
 src/leap/mail/imap/service/imap.py | 180 +--------------------------------
 2 files changed, 202 insertions(+), 177 deletions(-)
 create mode 100644 src/leap/mail/imap/server.py

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
new file mode 100644
index 0000000..8bd875b
--- /dev/null
+++ b/src/leap/mail/imap/server.py
@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+# server.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Leap IMAP4 Server Implementation.
+"""
+from copy import copy
+
+from twisted import cred
+from twisted.internet.defer import maybeDeferred
+from twisted.internet.task import deferLater
+from twisted.mail import imap4
+from twisted.python import log
+
+from leap.common import events as leap_events
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN
+from leap.soledad.client import Soledad
+
+
+class LeapIMAPServer(imap4.IMAP4Server):
+    """
+    An IMAP4 Server with mailboxes backed by soledad
+    """
+    def __init__(self, *args, **kwargs):
+        # pop extraneous arguments
+        soledad = kwargs.pop('soledad', None)
+        uuid = kwargs.pop('uuid', None)
+        userid = kwargs.pop('userid', None)
+        leap_assert(soledad, "need a soledad instance")
+        leap_assert_type(soledad, Soledad)
+        leap_assert(uuid, "need a user in the initialization")
+
+        self._userid = userid
+
+        # initialize imap server!
+        imap4.IMAP4Server.__init__(self, *args, **kwargs)
+
+        # we should initialize the account here,
+        # but we move it to the factory so we can
+        # populate the test account properly (and only once
+        # per session)
+
+    def lineReceived(self, line):
+        """
+        Attempt to parse a single line from the server.
+
+        :param line: the line from the server, without the line delimiter.
+        :type line: str
+        """
+        if self.theAccount.closed is True and self.state != "unauth":
+            log.msg("Closing the session. State: unauth")
+            self.state = "unauth"
+
+        if "login" in line.lower():
+            # avoid to log the pass, even though we are using a dummy auth
+            # by now.
+            msg = line[:7] + " [...]"
+        else:
+            msg = copy(line)
+        log.msg('rcv (%s): %s' % (self.state, msg))
+        imap4.IMAP4Server.lineReceived(self, line)
+
+    def authenticateLogin(self, username, password):
+        """
+        Lookup the account with the given parameters, and deny
+        the improper combinations.
+
+        :param username: the username that is attempting authentication.
+        :type username: str
+        :param password: the password to authenticate with.
+        :type password: str
+        """
+        # XXX this should use portal:
+        # return portal.login(cred.credentials.UsernamePassword(user, pass)
+        if username != self._userid:
+            # bad username, reject.
+            raise cred.error.UnauthorizedLogin()
+        # any dummy password is allowed so far. use realm instead!
+        leap_events.signal(IMAP_CLIENT_LOGIN, "1")
+        return imap4.IAccount, self.theAccount, lambda: None
+
+    def do_FETCH(self, tag, messages, query, uid=0):
+        """
+        Overwritten fetch dispatcher to use the fast fetch_flags
+        method
+        """
+        from twisted.internet import reactor
+        if not query:
+            self.sendPositiveResponse(tag, 'FETCH complete')
+            return  # XXX ???
+
+        cbFetch = self._IMAP4Server__cbFetch
+        ebFetch = self._IMAP4Server__ebFetch
+
+        if len(query) == 1 and str(query[0]) == "flags":
+            self._oldTimeout = self.setTimeout(None)
+            # no need to call iter, we get a generator
+            maybeDeferred(
+                self.mbox.fetch_flags, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(ebFetch, tag)
+        elif len(query) == 1 and str(query[0]) == "rfc822.header":
+            self._oldTimeout = self.setTimeout(None)
+            # no need to call iter, we get a generator
+            maybeDeferred(
+                self.mbox.fetch_headers, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(ebFetch, tag)
+        else:
+            self._oldTimeout = self.setTimeout(None)
+            # no need to call iter, we get a generator
+            maybeDeferred(
+                self.mbox.fetch, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(
+                ebFetch, tag)
+
+        deferLater(reactor,
+                   2, self.mbox.unset_recent_flags, messages)
+        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
+
+    select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
+                    imap4.IMAP4Server.arg_fetchatt)
+
+    def do_COPY(self, tag, messages, mailbox, uid=0):
+        from twisted.internet import reactor
+        imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
+        deferLater(reactor,
+                   2, self.mbox.unset_recent_flags, messages)
+        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
+
+    select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
+                   imap4.IMAP4Server.arg_astring)
+
+    def notifyNew(self, ignored):
+        """
+        Notify new messages to listeners.
+        """
+        self.mbox.notify_new()
+
+    def _cbSelectWork(self, mbox, cmdName, tag):
+        """
+        Callback for selectWork, patched to avoid conformance errors due to
+        incomplete UIDVALIDITY line.
+        """
+        if mbox is None:
+            self.sendNegativeResponse(tag, 'No such mailbox')
+            return
+        if '\\noselect' in [s.lower() for s in mbox.getFlags()]:
+            self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
+            return
+
+        flags = mbox.getFlags()
+        self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS')
+        self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT')
+        self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
+
+        # Patched -------------------------------------------------------
+        # imaptest was complaining about the incomplete line, we're adding
+        # "UIDs valid" here.
+        self.sendPositiveResponse(
+            None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity())
+        # ----------------------------------------------------------------
+
+        s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY'
+        mbox.addListener(self)
+        self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName))
+        self.state = 'select'
+        self.mbox = mbox
+
+    def checkpoint(self):
+        """
+        Called when the client issues a CHECK command.
+
+        This should perform any checkpoint operations required by the server.
+        It may be a long running operation, but may not block.  If it returns
+        a deferred, the client will only be informed of success (or failure)
+        when the deferred's callback (or errback) is invoked.
+        """
+        # TODO return the output of _memstore.is_writing
+        # XXX and that should return a deferred!
+        return None
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 71b9950..3f99da6 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -17,17 +17,11 @@
 """
 Imap service initialization
 """
-from copy import copy
-
 import logging
 
 from twisted.internet.protocol import ServerFactory
-from twisted.internet.defer import maybeDeferred
 from twisted.internet.error import CannotListenError
-from twisted.internet.task import deferLater
 from twisted.mail import imap4
-from twisted.python import log
-from twisted import cred
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +31,7 @@ from leap.keymanager import KeyManager
 from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
 from leap.mail.imap.memorystore import MemoryStore
+from leap.mail.imap.server import LeapIMAPServer
 from leap.soledad.client import Soledad
 
 # The default port in which imap service will run
@@ -48,7 +43,6 @@ INCOMING_CHECK_PERIOD = 60
 
 from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED
 from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START
-from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN
 
 ######################################################
 # Temporary workaround for RecursionLimit when using
@@ -70,163 +64,6 @@ except Exception:
 ######################################################
 
 
-# TODO move this to imap.server
-
-class LeapIMAPServer(imap4.IMAP4Server):
-    """
-    An IMAP4 Server with mailboxes backed by soledad
-    """
-    def __init__(self, *args, **kwargs):
-        # pop extraneous arguments
-        soledad = kwargs.pop('soledad', None)
-        uuid = kwargs.pop('uuid', None)
-        userid = kwargs.pop('userid', None)
-        leap_assert(soledad, "need a soledad instance")
-        leap_assert_type(soledad, Soledad)
-        leap_assert(uuid, "need a user in the initialization")
-
-        self._userid = userid
-
-        # initialize imap server!
-        imap4.IMAP4Server.__init__(self, *args, **kwargs)
-
-        # we should initialize the account here,
-        # but we move it to the factory so we can
-        # populate the test account properly (and only once
-        # per session)
-
-    def lineReceived(self, line):
-        """
-        Attempt to parse a single line from the server.
-
-        :param line: the line from the server, without the line delimiter.
-        :type line: str
-        """
-        if self.theAccount.closed is True and self.state != "unauth":
-            log.msg("Closing the session. State: unauth")
-            self.state = "unauth"
-
-        if "login" in line.lower():
-            # avoid to log the pass, even though we are using a dummy auth
-            # by now.
-            msg = line[:7] + " [...]"
-        else:
-            msg = copy(line)
-        log.msg('rcv (%s): %s' % (self.state, msg))
-        imap4.IMAP4Server.lineReceived(self, line)
-
-    def authenticateLogin(self, username, password):
-        """
-        Lookup the account with the given parameters, and deny
-        the improper combinations.
-
-        :param username: the username that is attempting authentication.
-        :type username: str
-        :param password: the password to authenticate with.
-        :type password: str
-        """
-        # XXX this should use portal:
-        # return portal.login(cred.credentials.UsernamePassword(user, pass)
-        if username != self._userid:
-            # bad username, reject.
-            raise cred.error.UnauthorizedLogin()
-        # any dummy password is allowed so far. use realm instead!
-        leap_events.signal(IMAP_CLIENT_LOGIN, "1")
-        return imap4.IAccount, self.theAccount, lambda: None
-
-    def do_FETCH(self, tag, messages, query, uid=0):
-        """
-        Overwritten fetch dispatcher to use the fast fetch_flags
-        method
-        """
-        from twisted.internet import reactor
-        if not query:
-            self.sendPositiveResponse(tag, 'FETCH complete')
-            return  # XXX ???
-
-        cbFetch = self._IMAP4Server__cbFetch
-        ebFetch = self._IMAP4Server__ebFetch
-
-        if len(query) == 1 and str(query[0]) == "flags":
-            self._oldTimeout = self.setTimeout(None)
-            # no need to call iter, we get a generator
-            maybeDeferred(
-                self.mbox.fetch_flags, messages, uid=uid
-            ).addCallback(
-                cbFetch, tag, query, uid
-            ).addErrback(ebFetch, tag)
-        elif len(query) == 1 and str(query[0]) == "rfc822.header":
-            self._oldTimeout = self.setTimeout(None)
-            # no need to call iter, we get a generator
-            maybeDeferred(
-                self.mbox.fetch_headers, messages, uid=uid
-            ).addCallback(
-                cbFetch, tag, query, uid
-            ).addErrback(ebFetch, tag)
-        else:
-            self._oldTimeout = self.setTimeout(None)
-            # no need to call iter, we get a generator
-            maybeDeferred(
-                self.mbox.fetch, messages, uid=uid
-            ).addCallback(
-                cbFetch, tag, query, uid
-            ).addErrback(
-                ebFetch, tag)
-
-        deferLater(reactor,
-                   2, self.mbox.unset_recent_flags, messages)
-        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
-
-    select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
-                    imap4.IMAP4Server.arg_fetchatt)
-
-    def do_COPY(self, tag, messages, mailbox, uid=0):
-        from twisted.internet import reactor
-        imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
-        deferLater(reactor,
-                   2, self.mbox.unset_recent_flags, messages)
-        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
-
-    select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
-                   imap4.IMAP4Server.arg_astring)
-
-    def notifyNew(self, ignored):
-        """
-        Notify new messages to listeners.
-        """
-        self.mbox.notify_new()
-
-    def _cbSelectWork(self, mbox, cmdName, tag):
-        """
-        Callback for selectWork, patched to avoid conformance errors due to
-        incomplete UIDVALIDITY line.
-        """
-        if mbox is None:
-            self.sendNegativeResponse(tag, 'No such mailbox')
-            return
-        if '\\noselect' in [s.lower() for s in mbox.getFlags()]:
-            self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
-            return
-
-        flags = mbox.getFlags()
-        self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS')
-        self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT')
-        self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
-
-        # Patched -------------------------------------------------------
-        # imaptest was complaining about the incomplete line, we're adding
-        # "UIDs valid" here.
-        self.sendPositiveResponse(
-            None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity())
-        # ----------------------------------------------------------------
-
-        s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY'
-        mbox.addListener(self)
-        self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName))
-        self.state = 'select'
-        self.mbox = mbox
-
-
 class IMAPAuthRealm(object):
     """
     Dummy authentication realm. Do not use in production!
@@ -288,6 +125,8 @@ def run_service(*args, **kwargs):
               the reactor when starts listening, and the factory for
               the protocol.
     """
+    from twisted.internet import reactor
+
     leap_assert(len(args) == 2)
     soledad, keymanager = args
     leap_assert_type(soledad, Soledad)
@@ -302,8 +141,6 @@ def run_service(*args, **kwargs):
     uuid = soledad._get_uuid()
     factory = LeapIMAPFactory(uuid, userid, soledad)
 
-    from twisted.internet import reactor
-
     try:
         tport = reactor.listenTCP(port, factory,
                                   interface="localhost")
@@ -330,14 +167,3 @@ def run_service(*args, **kwargs):
 
     # not ok, signal error.
     leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port))
-
-    def checkpoint(self):
-        """
-        Called when the client issues a CHECK command.
-
-        This should perform any checkpoint operations required by the server.
-        It may be a long running operation, but may not block.  If it returns
-        a deferred, the client will only be informed of success (or failure)
-        when the deferred's callback (or errback) is invoked.
-        """
-        return None
-- 
cgit v1.2.3


From f3e23e9f3f41bc30d92e88b4ed7eb56b2aeb40ff Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 02:32:12 -0400
Subject: add enum dependency

---
 pkg/requirements.pip | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index dc0635c..603eaf6 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -4,3 +4,4 @@ leap.common>=0.3.5
 leap.keymanager>=0.3.7
 twisted  # >= 12.0.3 ??
 zope.proxy
+enum
-- 
cgit v1.2.3


From eaa4bcb241d5d55c4fd2458cb05c74fcdc79368c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 02:32:52 -0400
Subject: split messageparts

---
 src/leap/mail/imap/messageparts.py | 262 +++++++++++++++++++++++
 src/leap/mail/imap/messages.py     | 423 +++----------------------------------
 2 files changed, 286 insertions(+), 399 deletions(-)
 create mode 100644 src/leap/mail/imap/messageparts.py

diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
new file mode 100644
index 0000000..a47ea1d
--- /dev/null
+++ b/src/leap/mail/imap/messageparts.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+# messageparts.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+MessagePart implementation. Used from LeapMessage.
+"""
+import logging
+import re
+import StringIO
+
+from enum import Enum
+from zope.interface import implements
+from twisted.mail import imap4
+
+from leap.common.decorators import memoized_method
+from leap.common.mail import get_email_charset
+from leap.mail.imap.fields import fields
+from leap.mail.utils import first
+
+MessagePartType = Enum("hdoc", "fdoc", "cdoc")
+
+
+logger = logging.getLogger(__name__)
+
+
+CHARSET_PATTERN = r"""charset=([\w-]+)"""
+CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
+
+
+class MessagePart(object):
+    """
+    IMessagePart implementor.
+    It takes a subpart message and is able to find
+    the inner parts.
+
+    Excusatio non petita: see the interface documentation.
+    """
+
+    implements(imap4.IMessagePart)
+
+    def __init__(self, soledad, part_map):
+        """
+        Initializes the MessagePart.
+
+        :param part_map: a dictionary containing the parts map for this
+                         message
+        :type part_map: dict
+        """
+        # TODO
+        # It would be good to pass the uid/mailbox also
+        # for references while debugging.
+
+        # We have a problem on bulk moves, and is
+        # that when the fetch on the new mailbox is done
+        # the parts maybe are not complete.
+        # So we should be able to fail with empty
+        # docs until we solve that. The ideal would be
+        # to gather the results of the deferred operations
+        # to signal the operation is complete.
+        #leap_assert(part_map, "part map dict cannot be null")
+        self._soledad = soledad
+        self._pmap = part_map
+
+    def getSize(self):
+        """
+        Return the total size, in octets, of this message part.
+
+        :return: size of the message, in octets
+        :rtype: int
+        """
+        if not self._pmap:
+            return 0
+        size = self._pmap.get('size', None)
+        if not size:
+            logger.error("Message part cannot find size in the partmap")
+        return size
+
+    def getBodyFile(self):
+        """
+        Retrieve a file object containing only the body of this message.
+
+        :return: file-like object opened for reading
+        :rtype: StringIO
+        """
+        fd = StringIO.StringIO()
+        if self._pmap:
+            multi = self._pmap.get('multi')
+            if not multi:
+                phash = self._pmap.get("phash", None)
+            else:
+                pmap = self._pmap.get('part_map')
+                first_part = pmap.get('1', None)
+                if first_part:
+                    phash = first_part['phash']
+
+            if not phash:
+                logger.warning("Could not find phash for this subpart!")
+                payload = str("")
+            else:
+                payload = self._get_payload_from_document(phash)
+
+        else:
+            logger.warning("Message with no part_map!")
+            payload = str("")
+
+        if payload:
+            content_type = self._get_ctype_from_document(phash)
+            charset = first(CHARSET_RE.findall(content_type))
+            logger.debug("Got charset from header: %s" % (charset,))
+            if not charset:
+                charset = self._get_charset(payload)
+            try:
+                payload = payload.encode(charset)
+            except UnicodeError as exc:
+                logger.error("Unicode error {0}".format(exc))
+                payload = payload.encode(charset, 'replace')
+
+        fd.write(payload)
+        fd.seek(0)
+        return fd
+
+    # TODO cache the phash retrieval
+    def _get_payload_from_document(self, phash):
+        """
+        Gets the message payload from the content document.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: basestring
+        """
+        cdocs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+
+        cdoc = first(cdocs)
+        if not cdoc:
+            logger.warning(
+                "Could not find the content doc "
+                "for phash %s" % (phash,))
+        payload = cdoc.content.get(fields.RAW_KEY, "")
+        return payload
+
+    # TODO cache the pahash retrieval
+    def _get_ctype_from_document(self, phash):
+        """
+        Gets the content-type from the content document.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: basestring
+        """
+        cdocs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+
+        cdoc = first(cdocs)
+        if not cdoc:
+            logger.warning(
+                "Could not find the content doc "
+                "for phash %s" % (phash,))
+        ctype = cdoc.content.get('ctype', "")
+        return ctype
+
+    @memoized_method
+    def _get_charset(self, stuff):
+        # TODO put in a common class with LeapMessage
+        """
+        Gets (guesses?) the charset of a payload.
+
+        :param stuff: the stuff to guess about.
+        :type stuff: basestring
+        :returns: charset
+        """
+        # XXX existential doubt 2. shouldn't we make the scope
+        # of the decorator somewhat more persistent?
+        # ah! yes! and put memory bounds.
+        return get_email_charset(unicode(stuff))
+
+    def getHeaders(self, negate, *names):
+        """
+        Retrieve a group of message headers.
+
+        :param names: The names of the headers to retrieve or omit.
+        :type names: tuple of str
+
+        :param negate: If True, indicates that the headers listed in names
+                       should be omitted from the return value, rather
+                       than included.
+        :type negate: bool
+
+        :return: A mapping of header field names to header field values
+        :rtype: dict
+        """
+        if not self._pmap:
+            logger.warning("No pmap in Subpart!")
+            return {}
+        headers = dict(self._pmap.get("headers", []))
+
+        # twisted imap server expects *some* headers to be lowercase
+        # We could use a CaseInsensitiveDict here...
+        headers = dict(
+            (str(key), str(value)) if key.lower() != "content-type"
+            else (str(key.lower()), str(value))
+            for (key, value) in headers.items())
+
+        names = map(lambda s: s.upper(), names)
+        if negate:
+            cond = lambda key: key.upper() not in names
+        else:
+            cond = lambda key: key.upper() in names
+
+        # unpack and filter original dict by negate-condition
+        filter_by_cond = [
+            map(str, (key, val)) for
+            key, val in headers.items()
+            if cond(key)]
+        filtered = dict(filter_by_cond)
+        return filtered
+
+    def isMultipart(self):
+        """
+        Return True if this message is multipart.
+        """
+        if not self._pmap:
+            logger.warning("Could not get part map!")
+            return False
+        multi = self._pmap.get("multi", False)
+        return multi
+
+    def getSubPart(self, part):
+        """
+        Retrieve a MIME submessage
+
+        :type part: C{int}
+        :param part: The number of the part to retrieve, indexed from 0.
+        :raise IndexError: Raised if the specified part does not exist.
+        :raise TypeError: Raised if this message is not multipart.
+        :rtype: Any object implementing C{IMessagePart}.
+        :return: The specified sub-part.
+        """
+        if not self.isMultipart():
+            raise TypeError
+        sub_pmap = self._pmap.get("part_map", {})
+        try:
+            part_map = sub_pmap[str(part + 1)]
+        except KeyError:
+            logger.debug("getSubpart for %s: KeyError" % (part,))
+            raise IndexError
+
+        # XXX check for validity
+        return MessagePart(self._soledad, part_map)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index ef0b0a1..67e5a41 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -24,13 +24,12 @@ import time
 import threading
 import StringIO
 
-from collections import defaultdict, namedtuple
+from collections import defaultdict
 from functools import partial
 
 from twisted.mail import imap4
 from twisted.internet import defer
 from twisted.python import log
-from u1db import errors as u1db_errors
 from zope.interface import implements
 from zope.proxy import sameProxiedObjects
 
@@ -38,13 +37,12 @@ from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail import walk
-from leap.mail.utils import first, find_charset
+from leap.mail.utils import first, find_charset, lowerdict
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.memorystore import MessageDict
 from leap.mail.imap.parser import MailParser, MBoxParser
-from leap.mail.messageflow import IMessageConsumer
 
 logger = logging.getLogger(__name__)
 
@@ -52,29 +50,18 @@ logger = logging.getLogger(__name__)
 
 # [ ] Add ref to incoming message during add_msg
 # [ ] Add linked-from info.
+#     * Need a new type of documents: linkage info.
+#     * HDOCS are linked from FDOCs (ref to chash)
+#     * CDOCS are linked from HDOCS (ref to chash)
+
 # [ ] Delete incoming mail only after successful write!
 # [ ] Remove UID from syncable db. Store only those indexes locally.
 
+CHARSET_PATTERN = r"""charset=([\w-]+)"""
+MSGID_PATTERN = r"""<([\w@.]+)>"""
 
-# XXX no longer needed, since i'm using proxies instead of direct weakrefs
-def maybe_call(thing):
-    """
-    Return the same thing, or the result of its invocation if it is a callable.
-    """
-    return thing() if callable(thing) else thing
-
-
-def lowerdict(_dict):
-    """
-    Return a dict with the keys in lowercase.
-
-    :param _dict: the dict to convert
-    :rtype: dict
-    """
-    # TODO should properly implement a CaseInsensitive dict.
-    # Look into requests code.
-    return dict((key.lower(), value)
-                for key, value in _dict.items())
+CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
+MSGID_RE = re.compile(MSGID_PATTERN)
 
 
 def try_unique_query(curried):
@@ -102,232 +89,6 @@ def try_unique_query(curried):
     except Exception as exc:
         logger.exception("Unhandled error %r" % exc)
 
-MSGID_PATTERN = r"""<([\w@.]+)>"""
-MSGID_RE = re.compile(MSGID_PATTERN)
-
-
-class MessagePart(object):
-    """
-    IMessagePart implementor.
-    It takes a subpart message and is able to find
-    the inner parts.
-
-    Excusatio non petita: see the interface documentation.
-    """
-
-    implements(imap4.IMessagePart)
-
-    def __init__(self, soledad, part_map):
-        """
-        Initializes the MessagePart.
-
-        :param part_map: a dictionary containing the parts map for this
-                         message
-        :type part_map: dict
-        """
-        # TODO
-        # It would be good to pass the uid/mailbox also
-        # for references while debugging.
-
-        # We have a problem on bulk moves, and is
-        # that when the fetch on the new mailbox is done
-        # the parts maybe are not complete.
-        # So we should be able to fail with empty
-        # docs until we solve that. The ideal would be
-        # to gather the results of the deferred operations
-        # to signal the operation is complete.
-        #leap_assert(part_map, "part map dict cannot be null")
-        self._soledad = soledad
-        self._pmap = part_map
-
-    def getSize(self):
-        """
-        Return the total size, in octets, of this message part.
-
-        :return: size of the message, in octets
-        :rtype: int
-        """
-        if not self._pmap:
-            return 0
-        size = self._pmap.get('size', None)
-        if not size:
-            logger.error("Message part cannot find size in the partmap")
-        return size
-
-    def getBodyFile(self):
-        """
-        Retrieve a file object containing only the body of this message.
-
-        :return: file-like object opened for reading
-        :rtype: StringIO
-        """
-        fd = StringIO.StringIO()
-        if self._pmap:
-            multi = self._pmap.get('multi')
-            if not multi:
-                phash = self._pmap.get("phash", None)
-            else:
-                pmap = self._pmap.get('part_map')
-                first_part = pmap.get('1', None)
-                if first_part:
-                    phash = first_part['phash']
-
-            if not phash:
-                logger.warning("Could not find phash for this subpart!")
-                payload = str("")
-            else:
-                payload = self._get_payload_from_document(phash)
-
-        else:
-            logger.warning("Message with no part_map!")
-            payload = str("")
-
-        if payload:
-            content_type = self._get_ctype_from_document(phash)
-            charset = find_charset(content_type)
-            logger.debug("Got charset from header: %s" % (charset,))
-            if charset is None:
-                charset = self._get_charset(payload)
-                logger.debug("Got charset: %s" % (charset,))
-            try:
-                payload = payload.encode(charset)
-            except (UnicodeEncodeError, UnicodeDecodeError) as e:
-                logger.error("Unicode error, using 'replace'. {0!r}".format(e))
-                payload = payload.encode(charset, 'replace')
-
-        fd.write(payload)
-        fd.seek(0)
-        return fd
-
-    # TODO cache the phash retrieval
-    def _get_payload_from_document(self, phash):
-        """
-        Gets the message payload from the content document.
-
-        :param phash: the payload hash to retrieve by.
-        :type phash: basestring
-        """
-        cdocs = self._soledad.get_from_index(
-            fields.TYPE_P_HASH_IDX,
-            fields.TYPE_CONTENT_VAL, str(phash))
-
-        cdoc = first(cdocs)
-        if not cdoc:
-            logger.warning(
-                "Could not find the content doc "
-                "for phash %s" % (phash,))
-        payload = cdoc.content.get(fields.RAW_KEY, "")
-        return payload
-
-    # TODO cache the pahash retrieval
-    def _get_ctype_from_document(self, phash):
-        """
-        Gets the content-type from the content document.
-
-        :param phash: the payload hash to retrieve by.
-        :type phash: basestring
-        """
-        cdocs = self._soledad.get_from_index(
-            fields.TYPE_P_HASH_IDX,
-            fields.TYPE_CONTENT_VAL, str(phash))
-
-        cdoc = first(cdocs)
-        if not cdoc:
-            logger.warning(
-                "Could not find the content doc "
-                "for phash %s" % (phash,))
-        ctype = cdoc.content.get('ctype', "")
-        return ctype
-
-    @memoized_method
-    def _get_charset(self, stuff):
-        # TODO put in a common class with LeapMessage
-        """
-        Gets (guesses?) the charset of a payload.
-
-        :param stuff: the stuff to guess about.
-        :type stuff: basestring
-        :returns: charset
-        """
-        # XXX existential doubt 2. shouldn't we make the scope
-        # of the decorator somewhat more persistent?
-        # ah! yes! and put memory bounds.
-        return get_email_charset(unicode(stuff))
-
-    def getHeaders(self, negate, *names):
-        """
-        Retrieve a group of message headers.
-
-        :param names: The names of the headers to retrieve or omit.
-        :type names: tuple of str
-
-        :param negate: If True, indicates that the headers listed in names
-                       should be omitted from the return value, rather
-                       than included.
-        :type negate: bool
-
-        :return: A mapping of header field names to header field values
-        :rtype: dict
-        """
-        if not self._pmap:
-            logger.warning("No pmap in Subpart!")
-            return {}
-        headers = dict(self._pmap.get("headers", []))
-
-        # twisted imap server expects *some* headers to be lowercase
-        # We could use a CaseInsensitiveDict here...
-        headers = dict(
-            (str(key), str(value)) if key.lower() != "content-type"
-            else (str(key.lower()), str(value))
-            for (key, value) in headers.items())
-
-        names = map(lambda s: s.upper(), names)
-        if negate:
-            cond = lambda key: key.upper() not in names
-        else:
-            cond = lambda key: key.upper() in names
-
-        # unpack and filter original dict by negate-condition
-        filter_by_cond = [
-            map(str, (key, val)) for
-            key, val in headers.items()
-            if cond(key)]
-        filtered = dict(filter_by_cond)
-        return filtered
-
-    def isMultipart(self):
-        """
-        Return True if this message is multipart.
-        """
-        if not self._pmap:
-            logger.warning("Could not get part map!")
-            return False
-        multi = self._pmap.get("multi", False)
-        return multi
-
-    def getSubPart(self, part):
-        """
-        Retrieve a MIME submessage
-
-        :type part: C{int}
-        :param part: The number of the part to retrieve, indexed from 0.
-        :raise IndexError: Raised if the specified part does not exist.
-        :raise TypeError: Raised if this message is not multipart.
-        :rtype: Any object implementing C{IMessagePart}.
-        :return: The specified sub-part.
-        """
-        if not self.isMultipart():
-            raise TypeError
-        sub_pmap = self._pmap.get("part_map", {})
-        try:
-            part_map = sub_pmap[str(part + 1)]
-        except KeyError:
-            logger.debug("getSubpart for %s: KeyError" % (part,))
-            raise IndexError
-
-        # XXX check for validity
-        return MessagePart(self._soledad, part_map)
-
 
 class LeapMessage(fields, MailParser, MBoxParser):
     """
@@ -380,7 +141,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             if not fdoc:
                 fdoc = self._get_flags_doc()
             if fdoc:
-                fdoc_content = maybe_call(fdoc.content)
+                fdoc_content = fdoc.content
                 self.__chash = fdoc_content.get(
                     fields.CONTENT_HASH_KEY, None)
             return fdoc
@@ -404,7 +165,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         if not self._fdoc:
             return None
         if not self.__chash and self._fdoc:
-            self.__chash = maybe_call(self._fdoc.content).get(
+            self.__chash = self._fdoc.content.get(
                 fields.CONTENT_HASH_KEY, None)
         return self.__chash
 
@@ -444,7 +205,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         flags = []
         fdoc = self._fdoc
         if fdoc:
-            flags = maybe_call(fdoc.content).get(self.FLAGS_KEY, None)
+            flags = fdoc.content.get(self.FLAGS_KEY, None)
 
         msgcol = self._collection
 
@@ -557,12 +318,12 @@ class LeapMessage(fields, MailParser, MBoxParser):
                 charset = self._get_charset(body)
             try:
                 body = body.encode(charset)
-            except UnicodeError as e:
-                logger.error("Unicode error {0}".format(e))
+            except UnicodeError as exc:
+                logger.error("Unicode error {0}".format(exc))
                 logger.debug("Attempted to encode with: %s" % charset)
                 try:
                     body = body.encode(charset, 'replace')
-                except UnicodeError as e:
+                except UnicodeError as exc:
                     try:
                         body = body.encode('utf-8', 'replace')
                     except:
@@ -601,7 +362,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         size = None
         if self._fdoc:
-            fdoc_content = maybe_call(self._fdoc.content)
+            fdoc_content = self._fdoc.content
             size = fdoc_content.get(self.SIZE_KEY, False)
         else:
             logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
@@ -667,7 +428,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the headers dict for this message.
         """
         if self._hdoc is not None:
-            hdoc_content = maybe_call(self._hdoc.content)
+            hdoc_content = self._hdoc.content
             headers = hdoc_content.get(self.HEADERS_KEY, {})
             return headers
 
@@ -682,7 +443,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return True if this message is multipart.
         """
         if self._fdoc:
-            fdoc_content = maybe_call(self._fdoc.content)
+            fdoc_content = self._fdoc.content
             is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
             return is_multipart
         else:
@@ -725,7 +486,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             logger.warning("Tried to get part but no HDOC found!")
             return None
 
-        hdoc_content = maybe_call(self._hdoc.content)
+        hdoc_content = self._hdoc.content
         pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
         return pmap[str(part)]
 
@@ -762,7 +523,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the document that keeps the body for this
         message.
         """
-        hdoc_content = maybe_call(self._hdoc.content)
+        hdoc_content = self._hdoc.content
         body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
         if not body_phash:
@@ -801,7 +562,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: The content value indexed by C{key} or None
         :rtype: str
         """
-        return maybe_call(self._fdoc.content).get(key, None)
+        return self._fdoc.content.get(key, None)
 
     # setters
 
@@ -874,143 +635,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         return self._fdoc is not None
 
 
-class ContentDedup(object):
-    """
-    Message deduplication.
-
-    We do a query for the content hashes before writing to our beloved
-    sqlcipher backend of Soledad. This means, by now, that:
-
-    1. We will not store the same attachment twice, only the hash of it.
-    2. We will not store the same message body twice, only the hash of it.
-
-    The first case is useful if you are always receiving the same old memes
-    from unwary friends that still have not discovered that 4chan is the
-    generator of the internet. The second will save your day if you have
-    initiated session with the same account in two different machines. I also
-    wonder why would you do that, but let's respect each other choices, like
-    with the religious celebrations, and assume that one day we'll be able
-    to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
-    Stack.
-    """
-
-    def _content_does_exist(self, doc):
-        """
-        Check whether we already have a content document for a payload
-        with this hash in our database.
-
-        :param doc: tentative body document
-        :type doc: dict
-        :returns: True if that happens, False otherwise.
-        """
-        if not doc:
-            return False
-        phash = doc[fields.PAYLOAD_HASH_KEY]
-        attach_docs = self._soledad.get_from_index(
-            fields.TYPE_P_HASH_IDX,
-            fields.TYPE_CONTENT_VAL, str(phash))
-        if not attach_docs:
-            return False
-
-        if len(attach_docs) != 1:
-            logger.warning("Found more than one copy of phash %s!"
-                           % (phash,))
-        logger.debug("Found attachment doc with that hash! Skipping save!")
-        return True
-
-
-SoledadWriterPayload = namedtuple(
-    'SoledadWriterPayload', ['mode', 'payload'])
-
-# TODO we could consider using enum here:
-# https://pypi.python.org/pypi/enum
-
-SoledadWriterPayload.CREATE = 1
-SoledadWriterPayload.PUT = 2
-SoledadWriterPayload.CONTENT_CREATE = 3
-
-
-"""
-SoledadDocWriter was used to avoid writing to the db from multiple threads.
-Its use here has been deprecated in favor of a local rw_lock in the client.
-But we might want to reuse in in the near future to implement priority queues.
-"""
-
-
-class SoledadDocWriter(object):
-    """
-    This writer will create docs serially in the local soledad database.
-    """
-
-    implements(IMessageConsumer)
-
-    def __init__(self, soledad):
-        """
-        Initialize the writer.
-
-        :param soledad: the soledad instance
-        :type soledad: Soledad
-        """
-        self._soledad = soledad
-
-    def _get_call_for_item(self, item):
-        """
-        Return the proper call type for a given item.
-
-        :param item: one of the types defined under the
-                     attributes of SoledadWriterPayload
-        :type item: int
-        """
-        call = None
-        payload = item.payload
-
-        if item.mode == SoledadWriterPayload.CREATE:
-            call = self._soledad.create_doc
-        elif (item.mode == SoledadWriterPayload.CONTENT_CREATE
-              and not self._content_does_exist(payload)):
-                call = self._soledad.create_doc
-        elif item.mode == SoledadWriterPayload.PUT:
-            call = self._soledad.put_doc
-        return call
-
-    def _process(self, queue):
-        """
-        Return the item and the proper call type for the next
-        item in the queue if any.
-
-        :param queue: the queue from where we'll pick item.
-        :type queue: Queue
-        """
-        item = queue.get()
-        call = self._get_call_for_item(item)
-        return item, call
-
-    def consume(self, queue):
-        """
-        Creates a new document in soledad db.
-
-        :param queue: queue to get item from, with content of the document
-                      to be inserted.
-        :type queue: Queue
-        """
-        empty = queue.empty()
-        while not empty:
-            item, call = self._process(queue)
-
-            if call:
-                # XXX should handle the delete case
-                # should handle errors
-                try:
-                    call(item.payload)
-                except u1db_errors.RevisionConflict as exc:
-                    logger.error("Error: %r" % (exc,))
-                    raise exc
-
-            empty = queue.empty()
-
-
-class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
-                        ContentDedup):
+class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     """
     A collection of messages, surprisingly.
 
@@ -1360,7 +985,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser,
         # TODO ---- add reference to original doc, to be deleted
         # after writes are done.
         msg_container = MessageDict(fd, hd, cdocs)
-        self._memstore.put(self.mbox, uid, msg_container)
+        self._memstore.create_message(self.mbox, uid, msg_container)
 
     def _remove_cb(self, result):
         return result
-- 
cgit v1.2.3


From d7a167e1ba5ea9bb8167e6255a81d4c96fdffef9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 02:33:32 -0400
Subject: move utilities

---
 src/leap/mail/utils.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 6c79227..64af04f 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -36,6 +36,14 @@ def first(things):
         return None
 
 
+def maybe_call(thing):
+    """
+    Return the same thing, or the result of its invocation if it is a
+    callable.
+    """
+    return thing() if callable(thing) else thing
+
+
 def find_charset(thing, default=None):
     """
     Looks into the object 'thing' for a charset specification.
@@ -46,16 +54,28 @@ def find_charset(thing, default=None):
     :param default: the dafault charset to return if no charset is found.
     :type default: str
 
-    :returns: the charset or 'default'
+    :return: the charset or 'default'
     :rtype: str or None
     """
     charset = first(CHARSET_RE.findall(repr(thing)))
     if charset is None:
         charset = default
-
     return charset
 
 
+def lowerdict(_dict):
+    """
+    Return a dict with the keys in lowercase.
+
+    :param _dict: the dict to convert
+    :rtype: dict
+    """
+    # TODO should properly implement a CaseInsensitive dict.
+    # Look into requests code.
+    return dict((key.lower(), value)
+                for key, value in _dict.items())
+
+
 class CustomJsonScanner(object):
     """
     This class is a context manager definition used to monkey patch the default
-- 
cgit v1.2.3


From e2218eec4fd91e4648160a05e3debc05efa0d0d9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 02:36:38 -0400
Subject: add soledadstore class move parts-related bits to messageparts pass
 soledad in initialization for memory messages

---
 src/leap/mail/imap/mailbox.py      |  29 +----
 src/leap/mail/imap/memorystore.py  | 185 ++---------------------------
 src/leap/mail/imap/messageparts.py | 183 +++++++++++++++++++++++++++-
 src/leap/mail/imap/messages.py     |  16 ++-
 src/leap/mail/imap/service/imap.py |   4 +-
 src/leap/mail/imap/soledadstore.py | 237 +++++++++++++++++++++++++++++++++++++
 6 files changed, 446 insertions(+), 208 deletions(-)
 create mode 100644 src/leap/mail/imap/soledadstore.py

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 9babe6b..5e16b4b 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -37,8 +37,8 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
 from leap.mail.decorators import deferred
 from leap.mail.imap.fields import WithMsgFields, fields
-from leap.mail.imap.memorystore import MessageDict
 from leap.mail.imap.messages import MessageCollection
+from leap.mail.imap.messageparts import MessageWrapper
 from leap.mail.imap.parser import MBoxParser
 
 logger = logging.getLogger(__name__)
@@ -549,10 +549,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         #sequence = True if uid == 0 else False
 
         messages_asked = self._bound_seq(messages_asked)
-        print "asked: ", messages_asked
         seq_messg = self._filter_msg_seq(messages_asked)
-
-        print "seq: ", seq_messg
         getmsg = lambda uid: self.messages.get_msg_by_uid(uid)
 
         # for sequence numbers (uid = 0)
@@ -791,29 +788,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.debug("Tried to copy a MSG with no fdoc")
             return
 
-        #old_mbox = fdoc.content[self.MBOX_KEY]
-        #old_uid = fdoc.content[self.UID_KEY]
-        #old_key = old_mbox, old_uid
-        #print "copying from OLD MBOX ", old_mbox
-
-        # XXX bit doubt... to duplicate in memory
-        # or not to...?
-        # I think it should be ok to duplicate as long as we're
-        # careful at the hour of writes...
-        # We could use also proxies, but it will break when
-        # the original mailbox is flushed.
-
-        # XXX DEBUG ----------------------------------------
-        #print "copying MESSAGE from %s (%s) to %s (%s)" % (
-            #msg._mbox, msg._uid, self.mbox, uid_next)
-
         new_fdoc = copy.deepcopy(fdoc.content)
         new_fdoc[self.UID_KEY] = uid_next
         new_fdoc[self.MBOX_KEY] = self.mbox
-        self._memstore.put(self.mbox, uid_next, MessageDict(
-            new_fdoc, hdoc.content))
+        self._memstore.create_message(
+            self.mbox, uid_next,
+            MessageWrapper(
+                new_fdoc, hdoc.content))
 
-        # XXX use memory store
+        # XXX use memory store !!!
         if hasattr(hdoc, 'doc_id'):
             self.messages.add_hdocset_docid(hdoc.doc_id)
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index b8829e0..7cb361f 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -21,187 +21,20 @@ import contextlib
 import logging
 import weakref
 
-from collections import namedtuple
-
 from twisted.internet.task import LoopingCall
 from zope.interface import implements
 
 from leap.mail import size
 from leap.mail.messageflow import MessageProducer
-from leap.mail.messageparts import MessagePartType
 from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
+from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc
+from leap.mail.imap.messageparts import MessageWrapper
+from leap.mail.imap.messageparts import ReferenciableDict
 
 logger = logging.getLogger(__name__)
 
 
-"""
-A MessagePartDoc is a light wrapper around the dictionary-like
-data that we pass along for message parts. It can be used almost everywhere
-that you would expect a SoledadDocument, since it has a dict under the
-`content` attribute.
-
-We also keep some metadata on it, relative in part to the message as a whole,
-and sometimes to a part in particular only.
-
-* `new` indicates that the document has just been created. SoledadStore
-  should just create a new doc for all the related message parts.
-* `store` indicates the type of store a given MessagePartDoc lives in.
-  We currently use this to indicate that  the document comes from memeory,
-  but we should probably get rid of it as soon as we extend the use of the
-  SoledadStore interface along LeapMessage, MessageCollection and Mailbox.
-* `part` is one of the MessagePartType enums.
-
-* `dirty` indicates that, while we already have the document in Soledad,
-  we have modified its state in memory, so we need to put_doc instead while
-  dumping the MemoryStore contents.
-  `dirty` attribute would only apply to flags-docs and linkage-docs.
-
-
-  XXX this is still not implemented!
-
-"""
-
-MessagePartDoc = namedtuple(
-    'MessagePartDoc',
-    ['new', 'dirty', 'part', 'store', 'content'])
-
-
-class ReferenciableDict(dict):
-    """
-    A dict that can be weak-referenced.
-
-    Some builtin objects are not weak-referenciable unless
-    subclassed. So we do.
-
-    Used to return pointers to the items in the MemoryStore.
-    """
-
-
-class MessageWrapper(object):
-    """
-    A simple nested dictionary container around the different message subparts.
-    """
-    implements(interfaces.IMessageContainer)
-
-    FDOC = "fdoc"
-    HDOC = "hdoc"
-    CDOCS = "cdocs"
-
-    # XXX can use this to limit the memory footprint,
-    # or is it too premature to optimize?
-    # Does it work well together with the interfaces.implements?
-
-    #__slots__ = ["_dict", "_new", "_dirty", "memstore"]
-
-    def __init__(self, fdoc=None, hdoc=None, cdocs=None,
-                 from_dict=None, memstore=None,
-                 new=True, dirty=False):
-        self._dict = {}
-
-        self._new = new
-        self._dirty = dirty
-        self.memstore = memstore
-
-        if from_dict is not None:
-            self.from_dict(from_dict)
-        else:
-            if fdoc is not None:
-                self._dict[self.FDOC] = ReferenciableDict(fdoc)
-            if hdoc is not None:
-                self._dict[self.HDOC] = ReferenciableDict(hdoc)
-            if cdocs is not None:
-                self._dict[self.CDOCS] = ReferenciableDict(cdocs)
-
-    # properties
-
-    @property
-    def new(self):
-        return self._new
-
-    def set_new(self, value=True):
-        self._new = value
-
-    @property
-    def dirty(self):
-        return self._dirty
-
-    def set_dirty(self, value=True):
-        self._dirty = value
-
-    # IMessageContainer
-
-    @property
-    def fdoc(self):
-        _fdoc = self._dict.get(self.FDOC, None)
-        if _fdoc:
-            content_ref = weakref.proxy(_fdoc)
-        else:
-            logger.warning("NO FDOC!!!")
-            content_ref = {}
-        return MessagePartDoc(new=self.new, dirty=self.dirty,
-                              store=self._storetype,
-                              part=MessagePartType.fdoc,
-                              content=content_ref)
-
-    @property
-    def hdoc(self):
-        _hdoc = self._dict.get(self.HDOC, None)
-        if _hdoc:
-            content_ref = weakref.proxy(_hdoc)
-        else:
-            logger.warning("NO HDOC!!!!")
-            content_ref = {}
-        return MessagePartDoc(new=self.new, dirty=self.dirty,
-                              store=self._storetype,
-                              part=MessagePartType.hdoc,
-                              content=content_ref)
-
-    @property
-    def cdocs(self):
-        _cdocs = self._dict.get(self.CDOCS, None)
-        if _cdocs:
-            return weakref.proxy(_cdocs)
-        else:
-            return {}
-
-    def walk(self):
-        """
-        Generator that iterates through all the parts, returning
-        MessagePartDoc.
-        """
-        yield self.fdoc
-        yield self.hdoc
-        for cdoc in self.cdocs.values():
-            # XXX this will break ----
-            content_ref = weakref.proxy(cdoc)
-            yield MessagePartDoc(new=self.new, dirty=self.dirty,
-                                 store=self._storetype,
-                                 part=MessagePartType.cdoc,
-                                 content=content_ref)
-
-    # i/o
-
-    def as_dict(self):
-        """
-        Return a dict representation of the parts contained.
-        """
-        return self._dict
-
-    def from_dict(self, msg_dict):
-        """
-        Populate MessageWrapper parts from a dictionary.
-        It expects the same format that we use in a
-        MessageWrapper.
-        """
-        fdoc, hdoc, cdocs = map(
-            lambda part: msg_dict.get(part, None),
-            [self.FDOC, self.HDOC, self.CDOCS])
-        self._dict[self.FDOC] = fdoc
-        self._dict[self.HDOC] = hdoc
-        self._dict[self.CDOCS] = cdocs
-
-
 @contextlib.contextmanager
 def set_bool_flag(obj, att):
     """
@@ -232,8 +65,8 @@ class MemoryStore(object):
     writes to the permanent storage is controled by the write_period parameter
     in the constructor.
     """
-    implements(interfaces.IMessageStore)
-    implements(interfaces.IMessageStoreWriter)
+    implements(interfaces.IMessageStore,
+               interfaces.IMessageStoreWriter)
 
     producer = None
 
@@ -332,7 +165,7 @@ class MemoryStore(object):
             print "saving cdoc"
             cdoc = self._msg_store[key]['cdocs'][cdoc_key]
 
-            # XXX this should be done in the MessageWrapper constructor
+            # FIXME this should be done in the MessageWrapper constructor
             # instead...
             # first we make it weak-referenciable
             referenciable_cdoc = ReferenciableDict(cdoc)
@@ -399,10 +232,8 @@ class MemoryStore(object):
         """
         Get the highest UID for a given mbox.
         """
-        # XXX should get from msg_store keys instead!
-        if not self._new:
-            return 0
-        return max(self.get_uids(mbox))
+        uids = self.get_uids(mbox)
+        return uids and max(uids) or 0
 
     def count_new_mbox(self, mbox):
         """
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index a47ea1d..3f89193 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -20,6 +20,9 @@ MessagePart implementation. Used from LeapMessage.
 import logging
 import re
 import StringIO
+import weakref
+
+from collections import namedtuple
 
 from enum import Enum
 from zope.interface import implements
@@ -27,6 +30,7 @@ from twisted.mail import imap4
 
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
+from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
 from leap.mail.utils import first
 
@@ -36,13 +40,188 @@ MessagePartType = Enum("hdoc", "fdoc", "cdoc")
 logger = logging.getLogger(__name__)
 
 
+# XXX not needed anymoar ...
 CHARSET_PATTERN = r"""charset=([\w-]+)"""
 CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
 
+"""
+A MessagePartDoc is a light wrapper around the dictionary-like
+data that we pass along for message parts. It can be used almost everywhere
+that you would expect a SoledadDocument, since it has a dict under the
+`content` attribute.
+
+We also keep some metadata on it, relative in part to the message as a whole,
+and sometimes to a part in particular only.
+
+* `new` indicates that the document has just been created. SoledadStore
+  should just create a new doc for all the related message parts.
+* `store` indicates the type of store a given MessagePartDoc lives in.
+  We currently use this to indicate that  the document comes from memeory,
+  but we should probably get rid of it as soon as we extend the use of the
+  SoledadStore interface along LeapMessage, MessageCollection and Mailbox.
+* `part` is one of the MessagePartType enums.
+
+* `dirty` indicates that, while we already have the document in Soledad,
+  we have modified its state in memory, so we need to put_doc instead while
+  dumping the MemoryStore contents.
+  `dirty` attribute would only apply to flags-docs and linkage-docs.
+
+
+  XXX this is still not implemented!
+
+"""
+
+MessagePartDoc = namedtuple(
+    'MessagePartDoc',
+    ['new', 'dirty', 'part', 'store', 'content'])
+
+
+class ReferenciableDict(dict):
+    """
+    A dict that can be weak-referenced.
+
+    Some builtin objects are not weak-referenciable unless
+    subclassed. So we do.
+
+    Used to return pointers to the items in the MemoryStore.
+    """
+
+
+class MessageWrapper(object):
+    """
+    A simple nested dictionary container around the different message subparts.
+    """
+    implements(interfaces.IMessageContainer)
+
+    FDOC = "fdoc"
+    HDOC = "hdoc"
+    CDOCS = "cdocs"
+
+    # XXX can use this to limit the memory footprint,
+    # or is it too premature to optimize?
+    # Does it work well together with the interfaces.implements?
+
+    #__slots__ = ["_dict", "_new", "_dirty", "memstore"]
+
+    def __init__(self, fdoc=None, hdoc=None, cdocs=None,
+                 from_dict=None, memstore=None,
+                 new=True, dirty=False):
+        self._dict = {}
+        self.memstore = memstore
+
+        self._new = new
+        self._dirty = dirty
+        self._storetype = "mem"
+
+        if from_dict is not None:
+            self.from_dict(from_dict)
+        else:
+            if fdoc is not None:
+                self._dict[self.FDOC] = ReferenciableDict(fdoc)
+            if hdoc is not None:
+                self._dict[self.HDOC] = ReferenciableDict(hdoc)
+            if cdocs is not None:
+                self._dict[self.CDOCS] = ReferenciableDict(cdocs)
+
+    # properties
+
+    @property
+    def new(self):
+        return self._new
+
+    def set_new(self, value=True):
+        self._new = value
+
+    @property
+    def dirty(self):
+        return self._dirty
+
+    def set_dirty(self, value=True):
+        self._dirty = value
+
+    # IMessageContainer
+
+    @property
+    def fdoc(self):
+        _fdoc = self._dict.get(self.FDOC, None)
+        if _fdoc:
+            content_ref = weakref.proxy(_fdoc)
+        else:
+            logger.warning("NO FDOC!!!")
+            content_ref = {}
+        return MessagePartDoc(new=self.new, dirty=self.dirty,
+                              store=self._storetype,
+                              part=MessagePartType.fdoc,
+                              content=content_ref)
+
+    @property
+    def hdoc(self):
+        _hdoc = self._dict.get(self.HDOC, None)
+        if _hdoc:
+            content_ref = weakref.proxy(_hdoc)
+        else:
+            logger.warning("NO HDOC!!!!")
+            content_ref = {}
+        return MessagePartDoc(new=self.new, dirty=self.dirty,
+                              store=self._storetype,
+                              part=MessagePartType.hdoc,
+                              content=content_ref)
+
+    @property
+    def cdocs(self):
+        _cdocs = self._dict.get(self.CDOCS, None)
+        if _cdocs:
+            return weakref.proxy(_cdocs)
+        else:
+            return {}
+
+    def walk(self):
+        """
+        Generator that iterates through all the parts, returning
+        MessagePartDoc.
+        """
+        yield self.fdoc
+        yield self.hdoc
+        for cdoc in self.cdocs.values():
+            # XXX this will break ----
+            #content_ref = weakref.proxy(cdoc)
+            #yield MessagePartDoc(new=self.new, dirty=self.dirty,
+                                 #store=self._storetype,
+                                 #part=MessagePartType.cdoc,
+                                 #content=content_ref)
+
+            # the put is handling this for us, so
+            # we already have stored a MessagePartDoc
+            # but we should really do it while adding in the
+            # constructor or the from_dict method
+            yield cdoc
+
+    # i/o
+
+    def as_dict(self):
+        """
+        Return a dict representation of the parts contained.
+        """
+        return self._dict
+
+    def from_dict(self, msg_dict):
+        """
+        Populate MessageWrapper parts from a dictionary.
+        It expects the same format that we use in a
+        MessageWrapper.
+        """
+        fdoc, hdoc, cdocs = map(
+            lambda part: msg_dict.get(part, None),
+            [self.FDOC, self.HDOC, self.CDOCS])
+        self._dict[self.FDOC] = fdoc
+        self._dict[self.HDOC] = hdoc
+        self._dict[self.CDOCS] = cdocs
+
 
 class MessagePart(object):
     """
-    IMessagePart implementor.
+    IMessagePart implementor, to be passed to several methods
+    of the IMAP4Server.
     It takes a subpart message and is able to find
     the inner parts.
 
@@ -117,6 +296,8 @@ class MessagePart(object):
             payload = str("")
 
         if payload:
+            # XXX use find_charset instead --------------------------
+            # bad rebase???
             content_type = self._get_ctype_from_document(phash)
             charset = first(CHARSET_RE.findall(content_type))
             logger.debug("Got charset from header: %s" % (charset,))
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 67e5a41..46c9dc9 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -41,7 +41,7 @@ from leap.mail.utils import first, find_charset, lowerdict
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.memorystore import MessageDict
+from leap.mail.imap.memorystore import MessageWrapper
 from leap.mail.imap.parser import MailParser, MBoxParser
 
 logger = logging.getLogger(__name__)
@@ -984,7 +984,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # TODO ---- add reference to original doc, to be deleted
         # after writes are done.
-        msg_container = MessageDict(fd, hd, cdocs)
+        msg_container = MessageWrapper(fd, hd, cdocs)
         self._memstore.create_message(self.mbox, uid, msg_container)
 
     def _remove_cb(self, result):
@@ -1215,6 +1215,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # and we cannot find it otherwise. This seems to be enough.
 
         # XXX do a deferLater instead ??
+        # FIXME this won't be needed after the CHECK command is implemented.
         time.sleep(0.3)
         return self._get_uid_from_msgidCb(msgid)
 
@@ -1233,11 +1234,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :rtype: LeapMessage
         """
         print "getting msg by id!"
-        msg_container = self._memstore.get(self.mbox, uid)
+        msg_container = self._memstore.get_message(self.mbox, uid)
         print "msg container", msg_container
         if msg_container is not None:
             print "getting LeapMessage (from memstore)"
-            msg = LeapMessage(None, uid, self.mbox, collection=self,
+            # We pass a reference to soledad just to be able to retrieve
+            # missing parts that cannot be found in the container, like
+            # the content docs after a copy.
+            msg = LeapMessage(self._soledad, uid, self.mbox, collection=self,
                               container=msg_container)
             print "got msg:", msg
         else:
@@ -1309,7 +1313,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         Return a dict with all flags documents for this mailbox.
         """
-        # XXX get all from memstore and cahce it there
+        # XXX get all from memstore and cache it there
         all_flags = dict(((
             doc.content[self.UID_KEY],
             doc.content[self.FLAGS_KEY]) for doc in
@@ -1319,7 +1323,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         if self._memstore is not None:
             # XXX
             uids = self._memstore.get_uids(self.mbox)
-            fdocs = [(uid, self._memstore.get(self.mbox, uid).fdoc)
+            fdocs = [(uid, self._memstore.get_message(self.mbox, uid).fdoc)
                      for uid in uids]
             for uid, doc in fdocs:
                 all_flags[uid] = doc.content[self.FLAGS_KEY]
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 3f99da6..8350988 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -32,6 +32,7 @@ from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
 from leap.mail.imap.memorystore import MemoryStore
 from leap.mail.imap.server import LeapIMAPServer
+from leap.mail.imap.soledadstore import SoledadStore
 from leap.soledad.client import Soledad
 
 # The default port in which imap service will run
@@ -96,7 +97,8 @@ class LeapIMAPFactory(ServerFactory):
         self._uuid = uuid
         self._userid = userid
         self._soledad = soledad
-        self._memstore = MemoryStore()
+        self._memstore = MemoryStore(
+            permanent_store=SoledadStore(soledad))
 
         theAccount = SoledadBackedAccount(
             uuid, soledad=soledad,
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
new file mode 100644
index 0000000..62a3c53
--- /dev/null
+++ b/src/leap/mail/imap/soledadstore.py
@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+# soledadstore.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+A MessageStore that writes to Soledad.
+"""
+import logging
+
+from u1db import errors as u1db_errors
+from zope.interface import implements
+
+from leap.mail.imap.messageparts import MessagePartType
+from leap.mail.imap.fields import fields
+from leap.mail.imap.interfaces import IMessageStore
+from leap.mail.messageflow import IMessageConsumer
+
+logger = logging.getLogger(__name__)
+
+
+class ContentDedup(object):
+    """
+    Message deduplication.
+
+    We do a query for the content hashes before writing to our beloved
+    sqlcipher backend of Soledad. This means, by now, that:
+
+    1. We will not store the same attachment twice, only the hash of it.
+    2. We will not store the same message body twice, only the hash of it.
+
+    The first case is useful if you are always receiving the same old memes
+    from unwary friends that still have not discovered that 4chan is the
+    generator of the internet. The second will save your day if you have
+    initiated session with the same account in two different machines. I also
+    wonder why would you do that, but let's respect each other choices, like
+    with the religious celebrations, and assume that one day we'll be able
+    to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
+    Stack.
+    """
+
+    def _header_does_exist(self, doc):
+        """
+        Check whether we already have a header document for this
+        content hash in our database.
+
+        :param doc: tentative header document
+        :type doc: dict
+        :returns: True if it exists, False otherwise.
+        """
+        if not doc:
+            return False
+        chash = doc[fields.CONTENT_HASH_KEY]
+        header_docs = self._soledad.get_from_index(
+            fields.TYPE_C_HASH_IDX,
+            fields.TYPE_HEADERS_VAL, str(chash))
+        if not header_docs:
+            return False
+
+        if len(header_docs) != 1:
+            logger.warning("Found more than one copy of chash %s!"
+                           % (chash,))
+        logger.debug("Found header doc with that hash! Skipping save!")
+        return True
+
+    def _content_does_exist(self, doc):
+        """
+        Check whether we already have a content document for a payload
+        with this hash in our database.
+
+        :param doc: tentative content document
+        :type doc: dict
+        :returns: True if it exists, False otherwise.
+        """
+        if not doc:
+            return False
+        phash = doc[fields.PAYLOAD_HASH_KEY]
+        attach_docs = self._soledad.get_from_index(
+            fields.TYPE_P_HASH_IDX,
+            fields.TYPE_CONTENT_VAL, str(phash))
+        if not attach_docs:
+            return False
+
+        if len(attach_docs) != 1:
+            logger.warning("Found more than one copy of phash %s!"
+                           % (phash,))
+        logger.debug("Found attachment doc with that hash! Skipping save!")
+        return True
+
+
+class SoledadStore(ContentDedup):
+    """
+    This will create docs in the local Soledad database.
+    """
+
+    implements(IMessageConsumer, IMessageStore)
+
+    def __init__(self, soledad):
+        """
+        Initialize the writer.
+
+        :param soledad: the soledad instance
+        :type soledad: Soledad
+        """
+        self._soledad = soledad
+
+    # IMessageStore
+
+    # -------------------------------------------------------------------
+    # We are not yet using this interface, but it would make sense
+    # to implement it.
+
+    def create_message(self, mbox, uid, message):
+        """
+        Create the passed message into this SoledadStore.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        :param message: a IMessageContainer implementor.
+        """
+
+    def put_message(self, mbox, uid, message):
+        """
+        Put the passed existing message into this SoledadStore.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        :param message: a IMessageContainer implementor.
+        """
+
+    def remove_message(self, mbox, uid):
+        """
+        Remove the given message from this SoledadStore.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        """
+
+    def get_message(self, mbox, uid):
+        """
+        Get a IMessageContainer for the given mbox and uid combination.
+
+        :param mbox: the mbox this message belongs.
+        :param uid: the UID that identifies this message in this mailbox.
+        """
+
+    # IMessageConsumer
+
+    def consume(self, queue):
+        """
+        Creates a new document in soledad db.
+
+        :param queue: queue to get item from, with content of the document
+                      to be inserted.
+        :type queue: Queue
+        """
+        # TODO should delete the original message from incoming after
+        # the writes are done.
+        # TODO should handle the delete case
+        # TODO should handle errors
+
+        empty = queue.empty()
+        while not empty:
+            for item, call in self._process(queue):
+                self._try_call(call, item)
+            empty = queue.empty()
+
+    #
+    # SoledadStore specific methods.
+    #
+
+    def _process(self, queue):
+        """
+        Return the item and the proper call type for the next
+        item in the queue if any.
+
+        :param queue: the queue from where we'll pick item.
+        :type queue: Queue
+        """
+        msg_wrapper = queue.get()
+        return self._get_calls_for_msg_parts(msg_wrapper)
+
+    def _try_call(self, call, item):
+        """
+        Try to invoke a given call with item as a parameter.
+        """
+        if not call:
+            return
+        try:
+            call(item)
+        except u1db_errors.RevisionConflict as exc:
+            logger.error("Error: %r" % (exc,))
+            raise exc
+
+    def _get_calls_for_msg_parts(self, msg_wrapper):
+        """
+        Return the proper call type for a given item.
+
+        :param msg_wrapper: A MessageWrapper
+        :type msg_wrapper: IMessageContainer
+        """
+        call = None
+
+        if msg_wrapper.new is True:
+            call = self._soledad.create_doc
+
+            # item is expected to be a MessagePartDoc
+            for item in msg_wrapper.walk():
+                if item.part == MessagePartType.fdoc:
+                    yield dict(item.content), call
+
+                if item.part == MessagePartType.hdoc:
+                    if not self._header_does_exist(item.content):
+                        yield dict(item.content), call
+
+                if item.part == MessagePartType.cdoc:
+                    if self._content_does_exist(item.content):
+                        yield dict(item.content), call
+
+        # TODO should check for elements with the dirty state
+        # TODO if new == False and dirty == True, put_doc
+        # XXX for puts, we will have to retrieve
+        # the document, change the content, and
+        # pass the whole document under "content"
+        else:
+            logger.error("Cannot put documents yet!")
-- 
cgit v1.2.3


From 0754dac293730b02942716991d5edc513c36ff7c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 04:35:10 -0400
Subject: debug info

---
 src/leap/mail/imap/messages.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 46c9dc9..3c30aa8 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -524,8 +524,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
         message.
         """
         hdoc_content = self._hdoc.content
+        print "hdoc: ", hdoc_content
         body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
+        print "body phash: ", body_phash
         if not body_phash:
             logger.warning("No body phash for this document!")
             return None
@@ -537,16 +539,19 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
         if self._container is not None:
             bdoc = self._container.memstore.get_by_phash(body_phash)
+            print "bdoc from container -->", bdoc
             if bdoc:
                 return bdoc
             else:
                 print "no doc for that phash found!"
 
+        print "nuthing. soledad?"
         # no memstore or no doc found there
         if self._soledad:
             body_docs = self._soledad.get_from_index(
                 fields.TYPE_P_HASH_IDX,
                 fields.TYPE_CONTENT_VAL, str(body_phash))
+            print "returning body docs,,,", body_docs
             return first(body_docs)
         else:
             logger.error("No phash in container, and no soledad found!")
-- 
cgit v1.2.3


From ff28e22977db802c87f0b7be99e37c6de29183e9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 23 Jan 2014 13:32:01 -0400
Subject: Unset new flag after successful write

---
 src/leap/mail/imap/memorystore.py  | 16 ++++++++
 src/leap/mail/imap/messageparts.py | 33 ++++++++++++---
 src/leap/mail/imap/messages.py     | 27 ++++++++----
 src/leap/mail/imap/server.py       |  5 +++
 src/leap/mail/imap/soledadstore.py | 84 +++++++++++++++++++++++++++++++-------
 src/leap/mail/load_tests.py        |  3 --
 src/leap/mail/walk.py              | 33 +++++++++++++++
 7 files changed, 169 insertions(+), 32 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 7cb361f..f0bdab5 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -271,12 +271,28 @@ class MemoryStore(object):
         return (self.get_message(*key)
                 for key in sorted(self._msg_store.keys()))
 
+    # new, dirty flags
+
     def _get_new_dirty_state(self, key):
         """
         Return `new` and `dirty` flags for a given message.
         """
         return map(lambda _set: key in _set, (self._new, self._dirty))
 
+    def set_new(self, key):
+        """
+        Add the key value to the `new` set.
+        """
+        self._new.add(key)
+
+    def unset_new(self, key):
+        """
+        Remove the key value from the `new` set.
+        """
+        print "******************"
+        print "UNSETTING NEW FOR: %s" % str(key)
+        self._new.discard(key)
+
     @property
     def is_writing(self):
         """
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 3f89193..42eef02 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -125,20 +125,41 @@ class MessageWrapper(object):
 
     # properties
 
-    @property
-    def new(self):
+    def _get_new(self):
+        """
+        Get the value for the `new` flag.
+        """
         return self._new
 
-    def set_new(self, value=True):
+    def _set_new(self, value=True):
+        """
+        Set the value for the `new` flag, and propagate it
+        to the memory store if any.
+        """
         self._new = value
+        if self.memstore:
+            mbox = self.fdoc.content['mbox']
+            uid = self.fdoc.content['uid']
+            key = mbox, uid
+            fun = [self.memstore.unset_new,
+                   self.memstore.set_new][int(value)]
+            fun(key)
+        else:
+            logger.warning("Could not find a memstore referenced from this "
+                           "MessageWrapper. The value for new will not be "
+                           "propagated")
 
-    @property
-    def dirty(self):
+    new = property(_get_new, _set_new,
+                   doc="The `new` flag for this MessageWrapper")
+
+    def _get_dirty(self):
         return self._dirty
 
-    def set_dirty(self, value=True):
+    def _set_dirty(self, value=True):
         self._dirty = value
 
+    dirty = property(_get_dirty, _set_dirty)
+
     # IMessageContainer
 
     @property
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 3c30aa8..94bd714 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -42,6 +42,7 @@ from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.memorystore import MessageWrapper
+from leap.mail.imap.messageparts import MessagePart
 from leap.mail.imap.parser import MailParser, MBoxParser
 
 logger = logging.getLogger(__name__)
@@ -306,15 +307,25 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: file-like object opened for reading
         :rtype: StringIO
         """
+        def write_fd(body):
+            fd.write(body)
+            fd.seek(0)
+            return fd
+
         # TODO refactor with getBodyFile in MessagePart
         fd = StringIO.StringIO()
         if self._bdoc is not None:
             bdoc_content = self._bdoc.content
+            if bdoc_content is None:
+                logger.warning("No BODC content found for message!!!")
+                return write_fd(str(""))
+
             body = bdoc_content.get(self.RAW_KEY, "")
             content_type = bdoc_content.get('content-type', "")
             charset = find_charset(content_type)
             logger.debug('got charset from content-type: %s' % charset)
             if charset is None:
+                # XXX change for find_charset utility
                 charset = self._get_charset(body)
             try:
                 body = body.encode(charset)
@@ -328,15 +339,13 @@ class LeapMessage(fields, MailParser, MBoxParser):
                         body = body.encode('utf-8', 'replace')
                     except:
                         pass
+            finally:
+                return write_fd(body)
 
         # We are still returning funky characters from here.
         else:
             logger.warning("No BDOC found for message.")
-            body = str("")
-
-        fd.write(body)
-        fd.seek(0)
-        return fd
+            return write_fd(str(""))
 
     @memoized_method
     def _get_charset(self, stuff):
@@ -524,7 +533,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         message.
         """
         hdoc_content = self._hdoc.content
-        print "hdoc: ", hdoc_content
+        #print "hdoc: ", hdoc_content
         body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
         print "body phash: ", body_phash
@@ -540,10 +549,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
         if self._container is not None:
             bdoc = self._container.memstore.get_by_phash(body_phash)
             print "bdoc from container -->", bdoc
-            if bdoc:
+            if bdoc and bdoc.content is not None:
                 return bdoc
             else:
-                print "no doc for that phash found!"
+                print "no doc or not bdoc content for that phash found!"
 
         print "nuthing. soledad?"
         # no memstore or no doc found there
@@ -551,7 +560,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             body_docs = self._soledad.get_from_index(
                 fields.TYPE_P_HASH_IDX,
                 fields.TYPE_CONTENT_VAL, str(body_phash))
-            print "returning body docs,,,", body_docs
+            print "returning body docs...", body_docs
             return first(body_docs)
         else:
             logger.error("No phash in container, and no soledad found!")
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 8bd875b..c95a9be 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -196,4 +196,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
         """
         # TODO return the output of _memstore.is_writing
         # XXX and that should return a deferred!
+
+        # XXX  fake a delayed operation, to debug problem with messages getting
+        # back to the source mailbox...
+        import time
+        time.sleep(2)
         return None
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 62a3c53..d36acae 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -19,6 +19,8 @@ A MessageStore that writes to Soledad.
 """
 import logging
 
+from itertools import chain
+
 from u1db import errors as u1db_errors
 from zope.interface import implements
 
@@ -30,6 +32,13 @@ from leap.mail.messageflow import IMessageConsumer
 logger = logging.getLogger(__name__)
 
 
+# TODO
+# [ ] Delete original message from the incoming queue after all successful
+#     writes.
+# [ ] Implement a retry queue.
+# [ ] Consider journaling of operations.
+
+
 class ContentDedup(object):
     """
     Message deduplication.
@@ -37,8 +46,8 @@ class ContentDedup(object):
     We do a query for the content hashes before writing to our beloved
     sqlcipher backend of Soledad. This means, by now, that:
 
-    1. We will not store the same attachment twice, only the hash of it.
-    2. We will not store the same message body twice, only the hash of it.
+    1. We will not store the same body/attachment twice, only the hash of it.
+    2. We will not store the same message header twice, only the hash of it.
 
     The first case is useful if you are always receiving the same old memes
     from unwary friends that still have not discovered that 4chan is the
@@ -49,6 +58,7 @@ class ContentDedup(object):
     to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
     Stack.
     """
+    # TODO refactor using unique_query
 
     def _header_does_exist(self, doc):
         """
@@ -99,6 +109,12 @@ class ContentDedup(object):
         return True
 
 
+class MsgWriteError(Exception):
+    """
+    Raised if any exception is found while saving message parts.
+    """
+
+
 class SoledadStore(ContentDedup):
     """
     This will create docs in the local Soledad database.
@@ -108,7 +124,7 @@ class SoledadStore(ContentDedup):
 
     def __init__(self, soledad):
         """
-        Initialize the writer.
+        Initialize the permanent store that writes to Soledad database.
 
         :param soledad: the soledad instance
         :type soledad: Soledad
@@ -165,15 +181,40 @@ class SoledadStore(ContentDedup):
                       to be inserted.
         :type queue: Queue
         """
-        # TODO should delete the original message from incoming after
+        # TODO should delete the original message from incoming only after
         # the writes are done.
         # TODO should handle the delete case
         # TODO should handle errors
+        # TODO could generalize this method into a generic consumer
+        # and only implement `process` here
 
         empty = queue.empty()
         while not empty:
-            for item, call in self._process(queue):
-                self._try_call(call, item)
+            items = self._process(queue)
+            # we prime the generator, that should return the
+            # item in the first place.
+            msg_wrapper = items.next()
+
+            # From here, we unpack the subpart items and
+            # the right soledad call.
+            try:
+                failed = False
+                for item, call in items:
+                    try:
+                        self._try_call(call, item)
+                    except Exception:
+                        failed = True
+                        continue
+                if failed:
+                    raise MsgWriteError
+
+            except MsgWriteError:
+                logger.error("Error while processing item.")
+                pass
+            else:
+                # If everything went well, we can unset the new flag
+                # in the source store (memory store)
+                msg_wrapper.new = False
             empty = queue.empty()
 
     #
@@ -182,14 +223,16 @@ class SoledadStore(ContentDedup):
 
     def _process(self, queue):
         """
-        Return the item and the proper call type for the next
-        item in the queue if any.
+        Return an iterator that will yield the msg_wrapper in the first place,
+        followed by the subparts item and the proper call type for every
+        item in the queue, if any.
 
         :param queue: the queue from where we'll pick item.
         :type queue: Queue
         """
         msg_wrapper = queue.get()
-        return self._get_calls_for_msg_parts(msg_wrapper)
+        return chain((msg_wrapper,),
+                     self._get_calls_for_msg_parts(msg_wrapper))
 
     def _try_call(self, call, item):
         """
@@ -205,7 +248,7 @@ class SoledadStore(ContentDedup):
 
     def _get_calls_for_msg_parts(self, msg_wrapper):
         """
-        Return the proper call type for a given item.
+        Generator that return the proper call type for a given item.
 
         :param msg_wrapper: A MessageWrapper
         :type msg_wrapper: IMessageContainer
@@ -220,18 +263,31 @@ class SoledadStore(ContentDedup):
                 if item.part == MessagePartType.fdoc:
                     yield dict(item.content), call
 
-                if item.part == MessagePartType.hdoc:
+                elif item.part == MessagePartType.hdoc:
                     if not self._header_does_exist(item.content):
                         yield dict(item.content), call
 
-                if item.part == MessagePartType.cdoc:
-                    if self._content_does_exist(item.content):
+                elif item.part == MessagePartType.cdoc:
+                    if not self._content_does_exist(item.content):
+
+                        # XXX DEBUG -------------------
+                        print "about to write content-doc ",
+                        #import pprint; pprint.pprint(item.content)
+
                         yield dict(item.content), call
 
+        # TODO should write back to the queue
+        # with the results of the operation.
+        # We can write there:
+        # (*) MsgWriteACK  --> Should remove from incoming queue.
+        #                      (We should do this here).
+
+        # Implement using callbacks for each operation.
+
         # TODO should check for elements with the dirty state
         # TODO if new == False and dirty == True, put_doc
         # XXX for puts, we will have to retrieve
         # the document, change the content, and
         # pass the whole document under "content"
         else:
-            logger.error("Cannot put documents yet!")
+            logger.error("Cannot put/delete documents yet!")
diff --git a/src/leap/mail/load_tests.py b/src/leap/mail/load_tests.py
index ee89fcc..be65b8d 100644
--- a/src/leap/mail/load_tests.py
+++ b/src/leap/mail/load_tests.py
@@ -14,12 +14,9 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
 """
 Provide a function for loading tests.
 """
-
 import unittest
 
 
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 30cb70a..49f2c22 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -176,3 +176,36 @@ def walk_msg_tree(parts, body_phash=None):
         pdoc = outer
     pdoc[BODY] = body_phash
     return pdoc
+
+"""
+Groucho Marx: Now pay particular attention to this first clause, because it's
+              most important. There's the party of the first part shall be
+              known in this contract as the party of the first part. How do you
+              like that, that's pretty neat eh?
+
+Chico Marx: No, that's no good.
+Groucho Marx: What's the matter with it?
+
+Chico Marx: I don't know, let's hear it again.
+Groucho Marx: So the party of the first part shall be known in this contract as
+              the party of the first part.
+
+Chico Marx: Well it sounds a little better this time.
+Groucho Marx: Well, it grows on you. Would you like to hear it once more?
+
+Chico Marx: Just the first part.
+Groucho Marx: All right. It says the first part of the party of the first part
+              shall be known in this contract as the first part of the party of
+              the first part, shall be known in this contract - look, why
+              should we quarrel about a thing like this, we'll take it right
+              out, eh?
+
+Chico Marx: Yes, it's too long anyhow. Now what have we got left?
+Groucho Marx: Well I've got about a foot and a half. Now what's the matter?
+
+Chico Marx: I don't like the second party either.
+"""
+
+"""
+I feel you deserved it after reading the above and try to debug your problem ;)
+"""
-- 
cgit v1.2.3


From e02db78b1b6d8fe021efd4adb250c64a1dd4bac4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 24 Jan 2014 05:39:13 -0400
Subject: flags use the memstore

* add new/dirty deferred dict to notify when written to disk
* fix eventual duplication after copy
* fix flag flickering on first retrieval.
---
 src/leap/mail/imap/mailbox.py      |  70 +++++++---
 src/leap/mail/imap/memorystore.py  | 265 ++++++++++++++++++++++++++++++++-----
 src/leap/mail/imap/messageparts.py |  72 ++++++----
 src/leap/mail/imap/messages.py     | 162 +++++++++++++++--------
 src/leap/mail/imap/soledadstore.py |  35 +++--
 src/leap/mail/messageflow.py       |   8 +-
 src/leap/mail/utils.py             |   9 ++
 7 files changed, 479 insertions(+), 142 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 5e16b4b..108d0da 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -36,6 +36,7 @@ from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
 from leap.mail.decorators import deferred
+from leap.mail.utils import empty
 from leap.mail.imap.fields import WithMsgFields, fields
 from leap.mail.imap.messages import MessageCollection
 from leap.mail.imap.messageparts import MessageWrapper
@@ -475,8 +476,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Remove all messages flagged \\Deleted
         """
+        print "EXPUNGE!"
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
+        mstore = self._memstore
+        if mstore is not None:
+            deleted = mstore.all_deleted_uid_iter(self.mbox)
+            print "deleted ", list(deleted)
+            for uid in deleted:
+                mstore.remove_message(self.mbox, uid)
+
+        print "now deleting from soledad"
         d = self.messages.remove_all_deleted()
         d.addCallback(self._expunge_cb)
         d.addCallback(self.messages.reset_last_uid)
@@ -709,21 +719,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             msg = self.messages.get_msg_by_uid(msg_id)
             if not msg:
                 continue
+            # We duplicate the set operations here
+            # to return the result because it's less costly than
+            # retrieving the flags again.
+            newflags = set(msg.getFlags())
+
             if mode == 1:
                 msg.addFlags(flags)
+                newflags = newflags.union(set(flags))
             elif mode == -1:
                 msg.removeFlags(flags)
+                newflags.difference_update(flags)
             elif mode == 0:
                 msg.setFlags(flags)
-            result[msg_id] = msg.getFlags()
-
-        # After changing flags, we want to signal again to the
-        # UI because the number of unread might have changed.
-        # Hoever, we should probably limit this to INBOX only?
-        # this should really be called as a final callback of
-        # the do_STORE method...
-        from twisted.internet import reactor
-        deferLater(reactor, 1, self.signal_unread_to_ui)
+                newflags = set(flags)
+            result[msg_id] = newflags
         return result
 
     # ISearchableMailbox
@@ -780,6 +790,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         from twisted.internet import reactor
         uid_next = self.getUIDNext()
         msg = messageObject
+        memstore = self._memstore
 
         # XXX should use a public api instead
         fdoc = msg._fdoc
@@ -787,20 +798,35 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not fdoc:
             logger.debug("Tried to copy a MSG with no fdoc")
             return
-
         new_fdoc = copy.deepcopy(fdoc.content)
-        new_fdoc[self.UID_KEY] = uid_next
-        new_fdoc[self.MBOX_KEY] = self.mbox
-        self._memstore.create_message(
-            self.mbox, uid_next,
-            MessageWrapper(
-                new_fdoc, hdoc.content))
-
-        # XXX use memory store !!!
-        if hasattr(hdoc, 'doc_id'):
-            self.messages.add_hdocset_docid(hdoc.doc_id)
-
-        deferLater(reactor, 1, self.notify_new)
+
+        fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
+        dest_fdoc = memstore.get_fdoc_from_chash(
+            fdoc_chash, self.mbox)
+        exist = dest_fdoc and not empty(dest_fdoc.content)
+
+        if exist:
+            print "Destination message already exists!"
+
+        else:
+            print "DO COPY MESSAGE!"
+            new_fdoc[self.UID_KEY] = uid_next
+            new_fdoc[self.MBOX_KEY] = self.mbox
+
+            # XXX set recent!
+
+            print "****************************"
+            print "copy message..."
+            print "new fdoc ", new_fdoc
+            print "hdoc: ", hdoc
+            print "****************************"
+
+            self._memstore.create_message(
+                self.mbox, uid_next,
+                MessageWrapper(
+                    new_fdoc, hdoc.content))
+
+            deferLater(reactor, 1, self.notify_new)
 
     # convenience fun
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index f0bdab5..f0c0d4b 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -21,10 +21,13 @@ import contextlib
 import logging
 import weakref
 
+from twisted.internet import defer
 from twisted.internet.task import LoopingCall
+from twisted.python import log
 from zope.interface import implements
 
 from leap.mail import size
+from leap.mail.utils import empty
 from leap.mail.messageflow import MessageProducer
 from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
@@ -34,6 +37,8 @@ from leap.mail.imap.messageparts import ReferenciableDict
 
 logger = logging.getLogger(__name__)
 
+SOLEDAD_WRITE_PERIOD = 20
+
 
 @contextlib.contextmanager
 def set_bool_flag(obj, att):
@@ -79,7 +84,8 @@ class MemoryStore(object):
 
     WRITING_FLAG = "_writing"
 
-    def __init__(self, permanent_store=None, write_period=60):
+    def __init__(self, permanent_store=None,
+                 write_period=SOLEDAD_WRITE_PERIOD):
         """
         Initialize a MemoryStore.
 
@@ -92,10 +98,23 @@ class MemoryStore(object):
         self._permanent_store = permanent_store
         self._write_period = write_period
 
-        # Internal Storage
+        # Internal Storage: messages
         self._msg_store = {}
+
+        # Internal Storage: payload-hash
+        """
+        {'phash': weakreaf.proxy(dict)}
+        """
         self._phash_store = {}
 
+        # Internal Storage: content-hash:fdoc
+        """
+        {'chash': {'mbox-a': weakref.proxy(dict),
+                   'mbox-b': weakref.proxy(dict)}
+        }
+        """
+        self._chash_fdoc_store = {}
+
         # TODO ----------------- implement mailbox-level flags store too! ----
         self._rflags_store = {}
         self._hdocset_store = {}
@@ -103,7 +122,9 @@ class MemoryStore(object):
 
         # New and dirty flags, to set MessageWrapper State.
         self._new = set([])
+        self._new_deferreds = {}
         self._dirty = set([])
+        self._dirty_deferreds = {}
 
         # Flag for signaling we're busy writing to the disk storage.
         setattr(self, self.WRITING_FLAG, False)
@@ -141,48 +162,141 @@ class MemoryStore(object):
     # We would have to add a put_flags operation to modify only
     # the flags doc (and set the dirty flag accordingly)
 
-    def create_message(self, mbox, uid, message):
+    def create_message(self, mbox, uid, message, notify_on_disk=True):
         """
         Create the passed message into this MemoryStore.
 
         By default we consider that any message is a new message.
+
+        :param mbox: the mailbox
+        :type mbox: basestring
+        :param uid: the UID for the message
+        :type uid: int
+        :param message: a to be added
+        :type message: MessageWrapper
+        :param notify_on_disk:
+        :type notify_on_disk: bool
+
+        :return: a Deferred. if notify_on_disk is True, will be fired
+                 when written to the db on disk.
+                 Otherwise will fire inmediately
+        :rtype: Deferred
         """
         print "adding new doc to memstore %s (%s)" % (mbox, uid)
         key = mbox, uid
+
+        d = defer.Deferred()
+        d.addCallback(lambda result: log.msg("message save: %s" % result))
+
         self._new.add(key)
+        self._new_deferreds[key] = d
+        self._add_message(mbox, uid, message, notify_on_disk)
+        print "create message: ", d
+        return d
 
-        msg_dict = message.as_dict()
-        self._msg_store[key] = msg_dict
+    def put_message(self, mbox, uid, message, notify_on_disk=True):
+        """
+        Put an existing message.
 
-        cdocs = message.cdocs
+        :param mbox: the mailbox
+        :type mbox: basestring
+        :param uid: the UID for the message
+        :type uid: int
+        :param message: a to be added
+        :type message: MessageWrapper
+        :param notify_on_disk:
+        :type notify_on_disk: bool
 
-        dirty = key in self._dirty
-        new = key in self._new
+        :return: a Deferred. if notify_on_disk is True, will be fired
+                 when written to the db on disk.
+                 Otherwise will fire inmediately
+        :rtype: Deferred
+        """
+        key = mbox, uid
+
+        d = defer.Deferred()
+        d.addCallback(lambda result: log.msg("message save: %s" % result))
+
+        self._dirty.add(key)
+        self._dirty_deferreds[key] = d
+        self._add_message(mbox, uid, message, notify_on_disk)
+        return d
 
-        # XXX should capture this in log...
+    def _add_message(self, mbox, uid, message, notify_on_disk=True):
+        # XXX have to differentiate between notify_new and notify_dirty
+
+        key = mbox, uid
+        msg_dict = message.as_dict()
+        print "ADDING MESSAGE..."
+        import pprint; pprint.pprint(msg_dict)
+
+        # XXX use the enum as keys
+
+        try:
+            store = self._msg_store[key]
+        except KeyError:
+            self._msg_store[key] = {'fdoc': {},
+                                    'hdoc': {},
+                                    'cdocs': {},
+                                    'docs_id': {}}
+            store = self._msg_store[key]
+
+        print "In store (before):"
+        import pprint; pprint.pprint(store)
+
+        #self._msg_store[key] = msg_dict
+        fdoc = msg_dict.get('fdoc', None)
+        if fdoc:
+            if not store.get('fdoc', None):
+                store['fdoc'] = ReferenciableDict({})
+            store['fdoc'].update(fdoc)
+
+            # content-hash indexing
+            chash = fdoc.get(fields.CONTENT_HASH_KEY)
+            chash_fdoc_store = self._chash_fdoc_store
+            if not chash in chash_fdoc_store:
+                chash_fdoc_store[chash] = {}
+
+            chash_fdoc_store[chash][mbox] = weakref.proxy(
+                store['fdoc'])
+
+        hdoc = msg_dict.get('hdoc', None)
+        if hdoc:
+            if not store.get('hdoc', None):
+                store['hdoc'] = ReferenciableDict({})
+            store['hdoc'].update(hdoc)
+
+        docs_id = msg_dict.get('docs_id', None)
+        if docs_id:
+            if not store.get('docs_id', None):
+                store['docs_id'] = {}
+            store['docs_id'].update(docs_id)
 
+        cdocs = message.cdocs
         for cdoc_key in cdocs.keys():
-            print "saving cdoc"
-            cdoc = self._msg_store[key]['cdocs'][cdoc_key]
+            if not store.get('cdocs', None):
+                store['cdocs'] = {}
 
-            # FIXME this should be done in the MessageWrapper constructor
-            # instead...
+            cdoc = cdocs[cdoc_key]
             # first we make it weak-referenciable
             referenciable_cdoc = ReferenciableDict(cdoc)
-            self._msg_store[key]['cdocs'][cdoc_key] = MessagePartDoc(
-                new=new, dirty=dirty, store="mem",
-                part=MessagePartType.cdoc,
-                content=referenciable_cdoc)
+            store['cdocs'][cdoc_key] = referenciable_cdoc
             phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None)
             if not phash:
                 continue
             self._phash_store[phash] = weakref.proxy(referenciable_cdoc)
 
-    def put_message(self, mbox, uid, msg):
-        """
-        Put an existing message.
-        """
-        return NotImplementedError()
+        def prune(seq, store):
+            for key in seq:
+                if key in store and empty(store.get(key)):
+                    store.pop(key)
+
+        prune(('fdoc', 'hdoc', 'cdocs', 'docs_id'), store)
+        #import ipdb; ipdb.set_trace()
+
+
+        print "after appending to store: ", key
+        import pprint; pprint.pprint(self._msg_store[key])
 
     def get_message(self, mbox, uid):
         """
@@ -203,7 +317,13 @@ class MemoryStore(object):
         """
         Remove a Message from this MemoryStore.
         """
-        raise NotImplementedError()
+        try:
+            key = mbox, uid
+            self._new.discard(key)
+            self._dirty.discard(key)
+            self._msg_store.pop(key, None)
+        except Exception as exc:
+            logger.exception(exc)
 
     # IMessageStoreWriter
 
@@ -211,12 +331,15 @@ class MemoryStore(object):
         """
         Write the message documents in this MemoryStore to a different store.
         """
-        # XXX pass if it's writing (ie, the queue is not empty...)
-        # See how to make the writing_flag aware of the queue state...
-        print "writing messages to producer..."
+        # For now, we pass if the queue is not empty, to avoid duplication.
+        # We would better use a flag to know when we've already enqueued an
+        # item.
+        if not self.producer.is_queue_empty():
+            return
 
+        print "Writing messages to Soledad..."
         with set_bool_flag(self, self.WRITING_FLAG):
-            for msg_wrapper in self.all_msg_iter():
+            for msg_wrapper in self.all_new_dirty_msg_iter():
                 self.producer.push(msg_wrapper)
 
     # MemoryStore specific methods.
@@ -247,12 +370,14 @@ class MemoryStore(object):
         """
         return len(self._new)
 
-    def get_by_phash(self, phash):
+    def get_cdoc_from_phash(self, phash):
         """
         Return a content-document by its payload-hash.
         """
         doc = self._phash_store.get(phash, None)
 
+        # XXX return None for consistency?
+
         # XXX have to keep a mapping between phash and its linkage
         # info, to know if this payload is been already saved or not.
         # We will be able to get this from the linkage-docs,
@@ -262,7 +387,40 @@ class MemoryStore(object):
         return MessagePartDoc(
             new=new, dirty=dirty, store="mem",
             part=MessagePartType.cdoc,
-            content=doc)
+            content=doc,
+            doc_id=None)
+
+    def get_fdoc_from_chash(self, chash, mbox):
+        """
+        Return a flags-document by its content-hash and a given mailbox.
+
+        :return: MessagePartDoc, or None.
+        """
+        docs_dict = self._chash_fdoc_store.get(chash, None)
+        fdoc = docs_dict.get(mbox, None) if docs_dict else None
+
+        print "GETTING FDOC BY CHASH:", fdoc
+
+        # a couple of special cases.
+        # 1. We might have a doc with empty content...
+        if empty(fdoc):
+            return None
+
+        # ...Or the message could exist, but being flagged for deletion.
+        # We want to create a new one in this case.
+        # Hmmm what if the deletion is un-done?? We would end with a
+        # duplicate...
+        if fdoc and fields.DELETED_FLAG in fdoc[fields.FLAGS_KEY]:
+            return None
+
+        # XXX get flags
+        new = True
+        dirty = False
+        return MessagePartDoc(
+            new=new, dirty=dirty, store="mem",
+            part=MessagePartType.fdoc,
+            content=fdoc,
+            doc_id=None)
 
     def all_msg_iter(self):
         """
@@ -271,6 +429,25 @@ class MemoryStore(object):
         return (self.get_message(*key)
                 for key in sorted(self._msg_store.keys()))
 
+    def all_new_dirty_msg_iter(self):
+        """
+        Return geneator that iterates through all new and dirty messages.
+        """
+        return (self.get_message(*key)
+                for key in sorted(self._msg_store.keys())
+                if key in self._new or key in self._dirty)
+
+    def all_deleted_uid_iter(self, mbox):
+        """
+        Return generator that iterates through the UIDs for all messags
+        with deleted flag in a given mailbox.
+        """
+        all_deleted = (
+            msg['fdoc']['uid'] for msg in self._msg_store.values()
+            if msg.get('fdoc', None)
+            and fields.DELETED_FLAG in msg['fdoc']['flags'])
+        return all_deleted
+
     # new, dirty flags
 
     def _get_new_dirty_state(self, key):
@@ -289,9 +466,35 @@ class MemoryStore(object):
         """
         Remove the key value from the `new` set.
         """
-        print "******************"
-        print "UNSETTING NEW FOR: %s" % str(key)
+        print "Unsetting NEW for: %s" % str(key)
         self._new.discard(key)
+        deferreds = self._new_deferreds
+        d = deferreds.get(key, None)
+        if d:
+            # XXX use a namedtuple for passing the result
+            # when we check it in the other side.
+            d.callback('%s, ok' % str(key))
+            deferreds.pop(key)
+
+    def set_dirty(self, key):
+        """
+        Add the key value to the `dirty` set.
+        """
+        self._dirty.add(key)
+
+    def unset_dirty(self, key):
+        """
+        Remove the key value from the `dirty` set.
+        """
+        print "Unsetting DIRTY for: %s" % str(key)
+        self._dirty.discard(key)
+        deferreds = self._dirty_deferreds
+        d = deferreds.get(key, None)
+        if d:
+            # XXX use a namedtuple for passing the result
+            # when we check it in the other side.
+            d.callback('%s, ok' % str(key))
+            deferreds.pop(key)
 
     @property
     def is_writing(self):
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 42eef02..b43bc37 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -65,15 +65,13 @@ and sometimes to a part in particular only.
   we have modified its state in memory, so we need to put_doc instead while
   dumping the MemoryStore contents.
   `dirty` attribute would only apply to flags-docs and linkage-docs.
-
-
-  XXX this is still not implemented!
+* `doc_id` is the identifier for the document in the u1db database, if any.
 
 """
 
 MessagePartDoc = namedtuple(
     'MessagePartDoc',
-    ['new', 'dirty', 'part', 'store', 'content'])
+    ['new', 'dirty', 'part', 'store', 'content', 'doc_id'])
 
 
 class ReferenciableDict(dict):
@@ -96,6 +94,7 @@ class MessageWrapper(object):
     FDOC = "fdoc"
     HDOC = "hdoc"
     CDOCS = "cdocs"
+    DOCS_ID = "docs_id"
 
     # XXX can use this to limit the memory footprint,
     # or is it too premature to optimize?
@@ -105,12 +104,17 @@ class MessageWrapper(object):
 
     def __init__(self, fdoc=None, hdoc=None, cdocs=None,
                  from_dict=None, memstore=None,
-                 new=True, dirty=False):
+                 new=True, dirty=False, docs_id={}):
+        """
+        Initialize a MessageWrapper.
+        """
+        # TODO add optional reference to original message in the incoming
         self._dict = {}
         self.memstore = memstore
 
         self._new = new
         self._dirty = dirty
+
         self._storetype = "mem"
 
         if from_dict is not None:
@@ -122,6 +126,7 @@ class MessageWrapper(object):
                 self._dict[self.HDOC] = ReferenciableDict(hdoc)
             if cdocs is not None:
                 self._dict[self.CDOCS] = ReferenciableDict(cdocs)
+        self._dict[self.DOCS_ID] = docs_id
 
     # properties
 
@@ -153,10 +158,28 @@ class MessageWrapper(object):
                    doc="The `new` flag for this MessageWrapper")
 
     def _get_dirty(self):
+        """
+        Get the value for the `dirty` flag.
+        """
         return self._dirty
 
     def _set_dirty(self, value=True):
+        """
+        Set the value for the `dirty` flag, and propagate it
+        to the memory store if any.
+        """
         self._dirty = value
+        if self.memstore:
+            mbox = self.fdoc.content['mbox']
+            uid = self.fdoc.content['uid']
+            key = mbox, uid
+            fun = [self.memstore.unset_dirty,
+                   self.memstore.set_dirty][int(value)]
+            fun(key)
+        else:
+            logger.warning("Could not find a memstore referenced from this "
+                           "MessageWrapper. The value for new will not be "
+                           "propagated")
 
     dirty = property(_get_dirty, _set_dirty)
 
@@ -173,7 +196,9 @@ class MessageWrapper(object):
         return MessagePartDoc(new=self.new, dirty=self.dirty,
                               store=self._storetype,
                               part=MessagePartType.fdoc,
-                              content=content_ref)
+                              content=content_ref,
+                              doc_id=self._dict[self.DOCS_ID].get(
+                                  self.FDOC, None))
 
     @property
     def hdoc(self):
@@ -186,7 +211,9 @@ class MessageWrapper(object):
         return MessagePartDoc(new=self.new, dirty=self.dirty,
                               store=self._storetype,
                               part=MessagePartType.hdoc,
-                              content=content_ref)
+                              content=content_ref,
+                              doc_id=self._dict[self.DOCS_ID].get(
+                                  self.HDOC, None))
 
     @property
     def cdocs(self):
@@ -201,21 +228,18 @@ class MessageWrapper(object):
         Generator that iterates through all the parts, returning
         MessagePartDoc.
         """
-        yield self.fdoc
-        yield self.hdoc
+        if self.fdoc is not None:
+            yield self.fdoc
+        if self.hdoc is not None:
+            yield self.hdoc
         for cdoc in self.cdocs.values():
-            # XXX this will break ----
-            #content_ref = weakref.proxy(cdoc)
-            #yield MessagePartDoc(new=self.new, dirty=self.dirty,
-                                 #store=self._storetype,
-                                 #part=MessagePartType.cdoc,
-                                 #content=content_ref)
-
-            # the put is handling this for us, so
-            # we already have stored a MessagePartDoc
-            # but we should really do it while adding in the
-            # constructor or the from_dict method
-            yield cdoc
+            if cdoc is not None:
+                content_ref = weakref.proxy(cdoc)
+                yield MessagePartDoc(new=self.new, dirty=self.dirty,
+                                     store=self._storetype,
+                                     part=MessagePartType.cdoc,
+                                     content=content_ref,
+                                     doc_id=None)
 
     # i/o
 
@@ -234,9 +258,9 @@ class MessageWrapper(object):
         fdoc, hdoc, cdocs = map(
             lambda part: msg_dict.get(part, None),
             [self.FDOC, self.HDOC, self.CDOCS])
-        self._dict[self.FDOC] = fdoc
-        self._dict[self.HDOC] = hdoc
-        self._dict[self.CDOCS] = cdocs
+        for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc),
+                       (self.CDOCS, cdocs)):
+            self._dict[t] = ReferenciableDict(doc) if doc else None
 
 
 class MessagePart(object):
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 94bd714..c212472 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -37,7 +37,7 @@ from leap.common.check import leap_assert, leap_assert_type
 from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail import walk
-from leap.mail.utils import first, find_charset, lowerdict
+from leap.mail.utils import first, find_charset, lowerdict, empty
 from leap.mail.decorators import deferred
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
@@ -130,6 +130,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         self.__chash = None
         self.__bdoc = None
 
+    # XXX make these properties public
+
     @property
     def _fdoc(self):
         """
@@ -154,8 +156,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         if self._container is not None:
             hdoc = self._container.hdoc
-            if hdoc:
+            if hdoc and not empty(hdoc.content):
                 return hdoc
+        # XXX cache this into the memory store !!!
         return self._get_headers_doc()
 
     @property
@@ -248,7 +251,13 @@ class LeapMessage(fields, MailParser, MBoxParser):
         doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
         doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
 
-        if getattr(doc, 'store', None) != "mem":
+        if self._collection.memstore is not None:
+            self._collection.memstore.put_message(
+                self._mbox, self._uid,
+                MessageWrapper(fdoc=doc.content, new=False, dirty=True,
+                               docs_id={'fdoc': doc.doc_id}))
+        else:
+            # fallback for non-memstore initializations.
             self._soledad.put_doc(doc)
 
     def addFlags(self, flags):
@@ -547,20 +556,18 @@ class LeapMessage(fields, MailParser, MBoxParser):
         # phash doc...
 
         if self._container is not None:
-            bdoc = self._container.memstore.get_by_phash(body_phash)
+            bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
             print "bdoc from container -->", bdoc
             if bdoc and bdoc.content is not None:
                 return bdoc
             else:
                 print "no doc or not bdoc content for that phash found!"
 
-        print "nuthing. soledad?"
         # no memstore or no doc found there
         if self._soledad:
             body_docs = self._soledad.get_from_index(
                 fields.TYPE_P_HASH_IDX,
                 fields.TYPE_CONTENT_VAL, str(body_phash))
-            print "returning body docs...", body_docs
             return first(body_docs)
         else:
             logger.error("No phash in container, and no soledad found!")
@@ -581,32 +588,32 @@ class LeapMessage(fields, MailParser, MBoxParser):
     # setters
 
     # XXX to be used in the messagecopier interface?!
-
-    def set_uid(self, uid):
-        """
-        Set new uid for this message.
-
-        :param uid: the new uid
-        :type uid: basestring
-        """
+#
+    #def set_uid(self, uid):
+        #"""
+        #Set new uid for this message.
+#
+        #:param uid: the new uid
+        #:type uid: basestring
+        #"""
         # XXX dangerous! lock?
-        self._uid = uid
-        d = self._fdoc
-        d.content[self.UID_KEY] = uid
-        self._soledad.put_doc(d)
-
-    def set_mbox(self, mbox):
-        """
-        Set new mbox for this message.
-
-        :param mbox: the new mbox
-        :type mbox: basestring
-        """
+        #self._uid = uid
+        #d = self._fdoc
+        #d.content[self.UID_KEY] = uid
+        #self._soledad.put_doc(d)
+#
+    #def set_mbox(self, mbox):
+        #"""
+        #Set new mbox for this message.
+#
+        #:param mbox: the new mbox
+        #:type mbox: basestring
+        #"""
         # XXX dangerous! lock?
-        self._mbox = mbox
-        d = self._fdoc
-        d.content[self.MBOX_KEY] = mbox
-        self._soledad.put_doc(d)
+        #self._mbox = mbox
+        #d = self._fdoc
+        #d.content[self.MBOX_KEY] = mbox
+        #self._soledad.put_doc(d)
 
     # destructor
 
@@ -614,14 +621,13 @@ class LeapMessage(fields, MailParser, MBoxParser):
     def remove(self):
         """
         Remove all docs associated with this message.
+        Currently it removes only the flags doc.
         """
         # XXX For the moment we are only removing the flags and headers
         # docs. The rest we leave there polluting your hard disk,
         # until we think about a good way of deorphaning.
         # Maybe a crawler of unreferenced docs.
 
-        # XXX remove from memory store!!!
-
         # XXX implement elijah's idea of using a PUT document as a
         # token to ensure consistency in the removal.
 
@@ -632,13 +638,35 @@ class LeapMessage(fields, MailParser, MBoxParser):
         #bd = self._get_body_doc()
         #docs = [fd, hd, bd]
 
-        docs = [fd]
+        try:
+            memstore = self._collection.memstore
+        except AttributeError:
+            memstore = False
+
+        if memstore and hasattr(fd, "store", None) == "mem":
+            key = self._mbox, self._uid
+            if fd.new:
+                # it's a new document, so we can remove it and it will not
+                # be writen. Watch out! We need to be sure it has not been
+                # just queued to write!
+                memstore.remove_message(*key)
+
+            if fd.dirty:
+                doc_id = fd.doc_id
+                doc = self._soledad.get_doc(doc_id)
+                try:
+                    self._soledad.delete_doc(doc)
+                except Exception as exc:
+                    logger.exception(exc)
 
-        for d in filter(None, docs):
+        else:
+            # we just got a soledad_doc
             try:
-                self._soledad.delete_doc(d)
+                doc_id = fd.doc_id
+                latest_doc = self._soledad.get_doc(doc_id)
+                self._soledad.delete_doc(latest_doc)
             except Exception as exc:
-                logger.error(exc)
+                logger.exception(exc)
         return uid
 
     def does_exist(self):
@@ -786,8 +814,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # okay, all in order, keep going...
         self.mbox = self._parse_mailbox_name(mbox)
+
+        # XXX get a SoledadStore passed instead
         self._soledad = soledad
-        self._memstore = memstore
+        self.memstore = memstore
 
         self.__rflags = None
         self.__hdocset = None
@@ -913,13 +943,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :type chash: basestring
         :return: False, if it does not exist, or UID.
         """
-        exist = self._get_fdoc_from_chash(chash)
+        exist = False
+        if self.memstore is not None:
+            exist = self.memstore.get_fdoc_from_chash(chash, self.mbox)
+
+        if not exist:
+            exist = self._get_fdoc_from_chash(chash)
+
+        print "FDOC EXIST?", exist
         if exist:
             return exist.content.get(fields.UID_KEY, "unknown-uid")
         else:
             return False
 
-    @deferred
+    # not deferring to thread cause this now uses deferred asa retval
+    #@deferred
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
         """
         Creates a new message document.
@@ -945,6 +983,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # TODO add the linked-from info !
         # TODO add reference to the original message
+        print "ADDING MESSAGE..."
 
         logger.debug('adding message')
         if flags is None:
@@ -956,11 +995,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # check for uniqueness.
         if self._fdoc_already_exists(chash):
+            print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+            print
+            print
             logger.warning("We already have that message in this mailbox.")
             # note that this operation will leave holes in the UID sequence,
             # but we're gonna change that all the same for a local-only table.
             # so not touch it by the moment.
-            return False
+            return defer.succeed('already_exists')
 
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
@@ -999,7 +1041,16 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # TODO ---- add reference to original doc, to be deleted
         # after writes are done.
         msg_container = MessageWrapper(fd, hd, cdocs)
-        self._memstore.create_message(self.mbox, uid, msg_container)
+
+        # XXX Should allow also to dump to disk directly,
+        # for no-memstore cases.
+
+        # we return a deferred that, by default, will be triggered when
+        # saved to disk
+        d = self.memstore.create_message(self.mbox, uid, msg_container)
+        print "defered-add", d
+        print "adding message", d
+        return d
 
     def _remove_cb(self, result):
         return result
@@ -1247,17 +1298,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                  or None if not found.
         :rtype: LeapMessage
         """
-        print "getting msg by id!"
-        msg_container = self._memstore.get_message(self.mbox, uid)
-        print "msg container", msg_container
+        msg_container = self.memstore.get_message(self.mbox, uid)
         if msg_container is not None:
-            print "getting LeapMessage (from memstore)"
             # We pass a reference to soledad just to be able to retrieve
             # missing parts that cannot be found in the container, like
             # the content docs after a copy.
             msg = LeapMessage(self._soledad, uid, self.mbox, collection=self,
                               container=msg_container)
-            print "got msg:", msg
         else:
             msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
         if not msg.does_exist():
@@ -1303,8 +1350,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                        self._soledad.get_from_index(
                            fields.TYPE_MBOX_IDX,
                            fields.TYPE_FLAGS_VAL, self.mbox)])
-        if self._memstore is not None:
-            mem_uids = self._memstore.get_uids(self.mbox)
+        if self.memstore is not None:
+            mem_uids = self.memstore.get_uids(self.mbox)
             uids = db_uids.union(set(mem_uids))
         else:
             uids = db_uids
@@ -1328,19 +1375,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         Return a dict with all flags documents for this mailbox.
         """
         # XXX get all from memstore and cache it there
+        # FIXME should get all uids, get them fro memstore,
+        # and get only the missing ones from disk.
+
         all_flags = dict(((
             doc.content[self.UID_KEY],
             doc.content[self.FLAGS_KEY]) for doc in
             self._soledad.get_from_index(
                 fields.TYPE_MBOX_IDX,
                 fields.TYPE_FLAGS_VAL, self.mbox)))
-        if self._memstore is not None:
+        if self.memstore is not None:
             # XXX
-            uids = self._memstore.get_uids(self.mbox)
-            fdocs = [(uid, self._memstore.get_message(self.mbox, uid).fdoc)
-                     for uid in uids]
-            for uid, doc in fdocs:
-                all_flags[uid] = doc.content[self.FLAGS_KEY]
+            uids = self.memstore.get_uids(self.mbox)
+            docs = ((uid, self.memstore.get_message(self.mbox, uid))
+                    for uid in uids)
+            for uid, doc in docs:
+                all_flags[uid] = doc.fdoc.content[self.FLAGS_KEY]
 
         return all_flags
 
@@ -1378,8 +1428,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         count = self._soledad.get_count_from_index(
             fields.TYPE_MBOX_IDX,
             fields.TYPE_FLAGS_VAL, self.mbox)
-        if self._memstore is not None:
-            count += self._memstore.count_new()
+        if self.memstore is not None:
+            count += self.memstore.count_new()
         return count
 
     # unseen messages
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index d36acae..b321da8 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -81,7 +81,8 @@ class ContentDedup(object):
         if len(header_docs) != 1:
             logger.warning("Found more than one copy of chash %s!"
                            % (chash,))
-        logger.debug("Found header doc with that hash! Skipping save!")
+        # XXX re-enable
+        #logger.debug("Found header doc with that hash! Skipping save!")
         return True
 
     def _content_does_exist(self, doc):
@@ -105,7 +106,8 @@ class ContentDedup(object):
         if len(attach_docs) != 1:
             logger.warning("Found more than one copy of phash %s!"
                            % (phash,))
-        logger.debug("Found attachment doc with that hash! Skipping save!")
+        # XXX re-enable
+        #logger.debug("Found attachment doc with that hash! Skipping save!")
         return True
 
 
@@ -215,6 +217,7 @@ class SoledadStore(ContentDedup):
                 # If everything went well, we can unset the new flag
                 # in the source store (memory store)
                 msg_wrapper.new = False
+                msg_wrapper.dirty = False
             empty = queue.empty()
 
     #
@@ -261,6 +264,9 @@ class SoledadStore(ContentDedup):
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
                 if item.part == MessagePartType.fdoc:
+
+                    # FIXME add content duplication for HEADERS too!
+                    # (only 1 chash per mailbox!)
                     yield dict(item.content), call
 
                 elif item.part == MessagePartType.hdoc:
@@ -276,18 +282,31 @@ class SoledadStore(ContentDedup):
 
                         yield dict(item.content), call
 
+        # For now, the only thing that will be dirty is
+        # the flags doc.
+
+        elif msg_wrapper.dirty is True:
+            print "DIRTY DOC! ----------------------"
+            call = self._soledad.put_doc
+
+            # item is expected to be a MessagePartDoc
+            for item in msg_wrapper.walk():
+                doc_id = item.doc_id  # defend!
+                doc = self._soledad.get_doc(doc_id)
+                doc.content = item.content
+
+                if item.part == MessagePartType.fdoc:
+                    print "Will PUT the doc: ", doc
+                    yield dict(doc), call
+
+                # XXX also for linkage-doc
+
         # TODO should write back to the queue
         # with the results of the operation.
         # We can write there:
         # (*) MsgWriteACK  --> Should remove from incoming queue.
         #                      (We should do this here).
-
         # Implement using callbacks for each operation.
 
-        # TODO should check for elements with the dirty state
-        # TODO if new == False and dirty == True, put_doc
-        # XXX for puts, we will have to retrieve
-        # the document, change the content, and
-        # pass the whole document under "content"
         else:
             logger.error("Cannot put/delete documents yet!")
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index ed6abcd..b7fc030 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -126,9 +126,15 @@ class MessageProducer(object):
         again after the addition of new items.
         """
         self._consumer.consume(self._queue)
-        if self._queue.empty():
+        if self.is_queue_empty():
             self.stop()
 
+    def is_queue_empty(self):
+        """
+        Return True if queue is empty, False otherwise.
+        """
+        return self._queue.empty()
+
     # public methods: IMessageProducer
 
     def push(self, item):
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 64af04f..bae2898 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -36,6 +36,15 @@ def first(things):
         return None
 
 
+def empty(thing):
+    """
+    Return True if a thing is None or its length is zero.
+    """
+    if thing is None:
+        return True
+    return len(thing) == 0
+
+
 def maybe_call(thing):
     """
     Return the same thing, or the result of its invocation if it is a
-- 
cgit v1.2.3


From b6f08b2fb731a4f3d1e6a04839bd3af71e9b2f5c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 24 Jan 2014 21:09:38 -0400
Subject: use enums for dict keys

---
 src/leap/mail/imap/memorystore.py  | 60 ++++++++++++++++----------------------
 src/leap/mail/imap/messageparts.py |  2 +-
 2 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index f0c0d4b..dcae6b0 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -224,32 +224,28 @@ class MemoryStore(object):
 
     def _add_message(self, mbox, uid, message, notify_on_disk=True):
         # XXX have to differentiate between notify_new and notify_dirty
-
         key = mbox, uid
         msg_dict = message.as_dict()
-        print "ADDING MESSAGE..."
-        import pprint; pprint.pprint(msg_dict)
 
-        # XXX use the enum as keys
+        FDOC = MessagePartType.fdoc.key
+        HDOC = MessagePartType.hdoc.key
+        CDOCS = MessagePartType.cdocs.key
+        DOCS_ID = MessagePartType.docs_id.key
 
         try:
             store = self._msg_store[key]
         except KeyError:
-            self._msg_store[key] = {'fdoc': {},
-                                    'hdoc': {},
-                                    'cdocs': {},
-                                    'docs_id': {}}
+            self._msg_store[key] = {FDOC: {},
+                                    HDOC: {},
+                                    CDOCS: {},
+                                    DOCS_ID: {}}
             store = self._msg_store[key]
 
-        print "In store (before):"
-        import pprint; pprint.pprint(store)
-
-        #self._msg_store[key] = msg_dict
-        fdoc = msg_dict.get('fdoc', None)
+        fdoc = msg_dict.get(FDOC, None)
         if fdoc:
-            if not store.get('fdoc', None):
-                store['fdoc'] = ReferenciableDict({})
-            store['fdoc'].update(fdoc)
+            if not store.get(FDOC, None):
+                store[FDOC] = ReferenciableDict({})
+            store[FDOC].update(fdoc)
 
             # content-hash indexing
             chash = fdoc.get(fields.CONTENT_HASH_KEY)
@@ -258,29 +254,29 @@ class MemoryStore(object):
                 chash_fdoc_store[chash] = {}
 
             chash_fdoc_store[chash][mbox] = weakref.proxy(
-                store['fdoc'])
+                store[FDOC])
 
-        hdoc = msg_dict.get('hdoc', None)
+        hdoc = msg_dict.get(HDOC, None)
         if hdoc:
-            if not store.get('hdoc', None):
-                store['hdoc'] = ReferenciableDict({})
-            store['hdoc'].update(hdoc)
+            if not store.get(HDOC, None):
+                store[HDOC] = ReferenciableDict({})
+            store[HDOC].update(hdoc)
 
-        docs_id = msg_dict.get('docs_id', None)
+        docs_id = msg_dict.get(DOCS_ID, None)
         if docs_id:
-            if not store.get('docs_id', None):
-                store['docs_id'] = {}
-            store['docs_id'].update(docs_id)
+            if not store.get(DOCS_ID, None):
+                store[DOCS_ID] = {}
+            store[DOCS_ID].update(docs_id)
 
         cdocs = message.cdocs
         for cdoc_key in cdocs.keys():
-            if not store.get('cdocs', None):
-                store['cdocs'] = {}
+            if not store.get(CDOCS, None):
+                store[CDOCS] = {}
 
             cdoc = cdocs[cdoc_key]
             # first we make it weak-referenciable
             referenciable_cdoc = ReferenciableDict(cdoc)
-            store['cdocs'][cdoc_key] = referenciable_cdoc
+            store[CDOCS][cdoc_key] = referenciable_cdoc
             phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None)
             if not phash:
                 continue
@@ -290,13 +286,7 @@ class MemoryStore(object):
             for key in seq:
                 if key in store and empty(store.get(key)):
                     store.pop(key)
-
-        prune(('fdoc', 'hdoc', 'cdocs', 'docs_id'), store)
-        #import ipdb; ipdb.set_trace()
-
-
-        print "after appending to store: ", key
-        import pprint; pprint.pprint(self._msg_store[key])
+        prune((FDOC, HDOC, CDOCS, DOCS_ID), store)
 
     def get_message(self, mbox, uid):
         """
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index b43bc37..055e6a5 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -34,7 +34,7 @@ from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
 from leap.mail.utils import first
 
-MessagePartType = Enum("hdoc", "fdoc", "cdoc")
+MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id")
 
 
 logger = logging.getLogger(__name__)
-- 
cgit v1.2.3


From a5508429b90e2e9b58c5d073610ee5a10274663f Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 24 Jan 2014 23:14:38 -0400
Subject: recent-flags use the memory store

---
 src/leap/mail/imap/memorystore.py  | 112 +++++++++++++++++++++++++++++++++++--
 src/leap/mail/imap/messageparts.py |   8 +++
 src/leap/mail/imap/messages.py     |  60 +++++++++++++-------
 src/leap/mail/imap/soledadstore.py |  59 +++++++++++++++----
 4 files changed, 205 insertions(+), 34 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index dcae6b0..232a2fb 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -21,6 +21,8 @@ import contextlib
 import logging
 import weakref
 
+from collections import defaultdict
+
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
 from twisted.python import log
@@ -32,6 +34,7 @@ from leap.mail.messageflow import MessageProducer
 from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
 from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc
+from leap.mail.imap.messageparts import RecentFlagsDoc
 from leap.mail.imap.messageparts import MessageWrapper
 from leap.mail.imap.messageparts import ReferenciableDict
 
@@ -109,16 +112,38 @@ class MemoryStore(object):
 
         # Internal Storage: content-hash:fdoc
         """
+        chash-fdoc-store keeps references to
+        the flag-documents indexed by content-hash.
+
         {'chash': {'mbox-a': weakref.proxy(dict),
                    'mbox-b': weakref.proxy(dict)}
         }
         """
         self._chash_fdoc_store = {}
 
-        # TODO ----------------- implement mailbox-level flags store too! ----
-        self._rflags_store = {}
+        # Internal Storage: recent-flags store
+        """
+        recent-flags store keeps one dict per mailbox,
+        with the document-id of the u1db document
+        and the set of the UIDs that have the recent flag.
+
+        {'mbox-a': {'doc_id': 'deadbeef',
+                    'set': {1,2,3,4}
+                    }
+        }
+        """
+        # TODO this will have to transition to content-hash
+        # indexes after we move to local-only UIDs.
+
+        self._rflags_store = defaultdict(
+            lambda: {'doc_id': None, 'set': set([])})
+
+        # TODO ----------------- implement mailbox-level flags store too?
+        # XXX maybe we don't need this anymore...
+        # let's see how good does it prefetch the headers if
+        # we cache them in the store.
         self._hdocset_store = {}
-        # TODO ----------------- implement mailbox-level flags store too! ----
+        # --------------------------------------------------------------
 
         # New and dirty flags, to set MessageWrapper State.
         self._new = set([])
@@ -224,6 +249,8 @@ class MemoryStore(object):
 
     def _add_message(self, mbox, uid, message, notify_on_disk=True):
         # XXX have to differentiate between notify_new and notify_dirty
+        # TODO defaultdict the hell outa here...
+
         key = mbox, uid
         msg_dict = message.as_dict()
 
@@ -331,6 +358,8 @@ class MemoryStore(object):
         with set_bool_flag(self, self.WRITING_FLAG):
             for msg_wrapper in self.all_new_dirty_msg_iter():
                 self.producer.push(msg_wrapper)
+            for rflags_doc_wrapper in self.all_rdocs_iter():
+                self.producer.push(rflags_doc_wrapper)
 
     # MemoryStore specific methods.
 
@@ -486,6 +515,79 @@ class MemoryStore(object):
             d.callback('%s, ok' % str(key))
             deferreds.pop(key)
 
+    # Recent Flags
+
+    # TODO --- nice but unused
+    def set_recent_flag(self, mbox, uid):
+        """
+        Set the `Recent` flag for a given mailbox and UID.
+        """
+        self._rflags_store[mbox]['set'].add(uid)
+
+    # TODO --- nice but unused
+    def unset_recent_flag(self, mbox, uid):
+        """
+        Unset the `Recent` flag for a given mailbox and UID.
+        """
+        self._rflags_store[mbox]['set'].discard(uid)
+
+    def set_recent_flags(self, mbox, value):
+        """
+        Set the value for the set of the recent flags.
+        Used from the property in the MessageCollection.
+        """
+        self._rflags_store[mbox]['set'] = set(value)
+
+    def load_recent_flags(self, mbox, flags_doc):
+        """
+        Load the passed flags document in the recent flags store, for a given
+        mailbox.
+
+        :param flags_doc: A dictionary containing the `doc_id` of the Soledad
+                          flags-document for this mailbox, and the `set`
+                          of uids marked with that flag.
+        """
+        self._rflags_store[mbox] = flags_doc
+
+    def get_recent_flags(self, mbox):
+        """
+        Get the set of UIDs with the `Recent` flag for this mailbox.
+
+        :return: set, or None
+        """
+        rflag_for_mbox = self._rflags_store.get(mbox, None)
+        if not rflag_for_mbox:
+            return None
+        return self._rflags_store[mbox]['set']
+
+    def all_rdocs_iter(self):
+        """
+        Return an iterator through all in-memory recent flag dicts, wrapped
+        under a RecentFlagsDoc namedtuple.
+        Used for saving to disk.
+
+        :rtype: generator
+        """
+        rflags_store = self._rflags_store
+
+        # XXX use enums
+        DOC_ID = "doc_id"
+        SET = "set"
+
+        print "LEN RFLAGS_STORE ------->", len(rflags_store)
+        return (
+            RecentFlagsDoc(
+                doc_id=rflags_store[mbox][DOC_ID],
+                content={
+                    fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
+                    fields.MBOX_KEY: mbox,
+                    fields.RECENTFLAGS_KEY: list(
+                        rflags_store[mbox][SET])
+                })
+            for mbox in rflags_store)
+
+    # Dump-to-disk controls.
+
     @property
     def is_writing(self):
         """
@@ -498,7 +600,9 @@ class MemoryStore(object):
 
         :rtype: bool
         """
-        # XXX this should probably return a deferred !!!
+        # FIXME this should return a deferred !!!
+        # XXX ----- can fire when all new + dirty deferreds
+        # are done (gatherResults)
         return getattr(self, self.WRITING_FLAG)
 
     def put_part(self, part_type, value):
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 055e6a5..257d3f0 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -73,6 +73,14 @@ MessagePartDoc = namedtuple(
     'MessagePartDoc',
     ['new', 'dirty', 'part', 'store', 'content', 'doc_id'])
 
+"""
+A RecentFlagsDoc is used to send the recent-flags document payload to the
+SoledadWriter during dumps.
+"""
+RecentFlagsDoc = namedtuple(
+    'RecentFlagsDoc',
+    ['content', 'doc_id'])
+
 
 class ReferenciableDict(dict):
     """
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index c212472..5de638b 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -813,6 +813,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         leap_assert(soledad, "Need a soledad instance to initialize")
 
         # okay, all in order, keep going...
+
         self.mbox = self._parse_mailbox_name(mbox)
 
         # XXX get a SoledadStore passed instead
@@ -996,8 +997,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # check for uniqueness.
         if self._fdoc_already_exists(chash):
             print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
-            print
-            print
             logger.warning("We already have that message in this mailbox.")
             # note that this operation will leave holes in the UID sequence,
             # but we're gonna change that all the same for a local-only table.
@@ -1023,21 +1022,16 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX review-me
         cdocs = dict((index, doc) for index, doc in
                      enumerate(walk.get_raw_docs(msg, parts)))
-        print "cdocs is", cdocs
 
-        # Saving ----------------------------------------
-        # XXX should check for content duplication on headers too
-        # but with chash. !!!
+        self.set_recent_flag(uid)
 
+        # Saving ----------------------------------------
         # XXX adapt hdocset to use memstore
         #hdoc = self._soledad.create_doc(hd)
         # We add the newly created hdoc to the fast-access set of
         # headers documents associated with the mailbox.
         #self.add_hdocset_docid(hdoc.doc_id)
 
-        # XXX move to memory store too
-        # self.set_recent_flag(uid)
-
         # TODO ---- add reference to original doc, to be deleted
         # after writes are done.
         msg_container = MessageWrapper(fd, hd, cdocs)
@@ -1088,24 +1082,48 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         An accessor for the recent-flags set for this mailbox.
         """
-        if not self.__rflags:
+        if self.__rflags is not None:
+            return self.__rflags
+
+        if self.memstore:
+            with self._rdoc_lock:
+                rflags = self.memstore.get_recent_flags(self.mbox)
+                if not rflags:
+                    # not loaded in the memory store yet.
+                    # let's fetch them from soledad...
+                    rdoc = self._get_recent_doc()
+                    rflags = set(rdoc.content.get(
+                        fields.RECENTFLAGS_KEY, []))
+                    # ...and cache them now.
+                    self.memstore.load_recent_flags(
+                        self.mbox,
+                        {'doc_id': rdoc.doc_id, 'set': rflags})
+            return rflags
+
+        else:
+            # fallback for cases without memory store
             with self._rdoc_lock:
                 rdoc = self._get_recent_doc()
                 self.__rflags = set(rdoc.content.get(
                     fields.RECENTFLAGS_KEY, []))
-        return self.__rflags
+            return self.__rflags
 
     def _set_recent_flags(self, value):
         """
         Setter for the recent-flags set for this mailbox.
         """
-        with self._rdoc_lock:
-            rdoc = self._get_recent_doc()
-            newv = set(value)
-            self.__rflags = newv
-            rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
-            # XXX should deferLater 0 it?
-            self._soledad.put_doc(rdoc)
+        if self.memstore:
+            self.memstore.set_recent_flags(self.mbox, value)
+
+        else:
+            # fallback for cases without memory store
+            with self._rdoc_lock:
+                rdoc = self._get_recent_doc()
+                newv = set(value)
+                self.__rflags = newv
+                rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
+                # XXX should deferLater 0 it?
+                self._soledad.put_doc(rdoc)
 
     recent_flags = property(
         _get_recent_flags, _set_recent_flags,
@@ -1131,15 +1149,17 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         Unset Recent flag for a sequence of uids.
         """
         with self._rdoc_property_lock:
-            self.recent_flags = self.recent_flags.difference(
+            self.recent_flags.difference_update(
                 set(uids))
 
+    # Individual flags operations
+
     def unset_recent_flag(self, uid):
         """
         Unset Recent flag for a given uid.
         """
         with self._rdoc_property_lock:
-            self.recent_flags = self.recent_flags.difference(
+            self.recent_flags.difference_update(
                 set([uid]))
 
     def set_recent_flag(self, uid):
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index b321da8..ea5b36e 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -25,6 +25,8 @@ from u1db import errors as u1db_errors
 from zope.interface import implements
 
 from leap.mail.imap.messageparts import MessagePartType
+from leap.mail.imap.messageparts import MessageWrapper
+from leap.mail.imap.messageparts import RecentFlagsDoc
 from leap.mail.imap.fields import fields
 from leap.mail.imap.interfaces import IMessageStore
 from leap.mail.messageflow import IMessageConsumer
@@ -193,9 +195,10 @@ class SoledadStore(ContentDedup):
         empty = queue.empty()
         while not empty:
             items = self._process(queue)
+
             # we prime the generator, that should return the
-            # item in the first place.
-            msg_wrapper = items.next()
+            # message or flags wrapper item in the first place.
+            doc_wrapper = items.next()
 
             # From here, we unpack the subpart items and
             # the right soledad call.
@@ -214,10 +217,11 @@ class SoledadStore(ContentDedup):
                 logger.error("Error while processing item.")
                 pass
             else:
-                # If everything went well, we can unset the new flag
-                # in the source store (memory store)
-                msg_wrapper.new = False
-                msg_wrapper.dirty = False
+                if isinstance(doc_wrapper, MessageWrapper):
+                    # If everything went well, we can unset the new flag
+                    # in the source store (memory store)
+                    doc_wrapper.new = False
+                    doc_wrapper.dirty = False
             empty = queue.empty()
 
     #
@@ -233,9 +237,20 @@ class SoledadStore(ContentDedup):
         :param queue: the queue from where we'll pick item.
         :type queue: Queue
         """
-        msg_wrapper = queue.get()
-        return chain((msg_wrapper,),
-                     self._get_calls_for_msg_parts(msg_wrapper))
+        doc_wrapper = queue.get()
+
+        if isinstance(doc_wrapper, MessageWrapper):
+            return chain((doc_wrapper,),
+                         self._get_calls_for_msg_parts(doc_wrapper))
+        elif isinstance(doc_wrapper, RecentFlagsDoc):
+            print "getting calls for rflags"
+            return chain((doc_wrapper,),
+                         self._get_calls_for_rflags_doc(doc_wrapper))
+        else:
+            print "********************"
+            print "CANNOT PROCESS ITEM!"
+            print "item --------------------->", doc_wrapper
+            return (i for i in [])
 
     def _try_call(self, call, item):
         """
@@ -309,4 +324,28 @@ class SoledadStore(ContentDedup):
         # Implement using callbacks for each operation.
 
         else:
-            logger.error("Cannot put/delete documents yet!")
+            logger.error("Cannot delete documents yet!")
+
+    def _get_calls_for_rflags_doc(self, rflags_wrapper):
+        """
+        We always put these documents.
+        """
+        call = self._soledad.put_doc
+        rdoc = self._soledad.get_doc(rflags_wrapper.doc_id)
+
+        payload = rflags_wrapper.content
+        print "rdoc", rdoc
+        print "SAVING RFLAGS TO SOLEDAD..."
+        import pprint; pprint.pprint(payload)
+
+        if payload:
+            rdoc.content = payload
+            print
+            print "YIELDING -----", rdoc
+            print "AND ----------", call
+            yield rdoc, call
+        else:
+            print ">>>>>>>>>>>>>>>>>"
+            print ">>>>>>>>>>>>>>>>>"
+            print ">>>>>>>>>>>>>>>>>"
+            print "No payload"
-- 
cgit v1.2.3


From f5365ae0c2edb8b3e879f876f2f7e42b25f4616a Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 27 Jan 2014 16:11:53 -0400
Subject: handle last_uid property in memory store

---
 src/leap/mail/imap/mailbox.py                | 131 +++++------
 src/leap/mail/imap/memorystore.py            | 236 +++++++++++++++----
 src/leap/mail/imap/messageparts.py           |  26 ++-
 src/leap/mail/imap/messages.py               | 336 ++++++++++++---------------
 src/leap/mail/imap/server.py                 |   1 +
 src/leap/mail/imap/soledadstore.py           | 129 +++++++---
 src/leap/mail/imap/tests/leap_tests_imap.zsh |   7 +-
 src/leap/mail/utils.py                       |   5 +-
 8 files changed, 532 insertions(+), 339 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 108d0da..b5c5719 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -26,7 +26,7 @@ import cStringIO
 from collections import defaultdict
 
 from twisted.internet import defer
-from twisted.internet.task import deferLater
+#from twisted.internet.task import deferLater
 from twisted.python import log
 
 from twisted.mail import imap4
@@ -119,6 +119,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
 
+        if self._memstore:
+            self.prime_last_uid_to_memstore()
+
     @property
     def listeners(self):
         """
@@ -132,6 +135,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         return self._listeners[self.mbox]
 
+    # TODO this grows too crazily when many instances are fired, like
+    # during imaptest stress testing. Should have a queue of limited size
+    # instead.
     def addListener(self, listener):
         """
         Add a listener to the listeners queue.
@@ -153,6 +159,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         self.listeners.remove(listener)
 
+    # TODO move completely to soledadstore, under memstore reponsibility.
     def _get_mbox(self):
         """
         Return mailbox document.
@@ -228,52 +235,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     def _get_last_uid(self):
         """
         Return the last uid for this mailbox.
+        If we have a memory store, the last UID will be the highest
+        recorded UID in the message store, or a counter cached from
+        the mailbox document in soledad if this is higher.
 
         :return: the last uid for messages in this mailbox
         :rtype: bool
         """
-        mbox = self._get_mbox()
-        if not mbox:
-            logger.error("We could not get a mbox!")
-            # XXX It looks like it has been corrupted.
-            # We need to be able to survive this.
-            return None
-        last = mbox.content.get(self.LAST_UID_KEY, 1)
-        if self._memstore:
-            last = max(last, self._memstore.get_last_uid(mbox))
+        last = self._memstore.get_last_uid(self.mbox)
+        print "last uid for %s: %s (from memstore)" % (self.mbox, last)
         return last
 
-    def _set_last_uid(self, uid):
-        """
-        Sets the last uid for this mailbox.
+    last_uid = property(
+        _get_last_uid, doc="Last_UID attribute.")
 
-        :param uid: the uid to be set
-        :type uid: int
+    def prime_last_uid_to_memstore(self):
         """
-        leap_assert(isinstance(uid, int), "uid has to be int")
-        mbox = self._get_mbox()
-        key = self.LAST_UID_KEY
-
-        count = self.getMessageCount()
-
-        # XXX safety-catch. If we do get duplicates,
-        # we want to avoid further duplication.
-
-        if uid >= count:
-            value = uid
-        else:
-            # something is wrong,
-            # just set the last uid
-            # beyond the max msg count.
-            logger.debug("WRONG uid < count. Setting last uid to %s", count)
-            value = count
-
-        mbox.content[key] = value
-        # XXX this should be set in the memorystore instead!!!
-        self._soledad.put_doc(mbox)
-
-    last_uid = property(
-        _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
+        Prime memstore with last_uid value
+        """
+        set_exist = set(self.messages.all_uid_iter())
+        last = max(set_exist) + 1 if set_exist else 1
+        logger.info("Priming Soledad last_uid to %s" % (last,))
+        self._memstore.set_last_soledad_uid(self.mbox, last)
 
     def getUIDValidity(self):
         """
@@ -315,8 +298,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: int
         """
         with self.next_uid_lock:
-            self.last_uid += 1
-            return self.last_uid
+            if self._memstore:
+                return self.last_uid + 1
+            else:
+                # XXX after lock, it should be safe to
+                # return just the increment here, and
+                # have a different method that actually increments
+                # the counter when really adding.
+                self.last_uid += 1
+                return self.last_uid
 
     def getMessageCount(self):
         """
@@ -397,26 +387,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         :return: a deferred that evals to None
         """
+        # TODO have a look at the cases for internal date in the rfc
         if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)):
             message = message.getvalue()
-        # XXX we should treat the message as an IMessage from here
+
+        # XXX we could treat the message as an IMessage from here
         leap_assert_type(message, basestring)
-        uid_next = self.getUIDNext()
-        logger.debug('Adding msg with UID :%s' % uid_next)
         if flags is None:
             flags = tuple()
         else:
             flags = tuple(str(flag) for flag in flags)
 
-        d = self._do_add_message(message, flags=flags, date=date, uid=uid_next)
+        d = self._do_add_message(message, flags=flags, date=date)
         return d
 
-    def _do_add_message(self, message, flags, date, uid):
+    def _do_add_message(self, message, flags, date):
         """
-        Calls to the messageCollection add_msg method (deferred to thread).
+        Calls to the messageCollection add_msg method.
         Invoked from addMessage.
         """
-        d = self.messages.add_msg(message, flags=flags, date=date, uid=uid)
+        d = self.messages.add_msg(message, flags=flags, date=date)
         # XXX Removing notify temporarily.
         # This is interfering with imaptest results. I'm not clear if it's
         # because we clutter the logging or because the set of listeners is
@@ -456,6 +446,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         # XXX removing the mailbox in situ for now,
         # we should postpone the removal
+
+        # XXX move to memory store??
         self._soledad.delete_doc(self._get_mbox())
 
     def _close_cb(self, result):
@@ -466,8 +458,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Expunge and mark as closed
         """
         d = self.expunge()
-        d.addCallback(self._close_cb)
-        return d
+        #d.addCallback(self._close_cb)
+        #return d
 
     def _expunge_cb(self, result):
         return result
@@ -479,22 +471,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         print "EXPUNGE!"
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
-        mstore = self._memstore
-        if mstore is not None:
-            deleted = mstore.all_deleted_uid_iter(self.mbox)
-            print "deleted ", list(deleted)
-            for uid in deleted:
-                mstore.remove_message(self.mbox, uid)
-
-        print "now deleting from soledad"
-        d = self.messages.remove_all_deleted()
-        d.addCallback(self._expunge_cb)
-        d.addCallback(self.messages.reset_last_uid)
-
-        # XXX DEBUG -------------------
-        # FIXME !!!
-        # XXX should remove the hdocset too!!!
-        return d
+
+        return self._memstore.expunge(self.mbox)
+
+        # TODO we can defer this back when it's correct
+        # but we should make sure the memstore has been synced.
+
+        #d = self._memstore.expunge(self.mbox)
+        #d.addCallback(self._expunge_cb)
+        #return d
 
     def _bound_seq(self, messages_asked):
         """
@@ -783,12 +768,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     # IMessageCopier
 
     @deferred
+    #@profile
     def copy(self, messageObject):
         """
         Copy the given message object into this mailbox.
         """
         from twisted.internet import reactor
-        uid_next = self.getUIDNext()
         msg = messageObject
         memstore = self._memstore
 
@@ -796,7 +781,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         fdoc = msg._fdoc
         hdoc = msg._hdoc
         if not fdoc:
-            logger.debug("Tried to copy a MSG with no fdoc")
+            logger.warning("Tried to copy a MSG with no fdoc")
             return
         new_fdoc = copy.deepcopy(fdoc.content)
 
@@ -807,11 +792,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         if exist:
             print "Destination message already exists!"
-
         else:
             print "DO COPY MESSAGE!"
+            mbox = self.mbox
+            uid_next = memstore.increment_last_soledad_uid(mbox)
             new_fdoc[self.UID_KEY] = uid_next
-            new_fdoc[self.MBOX_KEY] = self.mbox
+            new_fdoc[self.MBOX_KEY] = mbox
 
             # XXX set recent!
 
@@ -824,9 +810,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             self._memstore.create_message(
                 self.mbox, uid_next,
                 MessageWrapper(
-                    new_fdoc, hdoc.content))
-
-            deferLater(reactor, 1, self.notify_new)
+                    new_fdoc, hdoc.content),
+                notify_on_disk=False)
 
     # convenience fun
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 232a2fb..60e98c7 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -19,16 +19,20 @@ In-memory transient store for a LEAPIMAPServer.
 """
 import contextlib
 import logging
+import threading
 import weakref
 
 from collections import defaultdict
+from copy import copy
 
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
 from twisted.python import log
 from zope.interface import implements
 
+from leap.common.check import leap_assert_type
 from leap.mail import size
+from leap.mail.decorators import deferred
 from leap.mail.utils import empty
 from leap.mail.messageflow import MessageProducer
 from leap.mail.imap import interfaces
@@ -40,7 +44,10 @@ from leap.mail.imap.messageparts import ReferenciableDict
 
 logger = logging.getLogger(__name__)
 
-SOLEDAD_WRITE_PERIOD = 20
+
+# The default period to do writebacks to the permanent
+# soledad storage, in seconds.
+SOLEDAD_WRITE_PERIOD = 10
 
 
 @contextlib.contextmanager
@@ -76,16 +83,11 @@ class MemoryStore(object):
     implements(interfaces.IMessageStore,
                interfaces.IMessageStoreWriter)
 
-    producer = None
-
     # TODO We will want to index by chash when we transition to local-only
     # UIDs.
-    # TODO should store RECENT-FLAGS too
-    # TODO should store HDOCSET too (use weakrefs!) -- will need to subclass
-    # TODO do use dirty flag (maybe use namedtuples for that) so we can use it
-    # also as a read-cache.
 
     WRITING_FLAG = "_writing"
+    _last_uid_lock = threading.Lock()
 
     def __init__(self, permanent_store=None,
                  write_period=SOLEDAD_WRITE_PERIOD):
@@ -138,17 +140,20 @@ class MemoryStore(object):
         self._rflags_store = defaultdict(
             lambda: {'doc_id': None, 'set': set([])})
 
-        # TODO ----------------- implement mailbox-level flags store too?
-        # XXX maybe we don't need this anymore...
-        # let's see how good does it prefetch the headers if
-        # we cache them in the store.
-        self._hdocset_store = {}
-        # --------------------------------------------------------------
+        """
+        last-uid store keeps the count of the highest UID
+        per mailbox.
+
+        {'mbox-a': 42,
+         'mbox-b': 23}
+        """
+        self._last_uid = {}
 
         # New and dirty flags, to set MessageWrapper State.
         self._new = set([])
         self._new_deferreds = {}
         self._dirty = set([])
+        self._rflags_dirty = set([])
         self._dirty_deferreds = {}
 
         # Flag for signaling we're busy writing to the disk storage.
@@ -210,14 +215,25 @@ class MemoryStore(object):
         print "adding new doc to memstore %s (%s)" % (mbox, uid)
         key = mbox, uid
 
+        self._add_message(mbox, uid, message, notify_on_disk)
+
         d = defer.Deferred()
         d.addCallback(lambda result: log.msg("message save: %s" % result))
-
         self._new.add(key)
+
+        # We store this deferred so we can keep track of the pending
+        # operations internally.
         self._new_deferreds[key] = d
-        self._add_message(mbox, uid, message, notify_on_disk)
-        print "create message: ", d
-        return d
+
+        if notify_on_disk:
+            # Caller wants to be notified when the message is on disk
+            # so we pass the deferred that will be fired when the message
+            # has been written.
+            return d
+        else:
+            # Caller does not care, just fired and forgot, so we pass
+            # a defer that will inmediately have its callback triggered.
+            return defer.succeed('fire-and-forget:%s' % str(key))
 
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
@@ -238,13 +254,14 @@ class MemoryStore(object):
         :rtype: Deferred
         """
         key = mbox, uid
-
         d = defer.Deferred()
-        d.addCallback(lambda result: log.msg("message save: %s" % result))
+        d.addCallback(lambda result: log.msg("message PUT save: %s" % result))
 
         self._dirty.add(key)
         self._dirty_deferreds[key] = d
         self._add_message(mbox, uid, message, notify_on_disk)
+        #print "dirty ", self._dirty
+        #print "new ", self._new
         return d
 
     def _add_message(self, mbox, uid, message, notify_on_disk=True):
@@ -315,6 +332,19 @@ class MemoryStore(object):
                     store.pop(key)
         prune((FDOC, HDOC, CDOCS, DOCS_ID), store)
 
+        #print "after adding: "
+        #import pprint; pprint.pprint(self._msg_store[key])
+
+    def get_docid_for_fdoc(self, mbox, uid):
+        """
+        Get Soledad document id for the flags-doc for a given mbox and uid.
+        """
+        fdoc = self._permanent_store.get_flags_doc(mbox, uid)
+        if not fdoc:
+            return None
+        doc_id = fdoc.doc_id
+        return doc_id
+
     def get_message(self, mbox, uid):
         """
         Get a MessageWrapper for the given mbox and uid combination.
@@ -326,6 +356,8 @@ class MemoryStore(object):
         if msg_dict:
             new, dirty = self._get_new_dirty_state(key)
             return MessageWrapper(from_dict=msg_dict,
+                                  new=new,
+                                  dirty=dirty,
                                   memstore=weakref.proxy(self))
         else:
             return None
@@ -334,6 +366,13 @@ class MemoryStore(object):
         """
         Remove a Message from this MemoryStore.
         """
+        # XXX For the moment we are only removing the flags and headers
+        # docs. The rest we leave there polluting your hard disk,
+        # until we think about a good way of deorphaning.
+
+        # XXX implement elijah's idea of using a PUT document as a
+        # token to ensure consistency in the removal.
+
         try:
             key = mbox, uid
             self._new.discard(key)
@@ -348,18 +387,22 @@ class MemoryStore(object):
         """
         Write the message documents in this MemoryStore to a different store.
         """
-        # For now, we pass if the queue is not empty, to avoid duplication.
+        # For now, we pass if the queue is not empty, to avoid duplicate
+        # queuing.
         # We would better use a flag to know when we've already enqueued an
         # item.
+
+        # XXX this could return the deferred for all the enqueued operations
+
         if not self.producer.is_queue_empty():
             return
 
         print "Writing messages to Soledad..."
         with set_bool_flag(self, self.WRITING_FLAG):
-            for msg_wrapper in self.all_new_dirty_msg_iter():
-                self.producer.push(msg_wrapper)
             for rflags_doc_wrapper in self.all_rdocs_iter():
                 self.producer.push(rflags_doc_wrapper)
+            for msg_wrapper in self.all_new_dirty_msg_iter():
+                self.producer.push(msg_wrapper)
 
     # MemoryStore specific methods.
 
@@ -370,12 +413,61 @@ class MemoryStore(object):
         all_keys = self._msg_store.keys()
         return [uid for m, uid in all_keys if m == mbox]
 
+    # last_uid
+
     def get_last_uid(self, mbox):
         """
         Get the highest UID for a given mbox.
+        It will be the highest between the highest uid in the message store for
+        the mailbox, and the soledad integer cache.
         """
         uids = self.get_uids(mbox)
-        return uids and max(uids) or 0
+        last_mem_uid = uids and max(uids) or 0
+        last_soledad_uid = self.get_last_soledad_uid(mbox)
+        return max(last_mem_uid, last_soledad_uid)
+
+    def get_last_soledad_uid(self, mbox):
+        """
+        Get last uid for a given mbox from the soledad integer cache.
+        """
+        return self._last_uid.get(mbox, 0)
+
+    def set_last_soledad_uid(self, mbox, value):
+        """
+        Set last uid for a given mbox in the soledad integer cache.
+        SoledadMailbox should prime this value during initialization.
+        Other methods (during message adding) SHOULD call
+        `increment_last_soledad_uid` instead.
+        """
+        leap_assert_type(value, int)
+        print "setting last soledad uid for ", mbox, "to", value
+        # if we already have a vlue here, don't do anything
+        with self._last_uid_lock:
+            if not self._last_uid.get(mbox, None):
+                self._last_uid[mbox] = value
+
+    def increment_last_soledad_uid(self, mbox):
+        """
+        Increment by one the soledad integer cache for the last_uid for
+        this mbox, and fire a defer-to-thread to update the soledad value.
+        The caller should lock the call tho this method.
+        """
+        with self._last_uid_lock:
+            self._last_uid[mbox] += 1
+            value = self._last_uid[mbox]
+            self.write_last_uid(mbox, value)
+            return value
+
+    @deferred
+    def write_last_uid(self, mbox, value):
+        """
+        Increment the soledad cache,
+        """
+        leap_assert_type(value, int)
+        if self._permanent_store:
+            self._permanent_store.write_last_uid(mbox, value)
+
+    # Counting sheeps...
 
     def count_new_mbox(self, mbox):
         """
@@ -418,14 +510,12 @@ class MemoryStore(object):
         docs_dict = self._chash_fdoc_store.get(chash, None)
         fdoc = docs_dict.get(mbox, None) if docs_dict else None
 
-        print "GETTING FDOC BY CHASH:", fdoc
-
         # a couple of special cases.
         # 1. We might have a doc with empty content...
         if empty(fdoc):
             return None
 
-        # ...Or the message could exist, but being flagged for deletion.
+        # 2. ...Or the message could exist, but being flagged for deletion.
         # We want to create a new one in this case.
         # Hmmm what if the deletion is un-done?? We would end with a
         # duplicate...
@@ -456,15 +546,22 @@ class MemoryStore(object):
                 for key in sorted(self._msg_store.keys())
                 if key in self._new or key in self._dirty)
 
+    def all_msg_dict_for_mbox(self, mbox):
+        """
+        Return all the message dicts for a given mbox.
+        """
+        return [self._msg_store[(mb, uid)]
+                for mb, uid in self._msg_store if mb == mbox]
+
     def all_deleted_uid_iter(self, mbox):
         """
         Return generator that iterates through the UIDs for all messags
         with deleted flag in a given mailbox.
         """
-        all_deleted = (
-            msg['fdoc']['uid'] for msg in self._msg_store.values()
+        all_deleted = [
+            msg['fdoc']['uid'] for msg in self.all_msg_dict_for_mbox(mbox)
             if msg.get('fdoc', None)
-            and fields.DELETED_FLAG in msg['fdoc']['flags'])
+            and fields.DELETED_FLAG in msg['fdoc']['flags']]
         return all_deleted
 
     # new, dirty flags
@@ -473,6 +570,7 @@ class MemoryStore(object):
         """
         Return `new` and `dirty` flags for a given message.
         """
+        # XXX should return *first* the news, and *then* the dirty...
         return map(lambda _set: key in _set, (self._new, self._dirty))
 
     def set_new(self, key):
@@ -485,7 +583,7 @@ class MemoryStore(object):
         """
         Remove the key value from the `new` set.
         """
-        print "Unsetting NEW for: %s" % str(key)
+        #print "Unsetting NEW for: %s" % str(key)
         self._new.discard(key)
         deferreds = self._new_deferreds
         d = deferreds.get(key, None)
@@ -505,7 +603,7 @@ class MemoryStore(object):
         """
         Remove the key value from the `dirty` set.
         """
-        print "Unsetting DIRTY for: %s" % str(key)
+        #print "Unsetting DIRTY for: %s" % str(key)
         self._dirty.discard(key)
         deferreds = self._dirty_deferreds
         d = deferreds.get(key, None)
@@ -522,6 +620,7 @@ class MemoryStore(object):
         """
         Set the `Recent` flag for a given mailbox and UID.
         """
+        self._rflags_dirty.add(mbox)
         self._rflags_store[mbox]['set'].add(uid)
 
     # TODO --- nice but unused
@@ -536,6 +635,7 @@ class MemoryStore(object):
         Set the value for the set of the recent flags.
         Used from the property in the MessageCollection.
         """
+        self._rflags_dirty.add(mbox)
         self._rflags_store[mbox]['set'] = set(value)
 
     def load_recent_flags(self, mbox, flags_doc):
@@ -568,23 +668,81 @@ class MemoryStore(object):
 
         :rtype: generator
         """
-        rflags_store = self._rflags_store
-
         # XXX use enums
         DOC_ID = "doc_id"
         SET = "set"
 
-        print "LEN RFLAGS_STORE ------->", len(rflags_store)
-        return (
-            RecentFlagsDoc(
+        rflags_store = self._rflags_store
+
+        def get_rdoc(mbox, rdict):
+            mbox_rflag_set = rdict[SET]
+            recent_set = copy(mbox_rflag_set)
+            # zero it!
+            mbox_rflag_set.difference_update(mbox_rflag_set)
+            return RecentFlagsDoc(
                 doc_id=rflags_store[mbox][DOC_ID],
                 content={
                     fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
                     fields.MBOX_KEY: mbox,
-                    fields.RECENTFLAGS_KEY: list(
-                        rflags_store[mbox][SET])
+                    fields.RECENTFLAGS_KEY: list(recent_set)
                 })
-            for mbox in rflags_store)
+
+        return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items()
+                if not empty(rdict[SET]))
+
+    # Methods that mirror the IMailbox interface
+
+    def remove_all_deleted(self, mbox):
+        """
+        Remove all messages flagged \\Deleted from this Memory Store only.
+        Called from `expunge`
+        """
+        mem_deleted = self.all_deleted_uid_iter(mbox)
+        for uid in mem_deleted:
+            self.remove_message(mbox, uid)
+        return mem_deleted
+
+    def expunge(self, mbox):
+        """
+        Remove all messages flagged \\Deleted, from the Memory Store
+        and from the permanent store also.
+        """
+        # TODO expunge should add itself as a callback to the ongoing
+        # writes.
+        soledad_store = self._permanent_store
+
+        try:
+            # 1. Stop the writing call
+            self._stop_write_loop()
+            # 2. Enqueue a last write.
+            #self.write_messages(soledad_store)
+            # 3. Should wait on the writebacks to finish ???
+            # FIXME wait for this, and add all the rest of the method
+            # as a callback!!!
+        except Exception as exc:
+            logger.exception(exc)
+
+        # Now, we...:
+
+        try:
+            # 1. Delete all messages marked as deleted in soledad.
+
+            # XXX this could be deferred for faster operation.
+            if soledad_store:
+                sol_deleted = soledad_store.remove_all_deleted(mbox)
+            else:
+                sol_deleted = []
+
+            # 2. Delete all messages marked as deleted in memory.
+            mem_deleted = self.remove_all_deleted(mbox)
+
+            all_deleted = set(mem_deleted).union(set(sol_deleted))
+            print "deleted ", all_deleted
+        except Exception as exc:
+            logger.exception(exc)
+        finally:
+            self._start_write_loop()
+        return all_deleted
 
     # Dump-to-disk controls.
 
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 257d3f0..6d8631a 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -32,7 +32,7 @@ from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
-from leap.mail.utils import first
+from leap.mail.utils import empty, first
 
 MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id")
 
@@ -134,6 +134,13 @@ class MessageWrapper(object):
                 self._dict[self.HDOC] = ReferenciableDict(hdoc)
             if cdocs is not None:
                 self._dict[self.CDOCS] = ReferenciableDict(cdocs)
+
+        # This will keep references to the doc_ids to be able to put
+        # messages to soledad. It will be populated during the walk() to avoid
+        # the overhead of reading from the db.
+
+        # XXX it really *only* make sense for the FDOC, the other parts
+        # should not be "dirty", just new...!!!
         self._dict[self.DOCS_ID] = docs_id
 
     # properties
@@ -201,6 +208,7 @@ class MessageWrapper(object):
         else:
             logger.warning("NO FDOC!!!")
             content_ref = {}
+
         return MessagePartDoc(new=self.new, dirty=self.dirty,
                               store=self._storetype,
                               part=MessagePartType.fdoc,
@@ -214,7 +222,6 @@ class MessageWrapper(object):
         if _hdoc:
             content_ref = weakref.proxy(_hdoc)
         else:
-            logger.warning("NO HDOC!!!!")
             content_ref = {}
         return MessagePartDoc(new=self.new, dirty=self.dirty,
                               store=self._storetype,
@@ -234,14 +241,21 @@ class MessageWrapper(object):
     def walk(self):
         """
         Generator that iterates through all the parts, returning
-        MessagePartDoc.
+        MessagePartDoc. Used for writing to SoledadStore.
         """
-        if self.fdoc is not None:
+        if self._dirty:
+            mbox = self.fdoc.content[fields.MBOX_KEY]
+            uid = self.fdoc.content[fields.UID_KEY]
+            docid_dict = self._dict[self.DOCS_ID]
+            docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc(
+                mbox, uid)
+
+        if not empty(self.fdoc.content):
             yield self.fdoc
-        if self.hdoc is not None:
+        if not empty(self.hdoc.content):
             yield self.hdoc
         for cdoc in self.cdocs.values():
-            if cdoc is not None:
+            if not empty(cdoc):
                 content_ref = weakref.proxy(cdoc)
                 yield MessagePartDoc(new=self.new, dirty=self.dirty,
                                      store=self._storetype,
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 5de638b..35c07f5 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -202,21 +202,21 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: The flags, represented as strings
         :rtype: tuple
         """
-        if self._uid is None:
-            return []
+        #if self._uid is None:
+            #return []
         uid = self._uid
 
-        flags = []
+        flags = set([])
         fdoc = self._fdoc
         if fdoc:
-            flags = fdoc.content.get(self.FLAGS_KEY, None)
+            flags = set(fdoc.content.get(self.FLAGS_KEY, None))
 
         msgcol = self._collection
 
         # We treat the recent flag specially: gotten from
         # a mailbox-level document.
         if msgcol and uid in msgcol.recent_flags:
-            flags.append(fields.RECENT_FLAG)
+            flags.add(fields.RECENT_FLAG)
         if flags:
             flags = map(str, flags)
         return tuple(flags)
@@ -236,7 +236,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: a SoledadDocument instance
         :rtype: SoledadDocument
         """
-        # XXX use memory store ...!
+        # XXX Move logic to memory store ...
 
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         log.msg('setting flags: %s (%s)' % (self._uid, flags))
@@ -252,6 +252,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
 
         if self._collection.memstore is not None:
+            print "putting message in collection"
             self._collection.memstore.put_message(
                 self._mbox, self._uid,
                 MessageWrapper(fdoc=doc.content, new=False, dirty=True,
@@ -508,6 +509,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
         return pmap[str(part)]
 
+    # XXX moved to memory store
+    # move the rest too. ------------------------------------------
     def _get_flags_doc(self):
         """
         Return the document that keeps the flags for this
@@ -617,57 +620,38 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     # destructor
 
-    @deferred
-    def remove(self):
-        """
-        Remove all docs associated with this message.
-        Currently it removes only the flags doc.
-        """
-        # XXX For the moment we are only removing the flags and headers
-        # docs. The rest we leave there polluting your hard disk,
-        # until we think about a good way of deorphaning.
-        # Maybe a crawler of unreferenced docs.
-
-        # XXX implement elijah's idea of using a PUT document as a
-        # token to ensure consistency in the removal.
-
-        uid = self._uid
-
-        fd = self._get_flags_doc()
-        #hd = self._get_headers_doc()
-        #bd = self._get_body_doc()
-        #docs = [fd, hd, bd]
-
-        try:
-            memstore = self._collection.memstore
-        except AttributeError:
-            memstore = False
-
-        if memstore and hasattr(fd, "store", None) == "mem":
-            key = self._mbox, self._uid
-            if fd.new:
-                # it's a new document, so we can remove it and it will not
-                # be writen. Watch out! We need to be sure it has not been
-                # just queued to write!
-                memstore.remove_message(*key)
-
-            if fd.dirty:
-                doc_id = fd.doc_id
-                doc = self._soledad.get_doc(doc_id)
-                try:
-                    self._soledad.delete_doc(doc)
-                except Exception as exc:
-                    logger.exception(exc)
-
-        else:
+    # XXX this logic moved to remove_message in memory store...
+    #@deferred
+    #def remove(self):
+        #"""
+        #Remove all docs associated with this message.
+        #Currently it removes only the flags doc.
+        #"""
+        #fd = self._get_flags_doc()
+#
+        #if fd.new:
+            # it's a new document, so we can remove it and it will not
+            # be writen. Watch out! We need to be sure it has not been
+            # just queued to write!
+            #memstore.remove_message(*key)
+#
+        #if fd.dirty:
+            #doc_id = fd.doc_id
+            #doc = self._soledad.get_doc(doc_id)
+            #try:
+                #self._soledad.delete_doc(doc)
+            #except Exception as exc:
+                #logger.exception(exc)
+#
+        #else:
             # we just got a soledad_doc
-            try:
-                doc_id = fd.doc_id
-                latest_doc = self._soledad.get_doc(doc_id)
-                self._soledad.delete_doc(latest_doc)
-            except Exception as exc:
-                logger.exception(exc)
-        return uid
+            #try:
+                #doc_id = fd.doc_id
+                #latest_doc = self._soledad.get_doc(doc_id)
+                #self._soledad.delete_doc(latest_doc)
+            #except Exception as exc:
+                #logger.exception(exc)
+        #return uid
 
     def does_exist(self):
         """
@@ -826,7 +810,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # ensure that we have a recent-flags and a hdocs-sec doc
         self._get_or_create_rdoc()
-        self._get_or_create_hdocset()
+
+        # Not for now...
+        #self._get_or_create_hdocset()
 
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
@@ -959,7 +945,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # not deferring to thread cause this now uses deferred asa retval
     #@deferred
-    def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
+    #@profile
+    def add_msg(self, raw, subject=None, flags=None, date=None, uid=None,
+                notify_on_disk=False):
         """
         Creates a new message document.
         Here lives the magic of the leap mail. Well, in soledad, really.
@@ -994,7 +982,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # parse
         msg, chash, size, multi = self._do_parse(raw)
 
-        # check for uniqueness.
+        # check for uniqueness --------------------------------
+        # XXX profiler says that this test is costly.
+        # So we probably should just do an in-memory check and
+        # move the complete check to the soledad writer?
+        # Watch out! We're reserving a UID right after this!
         if self._fdoc_already_exists(chash):
             print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
             logger.warning("We already have that message in this mailbox.")
@@ -1003,6 +995,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             # so not touch it by the moment.
             return defer.succeed('already_exists')
 
+        uid = self.memstore.increment_last_soledad_uid(self.mbox)
+        print "ADDING MSG WITH UID: %s" % uid
+
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
 
@@ -1039,36 +1034,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX Should allow also to dump to disk directly,
         # for no-memstore cases.
 
-        # we return a deferred that, by default, will be triggered when
-        # saved to disk
-        d = self.memstore.create_message(self.mbox, uid, msg_container)
-        print "defered-add", d
+        # we return a deferred that by default will be triggered
+        # inmediately.
+        d = self.memstore.create_message(self.mbox, uid, msg_container,
+                                         notify_on_disk=notify_on_disk)
         print "adding message", d
         return d
 
-    def _remove_cb(self, result):
-        return result
-
-    def remove_all_deleted(self):
-        """
-        Removes all messages flagged as deleted.
-        """
-        delete_deferl = []
-        for msg in self.get_deleted():
-            delete_deferl.append(msg.remove())
-        d1 = defer.gatherResults(delete_deferl, consumeErrors=True)
-        d1.addCallback(self._remove_cb)
-        return d1
-
-    def remove(self, msg):
-        """
-        Remove a given msg.
-        :param msg: the message to be removed
-        :type msg: LeapMessage
-        """
-        d = msg.remove()
-        d.addCallback(self._remove_cb)
-        return d
+    #def remove(self, msg):
+        #"""
+        #Remove a given msg.
+        #:param msg: the message to be removed
+        #:type msg: LeapMessage
+        #"""
+        #d = msg.remove()
+        #d.addCallback(self._remove_cb)
+        #return d
 
     #
     # getters: specific queries
@@ -1175,76 +1156,76 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     # XXX FIXME -------------------------------------
     # This should be rewritten to use memory store.
 
-    def _get_hdocset(self):
-        """
-        An accessor for the hdocs-set for this mailbox.
-        """
-        if not self.__hdocset:
-            with self._hdocset_lock:
-                hdocset_doc = self._get_hdocset_doc()
-                value = set(hdocset_doc.content.get(
-                    fields.HDOCS_SET_KEY, []))
-                self.__hdocset = value
-        return self.__hdocset
-
-    def _set_hdocset(self, value):
-        """
-        Setter for the hdocs-set for this mailbox.
-        """
-        with self._hdocset_lock:
-            hdocset_doc = self._get_hdocset_doc()
-            newv = set(value)
-            self.__hdocset = newv
-            hdocset_doc.content[fields.HDOCS_SET_KEY] = list(newv)
+    #def _get_hdocset(self):
+        #"""
+        #An accessor for the hdocs-set for this mailbox.
+        #"""
+        #if not self.__hdocset:
+            #with self._hdocset_lock:
+                #hdocset_doc = self._get_hdocset_doc()
+                #value = set(hdocset_doc.content.get(
+                    #fields.HDOCS_SET_KEY, []))
+                #self.__hdocset = value
+        #return self.__hdocset
+#
+    #def _set_hdocset(self, value):
+        #"""
+        #Setter for the hdocs-set for this mailbox.
+        #"""
+        #with self._hdocset_lock:
+            #hdocset_doc = self._get_hdocset_doc()
+            #newv = set(value)
+            #self.__hdocset = newv
+            #hdocset_doc.content[fields.HDOCS_SET_KEY] = list(newv)
             # XXX should deferLater 0 it?
-            self._soledad.put_doc(hdocset_doc)
-
-    _hdocset = property(
-        _get_hdocset, _set_hdocset,
-        doc="Set of Document-IDs for the headers docs associated "
-            "with this mailbox.")
-
-    def _get_hdocset_doc(self):
-        """
-        Get hdocs-set document for this mailbox.
-        """
-        curried = partial(
-            self._soledad.get_from_index,
-            fields.TYPE_MBOX_IDX,
-            fields.TYPE_HDOCS_SET_VAL, self.mbox)
-        curried.expected = "hdocset"
-        hdocset_doc = try_unique_query(curried)
-        return hdocset_doc
-
+            #self._soledad.put_doc(hdocset_doc)
+#
+    #_hdocset = property(
+        #_get_hdocset, _set_hdocset,
+        #doc="Set of Document-IDs for the headers docs associated "
+            #"with this mailbox.")
+#
+    #def _get_hdocset_doc(self):
+        #"""
+        #Get hdocs-set document for this mailbox.
+        #"""
+        #curried = partial(
+            #self._soledad.get_from_index,
+            #fields.TYPE_MBOX_IDX,
+            #fields.TYPE_HDOCS_SET_VAL, self.mbox)
+        #curried.expected = "hdocset"
+        #hdocset_doc = try_unique_query(curried)
+        #return hdocset_doc
+#
     # Property-set modification (protected by a different
     # lock to give atomicity to the read/write operation)
-
-    def remove_hdocset_docids(self, docids):
-        """
-        Remove the given document IDs from the set of
-        header-documents associated with this mailbox.
-        """
-        with self._hdocset_property_lock:
-            self._hdocset = self._hdocset.difference(
-                set(docids))
-
-    def remove_hdocset_docid(self, docid):
-        """
-        Remove the given document ID from the set of
-        header-documents associated with this mailbox.
-        """
-        with self._hdocset_property_lock:
-            self._hdocset = self._hdocset.difference(
-                set([docid]))
-
-    def add_hdocset_docid(self, docid):
-        """
-        Add the given document ID to the set of
-        header-documents associated with this mailbox.
-        """
-        with self._hdocset_property_lock:
-            self._hdocset = self._hdocset.union(
-                set([docid]))
+#
+    #def remove_hdocset_docids(self, docids):
+        #"""
+        #Remove the given document IDs from the set of
+        #header-documents associated with this mailbox.
+        #"""
+        #with self._hdocset_property_lock:
+            #self._hdocset = self._hdocset.difference(
+                #set(docids))
+#
+    #def remove_hdocset_docid(self, docid):
+        #"""
+        #Remove the given document ID from the set of
+        #header-documents associated with this mailbox.
+        #"""
+        #with self._hdocset_property_lock:
+            #self._hdocset = self._hdocset.difference(
+                #set([docid]))
+#
+    #def add_hdocset_docid(self, docid):
+        #"""
+        #Add the given document ID to the set of
+        #header-documents associated with this mailbox.
+        #"""
+        #with self._hdocset_property_lock:
+            #self._hdocset = self._hdocset.union(
+                #set([docid]))
 
     # individual doc getters, message layer.
 
@@ -1378,18 +1359,20 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         return (u for u in sorted(uids))
 
-    def reset_last_uid(self, param):
-        """
-        Set the last uid to the highest uid found.
-        Used while expunging, passed as a callback.
-        """
-        try:
-            self.last_uid = max(self.all_uid_iter()) + 1
-        except ValueError:
+    # XXX Should be moved to memstore
+    #def reset_last_uid(self, param):
+        #"""
+        #Set the last uid to the highest uid found.
+        #Used while expunging, passed as a callback.
+        #"""
+        #try:
+            #self.last_uid = max(self.all_uid_iter()) + 1
+        #except ValueError:
             # empty sequence
-            pass
-        return param
+            #pass
+        #return param
 
+    # XXX MOVE to memstore
     def all_flags(self):
         """
         Return a dict with all flags documents for this mailbox.
@@ -1444,7 +1427,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         :rtype: int
         """
-        # XXX We could cache this in memstore too until next write...
+        # XXX We should cache this in memstore too until next write...
         count = self._soledad.get_count_from_index(
             fields.TYPE_MBOX_IDX,
             fields.TYPE_FLAGS_VAL, self.mbox)
@@ -1491,6 +1474,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # recent messages
 
+    # XXX take it from memstore
     def count_recent(self):
         """
         Count all messages with the `Recent` flag.
@@ -1503,30 +1487,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         return len(self.recent_flags)
 
-    # deleted messages
-
-    def deleted_iter(self):
-        """
-        Get an iterator for the message UIDs with `deleted` flag.
-
-        :return: iterator through deleted message docs
-        :rtype: iterable
-        """
-        return (doc.content[self.UID_KEY] for doc in
-                self._soledad.get_from_index(
-                    fields.TYPE_MBOX_DEL_IDX,
-                    fields.TYPE_FLAGS_VAL, self.mbox, '1'))
-
-    def get_deleted(self):
-        """
-        Get all messages with the `Deleted` flag.
-
-        :returns: a generator of LeapMessages
-        :rtype: generator
-        """
-        return (LeapMessage(self._soledad, docid, self.mbox)
-                for docid in self.deleted_iter())
-
     def __len__(self):
         """
         Returns the number of messages on this mailbox.
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index c95a9be..3a6ac9a 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -199,6 +199,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
 
         # XXX  fake a delayed operation, to debug problem with messages getting
         # back to the source mailbox...
+        print "faking checkpoint..."
         import time
         time.sleep(2)
         return None
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index ea5b36e..60576a3 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -18,18 +18,22 @@
 A MessageStore that writes to Soledad.
 """
 import logging
+import threading
 
 from itertools import chain
 
+#from twisted.internet import defer
 from u1db import errors as u1db_errors
 from zope.interface import implements
 
+from leap.common.check import leap_assert_type
 from leap.mail.imap.messageparts import MessagePartType
 from leap.mail.imap.messageparts import MessageWrapper
 from leap.mail.imap.messageparts import RecentFlagsDoc
 from leap.mail.imap.fields import fields
 from leap.mail.imap.interfaces import IMessageStore
 from leap.mail.messageflow import IMessageConsumer
+from leap.mail.utils import first
 
 logger = logging.getLogger(__name__)
 
@@ -123,6 +127,7 @@ class SoledadStore(ContentDedup):
     """
     This will create docs in the local Soledad database.
     """
+    _last_uid_lock = threading.Lock()
 
     implements(IMessageConsumer, IMessageStore)
 
@@ -177,6 +182,7 @@ class SoledadStore(ContentDedup):
 
     # IMessageConsumer
 
+    #@profile
     def consume(self, queue):
         """
         Creates a new document in soledad db.
@@ -220,6 +226,7 @@ class SoledadStore(ContentDedup):
                 if isinstance(doc_wrapper, MessageWrapper):
                     # If everything went well, we can unset the new flag
                     # in the source store (memory store)
+                    print "unsetting new flag!"
                     doc_wrapper.new = False
                     doc_wrapper.dirty = False
             empty = queue.empty()
@@ -243,13 +250,11 @@ class SoledadStore(ContentDedup):
             return chain((doc_wrapper,),
                          self._get_calls_for_msg_parts(doc_wrapper))
         elif isinstance(doc_wrapper, RecentFlagsDoc):
-            print "getting calls for rflags"
             return chain((doc_wrapper,),
                          self._get_calls_for_rflags_doc(doc_wrapper))
         else:
             print "********************"
             print "CANNOT PROCESS ITEM!"
-            print "item --------------------->", doc_wrapper
             return (i for i in [])
 
     def _try_call(self, call, item):
@@ -275,6 +280,7 @@ class SoledadStore(ContentDedup):
 
         if msg_wrapper.new is True:
             call = self._soledad.create_doc
+            print "NEW DOC ----------------------"
 
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
@@ -301,30 +307,22 @@ class SoledadStore(ContentDedup):
         # the flags doc.
 
         elif msg_wrapper.dirty is True:
-            print "DIRTY DOC! ----------------------"
             call = self._soledad.put_doc
-
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
+                # XXX FIXME Give error if dirty and not doc_id !!!
                 doc_id = item.doc_id  # defend!
+                if not doc_id:
+                    continue
                 doc = self._soledad.get_doc(doc_id)
-                doc.content = item.content
-
+                doc.content = dict(item.content)
                 if item.part == MessagePartType.fdoc:
-                    print "Will PUT the doc: ", doc
-                    yield dict(doc), call
-
-                # XXX also for linkage-doc
-
-        # TODO should write back to the queue
-        # with the results of the operation.
-        # We can write there:
-        # (*) MsgWriteACK  --> Should remove from incoming queue.
-        #                      (We should do this here).
-        # Implement using callbacks for each operation.
+                    logger.debug("PUT dirty fdoc")
+                    yield doc, call
 
+                # XXX also for linkage-doc !!!
         else:
-            logger.error("Cannot delete documents yet!")
+            logger.error("Cannot delete documents yet from the queue...!")
 
     def _get_calls_for_rflags_doc(self, rflags_wrapper):
         """
@@ -334,18 +332,91 @@ class SoledadStore(ContentDedup):
         rdoc = self._soledad.get_doc(rflags_wrapper.doc_id)
 
         payload = rflags_wrapper.content
-        print "rdoc", rdoc
-        print "SAVING RFLAGS TO SOLEDAD..."
-        import pprint; pprint.pprint(payload)
+        logger.debug("Saving RFLAGS to Soledad...")
 
         if payload:
             rdoc.content = payload
-            print
-            print "YIELDING -----", rdoc
-            print "AND ----------", call
             yield rdoc, call
-        else:
-            print ">>>>>>>>>>>>>>>>>"
-            print ">>>>>>>>>>>>>>>>>"
-            print ">>>>>>>>>>>>>>>>>"
-            print "No payload"
+
+    def _get_mbox_document(self, mbox):
+        """
+        Return mailbox document.
+
+        :return: A SoledadDocument containing this mailbox, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
+        """
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MBOX_IDX,
+                fields.TYPE_MBOX_VAL, mbox)
+            if query:
+                return query.pop()
+        except Exception as exc:
+            logger.exception("Unhandled error %r" % exc)
+
+    def get_flags_doc(self, mbox, uid):
+        """
+        Return the SoledadDocument for the given mbox and uid.
+        """
+        try:
+            flag_docs = self._soledad.get_from_index(
+                fields.TYPE_MBOX_UID_IDX,
+                fields.TYPE_FLAGS_VAL, mbox, str(uid))
+            result = first(flag_docs)
+        except Exception as exc:
+            # ugh! Something's broken down there!
+            logger.warning("ERROR while getting flags for UID: %s" % uid)
+            logger.exception(exc)
+        finally:
+            return result
+
+    def write_last_uid(self, mbox, value):
+        """
+        Write the `last_uid` integer to the proper mailbox document
+        in Soledad.
+        This is called from the deferred triggered by
+        memorystore.increment_last_soledad_uid, which is expected to
+        run in a separate thread.
+        """
+        leap_assert_type(value, int)
+        key = fields.LAST_UID_KEY
+
+        with self._last_uid_lock:
+            mbox_doc = self._get_mbox_document(mbox)
+            old_val = mbox_doc.content[key]
+            if value < old_val:
+                logger.error("%s:%s Tried to write a UID lesser than what's "
+                             "stored!" % (mbox, value))
+            mbox_doc.content[key] = value
+            self._soledad.put_doc(mbox_doc)
+
+    # deleted messages
+
+    def deleted_iter(self, mbox):
+        """
+        Get an iterator for the SoledadDocuments for messages
+        with \\Deleted flag for a given mailbox.
+
+        :return: iterator through deleted message docs
+        :rtype: iterable
+        """
+        return (doc for doc in self._soledad.get_from_index(
+                fields.TYPE_MBOX_DEL_IDX,
+                fields.TYPE_FLAGS_VAL, mbox, '1'))
+
+    # TODO can deferToThread this?
+    def remove_all_deleted(self, mbox):
+        """
+        Remove from Soledad all messages flagged as deleted for a given
+        mailbox.
+        """
+        print "DELETING ALL DOCS FOR -------", mbox
+        deleted = []
+        for doc in self.deleted_iter(mbox):
+            deleted.append(doc.content[fields.UID_KEY])
+            print
+            print ">>>>>>>>>>>>>>>>>>>>"
+            print "deleting doc: ", doc.doc_id, doc.content
+            self._soledad.delete_doc(doc)
+        return deleted
diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh
index 676d1a8..8f0df9f 100755
--- a/src/leap/mail/imap/tests/leap_tests_imap.zsh
+++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh
@@ -61,7 +61,8 @@ IMAPTEST="imaptest"
 # These should be kept constant across benchmarking
 # runs across different machines, for comparability.
 
-DURATION=200
+#DURATION=200
+DURATION=60
 NUM_MSG=200
 
 
@@ -76,7 +77,7 @@ imaptest_cmd() {
 }
 
 stress_imap() 	{
-  mknod imap_pipe p
+  mkfifo imap_pipe
   cat imap_pipe | tee output &
   imaptest_cmd >> imap_pipe
 }
@@ -99,7 +100,7 @@ print_results() {
 	echo "----------------------"
 	echo "\tavg\tstdev"
 	$GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \
-	awk '
+	gawk '
 function avg(data, count) {
     sum=0;
     for( x=0; x <= count-1; x++) {
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index bae2898..1f43947 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -42,7 +42,10 @@ def empty(thing):
     """
     if thing is None:
         return True
-    return len(thing) == 0
+    try:
+        return len(thing) == 0
+    except ReferenceError:
+        return True
 
 
 def maybe_call(thing):
-- 
cgit v1.2.3


From f096368cfbc49caab52811ae50388aae74272a1a Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 28 Jan 2014 10:24:04 -0400
Subject: fix find_charset rebase

---
 src/leap/mail/imap/messageparts.py | 16 ++++++----------
 src/leap/mail/imap/messages.py     |  8 +++++---
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 6d8631a..10672ed 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -32,7 +32,7 @@ from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
-from leap.mail.utils import empty, first
+from leap.mail.utils import empty, first, find_charset
 
 MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id")
 
@@ -40,10 +40,6 @@ MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id")
 logger = logging.getLogger(__name__)
 
 
-# XXX not needed anymoar ...
-CHARSET_PATTERN = r"""charset=([\w-]+)"""
-CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
-
 """
 A MessagePartDoc is a light wrapper around the dictionary-like
 data that we pass along for message parts. It can be used almost everywhere
@@ -363,17 +359,17 @@ class MessagePart(object):
             payload = str("")
 
         if payload:
-            # XXX use find_charset instead --------------------------
-            # bad rebase???
             content_type = self._get_ctype_from_document(phash)
-            charset = first(CHARSET_RE.findall(content_type))
+            charset = find_charset(content_type)
             logger.debug("Got charset from header: %s" % (charset,))
-            if not charset:
+            if charset is None:
                 charset = self._get_charset(payload)
+                logger.debug("Got charset: %s" % (charset,))
             try:
                 payload = payload.encode(charset)
             except UnicodeError as exc:
-                logger.error("Unicode error {0}".format(exc))
+                logger.error(
+                    "Unicode error, using 'replace'. {0!r}".format(exc))
                 payload = payload.encode(charset, 'replace')
 
         fd.write(payload)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 35c07f5..7617fb8 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -335,16 +335,18 @@ class LeapMessage(fields, MailParser, MBoxParser):
             charset = find_charset(content_type)
             logger.debug('got charset from content-type: %s' % charset)
             if charset is None:
-                # XXX change for find_charset utility
                 charset = self._get_charset(body)
             try:
                 body = body.encode(charset)
             except UnicodeError as exc:
-                logger.error("Unicode error {0}".format(exc))
+                logger.error(
+                    "Unicode error, using 'replace'. {0!r}".format(exc))
                 logger.debug("Attempted to encode with: %s" % charset)
                 try:
                     body = body.encode(charset, 'replace')
-                except UnicodeError as exc:
+
+                # XXX desperate attempt. I've seen things you wouldn't believe
+                except UnicodeError:
                     try:
                         body = body.encode('utf-8', 'replace')
                     except:
-- 
cgit v1.2.3


From a7e0054b595822325f749b0b1df7d25cab4e6486 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 28 Jan 2014 18:39:59 -0400
Subject: docstring fixes

Also some fixes for None comparisons.
---
 src/leap/mail/imap/account.py                |   4 +-
 src/leap/mail/imap/interfaces.py             |   1 +
 src/leap/mail/imap/mailbox.py                |  31 ++--
 src/leap/mail/imap/memorystore.py            | 215 +++++++++++++++++++-----
 src/leap/mail/imap/messageparts.py           | 129 +++++++++-----
 src/leap/mail/imap/messages.py               | 240 +++------------------------
 src/leap/mail/imap/server.py                 |  17 +-
 src/leap/mail/imap/soledadstore.py           |  87 ++++++----
 src/leap/mail/imap/tests/leap_tests_imap.zsh |   3 +-
 src/leap/mail/size.py                        |   2 +-
 src/leap/mail/utils.py                       |   4 +
 11 files changed, 373 insertions(+), 360 deletions(-)

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 7641ea8..f985c04 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -48,7 +48,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     selected = None
     closed = False
 
-    def __init__(self, account_name, soledad=None, memstore=None):
+    def __init__(self, account_name, soledad, memstore=None):
         """
         Creates a SoledadAccountIndex that keeps track of the mailboxes
         and subscriptions handled by this account.
@@ -134,7 +134,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         if name not in self.mailboxes:
             raise imap4.MailboxException("No such mailbox: %r" % name)
 
-        return SoledadMailbox(name, soledad=self._soledad,
+        return SoledadMailbox(name, self._soledad,
                               memstore=self._memstore)
 
     ##
diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py
index 585165a..c906278 100644
--- a/src/leap/mail/imap/interfaces.py
+++ b/src/leap/mail/imap/interfaces.py
@@ -75,6 +75,7 @@ class IMessageStore(Interface):
 
         :param mbox: the mbox this message belongs.
         :param uid: the UID that identifies this message in this mailbox.
+        :return: IMessageContainer
         """
 
 
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index b5c5719..a0eb0a9 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -26,7 +26,6 @@ import cStringIO
 from collections import defaultdict
 
 from twisted.internet import defer
-#from twisted.internet.task import deferLater
 from twisted.python import log
 
 from twisted.mail import imap4
@@ -99,7 +98,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param rw: read-and-write flag for this mailbox
         :type rw: int
         """
-        print "got memstore: ", memstore
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
@@ -240,10 +238,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         the mailbox document in soledad if this is higher.
 
         :return: the last uid for messages in this mailbox
-        :rtype: bool
+        :rtype: int
         """
         last = self._memstore.get_last_uid(self.mbox)
-        print "last uid for %s: %s (from memstore)" % (self.mbox, last)
+        logger.debug("last uid for %s: %s (from memstore)" % (
+            repr(self.mbox), last))
         return last
 
     last_uid = property(
@@ -468,7 +467,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Remove all messages flagged \\Deleted
         """
-        print "EXPUNGE!"
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
 
@@ -537,8 +535,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # can treat them all the same.
         # Change this to the flag that twisted expects when we
         # switch to content-hash based index + local UID table.
-        print
-        print "FETCHING..."
 
         sequence = False
         #sequence = True if uid == 0 else False
@@ -648,9 +644,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             for msgid in seq_messg)
         return result
 
-    def signal_unread_to_ui(self):
+    def signal_unread_to_ui(self, *args, **kwargs):
         """
         Sends unread event to ui.
+
+        :param args: ignored
+        :param kwargs: ignored
         """
         unseen = self.getUnseenCount()
         leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
@@ -767,13 +766,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     # IMessageCopier
 
-    @deferred
+    #@deferred
     #@profile
     def copy(self, messageObject):
         """
         Copy the given message object into this mailbox.
         """
-        from twisted.internet import reactor
         msg = messageObject
         memstore = self._memstore
 
@@ -791,23 +789,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         exist = dest_fdoc and not empty(dest_fdoc.content)
 
         if exist:
-            print "Destination message already exists!"
+            logger.warning("Destination message already exists!")
         else:
-            print "DO COPY MESSAGE!"
             mbox = self.mbox
             uid_next = memstore.increment_last_soledad_uid(mbox)
             new_fdoc[self.UID_KEY] = uid_next
             new_fdoc[self.MBOX_KEY] = mbox
 
-            # XXX set recent!
-
-            print "****************************"
-            print "copy message..."
-            print "new fdoc ", new_fdoc
-            print "hdoc: ", hdoc
-            print "****************************"
+            # FIXME set recent!
 
-            self._memstore.create_message(
+            return self._memstore.create_message(
                 self.mbox, uid_next,
                 MessageWrapper(
                     new_fdoc, hdoc.content),
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 60e98c7..2d60b13 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -199,12 +199,14 @@ class MemoryStore(object):
         By default we consider that any message is a new message.
 
         :param mbox: the mailbox
-        :type mbox: basestring
+        :type mbox: str or unicode
         :param uid: the UID for the message
         :type uid: int
-        :param message: a to be added
+        :param message: a message to be added
         :type message: MessageWrapper
-        :param notify_on_disk:
+        :param notify_on_disk: whether the deferred that is returned should
+                               wait until the message is written to disk to
+                               be fired.
         :type notify_on_disk: bool
 
         :return: a Deferred. if notify_on_disk is True, will be fired
@@ -212,7 +214,7 @@ class MemoryStore(object):
                  Otherwise will fire inmediately
         :rtype: Deferred
         """
-        print "adding new doc to memstore %s (%s)" % (mbox, uid)
+        log.msg("adding new doc to memstore %r (%r)" % (mbox, uid))
         key = mbox, uid
 
         self._add_message(mbox, uid, message, notify_on_disk)
@@ -239,13 +241,17 @@ class MemoryStore(object):
         """
         Put an existing message.
 
+        This will set the dirty flag on the MemoryStore.
+
         :param mbox: the mailbox
-        :type mbox: basestring
+        :type mbox: str or unicode
         :param uid: the UID for the message
         :type uid: int
-        :param message: a to be added
+        :param message: a message to be added
         :type message: MessageWrapper
-        :param notify_on_disk:
+        :param notify_on_disk: whether the deferred that is returned should
+                               wait until the message is written to disk to
+                               be fired.
         :type notify_on_disk: bool
 
         :return: a Deferred. if notify_on_disk is True, will be fired
@@ -260,11 +266,13 @@ class MemoryStore(object):
         self._dirty.add(key)
         self._dirty_deferreds[key] = d
         self._add_message(mbox, uid, message, notify_on_disk)
-        #print "dirty ", self._dirty
-        #print "new ", self._new
         return d
 
     def _add_message(self, mbox, uid, message, notify_on_disk=True):
+        """
+        Helper method, called by both create_message and put_message.
+        See those for parameter documentation.
+        """
         # XXX have to differentiate between notify_new and notify_dirty
         # TODO defaultdict the hell outa here...
 
@@ -332,15 +340,19 @@ class MemoryStore(object):
                     store.pop(key)
         prune((FDOC, HDOC, CDOCS, DOCS_ID), store)
 
-        #print "after adding: "
-        #import pprint; pprint.pprint(self._msg_store[key])
-
     def get_docid_for_fdoc(self, mbox, uid):
         """
-        Get Soledad document id for the flags-doc for a given mbox and uid.
+        Return Soledad document id for the flags-doc for a given mbox and uid,
+        or None of no flags document could be found.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the message UID
+        :type uid: int
+        :rtype: unicode or None
         """
         fdoc = self._permanent_store.get_flags_doc(mbox, uid)
-        if not fdoc:
+        if empty(fdoc):
             return None
         doc_id = fdoc.doc_id
         return doc_id
@@ -349,22 +361,30 @@ class MemoryStore(object):
         """
         Get a MessageWrapper for the given mbox and uid combination.
 
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the message UID
+        :type uid: int
+
         :return: MessageWrapper or None
         """
         key = mbox, uid
         msg_dict = self._msg_store.get(key, None)
-        if msg_dict:
-            new, dirty = self._get_new_dirty_state(key)
-            return MessageWrapper(from_dict=msg_dict,
-                                  new=new,
-                                  dirty=dirty,
-                                  memstore=weakref.proxy(self))
-        else:
+        if empty(msg_dict):
             return None
+        new, dirty = self._get_new_dirty_state(key)
+        return MessageWrapper(from_dict=msg_dict,
+                              new=new, dirty=dirty,
+                              memstore=weakref.proxy(self))
 
     def remove_message(self, mbox, uid):
         """
         Remove a Message from this MemoryStore.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the message UID
+        :type uid: int
         """
         # XXX For the moment we are only removing the flags and headers
         # docs. The rest we leave there polluting your hard disk,
@@ -386,6 +406,8 @@ class MemoryStore(object):
     def write_messages(self, store):
         """
         Write the message documents in this MemoryStore to a different store.
+
+        :param store: the IMessageStore to write to
         """
         # For now, we pass if the queue is not empty, to avoid duplicate
         # queuing.
@@ -397,7 +419,10 @@ class MemoryStore(object):
         if not self.producer.is_queue_empty():
             return
 
-        print "Writing messages to Soledad..."
+        logger.info("Writing messages to Soledad...")
+
+        # TODO change for lock, and make the property access
+        # is accquired
         with set_bool_flag(self, self.WRITING_FLAG):
             for rflags_doc_wrapper in self.all_rdocs_iter():
                 self.producer.push(rflags_doc_wrapper)
@@ -409,6 +434,9 @@ class MemoryStore(object):
     def get_uids(self, mbox):
         """
         Get all uids for a given mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         """
         all_keys = self._msg_store.keys()
         return [uid for m, uid in all_keys if m == mbox]
@@ -420,6 +448,9 @@ class MemoryStore(object):
         Get the highest UID for a given mbox.
         It will be the highest between the highest uid in the message store for
         the mailbox, and the soledad integer cache.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         """
         uids = self.get_uids(mbox)
         last_mem_uid = uids and max(uids) or 0
@@ -429,6 +460,9 @@ class MemoryStore(object):
     def get_last_soledad_uid(self, mbox):
         """
         Get last uid for a given mbox from the soledad integer cache.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         """
         return self._last_uid.get(mbox, 0)
 
@@ -438,10 +472,16 @@ class MemoryStore(object):
         SoledadMailbox should prime this value during initialization.
         Other methods (during message adding) SHOULD call
         `increment_last_soledad_uid` instead.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: the value to set
+        :type value: int
         """
         leap_assert_type(value, int)
-        print "setting last soledad uid for ", mbox, "to", value
-        # if we already have a vlue here, don't do anything
+        logger.info("setting last soledad uid for %s to %s" %
+                    (mbox, value))
+        # if we already have a value here, don't do anything
         with self._last_uid_lock:
             if not self._last_uid.get(mbox, None):
                 self._last_uid[mbox] = value
@@ -451,6 +491,9 @@ class MemoryStore(object):
         Increment by one the soledad integer cache for the last_uid for
         this mbox, and fire a defer-to-thread to update the soledad value.
         The caller should lock the call tho this method.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         """
         with self._last_uid_lock:
             self._last_uid[mbox] += 1
@@ -461,7 +504,12 @@ class MemoryStore(object):
     @deferred
     def write_last_uid(self, mbox, value):
         """
-        Increment the soledad cache,
+        Increment the soledad integer cache for the highest uid value.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: the value to set
+        :type value: int
         """
         leap_assert_type(value, int)
         if self._permanent_store:
@@ -472,18 +520,30 @@ class MemoryStore(object):
     def count_new_mbox(self, mbox):
         """
         Count the new messages by inbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: number of new messages
+        :rtype: int
         """
         return len([(m, uid) for m, uid in self._new if mbox == mbox])
 
+    # XXX used at all?
     def count_new(self):
         """
         Count all the new messages in the MemoryStore.
+
+        :rtype: int
         """
         return len(self._new)
 
     def get_cdoc_from_phash(self, phash):
         """
         Return a content-document by its payload-hash.
+
+        :param phash: the payload hash to check against
+        :type phash: str or unicode
+        :rtype: MessagePartDoc
         """
         doc = self._phash_store.get(phash, None)
 
@@ -504,8 +564,16 @@ class MemoryStore(object):
     def get_fdoc_from_chash(self, chash, mbox):
         """
         Return a flags-document by its content-hash and a given mailbox.
+        Used during content-duplication detection while copying or adding a
+        message.
+
+        :param chash: the content hash to check against
+        :type chash: str or unicode
+        :param mbox: the mailbox
+        :type mbox: str or unicode
 
-        :return: MessagePartDoc, or None.
+        :return: MessagePartDoc. It will return None if the flags document
+                 has empty content or it is flagged as \\Deleted.
         """
         docs_dict = self._chash_fdoc_store.get(chash, None)
         fdoc = docs_dict.get(mbox, None) if docs_dict else None
@@ -522,9 +590,10 @@ class MemoryStore(object):
         if fdoc and fields.DELETED_FLAG in fdoc[fields.FLAGS_KEY]:
             return None
 
-        # XXX get flags
-        new = True
-        dirty = False
+        uid = fdoc.content[fields.UID_KEY]
+        key = mbox, uid
+        new = key in self._new
+        dirty = key in self._dirty
         return MessagePartDoc(
             new=new, dirty=dirty, store="mem",
             part=MessagePartType.fdoc,
@@ -534,13 +603,19 @@ class MemoryStore(object):
     def all_msg_iter(self):
         """
         Return generator that iterates through all messages in the store.
+
+        :return: generator of MessageWrappers
+        :rtype: generator
         """
         return (self.get_message(*key)
                 for key in sorted(self._msg_store.keys()))
 
     def all_new_dirty_msg_iter(self):
         """
-        Return geneator that iterates through all new and dirty messages.
+        Return generator that iterates through all new and dirty messages.
+
+        :return: generator of MessageWrappers
+        :rtype: generator
         """
         return (self.get_message(*key)
                 for key in sorted(self._msg_store.keys())
@@ -549,15 +624,29 @@ class MemoryStore(object):
     def all_msg_dict_for_mbox(self, mbox):
         """
         Return all the message dicts for a given mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: list of dictionaries
+        :rtype: list
         """
+        # This *needs* to return a fixed sequence. Otherwise the dictionary len
+        # will change during iteration, when we modify it
         return [self._msg_store[(mb, uid)]
                 for mb, uid in self._msg_store if mb == mbox]
 
     def all_deleted_uid_iter(self, mbox):
         """
-        Return generator that iterates through the UIDs for all messags
+        Return a list with the UIDs for all messags
         with deleted flag in a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: list of integers
+        :rtype: list
         """
+        # This *needs* to return a fixed sequence. Otherwise the dictionary len
+        # will change during iteration, when we modify it
         all_deleted = [
             msg['fdoc']['uid'] for msg in self.all_msg_dict_for_mbox(mbox)
             if msg.get('fdoc', None)
@@ -569,6 +658,11 @@ class MemoryStore(object):
     def _get_new_dirty_state(self, key):
         """
         Return `new` and `dirty` flags for a given message.
+
+        :param key: the key for the message, in the form mbox, uid
+        :type key: tuple
+        :return: tuple of bools
+        :rtype: tuple
         """
         # XXX should return *first* the news, and *then* the dirty...
         return map(lambda _set: key in _set, (self._new, self._dirty))
@@ -576,14 +670,19 @@ class MemoryStore(object):
     def set_new(self, key):
         """
         Add the key value to the `new` set.
+
+        :param key: the key for the message, in the form mbox, uid
+        :type key: tuple
         """
         self._new.add(key)
 
     def unset_new(self, key):
         """
         Remove the key value from the `new` set.
+
+        :param key: the key for the message, in the form mbox, uid
+        :type key: tuple
         """
-        #print "Unsetting NEW for: %s" % str(key)
         self._new.discard(key)
         deferreds = self._new_deferreds
         d = deferreds.get(key, None)
@@ -596,14 +695,19 @@ class MemoryStore(object):
     def set_dirty(self, key):
         """
         Add the key value to the `dirty` set.
+
+        :param key: the key for the message, in the form mbox, uid
+        :type key: tuple
         """
         self._dirty.add(key)
 
     def unset_dirty(self, key):
         """
         Remove the key value from the `dirty` set.
+
+        :param key: the key for the message, in the form mbox, uid
+        :type key: tuple
         """
-        #print "Unsetting DIRTY for: %s" % str(key)
         self._dirty.discard(key)
         deferreds = self._dirty_deferreds
         d = deferreds.get(key, None)
@@ -619,6 +723,11 @@ class MemoryStore(object):
     def set_recent_flag(self, mbox, uid):
         """
         Set the `Recent` flag for a given mailbox and UID.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the message UID
+        :type uid: int
         """
         self._rflags_dirty.add(mbox)
         self._rflags_store[mbox]['set'].add(uid)
@@ -627,6 +736,11 @@ class MemoryStore(object):
     def unset_recent_flag(self, mbox, uid):
         """
         Unset the `Recent` flag for a given mailbox and UID.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the message UID
+        :type uid: int
         """
         self._rflags_store[mbox]['set'].discard(uid)
 
@@ -634,6 +748,11 @@ class MemoryStore(object):
         """
         Set the value for the set of the recent flags.
         Used from the property in the MessageCollection.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: a sequence of flags to set
+        :type value: sequence
         """
         self._rflags_dirty.add(mbox)
         self._rflags_store[mbox]['set'] = set(value)
@@ -643,6 +762,8 @@ class MemoryStore(object):
         Load the passed flags document in the recent flags store, for a given
         mailbox.
 
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         :param flags_doc: A dictionary containing the `doc_id` of the Soledad
                           flags-document for this mailbox, and the `set`
                           of uids marked with that flag.
@@ -651,9 +772,11 @@ class MemoryStore(object):
 
     def get_recent_flags(self, mbox):
         """
-        Get the set of UIDs with the `Recent` flag for this mailbox.
+        Return the set of UIDs with the `Recent` flag for this mailbox.
 
-        :return: set, or None
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: set, or None
         """
         rflag_for_mbox = self._rflags_store.get(mbox, None)
         if not rflag_for_mbox:
@@ -666,6 +789,7 @@ class MemoryStore(object):
         under a RecentFlagsDoc namedtuple.
         Used for saving to disk.
 
+        :return: a generator of RecentFlagDoc
         :rtype: generator
         """
         # XXX use enums
@@ -696,6 +820,11 @@ class MemoryStore(object):
         """
         Remove all messages flagged \\Deleted from this Memory Store only.
         Called from `expunge`
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: a list of UIDs
+        :rtype: list
         """
         mem_deleted = self.all_deleted_uid_iter(mbox)
         for uid in mem_deleted:
@@ -706,6 +835,11 @@ class MemoryStore(object):
         """
         Remove all messages flagged \\Deleted, from the Memory Store
         and from the permanent store also.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: a list of UIDs
+        :rtype: list
         """
         # TODO expunge should add itself as a callback to the ongoing
         # writes.
@@ -737,7 +871,7 @@ class MemoryStore(object):
             mem_deleted = self.remove_all_deleted(mbox)
 
             all_deleted = set(mem_deleted).union(set(sol_deleted))
-            print "deleted ", all_deleted
+            logger.debug("deleted %r" % all_deleted)
         except Exception as exc:
             logger.exception(exc)
         finally:
@@ -763,18 +897,13 @@ class MemoryStore(object):
         # are done (gatherResults)
         return getattr(self, self.WRITING_FLAG)
 
-    def put_part(self, part_type, value):
-        """
-        Put the passed part into this IMessageStore.
-        `part` should be one of: fdoc, hdoc, cdoc
-        """
-        # XXX turn that into a enum
-
     # Memory management.
 
     def get_size(self):
         """
         Return the size of the internal storage.
         Use for calculating the limit beyond which we should flush the store.
+
+        :rtype: int
         """
         return size.get_size(self._msg_store)
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 10672ed..5067263 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -18,7 +18,6 @@
 MessagePart implementation. Used from LeapMessage.
 """
 import logging
-import re
 import StringIO
 import weakref
 
@@ -100,11 +99,10 @@ class MessageWrapper(object):
     CDOCS = "cdocs"
     DOCS_ID = "docs_id"
 
-    # XXX can use this to limit the memory footprint,
-    # or is it too premature to optimize?
-    # Does it work well together with the interfaces.implements?
+    # Using slots to limit some the memory footprint,
+    # Add your attribute here.
 
-    #__slots__ = ["_dict", "_new", "_dirty", "memstore"]
+    __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"]
 
     def __init__(self, fdoc=None, hdoc=None, cdocs=None,
                  from_dict=None, memstore=None,
@@ -141,9 +139,13 @@ class MessageWrapper(object):
 
     # properties
 
+    # TODO Could refactor new and dirty properties together.
+
     def _get_new(self):
         """
         Get the value for the `new` flag.
+
+        :rtype: bool
         """
         return self._new
 
@@ -151,6 +153,9 @@ class MessageWrapper(object):
         """
         Set the value for the `new` flag, and propagate it
         to the memory store if any.
+
+        :param value: the value to set
+        :type value: bool
         """
         self._new = value
         if self.memstore:
@@ -171,6 +176,8 @@ class MessageWrapper(object):
     def _get_dirty(self):
         """
         Get the value for the `dirty` flag.
+
+        :rtype: bool
         """
         return self._dirty
 
@@ -178,6 +185,9 @@ class MessageWrapper(object):
         """
         Set the value for the `dirty` flag, and propagate it
         to the memory store if any.
+
+        :param value: the value to set
+        :type value: bool
         """
         self._dirty = value
         if self.memstore:
@@ -198,6 +208,12 @@ class MessageWrapper(object):
 
     @property
     def fdoc(self):
+        """
+        Return a MessagePartDoc wrapping around a weak reference to
+        the flags-document in this MemoryStore, if any.
+
+        :rtype: MessagePartDoc
+        """
         _fdoc = self._dict.get(self.FDOC, None)
         if _fdoc:
             content_ref = weakref.proxy(_fdoc)
@@ -214,6 +230,12 @@ class MessageWrapper(object):
 
     @property
     def hdoc(self):
+        """
+        Return a MessagePartDoc wrapping around a weak reference to
+        the headers-document in this MemoryStore, if any.
+
+        :rtype: MessagePartDoc
+        """
         _hdoc = self._dict.get(self.HDOC, None)
         if _hdoc:
             content_ref = weakref.proxy(_hdoc)
@@ -228,6 +250,14 @@ class MessageWrapper(object):
 
     @property
     def cdocs(self):
+        """
+        Return a weak reference to a zero-indexed dict containing
+        the content-documents, or an empty dict if none found.
+        If you want access to the MessagePartDoc for the individual
+        parts, use the generator returned by `walk` instead.
+
+        :rtype: dict
+        """
         _cdocs = self._dict.get(self.CDOCS, None)
         if _cdocs:
             return weakref.proxy(_cdocs)
@@ -238,6 +268,8 @@ class MessageWrapper(object):
         """
         Generator that iterates through all the parts, returning
         MessagePartDoc. Used for writing to SoledadStore.
+
+        :rtype: generator
         """
         if self._dirty:
             mbox = self.fdoc.content[fields.MBOX_KEY]
@@ -264,6 +296,8 @@ class MessageWrapper(object):
     def as_dict(self):
         """
         Return a dict representation of the parts contained.
+
+        :rtype: dict
         """
         return self._dict
 
@@ -272,6 +306,11 @@ class MessageWrapper(object):
         Populate MessageWrapper parts from a dictionary.
         It expects the same format that we use in a
         MessageWrapper.
+
+
+        :param msg_dict: a dictionary containing the parts to populate
+                         the MessageWrapper from
+        :type msg_dict: dict
         """
         fdoc, hdoc, cdocs = map(
             lambda part: msg_dict.get(part, None),
@@ -288,7 +327,7 @@ class MessagePart(object):
     It takes a subpart message and is able to find
     the inner parts.
 
-    Excusatio non petita: see the interface documentation.
+    See the interface documentation.
     """
 
     implements(imap4.IMessagePart)
@@ -297,6 +336,8 @@ class MessagePart(object):
         """
         Initializes the MessagePart.
 
+        :param soledad: Soledad instance.
+        :type soledad: Soledad
         :param part_map: a dictionary containing the parts map for this
                          message
         :type part_map: dict
@@ -313,6 +354,7 @@ class MessagePart(object):
         # to gather the results of the deferred operations
         # to signal the operation is complete.
         #leap_assert(part_map, "part map dict cannot be null")
+
         self._soledad = soledad
         self._pmap = part_map
 
@@ -323,11 +365,12 @@ class MessagePart(object):
         :return: size of the message, in octets
         :rtype: int
         """
-        if not self._pmap:
+        if empty(self._pmap):
             return 0
         size = self._pmap.get('size', None)
-        if not size:
+        if size is None:
             logger.error("Message part cannot find size in the partmap")
+            size = 0
         return size
 
     def getBodyFile(self):
@@ -338,25 +381,25 @@ class MessagePart(object):
         :rtype: StringIO
         """
         fd = StringIO.StringIO()
-        if self._pmap:
+        if not empty(self._pmap):
             multi = self._pmap.get('multi')
             if not multi:
                 phash = self._pmap.get("phash", None)
             else:
                 pmap = self._pmap.get('part_map')
                 first_part = pmap.get('1', None)
-                if first_part:
+                if not empty(first_part):
                     phash = first_part['phash']
 
             if not phash:
                 logger.warning("Could not find phash for this subpart!")
-                payload = str("")
+                payload = ""
             else:
                 payload = self._get_payload_from_document(phash)
 
         else:
             logger.warning("Message with no part_map!")
-            payload = str("")
+            payload = ""
 
         if payload:
             content_type = self._get_ctype_from_document(phash)
@@ -366,7 +409,8 @@ class MessagePart(object):
                 charset = self._get_charset(payload)
                 logger.debug("Got charset: %s" % (charset,))
             try:
-                payload = payload.encode(charset)
+                if isinstance(payload, unicode):
+                    payload = payload.encode(charset)
             except UnicodeError as exc:
                 logger.error(
                     "Unicode error, using 'replace'. {0!r}".format(exc))
@@ -376,13 +420,15 @@ class MessagePart(object):
         fd.seek(0)
         return fd
 
-    # TODO cache the phash retrieval
+    # TODO should memory-bound this memoize!!!
+    @memoized_method
     def _get_payload_from_document(self, phash):
         """
-        Gets the message payload from the content document.
+        Return the message payload from the content document.
 
         :param phash: the payload hash to retrieve by.
-        :type phash: basestring
+        :type phash: str or unicode
+        :rtype: str or unicode
         """
         cdocs = self._soledad.get_from_index(
             fields.TYPE_P_HASH_IDX,
@@ -396,13 +442,15 @@ class MessagePart(object):
         payload = cdoc.content.get(fields.RAW_KEY, "")
         return payload
 
-    # TODO cache the pahash retrieval
+    # TODO should memory-bound this memoize!!!
+    @memoized_method
     def _get_ctype_from_document(self, phash):
         """
-        Gets the content-type from the content document.
+        Reeturn the content-type from the content document.
 
         :param phash: the payload hash to retrieve by.
-        :type phash: basestring
+        :type phash: str or unicode
+        :rtype: str or unicode
         """
         cdocs = self._soledad.get_from_index(
             fields.TYPE_P_HASH_IDX,
@@ -423,13 +471,14 @@ class MessagePart(object):
         Gets (guesses?) the charset of a payload.
 
         :param stuff: the stuff to guess about.
-        :type stuff: basestring
-        :returns: charset
+        :type stuff: str or unicode
+        :return: charset
+        :rtype: unicode
         """
         # XXX existential doubt 2. shouldn't we make the scope
         # of the decorator somewhat more persistent?
         # ah! yes! and put memory bounds.
-        return get_email_charset(unicode(stuff))
+        return get_email_charset(stuff)
 
     def getHeaders(self, negate, *names):
         """
@@ -446,37 +495,42 @@ class MessagePart(object):
         :return: A mapping of header field names to header field values
         :rtype: dict
         """
+        # XXX refactor together with MessagePart method
         if not self._pmap:
             logger.warning("No pmap in Subpart!")
             return {}
         headers = dict(self._pmap.get("headers", []))
 
-        # twisted imap server expects *some* headers to be lowercase
-        # We could use a CaseInsensitiveDict here...
-        headers = dict(
-            (str(key), str(value)) if key.lower() != "content-type"
-            else (str(key.lower()), str(value))
-            for (key, value) in headers.items())
-
         names = map(lambda s: s.upper(), names)
         if negate:
             cond = lambda key: key.upper() not in names
         else:
             cond = lambda key: key.upper() in names
 
-        # unpack and filter original dict by negate-condition
-        filter_by_cond = [
-            map(str, (key, val)) for
-            key, val in headers.items()
-            if cond(key)]
-        filtered = dict(filter_by_cond)
-        return filtered
+        # default to most likely standard
+        charset = find_charset(headers, "utf-8")
+        headers2 = dict()
+        for key, value in headers.items():
+            # twisted imap server expects *some* headers to be lowercase
+            # We could use a CaseInsensitiveDict here...
+            if key.lower() == "content-type":
+                key = key.lower()
+
+            if not isinstance(key, str):
+                key = key.encode(charset, 'replace')
+            if not isinstance(value, str):
+                value = value.encode(charset, 'replace')
+
+            # filter original dict by negate-condition
+            if cond(key):
+                headers2[key] = value
+        return headers2
 
     def isMultipart(self):
         """
         Return True if this message is multipart.
         """
-        if not self._pmap:
+        if empty(self._pmap):
             logger.warning("Could not get part map!")
             return False
         multi = self._pmap.get("multi", False)
@@ -495,6 +549,7 @@ class MessagePart(object):
         """
         if not self.isMultipart():
             raise TypeError
+
         sub_pmap = self._pmap.get("part_map", {})
         try:
             part_map = sub_pmap[str(part + 1)]
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 7617fb8..315cdda 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -58,10 +58,7 @@ logger = logging.getLogger(__name__)
 # [ ] Delete incoming mail only after successful write!
 # [ ] Remove UID from syncable db. Store only those indexes locally.
 
-CHARSET_PATTERN = r"""charset=([\w-]+)"""
 MSGID_PATTERN = r"""<([\w@.]+)>"""
-
-CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
 MSGID_RE = re.compile(MSGID_PATTERN)
 
 
@@ -202,8 +199,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: The flags, represented as strings
         :rtype: tuple
         """
-        #if self._uid is None:
-            #return []
         uid = self._uid
 
         flags = set([])
@@ -252,7 +247,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
 
         if self._collection.memstore is not None:
-            print "putting message in collection"
+            log.msg("putting message in collection")
             self._collection.memstore.put_message(
                 self._mbox, self._uid,
                 MessageWrapper(fdoc=doc.content, new=False, dirty=True,
@@ -327,8 +322,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         if self._bdoc is not None:
             bdoc_content = self._bdoc.content
             if bdoc_content is None:
-                logger.warning("No BODC content found for message!!!")
-                return write_fd(str(""))
+                logger.warning("No BDOC content found for message!!!")
+                return write_fd("")
 
             body = bdoc_content.get(self.RAW_KEY, "")
             content_type = bdoc_content.get('content-type', "")
@@ -337,20 +332,13 @@ class LeapMessage(fields, MailParser, MBoxParser):
             if charset is None:
                 charset = self._get_charset(body)
             try:
-                body = body.encode(charset)
+                if isinstance(body, unicode):
+                    body = body.encode(charset)
             except UnicodeError as exc:
                 logger.error(
                     "Unicode error, using 'replace'. {0!r}".format(exc))
                 logger.debug("Attempted to encode with: %s" % charset)
-                try:
-                    body = body.encode(charset, 'replace')
-
-                # XXX desperate attempt. I've seen things you wouldn't believe
-                except UnicodeError:
-                    try:
-                        body = body.encode('utf-8', 'replace')
-                    except:
-                        pass
+                body = body.encode(charset, 'replace')
             finally:
                 return write_fd(body)
 
@@ -409,6 +397,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: dict
         """
         # TODO split in smaller methods
+        # XXX refactor together with MessagePart method
+
         headers = self._get_headers()
         if not headers:
             logger.warning("No headers found")
@@ -425,11 +415,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
         # default to most likely standard
         charset = find_charset(headers, "utf-8")
-
-        # twisted imap server expects *some* headers to be lowercase
-        # XXX refactor together with MessagePart method
         headers2 = dict()
         for key, value in headers.items():
+            # twisted imap server expects *some* headers to be lowercase
+            # We could use a CaseInsensitiveDict here...
             if key.lower() == "content-type":
                 key = key.lower()
 
@@ -441,7 +430,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
             # filter original dict by negate-condition
             if cond(key):
                 headers2[key] = value
-
         return headers2
 
     def _get_headers(self):
@@ -547,10 +535,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         message.
         """
         hdoc_content = self._hdoc.content
-        #print "hdoc: ", hdoc_content
         body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
-        print "body phash: ", body_phash
         if not body_phash:
             logger.warning("No body phash for this document!")
             return None
@@ -562,11 +548,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
         if self._container is not None:
             bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
-            print "bdoc from container -->", bdoc
             if bdoc and bdoc.content is not None:
                 return bdoc
-            else:
-                print "no doc or not bdoc content for that phash found!"
 
         # no memstore or no doc found there
         if self._soledad:
@@ -590,77 +573,12 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         return self._fdoc.content.get(key, None)
 
-    # setters
-
-    # XXX to be used in the messagecopier interface?!
-#
-    #def set_uid(self, uid):
-        #"""
-        #Set new uid for this message.
-#
-        #:param uid: the new uid
-        #:type uid: basestring
-        #"""
-        # XXX dangerous! lock?
-        #self._uid = uid
-        #d = self._fdoc
-        #d.content[self.UID_KEY] = uid
-        #self._soledad.put_doc(d)
-#
-    #def set_mbox(self, mbox):
-        #"""
-        #Set new mbox for this message.
-#
-        #:param mbox: the new mbox
-        #:type mbox: basestring
-        #"""
-        # XXX dangerous! lock?
-        #self._mbox = mbox
-        #d = self._fdoc
-        #d.content[self.MBOX_KEY] = mbox
-        #self._soledad.put_doc(d)
-
-    # destructor
-
-    # XXX this logic moved to remove_message in memory store...
-    #@deferred
-    #def remove(self):
-        #"""
-        #Remove all docs associated with this message.
-        #Currently it removes only the flags doc.
-        #"""
-        #fd = self._get_flags_doc()
-#
-        #if fd.new:
-            # it's a new document, so we can remove it and it will not
-            # be writen. Watch out! We need to be sure it has not been
-            # just queued to write!
-            #memstore.remove_message(*key)
-#
-        #if fd.dirty:
-            #doc_id = fd.doc_id
-            #doc = self._soledad.get_doc(doc_id)
-            #try:
-                #self._soledad.delete_doc(doc)
-            #except Exception as exc:
-                #logger.exception(exc)
-#
-        #else:
-            # we just got a soledad_doc
-            #try:
-                #doc_id = fd.doc_id
-                #latest_doc = self._soledad.get_doc(doc_id)
-                #self._soledad.delete_doc(latest_doc)
-            #except Exception as exc:
-                #logger.exception(exc)
-        #return uid
-
     def does_exist(self):
         """
-        Return True if there is actually a flags message for this
+        Return True if there is actually a flags document for this
         UID and mbox.
         """
-        return self._fdoc is not None
+        return not empty(self._fdoc)
 
 
 class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
@@ -938,8 +856,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         if not exist:
             exist = self._get_fdoc_from_chash(chash)
-
-        print "FDOC EXIST?", exist
         if exist:
             return exist.content.get(fields.UID_KEY, "unknown-uid")
         else:
@@ -974,7 +890,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # TODO add the linked-from info !
         # TODO add reference to the original message
-        print "ADDING MESSAGE..."
 
         logger.debug('adding message')
         if flags is None:
@@ -990,15 +905,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # move the complete check to the soledad writer?
         # Watch out! We're reserving a UID right after this!
         if self._fdoc_already_exists(chash):
-            print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
             logger.warning("We already have that message in this mailbox.")
-            # note that this operation will leave holes in the UID sequence,
-            # but we're gonna change that all the same for a local-only table.
-            # so not touch it by the moment.
             return defer.succeed('already_exists')
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
-        print "ADDING MSG WITH UID: %s" % uid
+        logger.info("ADDING MSG WITH UID: %s" % uid)
 
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
@@ -1017,58 +928,36 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         # The MessageContainer expects a dict, zero-indexed
         # XXX review-me
-        cdocs = dict((index, doc) for index, doc in
-                     enumerate(walk.get_raw_docs(msg, parts)))
+        cdocs = dict(enumerate(walk.get_raw_docs(msg, parts)))
 
         self.set_recent_flag(uid)
 
         # Saving ----------------------------------------
-        # XXX adapt hdocset to use memstore
-        #hdoc = self._soledad.create_doc(hd)
-        # We add the newly created hdoc to the fast-access set of
-        # headers documents associated with the mailbox.
-        #self.add_hdocset_docid(hdoc.doc_id)
-
         # TODO ---- add reference to original doc, to be deleted
         # after writes are done.
         msg_container = MessageWrapper(fd, hd, cdocs)
 
-        # XXX Should allow also to dump to disk directly,
-        # for no-memstore cases.
-
         # we return a deferred that by default will be triggered
         # inmediately.
         d = self.memstore.create_message(self.mbox, uid, msg_container,
                                          notify_on_disk=notify_on_disk)
-        print "adding message", d
         return d
 
-    #def remove(self, msg):
-        #"""
-        #Remove a given msg.
-        #:param msg: the message to be removed
-        #:type msg: LeapMessage
-        #"""
-        #d = msg.remove()
-        #d.addCallback(self._remove_cb)
-        #return d
-
     #
     # getters: specific queries
     #
 
     # recent flags
-    # XXX FIXME -------------------------------------
-    # This should be rewritten to use memory store.
 
     def _get_recent_flags(self):
         """
         An accessor for the recent-flags set for this mailbox.
         """
+        # XXX check if we should remove this
         if self.__rflags is not None:
             return self.__rflags
 
-        if self.memstore:
+        if self.memstore is not None:
             with self._rdoc_lock:
                 rflags = self.memstore.get_recent_flags(self.mbox)
                 if not rflags:
@@ -1091,11 +980,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                     fields.RECENTFLAGS_KEY, []))
             return self.__rflags
 
+    @profile
     def _set_recent_flags(self, value):
         """
         Setter for the recent-flags set for this mailbox.
         """
-        if self.memstore:
+        if self.memstore is not None:
             self.memstore.set_recent_flags(self.mbox, value)
 
         else:
@@ -1112,9 +1002,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
 
+    # XXX change naming, indicate soledad query.
     def _get_recent_doc(self):
         """
-        Get recent-flags document for this mailbox.
+        Get recent-flags document from Soledad for this mailbox.
+        :rtype: SoledadDocument or None
         """
         curried = partial(
             self._soledad.get_from_index,
@@ -1153,82 +1045,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             self.recent_flags = self.recent_flags.union(
                 set([uid]))
 
-    # headers-docs-set
-
-    # XXX FIXME -------------------------------------
-    # This should be rewritten to use memory store.
-
-    #def _get_hdocset(self):
-        #"""
-        #An accessor for the hdocs-set for this mailbox.
-        #"""
-        #if not self.__hdocset:
-            #with self._hdocset_lock:
-                #hdocset_doc = self._get_hdocset_doc()
-                #value = set(hdocset_doc.content.get(
-                    #fields.HDOCS_SET_KEY, []))
-                #self.__hdocset = value
-        #return self.__hdocset
-#
-    #def _set_hdocset(self, value):
-        #"""
-        #Setter for the hdocs-set for this mailbox.
-        #"""
-        #with self._hdocset_lock:
-            #hdocset_doc = self._get_hdocset_doc()
-            #newv = set(value)
-            #self.__hdocset = newv
-            #hdocset_doc.content[fields.HDOCS_SET_KEY] = list(newv)
-            # XXX should deferLater 0 it?
-            #self._soledad.put_doc(hdocset_doc)
-#
-    #_hdocset = property(
-        #_get_hdocset, _set_hdocset,
-        #doc="Set of Document-IDs for the headers docs associated "
-            #"with this mailbox.")
-#
-    #def _get_hdocset_doc(self):
-        #"""
-        #Get hdocs-set document for this mailbox.
-        #"""
-        #curried = partial(
-            #self._soledad.get_from_index,
-            #fields.TYPE_MBOX_IDX,
-            #fields.TYPE_HDOCS_SET_VAL, self.mbox)
-        #curried.expected = "hdocset"
-        #hdocset_doc = try_unique_query(curried)
-        #return hdocset_doc
-#
-    # Property-set modification (protected by a different
-    # lock to give atomicity to the read/write operation)
-#
-    #def remove_hdocset_docids(self, docids):
-        #"""
-        #Remove the given document IDs from the set of
-        #header-documents associated with this mailbox.
-        #"""
-        #with self._hdocset_property_lock:
-            #self._hdocset = self._hdocset.difference(
-                #set(docids))
-#
-    #def remove_hdocset_docid(self, docid):
-        #"""
-        #Remove the given document ID from the set of
-        #header-documents associated with this mailbox.
-        #"""
-        #with self._hdocset_property_lock:
-            #self._hdocset = self._hdocset.difference(
-                #set([docid]))
-#
-    #def add_hdocset_docid(self, docid):
-        #"""
-        #Add the given document ID to the set of
-        #header-documents associated with this mailbox.
-        #"""
-        #with self._hdocset_property_lock:
-            #self._hdocset = self._hdocset.union(
-                #set([docid]))
-
     # individual doc getters, message layer.
 
     def _get_fdoc_from_chash(self, chash):
@@ -1361,19 +1177,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         return (u for u in sorted(uids))
 
-    # XXX Should be moved to memstore
-    #def reset_last_uid(self, param):
-        #"""
-        #Set the last uid to the highest uid found.
-        #Used while expunging, passed as a callback.
-        #"""
-        #try:
-            #self.last_uid = max(self.all_uid_iter()) + 1
-        #except ValueError:
-            # empty sequence
-            #pass
-        #return param
-
     # XXX MOVE to memstore
     def all_flags(self):
         """
@@ -1390,7 +1193,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                 fields.TYPE_MBOX_IDX,
                 fields.TYPE_FLAGS_VAL, self.mbox)))
         if self.memstore is not None:
-            # XXX
             uids = self.memstore.get_uids(self.mbox)
             docs = ((uid, self.memstore.get_message(self.mbox, uid))
                     for uid in uids)
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 3a6ac9a..b77678a 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation.
 from copy import copy
 
 from twisted import cred
+from twisted.internet import defer
 from twisted.internet.defer import maybeDeferred
 from twisted.internet.task import deferLater
 from twisted.mail import imap4
@@ -132,6 +133,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
             ).addErrback(
                 ebFetch, tag)
 
+        # XXX should be a callback
         deferLater(reactor,
                    2, self.mbox.unset_recent_flags, messages)
         deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
@@ -139,12 +141,17 @@ class LeapIMAPServer(imap4.IMAP4Server):
     select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
                     imap4.IMAP4Server.arg_fetchatt)
 
+    def on_copy_finished(self, defers):
+        d = defer.gatherResults(filter(None, defers))
+        d.addCallback(self.notifyNew)
+        d.addCallback(self.mbox.signal_unread_to_ui)
+
     def do_COPY(self, tag, messages, mailbox, uid=0):
         from twisted.internet import reactor
-        imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
-        deferLater(reactor,
-                   2, self.mbox.unset_recent_flags, messages)
-        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
+        defers = []
+        d = imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
+        defers.append(d)
+        deferLater(reactor, 0, self.on_copy_finished, defers)
 
     select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
                    imap4.IMAP4Server.arg_astring)
@@ -201,5 +208,5 @@ class LeapIMAPServer(imap4.IMAP4Server):
         # back to the source mailbox...
         print "faking checkpoint..."
         import time
-        time.sleep(2)
+        time.sleep(5)
         return None
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 60576a3..f64ed23 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -22,7 +22,6 @@ import threading
 
 from itertools import chain
 
-#from twisted.internet import defer
 from u1db import errors as u1db_errors
 from zope.interface import implements
 
@@ -71,7 +70,7 @@ class ContentDedup(object):
         Check whether we already have a header document for this
         content hash in our database.
 
-        :param doc: tentative header document
+        :param doc: tentative header for document
         :type doc: dict
         :returns: True if it exists, False otherwise.
         """
@@ -87,8 +86,7 @@ class ContentDedup(object):
         if len(header_docs) != 1:
             logger.warning("Found more than one copy of chash %s!"
                            % (chash,))
-        # XXX re-enable
-        #logger.debug("Found header doc with that hash! Skipping save!")
+        logger.debug("Found header doc with that hash! Skipping save!")
         return True
 
     def _content_does_exist(self, doc):
@@ -96,7 +94,7 @@ class ContentDedup(object):
         Check whether we already have a content document for a payload
         with this hash in our database.
 
-        :param doc: tentative content document
+        :param doc: tentative content for document
         :type doc: dict
         :returns: True if it exists, False otherwise.
         """
@@ -112,8 +110,7 @@ class ContentDedup(object):
         if len(attach_docs) != 1:
             logger.warning("Found more than one copy of phash %s!"
                            % (phash,))
-        # XXX re-enable
-        #logger.debug("Found attachment doc with that hash! Skipping save!")
+        logger.debug("Found attachment doc with that hash! Skipping save!")
         return True
 
 
@@ -151,38 +148,49 @@ class SoledadStore(ContentDedup):
         Create the passed message into this SoledadStore.
 
         :param mbox: the mbox this message belongs.
+        :type mbox: str or unicode
         :param uid: the UID that identifies this message in this mailbox.
+        :type uid: int
         :param message: a IMessageContainer implementor.
         """
+        raise NotImplementedError()
 
     def put_message(self, mbox, uid, message):
         """
         Put the passed existing message into this SoledadStore.
 
         :param mbox: the mbox this message belongs.
+        :type mbox: str or unicode
         :param uid: the UID that identifies this message in this mailbox.
+        :type uid: int
         :param message: a IMessageContainer implementor.
         """
+        raise NotImplementedError()
 
     def remove_message(self, mbox, uid):
         """
         Remove the given message from this SoledadStore.
 
         :param mbox: the mbox this message belongs.
+        :type mbox: str or unicode
         :param uid: the UID that identifies this message in this mailbox.
+        :type uid: int
         """
+        raise NotImplementedError()
 
     def get_message(self, mbox, uid):
         """
         Get a IMessageContainer for the given mbox and uid combination.
 
         :param mbox: the mbox this message belongs.
+        :type mbox: str or unicode
         :param uid: the UID that identifies this message in this mailbox.
+        :type uid: int
         """
+        raise NotImplementedError()
 
     # IMessageConsumer
 
-    #@profile
     def consume(self, queue):
         """
         Creates a new document in soledad db.
@@ -198,8 +206,7 @@ class SoledadStore(ContentDedup):
         # TODO could generalize this method into a generic consumer
         # and only implement `process` here
 
-        empty = queue.empty()
-        while not empty:
+        while not queue.empty():
             items = self._process(queue)
 
             # we prime the generator, that should return the
@@ -213,23 +220,22 @@ class SoledadStore(ContentDedup):
                 for item, call in items:
                     try:
                         self._try_call(call, item)
-                    except Exception:
-                        failed = True
+                    except Exception as exc:
+                        failed = exc
                         continue
                 if failed:
                     raise MsgWriteError
 
             except MsgWriteError:
                 logger.error("Error while processing item.")
-                pass
+                logger.exception(failed)
             else:
                 if isinstance(doc_wrapper, MessageWrapper):
                     # If everything went well, we can unset the new flag
                     # in the source store (memory store)
-                    print "unsetting new flag!"
+                    logger.info("unsetting new flag!")
                     doc_wrapper.new = False
                     doc_wrapper.dirty = False
-            empty = queue.empty()
 
     #
     # SoledadStore specific methods.
@@ -253,20 +259,24 @@ class SoledadStore(ContentDedup):
             return chain((doc_wrapper,),
                          self._get_calls_for_rflags_doc(doc_wrapper))
         else:
-            print "********************"
-            print "CANNOT PROCESS ITEM!"
+            logger.warning("CANNOT PROCESS ITEM!")
             return (i for i in [])
 
     def _try_call(self, call, item):
         """
         Try to invoke a given call with item as a parameter.
+
+        :param call: the function to call
+        :type call: callable
+        :param item: the payload to pass to the call as argument
+        :type item: object
         """
-        if not call:
+        if call is None:
             return
         try:
             call(item)
         except u1db_errors.RevisionConflict as exc:
-            logger.error("Error: %r" % (exc,))
+            logger.exception("Error: %r" % (exc,))
             raise exc
 
     def _get_calls_for_msg_parts(self, msg_wrapper):
@@ -275,12 +285,14 @@ class SoledadStore(ContentDedup):
 
         :param msg_wrapper: A MessageWrapper
         :type msg_wrapper: IMessageContainer
+        :return: a generator of tuples with recent-flags doc payload
+                 and callable
+        :rtype: generator
         """
         call = None
 
-        if msg_wrapper.new is True:
+        if msg_wrapper.new:
             call = self._soledad.create_doc
-            print "NEW DOC ----------------------"
 
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
@@ -296,17 +308,12 @@ class SoledadStore(ContentDedup):
 
                 elif item.part == MessagePartType.cdoc:
                     if not self._content_does_exist(item.content):
-
-                        # XXX DEBUG -------------------
-                        print "about to write content-doc ",
-                        #import pprint; pprint.pprint(item.content)
-
                         yield dict(item.content), call
 
         # For now, the only thing that will be dirty is
         # the flags doc.
 
-        elif msg_wrapper.dirty is True:
+        elif msg_wrapper.dirty:
             call = self._soledad.put_doc
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
@@ -327,6 +334,11 @@ class SoledadStore(ContentDedup):
     def _get_calls_for_rflags_doc(self, rflags_wrapper):
         """
         We always put these documents.
+
+        :param rflags_wrapper: A wrapper around recent flags doc.
+        :type rflags_wrapper: RecentFlagsWrapper
+        :return: a tuple with recent-flags doc payload and callable
+        :rtype: tuple
         """
         call = self._soledad.put_doc
         rdoc = self._soledad.get_doc(rflags_wrapper.doc_id)
@@ -342,6 +354,8 @@ class SoledadStore(ContentDedup):
         """
         Return mailbox document.
 
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         :return: A SoledadDocument containing this mailbox, or None if
                  the query failed.
         :rtype: SoledadDocument or None.
@@ -358,6 +372,11 @@ class SoledadStore(ContentDedup):
     def get_flags_doc(self, mbox, uid):
         """
         Return the SoledadDocument for the given mbox and uid.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the UID for the message
+        :type uid: int
         """
         try:
             flag_docs = self._soledad.get_from_index(
@@ -378,6 +397,11 @@ class SoledadStore(ContentDedup):
         This is called from the deferred triggered by
         memorystore.increment_last_soledad_uid, which is expected to
         run in a separate thread.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: the value to set
+        :type value: int
         """
         leap_assert_type(value, int)
         key = fields.LAST_UID_KEY
@@ -398,6 +422,8 @@ class SoledadStore(ContentDedup):
         Get an iterator for the SoledadDocuments for messages
         with \\Deleted flag for a given mailbox.
 
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         :return: iterator through deleted message docs
         :rtype: iterable
         """
@@ -410,13 +436,12 @@ class SoledadStore(ContentDedup):
         """
         Remove from Soledad all messages flagged as deleted for a given
         mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
         """
-        print "DELETING ALL DOCS FOR -------", mbox
         deleted = []
         for doc in self.deleted_iter(mbox):
             deleted.append(doc.content[fields.UID_KEY])
-            print
-            print ">>>>>>>>>>>>>>>>>>>>"
-            print "deleting doc: ", doc.doc_id, doc.content
             self._soledad.delete_doc(doc)
         return deleted
diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh
index 8f0df9f..544faca 100755
--- a/src/leap/mail/imap/tests/leap_tests_imap.zsh
+++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh
@@ -61,8 +61,7 @@ IMAPTEST="imaptest"
 # These should be kept constant across benchmarking
 # runs across different machines, for comparability.
 
-#DURATION=200
-DURATION=60
+DURATION=200
 NUM_MSG=200
 
 
diff --git a/src/leap/mail/size.py b/src/leap/mail/size.py
index 4880d71..c9eaabd 100644
--- a/src/leap/mail/size.py
+++ b/src/leap/mail/size.py
@@ -48,10 +48,10 @@ def get_size(item):
     some memory, so use with care.
 
     :param item: the item which size wants to be computed
+    :rtype: int
     """
     seen = set()
     size = _get_size(item, seen)
-    #print "len(seen) ", len(seen)
     del seen
     collect()
     return size
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 1f43947..6a1fcde 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -21,6 +21,8 @@ import json
 import re
 import traceback
 
+from leap.soledad.common.document import SoledadDocument
+
 
 CHARSET_PATTERN = r"""charset=([\w-]+)"""
 CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
@@ -42,6 +44,8 @@ def empty(thing):
     """
     if thing is None:
         return True
+    if isinstance(thing, SoledadDocument):
+        thing = thing.content
     try:
         return len(thing) == 0
     except ReferenceError:
-- 
cgit v1.2.3


From 3243d37fcc3703bc9428717ffc72c4e680831813 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 28 Jan 2014 19:52:20 -0400
Subject: changes file

---
 changes/feature_in-memory-store | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/feature_in-memory-store

diff --git a/changes/feature_in-memory-store b/changes/feature_in-memory-store
new file mode 100644
index 0000000..a7a4d7a
--- /dev/null
+++ b/changes/feature_in-memory-store
@@ -0,0 +1 @@
+  o Use a memory store as write-buffer and read-cache.
-- 
cgit v1.2.3


From 1b71ba510a2e6680f1ecc84eacfc492b0bbe24fc Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 29 Jan 2014 00:54:20 -0400
Subject: Fix copy and deletion problems

* reorganize and simplify STORE command processing
* add the notification after the processing of the whole sequence
---
 src/leap/mail/imap/mailbox.py      |  24 +-----
 src/leap/mail/imap/memorystore.py  |  20 +++--
 src/leap/mail/imap/messages.py     | 156 +++++++++++++++++++++----------------
 src/leap/mail/imap/server.py       |  26 +++----
 src/leap/mail/imap/soledadstore.py |   5 +-
 5 files changed, 118 insertions(+), 113 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index a0eb0a9..3a6937f 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -654,7 +654,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         unseen = self.getUnseenCount()
         leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
 
-    @deferred
     def store(self, messages_asked, flags, mode, uid):
         """
         Sets the flags of one or more messages.
@@ -697,28 +696,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             log.msg('read only mailbox!')
             raise imap4.ReadOnlyMailbox
 
-        result = {}
-        for msg_id in seq_messg:
-            log.msg("MSG ID = %s" % msg_id)
-            msg = self.messages.get_msg_by_uid(msg_id)
-            if not msg:
-                continue
-            # We duplicate the set operations here
-            # to return the result because it's less costly than
-            # retrieving the flags again.
-            newflags = set(msg.getFlags())
-
-            if mode == 1:
-                msg.addFlags(flags)
-                newflags = newflags.union(set(flags))
-            elif mode == -1:
-                msg.removeFlags(flags)
-                newflags.difference_update(flags)
-            elif mode == 0:
-                msg.setFlags(flags)
-                newflags = set(flags)
-            result[msg_id] = newflags
-        return result
+        return self.messages.set_flags(self.mbox, seq_messg, flags, mode)
 
     # ISearchableMailbox
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 2d60b13..fac66ad 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -357,7 +357,7 @@ class MemoryStore(object):
         doc_id = fdoc.doc_id
         return doc_id
 
-    def get_message(self, mbox, uid):
+    def get_message(self, mbox, uid, flags_only=False):
         """
         Get a MessageWrapper for the given mbox and uid combination.
 
@@ -365,17 +365,27 @@ class MemoryStore(object):
         :type mbox: str or unicode
         :param uid: the message UID
         :type uid: int
+        :param flags_only: whether the message should carry only a reference
+                           to the flags document.
+        :type flags_only: bool
 
         :return: MessageWrapper or None
         """
         key = mbox, uid
+        FDOC = MessagePartType.fdoc.key
+
         msg_dict = self._msg_store.get(key, None)
         if empty(msg_dict):
             return None
         new, dirty = self._get_new_dirty_state(key)
-        return MessageWrapper(from_dict=msg_dict,
-                              new=new, dirty=dirty,
-                              memstore=weakref.proxy(self))
+        if flags_only:
+            return MessageWrapper(fdoc=msg_dict[FDOC],
+                                  new=new, dirty=dirty,
+                                  memstore=weakref.proxy(self))
+        else:
+            return MessageWrapper(from_dict=msg_dict,
+                                  new=new, dirty=dirty,
+                                  memstore=weakref.proxy(self))
 
     def remove_message(self, mbox, uid):
         """
@@ -590,7 +600,7 @@ class MemoryStore(object):
         if fdoc and fields.DELETED_FLAG in fdoc[fields.FLAGS_KEY]:
             return None
 
-        uid = fdoc.content[fields.UID_KEY]
+        uid = fdoc[fields.UID_KEY]
         key = mbox, uid
         new = key in self._new
         dirty = key in self._dirty
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 315cdda..5770868 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -20,7 +20,6 @@ LeapMessage and MessageCollection.
 import copy
 import logging
 import re
-import time
 import threading
 import StringIO
 
@@ -97,11 +96,13 @@ class LeapMessage(fields, MailParser, MBoxParser):
     """
 
     # TODO this has to change.
-    # Should index primarily by chash, and keep a local-lonly
+    # Should index primarily by chash, and keep a local-only
     # UID table.
 
     implements(imap4.IMessage)
 
+    flags_lock = threading.Lock()
+
     def __init__(self, soledad, uid, mbox, collection=None, container=None):
         """
         Initializes a LeapMessage.
@@ -111,7 +112,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :param uid: the UID for the message.
         :type uid: int or basestring
         :param mbox: the mbox this message belongs to
-        :type mbox: basestring
+        :type mbox: str or unicode
         :param collection: a reference to the parent collection object
         :type collection: MessageCollection
         :param container: a IMessageContainer implementor instance
@@ -216,23 +217,17 @@ class LeapMessage(fields, MailParser, MBoxParser):
             flags = map(str, flags)
         return tuple(flags)
 
-    # setFlags, addFlags, removeFlags are not in the interface spec
-    # but we use them with store command.
+    # setFlags not in the interface spec but we use it with store command.
 
-    def setFlags(self, flags):
+    def setFlags(self, flags, mode):
         """
         Sets the flags for this message
 
-        Returns a SoledadDocument that needs to be updated by the caller.
-
         :param flags: the flags to update in the message.
         :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
+        :param mode: the mode for setting. 1 is append, -1 is remove, 0 set.
+        :type mode: int
         """
-        # XXX Move logic to memory store ...
-
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         log.msg('setting flags: %s (%s)' % (self._uid, flags))
 
@@ -242,51 +237,36 @@ class LeapMessage(fields, MailParser, MBoxParser):
                 "Could not find FDOC for %s:%s while setting flags!" %
                 (self._mbox, self._uid))
             return
-        doc.content[self.FLAGS_KEY] = flags
-        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-        doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
-
-        if self._collection.memstore is not None:
-            log.msg("putting message in collection")
-            self._collection.memstore.put_message(
-                self._mbox, self._uid,
-                MessageWrapper(fdoc=doc.content, new=False, dirty=True,
-                               docs_id={'fdoc': doc.doc_id}))
-        else:
-            # fallback for non-memstore initializations.
-            self._soledad.put_doc(doc)
-
-    def addFlags(self, flags):
-        """
-        Adds flags to this message.
-
-        Returns a SoledadDocument that needs to be updated by the caller.
-
-        :param flags: the flags to add to the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        oldflags = self.getFlags()
-        self.setFlags(tuple(set(flags + oldflags)))
-
-    def removeFlags(self, flags):
-        """
-        Remove flags from this message.
-
-        Returns a SoledadDocument that needs to be updated by the caller.
 
-        :param flags: the flags to be removed from the message.
-        :type flags: tuple of str
-
-        :return: a SoledadDocument instance
-        :rtype: SoledadDocument
-        """
-        leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        oldflags = self.getFlags()
-        self.setFlags(tuple(set(oldflags) - set(flags)))
+        APPEND = 1
+        REMOVE = -1
+        SET = 0
+
+        with self.flags_lock:
+            current = doc.content[self.FLAGS_KEY]
+            if mode == APPEND:
+                newflags = tuple(set(tuple(current) + flags))
+            elif mode == REMOVE:
+                newflags = tuple(set(current).difference(set(flags)))
+            elif mode == SET:
+                newflags = flags
+
+            # We could defer this, but I think it's better
+            # to put it under the lock...
+            doc.content[self.FLAGS_KEY] = newflags
+            doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+            doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
+
+            if self._collection.memstore is not None:
+                log.msg("putting message in collection")
+                self._collection.memstore.put_message(
+                    self._mbox, self._uid,
+                    MessageWrapper(fdoc=doc.content, new=False, dirty=True,
+                                   docs_id={'fdoc': doc.doc_id}))
+            else:
+                # fallback for non-memstore initializations.
+                self._soledad.put_doc(doc)
+        return map(str, newflags)
 
     def getInternalDate(self):
         """
@@ -1022,6 +1002,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     def unset_recent_flags(self, uids):
         """
         Unset Recent flag for a sequence of uids.
+
+        :param uids: the uids to unset
+        :type uid: sequence
         """
         with self._rdoc_property_lock:
             self.recent_flags.difference_update(
@@ -1032,6 +1015,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     def unset_recent_flag(self, uid):
         """
         Unset Recent flag for a given uid.
+
+        :param uid: the uid to unset
+        :type uid: int
         """
         with self._rdoc_property_lock:
             self.recent_flags.difference_update(
@@ -1040,6 +1026,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     def set_recent_flag(self, uid):
         """
         Set Recent flag for a given uid.
+
+        :param uid: the uid to set
+        :type uid: int
         """
         with self._rdoc_property_lock:
             self.recent_flags = self.recent_flags.union(
@@ -1099,31 +1088,64 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # and we cannot find it otherwise. This seems to be enough.
 
         # XXX do a deferLater instead ??
-        # FIXME this won't be needed after the CHECK command is implemented.
-        time.sleep(0.3)
+        # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
+    def set_flags(self, mbox, messages, flags, mode):
+        """
+        Set flags for a sequence of messages.
+
+        :param mbox: the mbox this message belongs to
+        :type mbox: str or unicode
+        :param messages: the messages to iterate through
+        :type messages: sequence
+        :flags: the flags to be set
+        :type flags: tuple
+        :param mode: the mode for setting. 1 is append, -1 is remove, 0 set.
+        :type mode: int
+        """
+        result = {}
+        for msg_id in messages:
+            log.msg("MSG ID = %s" % msg_id)
+            msg = self.get_msg_by_uid(msg_id, mem_only=True, flags_only=True)
+            if not msg:
+                continue
+            result[msg_id] = msg.setFlags(flags, mode)
+
+        return result
+
     # getters: generic for a mailbox
 
-    def get_msg_by_uid(self, uid):
+    def get_msg_by_uid(self, uid, mem_only=False, flags_only=False):
         """
         Retrieves a LeapMessage by UID.
         This is used primarity in the Mailbox fetch and store methods.
 
         :param uid: the message uid to query by
         :type uid: int
+        :param mem_only: a flag that indicates whether this Message should
+                         pass a reference to soledad to retrieve missing pieces
+                         or not.
+        :type mem_only: bool
+        :param flags_only: whether the message should carry only a reference
+                           to the flags document.
+        :type flags_only: bool
 
         :return: A LeapMessage instance matching the query,
                  or None if not found.
         :rtype: LeapMessage
         """
-        msg_container = self.memstore.get_message(self.mbox, uid)
+        msg_container = self.memstore.get_message(self.mbox, uid, flags_only)
         if msg_container is not None:
-            # We pass a reference to soledad just to be able to retrieve
-            # missing parts that cannot be found in the container, like
-            # the content docs after a copy.
-            msg = LeapMessage(self._soledad, uid, self.mbox, collection=self,
-                              container=msg_container)
+            if mem_only:
+                msg = LeapMessage(None, uid, self.mbox, collection=self,
+                                  container=msg_container)
+            else:
+                # We pass a reference to soledad just to be able to retrieve
+                # missing parts that cannot be found in the container, like
+                # the content docs after a copy.
+                msg = LeapMessage(self._soledad, uid, self.mbox,
+                                  collection=self, container=msg_container)
         else:
             msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
         if not msg.does_exist():
@@ -1159,7 +1181,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     def all_uid_iter(self):
         """
-        Return an iterator trhough the UIDs of all messages, sorted in
+        Return an iterator through the UIDs of all messages, sorted in
         ascending order.
         """
         # XXX we should get this from the uid table, local-only
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index b77678a..7bca39d 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -99,10 +99,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
         Overwritten fetch dispatcher to use the fast fetch_flags
         method
         """
-        from twisted.internet import reactor
         if not query:
             self.sendPositiveResponse(tag, 'FETCH complete')
-            return  # XXX ???
+            return
 
         cbFetch = self._IMAP4Server__cbFetch
         ebFetch = self._IMAP4Server__ebFetch
@@ -131,16 +130,19 @@ class LeapIMAPServer(imap4.IMAP4Server):
             ).addCallback(
                 cbFetch, tag, query, uid
             ).addErrback(
-                ebFetch, tag)
-
-        # XXX should be a callback
-        deferLater(reactor,
-                   2, self.mbox.unset_recent_flags, messages)
-        deferLater(reactor, 1, self.mbox.signal_unread_to_ui)
+                ebFetch, tag
+            ).addCallback(
+                self.on_fetch_finished, messages)
 
     select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
                     imap4.IMAP4Server.arg_fetchatt)
 
+    def on_fetch_finished(self, _, messages):
+        from twisted.internet import reactor
+        deferLater(reactor, 0, self.notifyNew)
+        deferLater(reactor, 0, self.mbox.unset_recent_flags, messages)
+        deferLater(reactor, 0, self.mbox.signal_unread_to_ui)
+
     def on_copy_finished(self, defers):
         d = defer.gatherResults(filter(None, defers))
         d.addCallback(self.notifyNew)
@@ -156,7 +158,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
     select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
                    imap4.IMAP4Server.arg_astring)
 
-    def notifyNew(self, ignored):
+    def notifyNew(self, ignored=None):
         """
         Notify new messages to listeners.
         """
@@ -203,10 +205,4 @@ class LeapIMAPServer(imap4.IMAP4Server):
         """
         # TODO return the output of _memstore.is_writing
         # XXX and that should return a deferred!
-
-        # XXX  fake a delayed operation, to debug problem with messages getting
-        # back to the source mailbox...
-        print "faking checkpoint..."
-        import time
-        time.sleep(5)
         return None
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index f64ed23..ae5c583 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -26,6 +26,7 @@ from u1db import errors as u1db_errors
 from zope.interface import implements
 
 from leap.common.check import leap_assert_type
+from leap.mail.decorators import deferred
 from leap.mail.imap.messageparts import MessagePartType
 from leap.mail.imap.messageparts import MessageWrapper
 from leap.mail.imap.messageparts import RecentFlagsDoc
@@ -191,6 +192,7 @@ class SoledadStore(ContentDedup):
 
     # IMessageConsumer
 
+    @deferred
     def consume(self, queue):
         """
         Creates a new document in soledad db.
@@ -297,9 +299,6 @@ class SoledadStore(ContentDedup):
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
                 if item.part == MessagePartType.fdoc:
-
-                    # FIXME add content duplication for HEADERS too!
-                    # (only 1 chash per mailbox!)
                     yield dict(item.content), call
 
                 elif item.part == MessagePartType.hdoc:
-- 
cgit v1.2.3


From 37090301e633cdf9aa0c924ea3899d4c5d4fcdb9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 29 Jan 2014 16:18:27 -0400
Subject: allow to pass file as argument

---
 src/leap/mail/imap/tests/walktree.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py
index 1626f65..f3cbcb0 100644
--- a/src/leap/mail/imap/tests/walktree.py
+++ b/src/leap/mail/imap/tests/walktree.py
@@ -18,12 +18,14 @@
 Tests for the walktree module.
 """
 import os
+import sys
 from email import parser
 
 from leap.mail import walk as W
 
 DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
 
+
 p = parser.Parser()
 
 # TODO pass an argument of the type of message
@@ -31,9 +33,17 @@ p = parser.Parser()
 ##################################################
 # Input from hell
 
-#msg = p.parse(open('rfc822.multi-signed.message'))
-#msg = p.parse(open('rfc822.plain.message'))
-msg = p.parse(open('rfc822.multi-minimal.message'))
+if len(sys.argv) > 1:
+    FILENAME = sys.argv[1]
+else:
+    FILENAME = "rfc822.multi-minimal.message"
+
+"""
+FILENAME = "rfc822.multi-signed.message"
+FILENAME = "rfc822.plain.message"
+"""
+
+msg = p.parse(open(FILENAME))
 DO_CHECK = False
 #################################################
 
-- 
cgit v1.2.3


From bd06be63eac85a29e32768e55ab52a46043f3493 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 29 Jan 2014 16:19:37 -0400
Subject: Fix UIDVALIDITY command.

thanks to evolution for complaining about this.
---
 src/leap/mail/imap/mailbox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 3a6937f..2d1ab88 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -366,7 +366,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if self.CMD_UIDNEXT in names:
             r[self.CMD_UIDNEXT] = self.last_uid + 1
         if self.CMD_UIDVALIDITY in names:
-            r[self.CMD_UIDVALIDITY] = self.getUID()
+            r[self.CMD_UIDVALIDITY] = self.getUIDValidity()
         if self.CMD_UNSEEN in names:
             r[self.CMD_UNSEEN] = self.getUnseenCount()
         return defer.succeed(r)
-- 
cgit v1.2.3


From 5818a2e6826d84cd82cc578fbce95aa549d70e25 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 29 Jan 2014 16:43:15 -0400
Subject: Fix indexing error that was rendering attachments unusable

Also, check for empty body-doc
---
 src/leap/mail/imap/messages.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 5770868..2ace103 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -301,7 +301,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         fd = StringIO.StringIO()
         if self._bdoc is not None:
             bdoc_content = self._bdoc.content
-            if bdoc_content is None:
+            if empty(bdoc_content):
                 logger.warning("No BDOC content found for message!!!")
                 return write_fd("")
 
@@ -906,9 +906,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             hd[key] = parts_map[key]
         del parts_map
 
-        # The MessageContainer expects a dict, zero-indexed
+        # The MessageContainer expects a dict, one-indexed
         # XXX review-me
-        cdocs = dict(enumerate(walk.get_raw_docs(msg, parts)))
+        cdocs = dict(((key + 1, doc) for key, doc in
+                     enumerate(walk.get_raw_docs(msg, parts))))
 
         self.set_recent_flag(uid)
 
@@ -960,7 +961,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                     fields.RECENTFLAGS_KEY, []))
             return self.__rflags
 
-    @profile
     def _set_recent_flags(self, value):
         """
         Setter for the recent-flags set for this mailbox.
-- 
cgit v1.2.3


From 5cc82b3e8937c0e4488f79db79891c90a2ce3d47 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 30 Jan 2014 17:23:19 -0400
Subject: fix badly terminated headers

---
 src/leap/mail/imap/messages.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 2ace103..356145f 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -407,6 +407,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
             if not isinstance(value, str):
                 value = value.encode(charset, 'replace')
 
+            if value.endswith(";"):
+                # bastards
+                value = value[:-1]
+
             # filter original dict by negate-condition
             if cond(key):
                 headers2[key] = value
-- 
cgit v1.2.3


From 75da338c765ffb935290f5ca16ea2df406dc89d8 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 30 Jan 2014 17:23:27 -0400
Subject: skip notifications

---
 src/leap/mail/imap/mailbox.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 2d1ab88..6c8d78d 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -22,6 +22,7 @@ import threading
 import logging
 import StringIO
 import cStringIO
+import os
 
 from collections import defaultdict
 
@@ -43,6 +44,12 @@ from leap.mail.imap.parser import MBoxParser
 
 logger = logging.getLogger(__name__)
 
+"""
+If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
+notifying clients of new messages. Use during stress tests.
+"""
+NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False)
+
 
 class SoledadMailbox(WithMsgFields, MBoxParser):
     """
@@ -77,6 +84,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     CMD_UIDVALIDITY = "UIDVALIDITY"
     CMD_UNSEEN = "UNSEEN"
 
+    # FIXME we should turn this into a datastructure with limited capacity
     _listeners = defaultdict(set)
 
     next_uid_lock = threading.Lock()
@@ -145,6 +153,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param listener: listener to add
         :type listener: an object that implements IMailboxListener
         """
+        if not NOTIFY_NEW:
+            return
         logger.debug('adding mailbox listener: %s' % listener)
         self.listeners.add(listener)
 
@@ -421,6 +431,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         :param args: ignored.
         """
+        if not NOTIFY_NEW:
+            return
         exists = self.getMessageCount()
         recent = self.getRecentCount()
         logger.debug("NOTIFY: there are %s messages, %s recent" % (
-- 
cgit v1.2.3


From ff7de0c9bc760e097c0286d2d62a19095be3f35e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 30 Jan 2014 18:35:03 -0400
Subject: prime-uids

We pre-fetch the uids from soledad on mailbox initialization
---
 src/leap/mail/imap/mailbox.py      | 13 +++++++++-
 src/leap/mail/imap/memorystore.py  | 30 +++++++++++++++++++++
 src/leap/mail/imap/messages.py     | 53 ++++++++++++++++++++++----------------
 src/leap/mail/imap/soledadstore.py |  3 ++-
 4 files changed, 75 insertions(+), 24 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 6c8d78d..802ebf3 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -126,6 +126,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             self.setFlags(self.INIT_FLAGS)
 
         if self._memstore:
+            self.prime_known_uids_to_memstore()
             self.prime_last_uid_to_memstore()
 
     @property
@@ -263,10 +264,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Prime memstore with last_uid value
         """
         set_exist = set(self.messages.all_uid_iter())
-        last = max(set_exist) + 1 if set_exist else 1
+        last = max(set_exist) if set_exist else 0
         logger.info("Priming Soledad last_uid to %s" % (last,))
         self._memstore.set_last_soledad_uid(self.mbox, last)
 
+    def prime_known_uids_to_memstore(self):
+        """
+        Prime memstore with the set of all known uids.
+
+        We do this to be able to filter the requests efficiently.
+        """
+        known_uids = self.messages.all_soledad_uid_iter()
+        self._memstore.set_known_uids(self.mbox, known_uids)
+
     def getUIDValidity(self):
         """
         Return the unique validity identifier for this mailbox.
@@ -525,6 +535,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         return seq_messg
 
     @deferred
+    #@profile
     def fetch(self, messages_asked, uid):
         """
         Retrieve one or more messages in this mailbox.
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index fac66ad..217ad8e 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -149,6 +149,14 @@ class MemoryStore(object):
         """
         self._last_uid = {}
 
+        """
+        known-uids keeps a count of the uids that soledad knows for a given
+        mailbox
+
+        {'mbox-a': set([1,2,3])}
+        """
+        self._known_uids = defaultdict(set)
+
         # New and dirty flags, to set MessageWrapper State.
         self._new = set([])
         self._new_deferreds = {}
@@ -447,10 +455,20 @@ class MemoryStore(object):
 
         :param mbox: the mailbox
         :type mbox: str or unicode
+        :rtype: list
         """
         all_keys = self._msg_store.keys()
         return [uid for m, uid in all_keys if m == mbox]
 
+    def get_soledad_known_uids(self, mbox):
+        """
+        Get all uids that soledad knows about, from the memory cache.
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: list
+        """
+        return self._known_uids.get(mbox, [])
+
     # last_uid
 
     def get_last_uid(self, mbox):
@@ -496,6 +514,18 @@ class MemoryStore(object):
             if not self._last_uid.get(mbox, None):
                 self._last_uid[mbox] = value
 
+    def set_known_uids(self, mbox, value):
+        """
+        Set the value fo the known-uids set for this mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: a sequence of integers to be added to the set.
+        :type value: tuple
+        """
+        current = self._known_uids[mbox]
+        self._known_uids[mbox] = current.union(set(value))
+
     def increment_last_soledad_uid(self, mbox):
         """
         Increment by one the soledad integer cache for the last_uid for
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 356145f..0e5c74a 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -219,6 +219,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     # setFlags not in the interface spec but we use it with store command.
 
+    #@profile
     def setFlags(self, flags, mode):
         """
         Sets the flags for this message
@@ -934,6 +935,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # recent flags
 
+    #@profile
     def _get_recent_flags(self):
         """
         An accessor for the recent-flags set for this mailbox.
@@ -957,13 +959,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                         {'doc_id': rdoc.doc_id, 'set': rflags})
             return rflags
 
-        else:
+        #else:
             # fallback for cases without memory store
-            with self._rdoc_lock:
-                rdoc = self._get_recent_doc()
-                self.__rflags = set(rdoc.content.get(
-                    fields.RECENTFLAGS_KEY, []))
-            return self.__rflags
+            #with self._rdoc_lock:
+                #rdoc = self._get_recent_doc()
+                #self.__rflags = set(rdoc.content.get(
+                    #fields.RECENTFLAGS_KEY, []))
+            #return self.__rflags
 
     def _set_recent_flags(self, value):
         """
@@ -972,21 +974,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         if self.memstore is not None:
             self.memstore.set_recent_flags(self.mbox, value)
 
-        else:
+        #else:
             # fallback for cases without memory store
-            with self._rdoc_lock:
-                rdoc = self._get_recent_doc()
-                newv = set(value)
-                self.__rflags = newv
-                rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
+            #with self._rdoc_lock:
+                #rdoc = self._get_recent_doc()
+                #newv = set(value)
+                #self.__rflags = newv
+                #rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
                 # XXX should deferLater 0 it?
-                self._soledad.put_doc(rdoc)
+                #self._soledad.put_doc(rdoc)
 
     recent_flags = property(
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
 
     # XXX change naming, indicate soledad query.
+    #@profile
     def _get_recent_doc(self):
         """
         Get recent-flags document from Soledad for this mailbox.
@@ -1027,6 +1030,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             self.recent_flags.difference_update(
                 set([uid]))
 
+    @deferred
     def set_recent_flag(self, uid):
         """
         Set Recent flag for a given uid.
@@ -1095,6 +1099,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
+    #@profile
     def set_flags(self, mbox, messages, flags, mode):
         """
         Set flags for a sequence of messages.
@@ -1183,25 +1188,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # FIXME ----------------------------------------------
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
-    def all_uid_iter(self):
+    #@profile
+    def all_soledad_uid_iter(self):
         """
         Return an iterator through the UIDs of all messages, sorted in
         ascending order.
         """
-        # XXX we should get this from the uid table, local-only
-        # XXX FIXME -------------
-        # This should be cached in the memstoretoo
         db_uids = set([doc.content[self.UID_KEY] for doc in
                        self._soledad.get_from_index(
                            fields.TYPE_MBOX_IDX,
                            fields.TYPE_FLAGS_VAL, self.mbox)])
+        return db_uids
+
+    #@profile
+    def all_uid_iter(self):
+        """
+        Return an iterator through the UIDs of all messages, from memory.
+        """
         if self.memstore is not None:
             mem_uids = self.memstore.get_uids(self.mbox)
-            uids = db_uids.union(set(mem_uids))
-        else:
-            uids = db_uids
-
-        return (u for u in sorted(uids))
+            soledad_known_uids = self.memstore.get_soledad_known_uids(
+                self.mbox)
+            combined = tuple(set(mem_uids).union(soledad_known_uids))
+            return combined
 
     # XXX MOVE to memstore
     def all_flags(self):
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index ae5c583..ff5e03b 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -192,7 +192,8 @@ class SoledadStore(ContentDedup):
 
     # IMessageConsumer
 
-    @deferred
+    # It's not thread-safe to defer this to a different thread
+
     def consume(self, queue):
         """
         Creates a new document in soledad db.
-- 
cgit v1.2.3


From 0f6a8e1c83995cffec51e81f626d4bb29d4f7345 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 31 Jan 2014 03:34:03 -0400
Subject: properly implement deferreds in several commands

Passing along a deferred as an observer whose callback will be
called with the proper result.

Returning to thread in the appropiate points.
just let's remember that twisted APIs are not thread safe!

SoledadStore process_item also properly returned to thread.

Changed @deferred to @deferred_to_thread so it results less
confusing to read.

"know the territory". aha!
---
 src/leap/mail/decorators.py        |   2 +-
 src/leap/mail/imap/fetch.py        |   4 +-
 src/leap/mail/imap/mailbox.py      | 112 ++++++++++++++++++++++++-------
 src/leap/mail/imap/memorystore.py  |  43 ++++++------
 src/leap/mail/imap/messages.py     | 133 +++++++++++++++++++++++--------------
 src/leap/mail/imap/soledadstore.py |  99 ++++++++++++++++++---------
 6 files changed, 264 insertions(+), 129 deletions(-)

diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py
index d5eac97..ae115f8 100644
--- a/src/leap/mail/decorators.py
+++ b/src/leap/mail/decorators.py
@@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
 # See this answer: http://stackoverflow.com/a/19019648/1157664
 # And the notes by glyph and jpcalderone
 
-def deferred(f):
+def deferred_to_thread(f):
     """
     Decorator, for deferring methods to Threads.
 
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 817ad6a..40dadb3 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -45,7 +45,7 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
 from leap.keymanager.openpgp import OpenPGPKey
-from leap.mail.decorators import deferred
+from leap.mail.decorators import deferred_to_thread
 from leap.mail.utils import json_loads
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
@@ -199,7 +199,7 @@ class LeapIncomingMail(object):
         logger.exception(failure.value)
         traceback.print_tb(*sys.exc_info())
 
-    @deferred
+    @deferred_to_thread
     def _sync_soledad(self):
         """
         Synchronizes with remote soledad.
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 802ebf3..79fb476 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -27,6 +27,7 @@ import os
 from collections import defaultdict
 
 from twisted.internet import defer
+from twisted.internet.task import deferLater
 from twisted.python import log
 
 from twisted.mail import imap4
@@ -35,7 +36,7 @@ from zope.interface import implements
 from leap.common import events as leap_events
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
 from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.decorators import deferred
+from leap.mail.decorators import deferred_to_thread
 from leap.mail.utils import empty
 from leap.mail.imap.fields import WithMsgFields, fields
 from leap.mail.imap.messages import MessageCollection
@@ -51,6 +52,11 @@ notifying clients of new messages. Use during stress tests.
 NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False)
 
 
+class MessageCopyError(Exception):
+    """
+    """
+
+
 class SoledadMailbox(WithMsgFields, MBoxParser):
     """
     A Soledad-backed IMAP mailbox.
@@ -534,7 +540,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         seq_messg = set_asked.intersection(set_exist)
         return seq_messg
 
-    @deferred
+    @deferred_to_thread
     #@profile
     def fetch(self, messages_asked, uid):
         """
@@ -574,7 +580,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
         return result
 
-    @deferred
+    @deferred_to_thread
     def fetch_flags(self, messages_asked, uid):
         """
         A fast method to fetch all flags, tricking just the
@@ -615,10 +621,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         all_flags = self.messages.all_flags()
         result = ((msgid, flagsPart(
-            msgid, all_flags[msgid])) for msgid in seq_messg)
+            msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg)
         return result
 
-    @deferred
+    @deferred_to_thread
     def fetch_headers(self, messages_asked, uid):
         """
         A fast method to fetch all headers, tricking just the
@@ -698,28 +704,43 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                     otherwise they are message sequence IDs.
         :type uid: bool
 
-        :return: A dict mapping message sequence numbers to sequences of
-                 str representing the flags set on the message after this
-                 operation has been performed.
-        :rtype: dict
+        :return: A deferred, that will be called with a dict mapping message
+                 sequence numbers to sequences of str representing the flags
+                 set on the message after this operation has been performed.
+        :rtype: deferred
 
         :raise ReadOnlyMailbox: Raised if this mailbox is not open for
                                 read-write.
         """
+        from twisted.internet import reactor
+        if not self.isWriteable():
+            log.msg('read only mailbox!')
+            raise imap4.ReadOnlyMailbox
+
+        d = defer.Deferred()
+        deferLater(reactor, 0, self._do_store, messages_asked, flags,
+                   mode, uid, d)
+        return d
+
+    def _do_store(self, messages_asked, flags, mode, uid, observer):
+        """
+        Helper method, invoke set_flags method in the MessageCollection.
+
+        See the documentation for the `store` method for the parameters.
+
+        :param observer: a deferred that will be called with the dictionary
+                         mapping UIDs to flags after the operation has been
+                         done.
+        :type observer: deferred
+        """
         # XXX implement also sequence (uid = 0)
-        # XXX we should prevent cclient from setting Recent flag.
+        # XXX we should prevent cclient from setting Recent flag?
         leap_assert(not isinstance(flags, basestring),
                     "flags cannot be a string")
         flags = tuple(flags)
-
         messages_asked = self._bound_seq(messages_asked)
         seq_messg = self._filter_msg_seq(messages_asked)
-
-        if not self.isWriteable():
-            log.msg('read only mailbox!')
-            raise imap4.ReadOnlyMailbox
-
-        return self.messages.set_flags(self.mbox, seq_messg, flags, mode)
+        self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer)
 
     # ISearchableMailbox
 
@@ -767,13 +788,46 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     # IMessageCopier
 
-    #@deferred
-    #@profile
-    def copy(self, messageObject):
+    def copy(self, message):
         """
         Copy the given message object into this mailbox.
-        """
-        msg = messageObject
+
+        :param message: an IMessage implementor
+        :type message: LeapMessage
+        :return: a deferred that will be fired with the message
+                 uid when the copy succeed.
+        :rtype: Deferred
+        """
+        from twisted.internet import reactor
+        print "COPY :", message
+        d = defer.Deferred()
+
+        # XXX this should not happen ... track it down,
+        # probably to FETCH...
+        if message is None:
+            log.msg("BUG: COPY found a None in passed message")
+            d.calback(None)
+        deferLater(reactor, 0, self._do_copy, message, d)
+        return d
+
+    #@profile
+    def _do_copy(self, message, observer):
+        """
+        Call invoked from the deferLater in `copy`. This will
+        copy the flags and header documents, and pass them to the
+        `create_message` method in the MemoryStore, together with
+        the observer deferred that we've been passed along.
+
+        :param message: an IMessage implementor
+        :type message: LeapMessage
+        :param observer: the deferred that will fire with the
+                         UID of the message
+        :type observer: Deferred
+        """
+        # XXX  for clarity, this could be delegated to a
+        # MessageCollection mixin that implements copy too, and
+        # moved out of here.
+        msg = message
         memstore = self._memstore
 
         # XXX should use a public api instead
@@ -785,12 +839,23 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         new_fdoc = copy.deepcopy(fdoc.content)
 
         fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
+
+        # XXX is this hitting the db??? --- probably.
+        # We should profile after the pre-fetch.
         dest_fdoc = memstore.get_fdoc_from_chash(
             fdoc_chash, self.mbox)
         exist = dest_fdoc and not empty(dest_fdoc.content)
 
         if exist:
+            # Should we signal error on the callback?
             logger.warning("Destination message already exists!")
+
+            # XXX I'm still not clear if we should raise the
+            # callback. This actually rases an ugly warning
+            # in some muas like thunderbird. I guess the user does
+            # not deserve that.
+            #observer.errback(MessageCopyError("Already exists!"))
+            observer.callback(True)
         else:
             mbox = self.mbox
             uid_next = memstore.increment_last_soledad_uid(mbox)
@@ -799,10 +864,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
             # FIXME set recent!
 
-            return self._memstore.create_message(
+            self._memstore.create_message(
                 self.mbox, uid_next,
                 MessageWrapper(
                     new_fdoc, hdoc.content),
+                observer=observer,
                 notify_on_disk=False)
 
     # convenience fun
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 217ad8e..211d282 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -32,7 +32,7 @@ from zope.interface import implements
 
 from leap.common.check import leap_assert_type
 from leap.mail import size
-from leap.mail.decorators import deferred
+from leap.mail.decorators import deferred_to_thread
 from leap.mail.utils import empty
 from leap.mail.messageflow import MessageProducer
 from leap.mail.imap import interfaces
@@ -200,7 +200,8 @@ class MemoryStore(object):
     # We would have to add a put_flags operation to modify only
     # the flags doc (and set the dirty flag accordingly)
 
-    def create_message(self, mbox, uid, message, notify_on_disk=True):
+    def create_message(self, mbox, uid, message, observer,
+                       notify_on_disk=True):
         """
         Create the passed message into this MemoryStore.
 
@@ -212,38 +213,38 @@ class MemoryStore(object):
         :type uid: int
         :param message: a message to be added
         :type message: MessageWrapper
-        :param notify_on_disk: whether the deferred that is returned should
+        :param observer: the deferred that will fire with the
+                         UID of the message. If notify_on_disk is True,
+                         this will happen when the message is written to
+                         Soledad. Otherwise it will fire as soon as we've
+                         added the message to the memory store.
+        :type observer: Deferred
+        :param notify_on_disk: whether the `observer` deferred should
                                wait until the message is written to disk to
                                be fired.
         :type notify_on_disk: bool
-
-        :return: a Deferred. if notify_on_disk is True, will be fired
-                 when written to the db on disk.
-                 Otherwise will fire inmediately
-        :rtype: Deferred
         """
         log.msg("adding new doc to memstore %r (%r)" % (mbox, uid))
         key = mbox, uid
 
         self._add_message(mbox, uid, message, notify_on_disk)
-
-        d = defer.Deferred()
-        d.addCallback(lambda result: log.msg("message save: %s" % result))
         self._new.add(key)
 
-        # We store this deferred so we can keep track of the pending
-        # operations internally.
-        self._new_deferreds[key] = d
+        def log_add(result):
+            log.msg("message save: %s" % result)
+            return result
+        observer.addCallback(log_add)
 
         if notify_on_disk:
-            # Caller wants to be notified when the message is on disk
-            # so we pass the deferred that will be fired when the message
-            # has been written.
-            return d
-        else:
+            # We store this deferred so we can keep track of the pending
+            # operations internally.
+            # TODO this should fire with the UID !!! -- change that in
+            # the soledad store code.
+            self._new_deferreds[key] = observer
+        if not notify_on_disk:
             # Caller does not care, just fired and forgot, so we pass
             # a defer that will inmediately have its callback triggered.
-            return defer.succeed('fire-and-forget:%s' % str(key))
+            observer.callback(uid)
 
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
@@ -541,7 +542,7 @@ class MemoryStore(object):
             self.write_last_uid(mbox, value)
             return value
 
-    @deferred
+    @deferred_to_thread
     def write_last_uid(self, mbox, value):
         """
         Increment the soledad integer cache for the highest uid value.
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 0e5c74a..03dde29 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -37,7 +37,7 @@ from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail import walk
 from leap.mail.utils import first, find_charset, lowerdict, empty
-from leap.mail.decorators import deferred
+from leap.mail.decorators import deferred_to_thread
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.memorystore import MessageWrapper
@@ -243,30 +243,30 @@ class LeapMessage(fields, MailParser, MBoxParser):
         REMOVE = -1
         SET = 0
 
-        with self.flags_lock:
-            current = doc.content[self.FLAGS_KEY]
-            if mode == APPEND:
-                newflags = tuple(set(tuple(current) + flags))
-            elif mode == REMOVE:
-                newflags = tuple(set(current).difference(set(flags)))
-            elif mode == SET:
-                newflags = flags
-
-            # We could defer this, but I think it's better
-            # to put it under the lock...
-            doc.content[self.FLAGS_KEY] = newflags
-            doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-            doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
-
-            if self._collection.memstore is not None:
-                log.msg("putting message in collection")
-                self._collection.memstore.put_message(
-                    self._mbox, self._uid,
-                    MessageWrapper(fdoc=doc.content, new=False, dirty=True,
-                                   docs_id={'fdoc': doc.doc_id}))
-            else:
-                # fallback for non-memstore initializations.
-                self._soledad.put_doc(doc)
+        #with self.flags_lock:
+        current = doc.content[self.FLAGS_KEY]
+        if mode == APPEND:
+            newflags = tuple(set(tuple(current) + flags))
+        elif mode == REMOVE:
+            newflags = tuple(set(current).difference(set(flags)))
+        elif mode == SET:
+            newflags = flags
+
+        # We could defer this, but I think it's better
+        # to put it under the lock...
+        doc.content[self.FLAGS_KEY] = newflags
+        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+        doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
+
+        if self._collection.memstore is not None:
+            log.msg("putting message in collection")
+            self._collection.memstore.put_message(
+                self._mbox, self._uid,
+                MessageWrapper(fdoc=doc.content, new=False, dirty=True,
+                               docs_id={'fdoc': doc.doc_id}))
+        else:
+            # fallback for non-memstore initializations.
+            self._soledad.put_doc(doc)
         return map(str, newflags)
 
     def getInternalDate(self):
@@ -457,8 +457,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: Any object implementing C{IMessagePart}.
         :return: The specified sub-part.
         """
-        if not self.isMultipart():
-            raise TypeError
+        #if not self.isMultipart():
+            #raise TypeError
         try:
             pmap_dict = self._get_part_from_parts_map(part + 1)
         except KeyError:
@@ -846,14 +846,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         else:
             return False
 
-    # not deferring to thread cause this now uses deferred asa retval
-    #@deferred
     #@profile
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=None,
                 notify_on_disk=False):
         """
         Creates a new message document.
-        Here lives the magic of the leap mail. Well, in soledad, really.
 
         :param raw: the raw message
         :type raw: str
@@ -869,6 +866,31 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         :param uid: the message uid for this mailbox
         :type uid: int
+
+        :return: a deferred that will be fired with the message
+                 uid when the adding succeed.
+        :rtype: deferred
+        """
+        logger.debug('adding message')
+        if flags is None:
+            flags = tuple()
+        leap_assert_type(flags, tuple)
+
+        d = defer.Deferred()
+        self._do_add_msg(raw, flags, subject, date, notify_on_disk, d)
+        return d
+
+    @deferred_to_thread
+    def _do_add_msg(self, raw, flags, subject, date, notify_on_disk, observer):
+        """
+        Helper that creates a new message document.
+        Here lives the magic of the leap mail. Well, in soledad, really.
+
+        See `add_msg` docstring for parameter info.
+
+        :param observer: a deferred that will be fired with the message
+                         uid when the adding succeed.
+        :type observer: deferred
         """
         # TODO signal that we can delete the original message!-----
         # when all the processing is done.
@@ -876,11 +898,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # TODO add the linked-from info !
         # TODO add reference to the original message
 
-        logger.debug('adding message')
-        if flags is None:
-            flags = tuple()
-        leap_assert_type(flags, tuple)
-
         # parse
         msg, chash, size, multi = self._do_parse(raw)
 
@@ -918,16 +935,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         self.set_recent_flag(uid)
 
-        # Saving ----------------------------------------
         # TODO ---- add reference to original doc, to be deleted
         # after writes are done.
         msg_container = MessageWrapper(fd, hd, cdocs)
 
-        # we return a deferred that by default will be triggered
-        # inmediately.
-        d = self.memstore.create_message(self.mbox, uid, msg_container,
-                                         notify_on_disk=notify_on_disk)
-        return d
+        self.memstore.create_message(self.mbox, uid, msg_container,
+                                     observer=observer,
+                                     notify_on_disk=notify_on_disk)
 
     #
     # getters: specific queries
@@ -1030,7 +1044,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             self.recent_flags.difference_update(
                 set([uid]))
 
-    @deferred
+    @deferred_to_thread
     def set_recent_flag(self, uid):
         """
         Set Recent flag for a given uid.
@@ -1080,7 +1094,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             return None
         return fdoc.content.get(fields.UID_KEY, None)
 
-    @deferred
+    @deferred_to_thread
     def _get_uid_from_msgid(self, msgid):
         """
         Return a UID for a given message-id.
@@ -1100,7 +1114,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         return self._get_uid_from_msgidCb(msgid)
 
     #@profile
-    def set_flags(self, mbox, messages, flags, mode):
+    def set_flags(self, mbox, messages, flags, mode, observer):
         """
         Set flags for a sequence of messages.
 
@@ -1112,16 +1126,33 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :type flags: tuple
         :param mode: the mode for setting. 1 is append, -1 is remove, 0 set.
         :type mode: int
+        :param observer: a deferred that will be called with the dictionary
+                         mapping UIDs to flags after the operation has been
+                         done.
+        :type observer: deferred
         """
-        result = {}
+        # XXX we could defer *this* to thread pool, and gather results...
+        # XXX use deferredList
+
+        deferreds = []
         for msg_id in messages:
-            log.msg("MSG ID = %s" % msg_id)
-            msg = self.get_msg_by_uid(msg_id, mem_only=True, flags_only=True)
-            if not msg:
-                continue
-            result[msg_id] = msg.setFlags(flags, mode)
+            deferreds.append(
+                self._set_flag_for_uid(msg_id, flags, mode))
 
-        return result
+        def notify(result):
+            observer.callback(dict(result))
+        d1 = defer.gatherResults(deferreds, consumeErrors=True)
+        d1.addCallback(notify)
+
+    @deferred_to_thread
+    def _set_flag_for_uid(self, msg_id, flags, mode):
+        """
+        Run the set_flag operation in the thread pool.
+        """
+        log.msg("MSG ID = %s" % msg_id)
+        msg = self.get_msg_by_uid(msg_id, mem_only=True, flags_only=True)
+        if msg is not None:
+            return msg_id, msg.setFlags(flags, mode)
 
     # getters: generic for a mailbox
 
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index ff5e03b..82f27e7 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -23,10 +23,12 @@ import threading
 from itertools import chain
 
 from u1db import errors as u1db_errors
+from twisted.internet import defer
+from twisted.python import log
 from zope.interface import implements
 
 from leap.common.check import leap_assert_type
-from leap.mail.decorators import deferred
+from leap.mail.decorators import deferred_to_thread
 from leap.mail.imap.messageparts import MessagePartType
 from leap.mail.imap.messageparts import MessageWrapper
 from leap.mail.imap.messageparts import RecentFlagsDoc
@@ -209,52 +211,87 @@ class SoledadStore(ContentDedup):
         # TODO could generalize this method into a generic consumer
         # and only implement `process` here
 
+        def docWriteCallBack(doc_wrapper):
+            """
+            Callback for a successful write of a document wrapper.
+            """
+            if isinstance(doc_wrapper, MessageWrapper):
+                # If everything went well, we can unset the new flag
+                # in the source store (memory store)
+                self._unset_new_dirty(doc_wrapper)
+
+        def docWriteErrorBack(failure):
+            """
+            Errorback for write operations.
+            """
+            log.error("Error while processing item.")
+            log.msg(failure.getTraceBack())
+
         while not queue.empty():
-            items = self._process(queue)
+            doc_wrapper = queue.get()
+            d = defer.Deferred()
+            d.addCallbacks(docWriteCallBack, docWriteErrorBack)
+
+            self._consume_doc(doc_wrapper, d)
+
+    @deferred_to_thread
+    def _unset_new_dirty(self, doc_wrapper):
+        """
+        Unset the `new` and `dirty` flags for this document wrapper in the
+        memory store.
+
+        :param doc_wrapper: a MessageWrapper instance
+        :type doc_wrapper: MessageWrapper
+        """
+        # XXX debug msg id/mbox?
+        logger.info("unsetting new flag!")
+        doc_wrapper.new = False
+        doc_wrapper.dirty = False
 
-            # we prime the generator, that should return the
-            # message or flags wrapper item in the first place.
-            doc_wrapper = items.next()
+    @deferred_to_thread
+    def _consume_doc(self, doc_wrapper, deferred):
+        """
+        Consume each document wrapper in a separate thread.
+
+        :param doc_wrapper:
+        :type doc_wrapper:
+        :param deferred:
+        :type deferred: Deferred
+        """
+        items = self._process(doc_wrapper)
 
-            # From here, we unpack the subpart items and
-            # the right soledad call.
+        # we prime the generator, that should return the
+        # message or flags wrapper item in the first place.
+        doc_wrapper = items.next()
+
+        # From here, we unpack the subpart items and
+        # the right soledad call.
+        failed = False
+        for item, call in items:
             try:
-                failed = False
-                for item, call in items:
-                    try:
-                        self._try_call(call, item)
-                    except Exception as exc:
-                        failed = exc
-                        continue
-                if failed:
-                    raise MsgWriteError
-
-            except MsgWriteError:
-                logger.error("Error while processing item.")
-                logger.exception(failed)
-            else:
-                if isinstance(doc_wrapper, MessageWrapper):
-                    # If everything went well, we can unset the new flag
-                    # in the source store (memory store)
-                    logger.info("unsetting new flag!")
-                    doc_wrapper.new = False
-                    doc_wrapper.dirty = False
+                self._try_call(call, item)
+            except Exception as exc:
+                failed = exc
+                continue
+        if failed:
+            deferred.errback(MsgWriteError(
+                "There was an error writing the mesage"))
+        else:
+            deferred.callback(doc_wrapper)
 
     #
     # SoledadStore specific methods.
     #
 
-    def _process(self, queue):
+    def _process(self, doc_wrapper):
         """
-        Return an iterator that will yield the msg_wrapper in the first place,
+        Return an iterator that will yield the doc_wrapper in the first place,
         followed by the subparts item and the proper call type for every
         item in the queue, if any.
 
         :param queue: the queue from where we'll pick item.
         :type queue: Queue
         """
-        doc_wrapper = queue.get()
-
         if isinstance(doc_wrapper, MessageWrapper):
             return chain((doc_wrapper,),
                          self._get_calls_for_msg_parts(doc_wrapper))
-- 
cgit v1.2.3


From d9b37a7a1115b76ebc72413cf1ffe9a613b58d52 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 31 Jan 2014 17:32:27 -0400
Subject: remove wrong unicode conversion

---
 src/leap/mail/imap/messages.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 03dde29..6ff3967 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -337,11 +337,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :type stuff: basestring
         :returns: charset
         """
-        # TODO get from subpart headers
-        # XXX existential doubt 2. shouldn't we make the scope
+        # XXX shouldn't we make the scope
         # of the decorator somewhat more persistent?
         # ah! yes! and put memory bounds.
-        return get_email_charset(unicode(stuff))
+        return get_email_charset(stuff)
 
     def getSize(self):
         """
-- 
cgit v1.2.3


From bed4a7b6abffe9d8cb9178b9dc89d13d9d87c1e8 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 31 Jan 2014 20:27:28 -0400
Subject: Restore expected TypeError.

I must have removed this to get rid of a error with some test sample
during the testing of the branch, but it's absolutely needed so that
mime attachments get shown properly. If the TypeError raises
inapropiately due to some malformed part_map, then we will have to
catch it using a workaround somewhere else.
---
 src/leap/mail/imap/messages.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 6ff3967..4a07ef7 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -299,7 +299,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
             return fd
 
         # TODO refactor with getBodyFile in MessagePart
+
         fd = StringIO.StringIO()
+
         if self._bdoc is not None:
             bdoc_content = self._bdoc.content
             if empty(bdoc_content):
@@ -456,8 +458,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: Any object implementing C{IMessagePart}.
         :return: The specified sub-part.
         """
-        #if not self.isMultipart():
-            #raise TypeError
+        if not self.isMultipart():
+            raise TypeError
         try:
             pmap_dict = self._get_part_from_parts_map(part + 1)
         except KeyError:
-- 
cgit v1.2.3


From 3a8fda3aa4645adbba228e7d2f204bfe6d400321 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 31 Jan 2014 22:16:26 -0400
Subject: enable manhole for debugging

---
 src/leap/mail/imap/service/imap.py | 125 +++++++++++++++++++++++++++++++++++++
 1 file changed, 125 insertions(+)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 8350988..8b95f75 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -18,6 +18,7 @@
 Imap service initialization
 """
 import logging
+import os
 
 from twisted.internet.protocol import ServerFactory
 from twisted.internet.error import CannotListenError
@@ -64,6 +65,8 @@ except Exception:
 
 ######################################################
 
+DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
+
 
 class IMAPAuthRealm(object):
     """
@@ -118,6 +121,118 @@ class LeapIMAPFactory(ServerFactory):
         return imapProtocol
 
 
+MANHOLE_PORT = 2222
+
+
+def getManholeFactory(namespace, user, secret):
+    """
+    Get an administrative manhole into the application.
+
+    :param namespace: the namespace to show in the manhole
+    :type namespace: dict
+    :param user: the user to authenticate into the administrative shell.
+    :type user: str
+    :param secret: pass for this manhole
+    :type secret: str
+    """
+    import string
+
+    from twisted.cred.portal import Portal
+    from twisted.conch import manhole, manhole_ssh
+    from twisted.conch.insults import insults
+    from twisted.cred.checkers import (
+        InMemoryUsernamePasswordDatabaseDontUse as MemoryDB)
+
+    from rlcompleter import Completer
+
+    class EnhancedColoredManhole(manhole.ColoredManhole):
+        """
+        A Manhole with some primitive autocomplete support.
+        """
+        # TODO use introspection to make life easier
+
+        def find_common(self, l):
+            """
+            find common parts in thelist items
+            ex: 'ab' for ['abcd','abce','abf']
+            requires an ordered list
+            """
+            if len(l) == 1:
+                return l[0]
+
+            init = l[0]
+            for item in l[1:]:
+                for i, (x, y) in enumerate(zip(init, item)):
+                    if x != y:
+                        init = "".join(init[:i])
+                        break
+
+                if not init:
+                    return None
+            return init
+
+        def handle_TAB(self):
+            """
+            Trap the TAB keystroke
+            """
+            necessarypart = "".join(self.lineBuffer).split(' ')[-1]
+            completer = Completer(globals())
+            if completer.complete(necessarypart, 0):
+                matches = list(set(completer.matches))  # has multiples
+
+                if len(matches) == 1:
+                    length = len(necessarypart)
+                    self.lineBuffer = self.lineBuffer[:-length]
+                    self.lineBuffer.extend(matches[0])
+                    self.lineBufferIndex = len(self.lineBuffer)
+                else:
+                    matches.sort()
+                    commons = self.find_common(matches)
+                    if commons:
+                        length = len(necessarypart)
+                        self.lineBuffer = self.lineBuffer[:-length]
+                        self.lineBuffer.extend(commons)
+                        self.lineBufferIndex = len(self.lineBuffer)
+
+                    self.terminal.nextLine()
+                    while matches:
+                        matches, part = matches[4:], matches[:4]
+                        for item in part:
+                            self.terminal.write('%s' % item.ljust(30))
+                            self.terminal.write('\n')
+                            self.terminal.nextLine()
+
+                self.terminal.eraseLine()
+                self.terminal.cursorBackward(self.lineBufferIndex + 5)
+                self.terminal.write("%s %s" % (
+                    self.ps[self.pn], "".join(self.lineBuffer)))
+
+        def keystrokeReceived(self, keyID, modifier):
+            """
+            Act upon any keystroke received.
+            """
+            self.keyHandlers.update({'\b': self.handle_BACKSPACE})
+            m = self.keyHandlers.get(keyID)
+            if m is not None:
+                m()
+            elif keyID in string.printable:
+                self.characterReceived(keyID, False)
+
+    sshRealm = manhole_ssh.TerminalRealm()
+
+    def chainedProtocolFactory():
+        return insults.ServerProtocol(EnhancedColoredManhole, namespace)
+
+    sshRealm = manhole_ssh.TerminalRealm()
+    sshRealm.chainedProtocolFactory = chainedProtocolFactory
+
+    portal = Portal(
+        sshRealm, [MemoryDB(**{user: secret})])
+
+    f = manhole_ssh.ConchFactory(portal)
+    return f
+
+
 def run_service(*args, **kwargs):
     """
     Main entry point to run the service from the client.
@@ -163,6 +278,16 @@ def run_service(*args, **kwargs):
     else:
         # all good.
         # (the caller has still to call fetcher.start_loop)
+
+        if DO_MANHOLE:
+            # TODO get pass from env var.too.
+            manhole_factory = getManholeFactory(
+                {'f': factory,
+                 'a': factory.theAccount,
+                 'gm': factory.theAccount.getMailbox},
+                "boss", "leap")
+            reactor.listenTCP(MANHOLE_PORT, manhole_factory,
+                              interface="127.0.0.1")
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
         leap_events.signal(IMAP_SERVICE_STARTED, str(port))
         return fetcher, tport, factory
-- 
cgit v1.2.3


From 18fed49c4143eb764ae9e806882d24f8f4e95744 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Sun, 2 Feb 2014 09:26:37 -0400
Subject: fix missing content after in-memory add

because THE KEYS WILL BE STRINGS AFTER ADDED TO SOLEDAD
Can I remember that?

* Fix copy from local folders
* Fix copy when we already have a copy of the message in the inbox,
  marked as deleted.
* Fix also bad deferred.succeed in add_msg when it already exist.
---
 src/leap/mail/imap/mailbox.py      |  5 +--
 src/leap/mail/imap/memorystore.py  |  6 ++-
 src/leap/mail/imap/messageparts.py | 12 ++++--
 src/leap/mail/imap/messages.py     | 88 ++++++++++++++++++++------------------
 src/leap/mail/imap/server.py       | 13 +++++-
 src/leap/mail/utils.py             | 38 ++++++++++++++++
 6 files changed, 110 insertions(+), 52 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 79fb476..688f941 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -162,6 +162,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         if not NOTIFY_NEW:
             return
+
         logger.debug('adding mailbox listener: %s' % listener)
         self.listeners.add(listener)
 
@@ -801,7 +802,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         from twisted.internet import reactor
         print "COPY :", message
         d = defer.Deferred()
-
         # XXX this should not happen ... track it down,
         # probably to FETCH...
         if message is None:
@@ -810,7 +810,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         deferLater(reactor, 0, self._do_copy, message, d)
         return d
 
-    #@profile
     def _do_copy(self, message, observer):
         """
         Call invoked from the deferLater in `copy`. This will
@@ -851,7 +850,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.warning("Destination message already exists!")
 
             # XXX I'm still not clear if we should raise the
-            # callback. This actually rases an ugly warning
+            # errback. This actually rases an ugly warning
             # in some muas like thunderbird. I guess the user does
             # not deserve that.
             #observer.errback(MessageCopyError("Already exists!"))
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 211d282..542e227 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -318,7 +318,7 @@ class MemoryStore(object):
                 store[FDOC])
 
         hdoc = msg_dict.get(HDOC, None)
-        if hdoc:
+        if hdoc is not None:
             if not store.get(HDOC, None):
                 store[HDOC] = ReferenciableDict({})
             store[HDOC].update(hdoc)
@@ -438,7 +438,8 @@ class MemoryStore(object):
         if not self.producer.is_queue_empty():
             return
 
-        logger.info("Writing messages to Soledad...")
+        if any(map(lambda i: not empty(i), (self._new, self._dirty))):
+            logger.info("Writing messages to Soledad...")
 
         # TODO change for lock, and make the property access
         # is accquired
@@ -885,6 +886,7 @@ class MemoryStore(object):
         # TODO expunge should add itself as a callback to the ongoing
         # writes.
         soledad_store = self._permanent_store
+        all_deleted = []
 
         try:
             # 1. Stop the writing call
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 5067263..b07681b 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # messageparts.py
 # Copyright (C) 2014 LEAP
 #
@@ -315,6 +314,7 @@ class MessageWrapper(object):
         fdoc, hdoc, cdocs = map(
             lambda part: msg_dict.get(part, None),
             [self.FDOC, self.HDOC, self.CDOCS])
+
         for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc),
                        (self.CDOCS, cdocs)):
             self._dict[t] = ReferenciableDict(doc) if doc else None
@@ -390,8 +390,10 @@ class MessagePart(object):
                 first_part = pmap.get('1', None)
                 if not empty(first_part):
                     phash = first_part['phash']
+                else:
+                    phash = None
 
-            if not phash:
+            if phash is None:
                 logger.warning("Could not find phash for this subpart!")
                 payload = ""
             else:
@@ -435,11 +437,13 @@ class MessagePart(object):
             fields.TYPE_CONTENT_VAL, str(phash))
 
         cdoc = first(cdocs)
-        if not cdoc:
+        if cdoc is None:
             logger.warning(
                 "Could not find the content doc "
                 "for phash %s" % (phash,))
-        payload = cdoc.content.get(fields.RAW_KEY, "")
+            payload = ""
+        else:
+            payload = cdoc.content.get(fields.RAW_KEY, "")
         return payload
 
     # TODO should memory-bound this memoize!!!
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 4a07ef7..6f822db 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -37,6 +37,7 @@ from leap.common.decorators import memoized_method
 from leap.common.mail import get_email_charset
 from leap.mail import walk
 from leap.mail.utils import first, find_charset, lowerdict, empty
+from leap.mail.utils import stringify_parts_map
 from leap.mail.decorators import deferred_to_thread
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
@@ -219,7 +220,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     # setFlags not in the interface spec but we use it with store command.
 
-    #@profile
     def setFlags(self, flags, mode):
         """
         Sets the flags for this message
@@ -243,30 +243,30 @@ class LeapMessage(fields, MailParser, MBoxParser):
         REMOVE = -1
         SET = 0
 
-        #with self.flags_lock:
-        current = doc.content[self.FLAGS_KEY]
-        if mode == APPEND:
-            newflags = tuple(set(tuple(current) + flags))
-        elif mode == REMOVE:
-            newflags = tuple(set(current).difference(set(flags)))
-        elif mode == SET:
-            newflags = flags
-
-        # We could defer this, but I think it's better
-        # to put it under the lock...
-        doc.content[self.FLAGS_KEY] = newflags
-        doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-        doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
-
-        if self._collection.memstore is not None:
-            log.msg("putting message in collection")
-            self._collection.memstore.put_message(
-                self._mbox, self._uid,
-                MessageWrapper(fdoc=doc.content, new=False, dirty=True,
-                               docs_id={'fdoc': doc.doc_id}))
-        else:
-            # fallback for non-memstore initializations.
-            self._soledad.put_doc(doc)
+        with self.flags_lock:
+            current = doc.content[self.FLAGS_KEY]
+            if mode == APPEND:
+                newflags = tuple(set(tuple(current) + flags))
+            elif mode == REMOVE:
+                newflags = tuple(set(current).difference(set(flags)))
+            elif mode == SET:
+                newflags = flags
+
+            # We could defer this, but I think it's better
+            # to put it under the lock...
+            doc.content[self.FLAGS_KEY] = newflags
+            doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+            doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
+
+            if self._collection.memstore is not None:
+                log.msg("putting message in collection")
+                self._collection.memstore.put_message(
+                    self._mbox, self._uid,
+                    MessageWrapper(fdoc=doc.content, new=False, dirty=True,
+                                   docs_id={'fdoc': doc.doc_id}))
+            else:
+                # fallback for non-memstore initializations.
+                self._soledad.put_doc(doc)
         return map(str, newflags)
 
     def getInternalDate(self):
@@ -483,6 +483,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
         hdoc_content = self._hdoc.content
         pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
+
+        # remember, lads, soledad is using strings in its keys,
+        # not integers!
         return pmap[str(part)]
 
     # XXX moved to memory store
@@ -534,10 +537,10 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
         if self._container is not None:
             bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
-            if bdoc and bdoc.content is not None:
+            if not empty(bdoc) and not empty(bdoc.content):
                 return bdoc
 
-        # no memstore or no doc found there
+        # no memstore, or no body doc found there
         if self._soledad:
             body_docs = self._soledad.get_from_index(
                 fields.TYPE_P_HASH_IDX,
@@ -847,7 +850,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         else:
             return False
 
-    #@profile
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=None,
                 notify_on_disk=False):
         """
@@ -881,7 +883,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         self._do_add_msg(raw, flags, subject, date, notify_on_disk, d)
         return d
 
-    @deferred_to_thread
+    # We SHOULD defer this (or the heavy load here) to the thread pool,
+    # but it gives troubles with the QSocketNotifier used by Qt...
     def _do_add_msg(self, raw, flags, subject, date, notify_on_disk, observer):
         """
         Helper that creates a new message document.
@@ -907,9 +910,19 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # So we probably should just do an in-memory check and
         # move the complete check to the soledad writer?
         # Watch out! We're reserving a UID right after this!
-        if self._fdoc_already_exists(chash):
-            logger.warning("We already have that message in this mailbox.")
-            return defer.succeed('already_exists')
+        existing_uid = self._fdoc_already_exists(chash)
+        if existing_uid:
+            logger.warning("We already have that message in this "
+                           "mailbox, unflagging as deleted")
+            uid = existing_uid
+            msg = self.get_msg_by_uid(uid)
+            msg.setFlags((fields.DELETED_FLAG,), -1)
+
+            # XXX if this is deferred to thread again we should not use
+            # the callback in the deferred thread, but return and
+            # call the callback from the caller fun...
+            observer.callback(uid)
+            return
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
         logger.info("ADDING MSG WITH UID: %s" % uid)
@@ -929,17 +942,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             hd[key] = parts_map[key]
         del parts_map
 
+        hd = stringify_parts_map(hd)
+
         # The MessageContainer expects a dict, one-indexed
         # XXX review-me
         cdocs = dict(((key + 1, doc) for key, doc in
                      enumerate(walk.get_raw_docs(msg, parts))))
 
         self.set_recent_flag(uid)
-
-        # TODO ---- add reference to original doc, to be deleted
-        # after writes are done.
         msg_container = MessageWrapper(fd, hd, cdocs)
-
         self.memstore.create_message(self.mbox, uid, msg_container,
                                      observer=observer,
                                      notify_on_disk=notify_on_disk)
@@ -950,7 +961,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # recent flags
 
-    #@profile
     def _get_recent_flags(self):
         """
         An accessor for the recent-flags set for this mailbox.
@@ -1004,7 +1014,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         doc="Set of UIDs with the recent flag for this mailbox.")
 
     # XXX change naming, indicate soledad query.
-    #@profile
     def _get_recent_doc(self):
         """
         Get recent-flags document from Soledad for this mailbox.
@@ -1114,7 +1123,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
-    #@profile
     def set_flags(self, mbox, messages, flags, mode, observer):
         """
         Set flags for a sequence of messages.
@@ -1220,7 +1228,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # FIXME ----------------------------------------------
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
-    #@profile
     def all_soledad_uid_iter(self):
         """
         Return an iterator through the UIDs of all messages, sorted in
@@ -1232,7 +1239,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                            fields.TYPE_FLAGS_VAL, self.mbox)])
         return db_uids
 
-    #@profile
     def all_uid_iter(self):
         """
         Return an iterator through the UIDs of all messages, from memory.
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 7bca39d..ba63846 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -139,14 +139,22 @@ class LeapIMAPServer(imap4.IMAP4Server):
 
     def on_fetch_finished(self, _, messages):
         from twisted.internet import reactor
+
+        print "FETCH FINISHED -- NOTIFY NEW"
         deferLater(reactor, 0, self.notifyNew)
         deferLater(reactor, 0, self.mbox.unset_recent_flags, messages)
         deferLater(reactor, 0, self.mbox.signal_unread_to_ui)
 
     def on_copy_finished(self, defers):
         d = defer.gatherResults(filter(None, defers))
-        d.addCallback(self.notifyNew)
-        d.addCallback(self.mbox.signal_unread_to_ui)
+
+        def when_finished(result):
+            log.msg("COPY FINISHED")
+            self.notifyNew()
+            self.mbox.signal_unread_to_ui()
+        d.addCallback(when_finished)
+        #d.addCallback(self.notifyNew)
+        #d.addCallback(self.mbox.signal_unread_to_ui)
 
     def do_COPY(self, tag, messages, mailbox, uid=0):
         from twisted.internet import reactor
@@ -162,6 +170,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         """
         Notify new messages to listeners.
         """
+        print "TRYING TO NOTIFY NEW"
         self.mbox.notify_new()
 
     def _cbSelectWork(self, mbox, cmdName, tag):
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 6a1fcde..942acfb 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -17,6 +17,7 @@
 """
 Mail utilities.
 """
+import copy
 import json
 import re
 import traceback
@@ -92,6 +93,43 @@ def lowerdict(_dict):
                 for key, value in _dict.items())
 
 
+PART_MAP = "part_map"
+
+
+def _str_dict(d, k):
+    """
+    Convert the dictionary key to string if it was a string.
+
+    :param d: the dict
+    :type d: dict
+    :param k: the key
+    :type k: object
+    """
+    if isinstance(k, int):
+        val = d[k]
+        d[str(k)] = val
+        del(d[k])
+
+
+def stringify_parts_map(d):
+    """
+    Modify a dictionary making all the nested dicts under "part_map" keys
+    having strings as keys.
+
+    :param d: the dictionary to modify
+    :type d: dictionary
+    :rtype: dictionary
+    """
+    for k in d:
+        if k == PART_MAP:
+            pmap = d[k]
+            for kk in pmap.keys():
+                _str_dict(d[k], kk)
+            for kk in pmap.keys():
+                stringify_parts_map(d[k][str(kk)])
+    return d
+
+
 class CustomJsonScanner(object):
     """
     This class is a context manager definition used to monkey patch the default
-- 
cgit v1.2.3


From 8201146254a204fec92395bf497a2a6f76274b85 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Sun, 2 Feb 2014 16:26:58 -0400
Subject: re-add expunge deferred

---
 src/leap/mail/imap/mailbox.py     | 20 ++++++++------------
 src/leap/mail/imap/memorystore.py | 10 +++++++++-
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 688f941..40d3420 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -486,8 +486,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Expunge and mark as closed
         """
         d = self.expunge()
-        #d.addCallback(self._close_cb)
-        #return d
+        d.addCallback(self._close_cb)
+        return d
 
     def _expunge_cb(self, result):
         return result
@@ -498,15 +498,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
-
-        return self._memstore.expunge(self.mbox)
-
-        # TODO we can defer this back when it's correct
-        # but we should make sure the memstore has been synced.
-
-        #d = self._memstore.expunge(self.mbox)
-        #d.addCallback(self._expunge_cb)
-        #return d
+        d = defer.Deferred()
+        return self._memstore.expunge(self.mbox, d)
+        self._memstore.expunge(self.mbox)
+        d.addCallback(self._expunge_cb, d)
+        return d
 
     def _bound_seq(self, messages_asked):
         """
@@ -800,7 +796,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: Deferred
         """
         from twisted.internet import reactor
-        print "COPY :", message
+
         d = defer.Deferred()
         # XXX this should not happen ... track it down,
         # probably to FETCH...
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 542e227..0632d1c 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -873,13 +873,15 @@ class MemoryStore(object):
             self.remove_message(mbox, uid)
         return mem_deleted
 
-    def expunge(self, mbox):
+    def expunge(self, mbox, observer):
         """
         Remove all messages flagged \\Deleted, from the Memory Store
         and from the permanent store also.
 
         :param mbox: the mailbox
         :type mbox: str or unicode
+        :param observer: a deferred that will be fired when expunge is done
+        :type observer: Deferred
         :return: a list of UIDs
         :rtype: list
         """
@@ -910,6 +912,11 @@ class MemoryStore(object):
             else:
                 sol_deleted = []
 
+            try:
+                self._known_uids[mbox].difference_update(set(sol_deleted))
+            except Exception as exc:
+                logger.exception(exc)
+
             # 2. Delete all messages marked as deleted in memory.
             mem_deleted = self.remove_all_deleted(mbox)
 
@@ -919,6 +926,7 @@ class MemoryStore(object):
             logger.exception(exc)
         finally:
             self._start_write_loop()
+        observer.callback(True)
         return all_deleted
 
     # Dump-to-disk controls.
-- 
cgit v1.2.3


From 23e28bae2c3cb74e00e29ee8add0b73adeb65c2b Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 4 Feb 2014 10:57:49 -0400
Subject: fixes after review

* Some more docstring completion/fixes.
* Removed unneeded str coertion.
* Handle mailbox name in logs.
* Separate manhole boilerplate into its own file.
---
 src/leap/mail/imap/mailbox.py         |   8 +--
 src/leap/mail/imap/memorystore.py     |   3 +-
 src/leap/mail/imap/messages.py        |   6 +-
 src/leap/mail/imap/service/imap.py    | 118 ++----------------------------
 src/leap/mail/imap/service/manhole.py | 130 ++++++++++++++++++++++++++++++++++
 src/leap/mail/imap/soledadstore.py    |  11 +--
 6 files changed, 146 insertions(+), 130 deletions(-)
 create mode 100644 src/leap/mail/imap/service/manhole.py

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 40d3420..c682578 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -52,11 +52,6 @@ notifying clients of new messages. Use during stress tests.
 NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False)
 
 
-class MessageCopyError(Exception):
-    """
-    """
-
-
 class SoledadMailbox(WithMsgFields, MBoxParser):
     """
     A Soledad-backed IMAP mailbox.
@@ -802,7 +797,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # probably to FETCH...
         if message is None:
             log.msg("BUG: COPY found a None in passed message")
-            d.calback(None)
+            d.callback(None)
         deferLater(reactor, 0, self._do_copy, message, d)
         return d
 
@@ -849,7 +844,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             # errback. This actually rases an ugly warning
             # in some muas like thunderbird. I guess the user does
             # not deserve that.
-            #observer.errback(MessageCopyError("Already exists!"))
             observer.callback(True)
         else:
             mbox = self.mbox
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 0632d1c..195cef7 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -475,12 +475,13 @@ class MemoryStore(object):
 
     def get_last_uid(self, mbox):
         """
-        Get the highest UID for a given mbox.
+        Return the highest UID for a given mbox.
         It will be the highest between the highest uid in the message store for
         the mailbox, and the soledad integer cache.
 
         :param mbox: the mailbox
         :type mbox: str or unicode
+        :rtype: int
         """
         uids = self.get_uids(mbox)
         last_mem_uid = uids and max(uids) or 0
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 6f822db..25fc55f 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -328,7 +328,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         # We are still returning funky characters from here.
         else:
             logger.warning("No BDOC found for message.")
-            return write_fd(str(""))
+            return write_fd("")
 
     @memoized_method
     def _get_charset(self, stuff):
@@ -945,9 +945,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         hd = stringify_parts_map(hd)
 
         # The MessageContainer expects a dict, one-indexed
-        # XXX review-me
-        cdocs = dict(((key + 1, doc) for key, doc in
-                     enumerate(walk.get_raw_docs(msg, parts))))
+        cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
 
         self.set_recent_flag(uid)
         msg_container = MessageWrapper(fd, hd, cdocs)
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 8b95f75..5487cfc 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -66,6 +66,8 @@ except Exception:
 ######################################################
 
 DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
+if DO_MANHOLE:
+    from leap.mail.imap.service import manhole
 
 
 class IMAPAuthRealm(object):
@@ -121,118 +123,6 @@ class LeapIMAPFactory(ServerFactory):
         return imapProtocol
 
 
-MANHOLE_PORT = 2222
-
-
-def getManholeFactory(namespace, user, secret):
-    """
-    Get an administrative manhole into the application.
-
-    :param namespace: the namespace to show in the manhole
-    :type namespace: dict
-    :param user: the user to authenticate into the administrative shell.
-    :type user: str
-    :param secret: pass for this manhole
-    :type secret: str
-    """
-    import string
-
-    from twisted.cred.portal import Portal
-    from twisted.conch import manhole, manhole_ssh
-    from twisted.conch.insults import insults
-    from twisted.cred.checkers import (
-        InMemoryUsernamePasswordDatabaseDontUse as MemoryDB)
-
-    from rlcompleter import Completer
-
-    class EnhancedColoredManhole(manhole.ColoredManhole):
-        """
-        A Manhole with some primitive autocomplete support.
-        """
-        # TODO use introspection to make life easier
-
-        def find_common(self, l):
-            """
-            find common parts in thelist items
-            ex: 'ab' for ['abcd','abce','abf']
-            requires an ordered list
-            """
-            if len(l) == 1:
-                return l[0]
-
-            init = l[0]
-            for item in l[1:]:
-                for i, (x, y) in enumerate(zip(init, item)):
-                    if x != y:
-                        init = "".join(init[:i])
-                        break
-
-                if not init:
-                    return None
-            return init
-
-        def handle_TAB(self):
-            """
-            Trap the TAB keystroke
-            """
-            necessarypart = "".join(self.lineBuffer).split(' ')[-1]
-            completer = Completer(globals())
-            if completer.complete(necessarypart, 0):
-                matches = list(set(completer.matches))  # has multiples
-
-                if len(matches) == 1:
-                    length = len(necessarypart)
-                    self.lineBuffer = self.lineBuffer[:-length]
-                    self.lineBuffer.extend(matches[0])
-                    self.lineBufferIndex = len(self.lineBuffer)
-                else:
-                    matches.sort()
-                    commons = self.find_common(matches)
-                    if commons:
-                        length = len(necessarypart)
-                        self.lineBuffer = self.lineBuffer[:-length]
-                        self.lineBuffer.extend(commons)
-                        self.lineBufferIndex = len(self.lineBuffer)
-
-                    self.terminal.nextLine()
-                    while matches:
-                        matches, part = matches[4:], matches[:4]
-                        for item in part:
-                            self.terminal.write('%s' % item.ljust(30))
-                            self.terminal.write('\n')
-                            self.terminal.nextLine()
-
-                self.terminal.eraseLine()
-                self.terminal.cursorBackward(self.lineBufferIndex + 5)
-                self.terminal.write("%s %s" % (
-                    self.ps[self.pn], "".join(self.lineBuffer)))
-
-        def keystrokeReceived(self, keyID, modifier):
-            """
-            Act upon any keystroke received.
-            """
-            self.keyHandlers.update({'\b': self.handle_BACKSPACE})
-            m = self.keyHandlers.get(keyID)
-            if m is not None:
-                m()
-            elif keyID in string.printable:
-                self.characterReceived(keyID, False)
-
-    sshRealm = manhole_ssh.TerminalRealm()
-
-    def chainedProtocolFactory():
-        return insults.ServerProtocol(EnhancedColoredManhole, namespace)
-
-    sshRealm = manhole_ssh.TerminalRealm()
-    sshRealm.chainedProtocolFactory = chainedProtocolFactory
-
-    portal = Portal(
-        sshRealm, [MemoryDB(**{user: secret})])
-
-    f = manhole_ssh.ConchFactory(portal)
-    return f
-
-
 def run_service(*args, **kwargs):
     """
     Main entry point to run the service from the client.
@@ -281,12 +171,12 @@ def run_service(*args, **kwargs):
 
         if DO_MANHOLE:
             # TODO get pass from env var.too.
-            manhole_factory = getManholeFactory(
+            manhole_factory = manhole.getManholeFactory(
                 {'f': factory,
                  'a': factory.theAccount,
                  'gm': factory.theAccount.getMailbox},
                 "boss", "leap")
-            reactor.listenTCP(MANHOLE_PORT, manhole_factory,
+            reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory,
                               interface="127.0.0.1")
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
         leap_events.signal(IMAP_SERVICE_STARTED, str(port))
diff --git a/src/leap/mail/imap/service/manhole.py b/src/leap/mail/imap/service/manhole.py
new file mode 100644
index 0000000..c83ae89
--- /dev/null
+++ b/src/leap/mail/imap/service/manhole.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+# manhole.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for enabling the manhole administrative interface into the
+LEAP Mail application.
+"""
+MANHOLE_PORT = 2222
+
+
+def getManholeFactory(namespace, user, secret):
+    """
+    Get an administrative manhole into the application.
+
+    :param namespace: the namespace to show in the manhole
+    :type namespace: dict
+    :param user: the user to authenticate into the administrative shell.
+    :type user: str
+    :param secret: pass for this manhole
+    :type secret: str
+    """
+    import string
+
+    from twisted.cred.portal import Portal
+    from twisted.conch import manhole, manhole_ssh
+    from twisted.conch.insults import insults
+    from twisted.cred.checkers import (
+        InMemoryUsernamePasswordDatabaseDontUse as MemoryDB)
+
+    from rlcompleter import Completer
+
+    class EnhancedColoredManhole(manhole.ColoredManhole):
+        """
+        A Manhole with some primitive autocomplete support.
+        """
+        # TODO use introspection to make life easier
+
+        def find_common(self, l):
+            """
+            find common parts in thelist items
+            ex: 'ab' for ['abcd','abce','abf']
+            requires an ordered list
+            """
+            if len(l) == 1:
+                return l[0]
+
+            init = l[0]
+            for item in l[1:]:
+                for i, (x, y) in enumerate(zip(init, item)):
+                    if x != y:
+                        init = "".join(init[:i])
+                        break
+
+                if not init:
+                    return None
+            return init
+
+        def handle_TAB(self):
+            """
+            Trap the TAB keystroke.
+            """
+            necessarypart = "".join(self.lineBuffer).split(' ')[-1]
+            completer = Completer(globals())
+            if completer.complete(necessarypart, 0):
+                matches = list(set(completer.matches))  # has multiples
+
+                if len(matches) == 1:
+                    length = len(necessarypart)
+                    self.lineBuffer = self.lineBuffer[:-length]
+                    self.lineBuffer.extend(matches[0])
+                    self.lineBufferIndex = len(self.lineBuffer)
+                else:
+                    matches.sort()
+                    commons = self.find_common(matches)
+                    if commons:
+                        length = len(necessarypart)
+                        self.lineBuffer = self.lineBuffer[:-length]
+                        self.lineBuffer.extend(commons)
+                        self.lineBufferIndex = len(self.lineBuffer)
+
+                    self.terminal.nextLine()
+                    while matches:
+                        matches, part = matches[4:], matches[:4]
+                        for item in part:
+                            self.terminal.write('%s' % item.ljust(30))
+                            self.terminal.write('\n')
+                            self.terminal.nextLine()
+
+                self.terminal.eraseLine()
+                self.terminal.cursorBackward(self.lineBufferIndex + 5)
+                self.terminal.write("%s %s" % (
+                    self.ps[self.pn], "".join(self.lineBuffer)))
+
+        def keystrokeReceived(self, keyID, modifier):
+            """
+            Act upon any keystroke received.
+            """
+            self.keyHandlers.update({'\b': self.handle_BACKSPACE})
+            m = self.keyHandlers.get(keyID)
+            if m is not None:
+                m()
+            elif keyID in string.printable:
+                self.characterReceived(keyID, False)
+
+    sshRealm = manhole_ssh.TerminalRealm()
+
+    def chainedProtocolFactory():
+        return insults.ServerProtocol(EnhancedColoredManhole, namespace)
+
+    sshRealm = manhole_ssh.TerminalRealm()
+    sshRealm.chainedProtocolFactory = chainedProtocolFactory
+
+    portal = Portal(
+        sshRealm, [MemoryDB(**{user: secret})])
+
+    f = manhole_ssh.ConchFactory(portal)
+    return f
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 82f27e7..8e22f26 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -253,9 +253,11 @@ class SoledadStore(ContentDedup):
         """
         Consume each document wrapper in a separate thread.
 
-        :param doc_wrapper:
-        :type doc_wrapper:
-        :param deferred:
+        :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance
+        :type doc_wrapper: MessageWrapper or RecentFlagsDoc
+        :param deferred: a deferred that will be fired when the write operation
+                         has finished, either calling its callback or its
+                         errback depending on whether it succeed.
         :type deferred: Deferred
         """
         items = self._process(doc_wrapper)
@@ -415,6 +417,7 @@ class SoledadStore(ContentDedup):
         :param uid: the UID for the message
         :type uid: int
         """
+        result = None
         try:
             flag_docs = self._soledad.get_from_index(
                 fields.TYPE_MBOX_UID_IDX,
@@ -447,7 +450,7 @@ class SoledadStore(ContentDedup):
             mbox_doc = self._get_mbox_document(mbox)
             old_val = mbox_doc.content[key]
             if value < old_val:
-                logger.error("%s:%s Tried to write a UID lesser than what's "
+                logger.error("%r:%s Tried to write a UID lesser than what's "
                              "stored!" % (mbox, value))
             mbox_doc.content[key] = value
             self._soledad.put_doc(mbox_doc)
-- 
cgit v1.2.3


From 498c6745abd91652dfef94045dfe005be0422bf2 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 4 Feb 2014 15:39:52 -0400
Subject: Rebased dreb's commit to update sizes dictionary

for faster calculation of sizes.
https://github.com/andrejb/leap_mail/commit/8b88e85fab3c2b75da16b16c8d492c001b8076c6
---
 src/leap/mail/imap/memorystore.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 195cef7..a99148f 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -106,6 +106,12 @@ class MemoryStore(object):
         # Internal Storage: messages
         self._msg_store = {}
 
+        # Sizes
+        """
+        {'mbox, uid': <int>}
+        """
+        self._sizes = {}
+
         # Internal Storage: payload-hash
         """
         {'phash': weakreaf.proxy(dict)}
@@ -347,8 +353,12 @@ class MemoryStore(object):
             for key in seq:
                 if key in store and empty(store.get(key)):
                     store.pop(key)
+
         prune((FDOC, HDOC, CDOCS, DOCS_ID), store)
 
+        # Update memory store size
+        self._sizes[key] = size(self._msg_store[key])
+
     def get_docid_for_fdoc(self, mbox, uid):
         """
         Return Soledad document id for the flags-doc for a given mbox and uid,
@@ -417,6 +427,9 @@ class MemoryStore(object):
             self._new.discard(key)
             self._dirty.discard(key)
             self._msg_store.pop(key, None)
+            if key in self._sizes:
+                del self._sizes[key]
+
         except Exception as exc:
             logger.exception(exc)
 
@@ -958,4 +971,4 @@ class MemoryStore(object):
 
         :rtype: int
         """
-        return size.get_size(self._msg_store)
+        return reduce(lambda x, y: x + y, self._sizes, 0)
-- 
cgit v1.2.3


From 3511f7992e67bc49e9fc4771f4b2c0d9199822d7 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Feb 2014 11:47:43 -0400
Subject: minimal regression tests

---
 src/leap/mail/imap/tests/getmail     |   2 -
 src/leap/mail/imap/tests/regressions | 451 +++++++++++++++++++++++++++++++++++
 2 files changed, 451 insertions(+), 2 deletions(-)
 create mode 100755 src/leap/mail/imap/tests/regressions

diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail
index 17e195c..0fb00d2 100755
--- a/src/leap/mail/imap/tests/getmail
+++ b/src/leap/mail/imap/tests/getmail
@@ -5,8 +5,6 @@
 
 # Modifications by LEAP Developers 2014 to fit
 # Bitmask configuration settings.
-
-
 """
 Simple IMAP4 client which displays the subjects of all messages in a
 particular mailbox.
diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions
new file mode 100755
index 0000000..0a43398
--- /dev/null
+++ b/src/leap/mail/imap/tests/regressions
@@ -0,0 +1,451 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+# regressions
+# Copyright (C) 2014 LEAP
+# Copyright (c) Twisted Matrix Laboratories.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Simple Regression Tests using IMAP4 client.
+
+Iterates trough all mails under a given folder and tries to APPEND them to
+the server being tested. After FETCHING the pushed message, it compares
+the received version with the one that was saved, and exits with an error
+code if they do not match.
+"""
+import os
+import StringIO
+import sys
+
+from email.parser import Parser
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+
+REGRESSIONS_FOLDER = "regressions_test"
+
+parser = Parser()
+
+
+def get_msg_parts(raw):
+    """
+    Return a representation of the parts of a message suitable for
+    comparison.
+
+    :param raw: string for the message
+    :type raw: str
+    """
+    m = parser.parsestr(raw)
+    return [dict(part.items())
+            if part.is_multipart()
+            else part.get_payload()
+            for part in m.walk()]
+
+
+def compare_msg_parts(a, b):
+    """
+    Compare two sequences of parts of messages.
+
+    :param a: part sequence for message a
+    :param b: part sequence for message b
+
+    :return: True if both message sequences are equivalent.
+    :rtype: bool
+    """
+    # XXX This could be smarter and show the differences in the
+    # different parts when/where they differ.
+    #import pprint; pprint.pprint(a[0])
+    #import pprint; pprint.pprint(b[0])
+
+    def lowerkey(d):
+        return dict((k.lower(), v.replace('\r', ''))
+                    for k, v in d.iteritems())
+
+    def eq(x, y):
+        # For dicts, we compare a variation with their keys
+        # in lowercase, and \r removed from their values
+        if all(map(lambda i: isinstance(i, dict), (x, y))):
+            x, y = map(lowerkey, (x, y))
+        return x == y
+
+    compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b))
+    all_match = all(compare_vector)
+
+    if not all_match:
+        print "PARTS MISMATCH!"
+        print "vector: ", compare_vector
+        index = compare_vector.index(False)
+        from pprint import pprint
+        print "Expected:"
+        pprint(a[index])
+        print ("***")
+        print "Found:"
+        pprint(b[index])
+        print
+
+
+    return all_match
+
+
+def get_fd(string):
+    """
+    Return a file descriptor with the passed string
+    as content.
+    """
+    fd = StringIO.StringIO()
+    fd.write(string)
+    fd.seek(0)
+    return fd
+
+
+class TrivialPrompter(basic.LineReceiver):
+    promptDeferred = None
+
+    def prompt(self, msg):
+        assert self.promptDeferred is None
+        self.display(msg)
+        self.promptDeferred = defer.Deferred()
+        return self.promptDeferred
+
+    def display(self, msg):
+        self.transport.write(msg)
+
+    def lineReceived(self, line):
+        if self.promptDeferred is None:
+            return
+        d, self.promptDeferred = self.promptDeferred, None
+        d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+    """
+    A client with callbacks for greeting messages from an IMAP server.
+    """
+    greetDeferred = None
+
+    def serverGreeting(self, caps):
+        self.serverCapabilities = caps
+        if self.greetDeferred is not None:
+            d, self.greetDeferred = self.greetDeferred, None
+            d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+    usedUp = False
+    protocol = SimpleIMAP4Client
+
+    def __init__(self, username, onConn):
+        self.ctx = ssl.ClientContextFactory()
+
+        self.username = username
+        self.onConn = onConn
+
+    def buildProtocol(self, addr):
+        """
+        Initiate the protocol instance. Since we are building a simple IMAP
+        client, we don't bother checking what capabilities the server has. We
+        just add all the authenticators twisted.mail has.  Note: Gmail no
+        longer uses any of the methods below, it's been using XOAUTH since
+        2010.
+        """
+        assert not self.usedUp
+        self.usedUp = True
+
+        p = self.protocol(self.ctx)
+        p.factory = self
+        p.greetDeferred = self.onConn
+
+        p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+        p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+        p.registerAuthenticator(
+            imap4.CramMD5ClientAuthenticator(self.username))
+
+        return p
+
+    def clientConnectionFailed(self, connector, reason):
+        d, self.onConn = self.onConn, None
+        d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+    """
+    Initial callback - invoked after the server sends us its greet message.
+    """
+    # Hook up stdio
+    tp = TrivialPrompter()
+    stdio.StandardIO(tp)
+
+    # And make it easily accessible
+    proto.prompt = tp.prompt
+    proto.display = tp.display
+
+    # Try to authenticate securely
+    return proto.authenticate(
+        password).addCallback(
+        cbAuthentication,
+        proto).addErrback(
+        ebAuthentication, proto, username, password
+    )
+
+
+def ebConnection(reason):
+    """
+    Fallback error-handler. If anything goes wrong, log it and quit.
+    """
+    log.startLogging(sys.stdout)
+    log.err(reason)
+    return reason
+
+
+def cbAuthentication(result, proto):
+    """
+    Callback after authentication has succeeded.
+
+    Lists a bunch of mailboxes.
+    """
+    return proto.select(
+        REGRESSIONS_FOLDER
+    ).addCallback(
+        cbSelectMbox, proto
+    ).addErrback(
+        ebSelectMbox, proto, REGRESSIONS_FOLDER)
+
+
+def ebAuthentication(failure, proto, username, password):
+    """
+    Errback invoked when authentication fails.
+
+    If it failed because no SASL mechanisms match, offer the user the choice
+    of logging in insecurely.
+
+    If you are trying to connect to your Gmail account, you will be here!
+    """
+    failure.trap(imap4.NoSupportedAuthentication)
+    return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+    """
+    Raise insecure-login error.
+    """
+    return proto.login(
+        username, password
+    ).addCallback(
+        cbAuthentication, proto)
+
+
+def cbSelectMbox(result, proto):
+    """
+    Callback invoked when select command finishes successfully.
+
+    If any message is in the test folder, it will flag them as deleted and
+    expunge.
+    If no messages found, it will start with the APPEND tests.
+    """
+    print "SELECT: %s EXISTS " % result.get("EXISTS", "??")
+
+    if result["EXISTS"] != 0:
+        # Flag as deleted, expunge, and do an examine again.
+        #print "There is mail here, will delete..."
+        return cbDeleteAndExpungeTestFolder(proto)
+
+    else:
+        return cbAppendNextMessage(proto)
+
+
+def ebSelectMbox(failure, proto, folder):
+    """
+    Errback invoked when the examine command fails.
+
+    Creates the folder.
+    """
+    print failure.getTraceback()
+    log.msg("Folder %r does not exist. Creating..." % (folder,))
+    return proto.create(folder).addCallback(cbAuthentication, proto)
+
+
+def cbDeleteAndExpungeTestFolder(proto):
+    """
+    Callback invoked fom cbExamineMbox when the number of messages in the
+    mailbox is not zero. It flags all messages as deleted and expunge the
+    mailbox.
+    """
+    return proto.setFlags(
+        "1:*", ("\\Deleted",)
+    ).addCallback(
+        lambda r: proto.expunge()
+    ).addCallback(
+        cbExpunge, proto)
+
+
+def cbExpunge(result, proto):
+    return proto.select(
+        REGRESSIONS_FOLDER
+    ).addCallback(
+        cbSelectMbox, proto
+    ).addErrback(ebSettingDeleted, proto)
+
+
+def ebSettingDeleted(failure, proto):
+    """
+    Report errors during deletion of messages in the mailbox.
+    """
+    print failure.getTraceback()
+
+
+def cbAppendNextMessage(proto):
+    """
+    Appends the next message in the global queue to the test folder.
+    """
+    # 1. Get the next test message from global tuple.
+    try:
+        next_sample = SAMPLES.pop()
+    except IndexError:
+        # we're done!
+        return proto.logout()
+
+    print "\nAPPEND %s" % (next_sample,)
+    raw = open(next_sample).read()
+    msg = get_fd(raw)
+    return proto.append(
+        REGRESSIONS_FOLDER, msg
+    ).addCallback(
+        lambda r: proto.examine(REGRESSIONS_FOLDER)
+    ).addCallback(
+        cbAppend, proto, raw
+    ).addErrback(
+        ebAppend, proto, raw)
+
+
+def cbAppend(result, proto, orig_msg):
+    """
+    Fetches the message right after an append.
+    """
+    # XXX keep account of highest UID
+    uid = "1:*"
+
+    return proto.fetchSpecific(
+        '%s' % uid,
+        headerType='',
+        headerArgs=['BODY.PEEK[]'],
+    ).addCallback(
+        cbCompareMessage, proto, orig_msg
+    ).addErrback(ebAppend, proto, orig_msg)
+
+
+def ebAppend(failure, proto, raw):
+    """
+    Errorback for the append operation
+    """
+    print "ERROR WHILE APPENDING!"
+    print failure.getTraceback()
+
+
+def cbPickMessage(result, proto):
+    """
+    Pick a message.
+    """
+    return proto.fetchSpecific(
+        '%s' % result,
+        headerType='',
+        headerArgs=['BODY.PEEK[]'],
+        ).addCallback(cbCompareMessage, proto)
+
+
+def cbCompareMessage(result, proto, raw):
+    """
+    Display message and compare it with the original one.
+    """
+    parts_orig = get_msg_parts(raw)
+
+    if result:
+        keys = result.keys()
+        keys.sort()
+
+    latest = max(keys)
+
+    fetched_msg = result[latest][0][2]
+    parts_fetched = get_msg_parts(fetched_msg)
+
+    equal = compare_msg_parts(
+        parts_orig,
+        parts_fetched)
+
+    if equal:
+        print "[+] MESSAGES MATCH"
+        return cbAppendNextMessage(proto)
+    else:
+        print "[-] ERROR: MESSAGES DO NOT MATCH !!!"
+        print "    ABORTING COMPARISON..."
+        # FIXME logout and print the subject ...
+        return proto.logout()
+
+
+def cbClose(result):
+    """
+    Close the connection when we finish everything.
+    """
+    from twisted.internet import reactor
+    reactor.stop()
+
+
+def main():
+    import glob
+    import sys
+
+    if len(sys.argv) != 4:
+        print "Usage: regressions <user> <pass> <samples-folder>"
+        sys.exit()
+
+    hostname = "localhost"
+    port = "1984"
+    username = sys.argv[1]
+    password = sys.argv[2]
+
+    samplesdir = sys.argv[3]
+
+    if not os.path.isdir(samplesdir):
+        print ("Could not find samples folder! "
+               "Make sure of copying mail_breaker contents there.")
+        sys.exit()
+
+    samples = glob.glob(samplesdir + '/*')
+
+    global SAMPLES
+    SAMPLES = []
+    SAMPLES += samples
+
+    onConn = defer.Deferred(
+    ).addCallback(
+        cbServerGreeting, username, password
+    ).addErrback(
+        ebConnection
+    ).addBoth(cbClose)
+
+    factory = SimpleIMAP4ClientFactory(username, onConn)
+
+    from twisted.internet import reactor
+    reactor.connectTCP(hostname, int(port), factory)
+    reactor.run()
+
+
+if __name__ == '__main__':
+    main()
-- 
cgit v1.2.3


From 423624e5f2c4d3f8cfe8f15f4d6649ed3eea11dc Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Feb 2014 11:48:20 -0400
Subject: fix expunge deferreds so they wait

---
 src/leap/mail/imap/mailbox.py     |  7 +------
 src/leap/mail/imap/memorystore.py | 38 +++++++++++++++++++++++++-------------
 2 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index c682578..d8af0a5 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -484,9 +484,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d.addCallback(self._close_cb)
         return d
 
-    def _expunge_cb(self, result):
-        return result
-
     def expunge(self):
         """
         Remove all messages flagged \\Deleted
@@ -494,9 +491,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
         d = defer.Deferred()
-        return self._memstore.expunge(self.mbox, d)
-        self._memstore.expunge(self.mbox)
-        d.addCallback(self._expunge_cb, d)
+        self._memstore.expunge(self.mbox, d)
         return d
 
     def _bound_seq(self, messages_asked):
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 195cef7..f4a4522 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -879,30 +879,43 @@ class MemoryStore(object):
         Remove all messages flagged \\Deleted, from the Memory Store
         and from the permanent store also.
 
+        It first queues up a last write, and wait for the deferreds to be done
+        before continuing.
+
         :param mbox: the mailbox
         :type mbox: str or unicode
         :param observer: a deferred that will be fired when expunge is done
         :type observer: Deferred
-        :return: a list of UIDs
-        :rtype: list
         """
-        # TODO expunge should add itself as a callback to the ongoing
-        # writes.
         soledad_store = self._permanent_store
-        all_deleted = []
-
         try:
             # 1. Stop the writing call
             self._stop_write_loop()
             # 2. Enqueue a last write.
-            #self.write_messages(soledad_store)
-            # 3. Should wait on the writebacks to finish ???
-            # FIXME wait for this, and add all the rest of the method
-            # as a callback!!!
+            self.write_messages(soledad_store)
+            # 3. Wait on the writebacks to finish
+
+            pending_deferreds = (self._new_deferreds.get(mbox, []) +
+                                 self._dirty_deferreds.get(mbox, []))
+            d1 = defer.gatherResults(pending_deferreds, consumeErrors=True)
+            d1.addCallback(
+                self._delete_from_soledad_and_memory, mbox, observer)
         except Exception as exc:
             logger.exception(exc)
 
-        # Now, we...:
+    def _delete_from_soledad_and_memory(self, result, mbox, observer):
+        """
+        Remove all messages marked as deleted from soledad and memory.
+
+        :param result: ignored. the result of the deferredList that triggers
+                       this as a callback from `expunge`.
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param observer: a deferred that will be fired when expunge is done
+        :type observer: Deferred
+        """
+        all_deleted = []
+        soledad_store = self._permanent_store
 
         try:
             # 1. Delete all messages marked as deleted in soledad.
@@ -927,8 +940,7 @@ class MemoryStore(object):
             logger.exception(exc)
         finally:
             self._start_write_loop()
-        observer.callback(True)
-        return all_deleted
+        observer.callback(all_deleted)
 
     # Dump-to-disk controls.
 
-- 
cgit v1.2.3


From 3f9c3ab22523c553dc677d5273dc8d01394d74f7 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Feb 2014 12:37:50 -0400
Subject: fix memoized call returning always None

---
 src/leap/mail/imap/messageparts.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index b07681b..2d9b3a2 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -397,7 +397,9 @@ class MessagePart(object):
                 logger.warning("Could not find phash for this subpart!")
                 payload = ""
             else:
-                payload = self._get_payload_from_document(phash)
+                payload = self._get_payload_from_document_memoized(phash)
+                if payload is None:
+                    payload = self._get_payload_from_document(phash)
 
         else:
             logger.warning("Message with no part_map!")
@@ -424,13 +426,24 @@ class MessagePart(object):
 
     # TODO should memory-bound this memoize!!!
     @memoized_method
+    def _get_payload_from_document_memoized(self, phash):
+        """
+        Memoized method call around the regular method, to be able
+        to call the non-memoized method in case we got a None.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: str or unicode
+        :rtype: str or unicode or None
+        """
+        return self._get_payload_from_document(phash)
+
     def _get_payload_from_document(self, phash):
         """
         Return the message payload from the content document.
 
         :param phash: the payload hash to retrieve by.
         :type phash: str or unicode
-        :rtype: str or unicode
+        :rtype: str or unicode or None
         """
         cdocs = self._soledad.get_from_index(
             fields.TYPE_P_HASH_IDX,
-- 
cgit v1.2.3


From bf9db4b5381230b4e2a1e1d2d4b2acc31c29ff87 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Feb 2014 16:47:36 -0400
Subject: Fix the fallback for the memoized call for bodies/content.

Changed to "empty" to consider empty strings too.
---
 src/leap/mail/imap/memorystore.py  | 9 +++++----
 src/leap/mail/imap/messageparts.py | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index f4a4522..9c7973d 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -230,10 +230,11 @@ class MemoryStore(object):
         self._add_message(mbox, uid, message, notify_on_disk)
         self._new.add(key)
 
-        def log_add(result):
-            log.msg("message save: %s" % result)
-            return result
-        observer.addCallback(log_add)
+        # XXX use this while debugging the callback firing,
+        # remove after unittesting this.
+        #def log_add(result):
+            #return result
+        #observer.addCallback(log_add)
 
         if notify_on_disk:
             # We store this deferred so we can keep track of the pending
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 2d9b3a2..b1f333a 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -398,7 +398,7 @@ class MessagePart(object):
                 payload = ""
             else:
                 payload = self._get_payload_from_document_memoized(phash)
-                if payload is None:
+                if empty(payload):
                     payload = self._get_payload_from_document(phash)
 
         else:
-- 
cgit v1.2.3


From 362aaec0897261973e58b4282f5c054985d1f113 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Thu, 6 Feb 2014 15:46:01 -0200
Subject: Flush IMAP data to disk when stopping. Closes #5095.

---
 .../feature_5095_flush-data-to-disk-when-stopping  |  1 +
 src/leap/mail/imap/memorystore.py                  | 22 ++++++++++-----
 src/leap/mail/imap/service/imap.py                 | 31 ++++++++++++++++++++++
 src/leap/mail/messageflow.py                       | 11 ++++++++
 4 files changed, 59 insertions(+), 6 deletions(-)
 create mode 100644 changes/feature_5095_flush-data-to-disk-when-stopping

diff --git a/changes/feature_5095_flush-data-to-disk-when-stopping b/changes/feature_5095_flush-data-to-disk-when-stopping
new file mode 100644
index 0000000..d7c1ce7
--- /dev/null
+++ b/changes/feature_5095_flush-data-to-disk-when-stopping
@@ -0,0 +1 @@
+  o Flush IMAP data to disk when stopping. Closes #5095.
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 9c7973d..3eba59a 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -875,6 +875,15 @@ class MemoryStore(object):
             self.remove_message(mbox, uid)
         return mem_deleted
 
+    def stop_and_flush(self):
+        """
+        Stop the write loop and trigger a write to the producer.
+        """
+        self._stop_write_loop()
+        if self._permanent_store is not None:
+            self.write_messages(self._permanent_store)
+            self.producer.flush()
+
     def expunge(self, mbox, observer):
         """
         Remove all messages flagged \\Deleted, from the Memory Store
@@ -890,12 +899,9 @@ class MemoryStore(object):
         """
         soledad_store = self._permanent_store
         try:
-            # 1. Stop the writing call
-            self._stop_write_loop()
-            # 2. Enqueue a last write.
-            self.write_messages(soledad_store)
-            # 3. Wait on the writebacks to finish
-
+            # Stop and trigger last write
+            self.stop_and_flush()
+            # Wait on the writebacks to finish
             pending_deferreds = (self._new_deferreds.get(mbox, []) +
                                  self._dirty_deferreds.get(mbox, []))
             d1 = defer.gatherResults(pending_deferreds, consumeErrors=True)
@@ -962,6 +968,10 @@ class MemoryStore(object):
         # are done (gatherResults)
         return getattr(self, self.WRITING_FLAG)
 
+    @property
+    def permanent_store(self):
+        return self._permanent_store
+
     # Memory management.
 
     def get_size(self):
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 5487cfc..93df51d 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -19,7 +19,9 @@ Imap service initialization
 """
 import logging
 import os
+import time
 
+from twisted.internet import defer, threads
 from twisted.internet.protocol import ServerFactory
 from twisted.internet.error import CannotListenError
 from twisted.mail import imap4
@@ -122,6 +124,35 @@ class LeapIMAPFactory(ServerFactory):
         imapProtocol.factory = self
         return imapProtocol
 
+    def doStop(self, cv):
+        """
+        Stops imap service (fetcher, factory and port).
+
+        :param cv: A condition variable to which we can signal when imap
+                   indeed stops.
+        :type cv: threading.Condition
+        :return: a Deferred that stops and flushes the in memory store data to
+                 disk in another thread.
+        :rtype: Deferred
+        """
+        ServerFactory.doStop(self)
+
+        def _stop_imap_cb():
+            logger.debug('Stopping in memory store.')
+            self._memstore.stop_and_flush()
+            while not self._memstore.producer.is_queue_empty():
+                logger.debug('Waiting for queue to be empty.')
+                # TODO use a gatherResults over the new/dirty deferred list,
+                # as in memorystore's expunge() method.
+                time.sleep(1)
+            # notify that service has stopped
+            logger.debug('Notifying that service has stopped.')
+            cv.acquire()
+            cv.notify()
+            cv.release()
+
+        return threads.deferToThread(_stop_imap_cb)
+
 
 def run_service(*args, **kwargs):
     """
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index b7fc030..80121c8 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -64,6 +64,11 @@ class IMessageProducer(Interface):
         Stop producing items.
         """
 
+    def flush(self):
+        """
+        Flush queued messages to consumer.
+        """
+
 
 class DummyMsgConsumer(object):
 
@@ -162,6 +167,12 @@ class MessageProducer(object):
         if self._loop.running:
             self._loop.stop()
 
+    def flush(self):
+        """
+        Flush queued messages to consumer.
+        """
+        self._check_for_new()
+
 
 if __name__ == "__main__":
     from twisted.internet import reactor
-- 
cgit v1.2.3


From 12ffea333922d99ee7f7b4ab2cd46cfcec6a0d05 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 11:31:40 -0400
Subject: fix get_size call

---
 src/leap/mail/imap/memorystore.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index ed2b3f2..d0321ae 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -350,6 +350,12 @@ class MemoryStore(object):
                 continue
             self._phash_store[phash] = weakref.proxy(referenciable_cdoc)
 
+        # Update memory store size
+        # XXX this should use [mbox][uid]
+        key = mbox, uid
+        self._sizes[key] = size.get_size(self._fdoc_store[key])
+        # TODO add hdoc and cdocs sizes too
+
         def prune(seq, store):
             for key in seq:
                 if key in store and empty(store.get(key)):
-- 
cgit v1.2.3


From c955c7015b5986af40b2253ac98846f4547e5e00 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Feb 2014 21:40:20 -0400
Subject: lock document retrieval/put

---
 src/leap/mail/imap/soledadstore.py | 47 +++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 8e22f26..bfa53b6 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -128,6 +128,7 @@ class SoledadStore(ContentDedup):
     This will create docs in the local Soledad database.
     """
     _last_uid_lock = threading.Lock()
+    _soledad_rw_lock = threading.Lock()
 
     implements(IMessageConsumer, IMessageStore)
 
@@ -140,6 +141,10 @@ class SoledadStore(ContentDedup):
         """
         self._soledad = soledad
 
+        self._CREATE_DOC_FUN = self._soledad.create_doc
+        self._PUT_DOC_FUN = self._soledad.put_doc
+        self._GET_DOC_FUN = self._soledad.get_doc
+
     # IMessageStore
 
     # -------------------------------------------------------------------
@@ -224,7 +229,7 @@ class SoledadStore(ContentDedup):
             """
             Errorback for write operations.
             """
-            log.error("Error while processing item.")
+            log.msg("ERROR: Error while processing item.")
             log.msg(failure.getTraceBack())
 
         while not queue.empty():
@@ -234,6 +239,7 @@ class SoledadStore(ContentDedup):
 
             self._consume_doc(doc_wrapper, d)
 
+    # FIXME this should not run the callback in the deferred thred
     @deferred_to_thread
     def _unset_new_dirty(self, doc_wrapper):
         """
@@ -248,7 +254,8 @@ class SoledadStore(ContentDedup):
         doc_wrapper.new = False
         doc_wrapper.dirty = False
 
-    @deferred_to_thread
+    # FIXME this should not run the callback in the deferred thred
+    #@deferred_to_thread
     def _consume_doc(self, doc_wrapper, deferred):
         """
         Consume each document wrapper in a separate thread.
@@ -273,6 +280,7 @@ class SoledadStore(ContentDedup):
             try:
                 self._try_call(call, item)
             except Exception as exc:
+                logger.exception(exc)
                 failed = exc
                 continue
         if failed:
@@ -315,11 +323,18 @@ class SoledadStore(ContentDedup):
         """
         if call is None:
             return
-        try:
-            call(item)
-        except u1db_errors.RevisionConflict as exc:
-            logger.exception("Error: %r" % (exc,))
-            raise exc
+
+        with self._soledad_rw_lock:
+            if call == self._PUT_DOC_FUN:
+                doc_id = item.doc_id
+                doc = self._GET_DOC_FUN(doc_id)
+                doc.content = dict(item.content)
+                item = doc
+            try:
+                call(item)
+            except u1db_errors.RevisionConflict as exc:
+                logger.exception("Error: %r" % (exc,))
+                raise exc
 
     def _get_calls_for_msg_parts(self, msg_wrapper):
         """
@@ -334,7 +349,7 @@ class SoledadStore(ContentDedup):
         call = None
 
         if msg_wrapper.new:
-            call = self._soledad.create_doc
+            call = self._CREATE_DOC_FUN
 
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
@@ -353,18 +368,17 @@ class SoledadStore(ContentDedup):
         # the flags doc.
 
         elif msg_wrapper.dirty:
-            call = self._soledad.put_doc
+            call = self._PUT_DOC_FUN
             # item is expected to be a MessagePartDoc
             for item in msg_wrapper.walk():
                 # XXX FIXME Give error if dirty and not doc_id !!!
                 doc_id = item.doc_id  # defend!
                 if not doc_id:
                     continue
-                doc = self._soledad.get_doc(doc_id)
-                doc.content = dict(item.content)
+
                 if item.part == MessagePartType.fdoc:
                     logger.debug("PUT dirty fdoc")
-                    yield doc, call
+                    yield item, call
 
                 # XXX also for linkage-doc !!!
         else:
@@ -379,15 +393,12 @@ class SoledadStore(ContentDedup):
         :return: a tuple with recent-flags doc payload and callable
         :rtype: tuple
         """
-        call = self._soledad.put_doc
-        rdoc = self._soledad.get_doc(rflags_wrapper.doc_id)
+        call = self._CREATE_DOC_FUN
 
         payload = rflags_wrapper.content
-        logger.debug("Saving RFLAGS to Soledad...")
-
         if payload:
-            rdoc.content = payload
-            yield rdoc, call
+            logger.debug("Saving RFLAGS to Soledad...")
+            yield payload, call
 
     def _get_mbox_document(self, mbox):
         """
-- 
cgit v1.2.3


From 06556ec6dc56a4859736fc2782779ee2eb9c1f55 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Feb 2014 23:44:23 -0400
Subject: defer parse to thread

---
 src/leap/mail/imap/memorystore.py |  4 ++-
 src/leap/mail/imap/messages.py    | 72 ++++++++++++++-------------------------
 2 files changed, 29 insertions(+), 47 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index d0321ae..8deddda 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -230,6 +230,8 @@ class MemoryStore(object):
                                be fired.
         :type notify_on_disk: bool
         """
+        from twisted.internet import reactor
+
         log.msg("adding new doc to memstore %r (%r)" % (mbox, uid))
         key = mbox, uid
 
@@ -251,7 +253,7 @@ class MemoryStore(object):
         if not notify_on_disk:
             # Caller does not care, just fired and forgot, so we pass
             # a defer that will inmediately have its callback triggered.
-            observer.callback(uid)
+            reactor.callLater(0, observer.callback, uid)
 
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 25fc55f..89beaaa 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -78,7 +78,7 @@ def try_unique_query(curried):
                 # TODO we could take action, like trigger a background
                 # process to kill dupes.
                 name = getattr(curried, 'expected', 'doc')
-                logger.warning(
+                logger.debug(
                     "More than one %s found for this mbox, "
                     "we got a duplicate!!" % (name,))
             return query.pop()
@@ -720,9 +720,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # ensure that we have a recent-flags and a hdocs-sec doc
         self._get_or_create_rdoc()
 
-        # Not for now...
-        #self._get_or_create_hdocset()
-
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
         Returns an empty doc for storing different message parts.
@@ -758,21 +755,26 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                 hdocset[fields.MBOX_KEY] = self.mbox
             self._soledad.create_doc(hdocset)
 
+    @deferred_to_thread
     def _do_parse(self, raw):
         """
         Parse raw message and return it along with
         relevant information about its outer level.
 
+        This is done in a separate thread, and the callback is passed
+        to `_do_add_msg` method.
+
         :param raw: the raw message
         :type raw: StringIO or basestring
-        :return: msg, chash, size, multi
+        :return: msg, parts, chash, size, multi
         :rtype: tuple
         """
         msg = self._get_parsed_msg(raw)
         chash = self._get_hash(msg)
         size = len(msg.as_string())
         multi = msg.is_multipart()
-        return msg, chash, size, multi
+        parts = walk.get_parts(msg)
+        return msg, parts, chash, size, multi
 
     def _populate_flags(self, flags, uid, chash, size, multi):
         """
@@ -879,19 +881,25 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             flags = tuple()
         leap_assert_type(flags, tuple)
 
-        d = defer.Deferred()
-        self._do_add_msg(raw, flags, subject, date, notify_on_disk, d)
-        return d
+        observer = defer.Deferred()
+
+        d = self._do_parse(raw)
+        d.addCallback(self._do_add_msg, flags, subject, date,
+                      notify_on_disk, observer)
+        return observer
 
-    # We SHOULD defer this (or the heavy load here) to the thread pool,
+    # We SHOULD defer the heavy load here) to the thread pool,
     # but it gives troubles with the QSocketNotifier used by Qt...
-    def _do_add_msg(self, raw, flags, subject, date, notify_on_disk, observer):
+    def _do_add_msg(self, parse_result, flags, subject,
+                    date, notify_on_disk, observer):
         """
         Helper that creates a new message document.
         Here lives the magic of the leap mail. Well, in soledad, really.
 
         See `add_msg` docstring for parameter info.
 
+        :param parse_result: a tuple with the results of `self._do_parse`
+        :type parse_result: tuple
         :param observer: a deferred that will be fired with the message
                          uid when the adding succeed.
         :type observer: deferred
@@ -902,26 +910,17 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # TODO add the linked-from info !
         # TODO add reference to the original message
 
-        # parse
-        msg, chash, size, multi = self._do_parse(raw)
+        from twisted.internet import reactor
+        msg, parts, chash, size, multi = parse_result
 
         # check for uniqueness --------------------------------
-        # XXX profiler says that this test is costly.
-        # So we probably should just do an in-memory check and
-        # move the complete check to the soledad writer?
         # Watch out! We're reserving a UID right after this!
         existing_uid = self._fdoc_already_exists(chash)
         if existing_uid:
-            logger.warning("We already have that message in this "
-                           "mailbox, unflagging as deleted")
             uid = existing_uid
             msg = self.get_msg_by_uid(uid)
-            msg.setFlags((fields.DELETED_FLAG,), -1)
-
-            # XXX if this is deferred to thread again we should not use
-            # the callback in the deferred thread, but return and
-            # call the callback from the caller fun...
-            observer.callback(uid)
+            reactor.callLater(0, msg.setFlags, (fields.DELETED_FLAG,), -1)
+            reactor.callLater(0, observer.callback, uid)
             return
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
@@ -930,7 +929,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
 
-        parts = walk.get_parts(msg)
         body_phash_fun = [walk.get_body_phash_simple,
                           walk.get_body_phash_multi][int(multi)]
         body_phash = body_phash_fun(walk.get_payloads(msg))
@@ -949,9 +947,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         self.set_recent_flag(uid)
         msg_container = MessageWrapper(fd, hd, cdocs)
-        self.memstore.create_message(self.mbox, uid, msg_container,
-                                     observer=observer,
-                                     notify_on_disk=notify_on_disk)
+        self.memstore.create_message(
+            self.mbox, uid, msg_container,
+            observer=observer, notify_on_disk=notify_on_disk)
 
     #
     # getters: specific queries
@@ -982,14 +980,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                         {'doc_id': rdoc.doc_id, 'set': rflags})
             return rflags
 
-        #else:
-            # fallback for cases without memory store
-            #with self._rdoc_lock:
-                #rdoc = self._get_recent_doc()
-                #self.__rflags = set(rdoc.content.get(
-                    #fields.RECENTFLAGS_KEY, []))
-            #return self.__rflags
-
     def _set_recent_flags(self, value):
         """
         Setter for the recent-flags set for this mailbox.
@@ -997,16 +987,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         if self.memstore is not None:
             self.memstore.set_recent_flags(self.mbox, value)
 
-        #else:
-            # fallback for cases without memory store
-            #with self._rdoc_lock:
-                #rdoc = self._get_recent_doc()
-                #newv = set(value)
-                #self.__rflags = newv
-                #rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
-                # XXX should deferLater 0 it?
-                #self._soledad.put_doc(rdoc)
-
     recent_flags = property(
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
-- 
cgit v1.2.3


From 860e407ba0a86be30865a77ec29c6ecacf7899a4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 01:39:47 -0400
Subject: defer copy and soledad writes

---
 src/leap/mail/imap/mailbox.py      | 68 +++++++++++++++++++++++---------------
 src/leap/mail/imap/soledadstore.py | 61 ++++++++++++++++++++++------------
 2 files changed, 81 insertions(+), 48 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index d8af0a5..84bfa54 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -447,7 +447,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             return
         exists = self.getMessageCount()
         recent = self.getRecentCount()
-        logger.debug("NOTIFY: there are %s messages, %s recent" % (
+        logger.debug("NOTIFY (%r): there are %s messages, %s recent" % (
+            self.mbox,
             exists,
             recent))
 
@@ -528,7 +529,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         return seq_messg
 
     @deferred_to_thread
-    #@profile
     def fetch(self, messages_asked, uid):
         """
         Retrieve one or more messages in this mailbox.
@@ -809,6 +809,44 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                          UID of the message
         :type observer: Deferred
         """
+        memstore = self._memstore
+
+        def createCopy(result):
+            exist, new_fdoc, hdoc = result
+            if exist:
+                # Should we signal error on the callback?
+                logger.warning("Destination message already exists!")
+
+                # XXX I'm still not clear if we should raise the
+                # errback. This actually rases an ugly warning
+                # in some muas like thunderbird. I guess the user does
+                # not deserve that.
+                observer.callback(True)
+            else:
+                mbox = self.mbox
+                uid_next = memstore.increment_last_soledad_uid(mbox)
+                new_fdoc[self.UID_KEY] = uid_next
+                new_fdoc[self.MBOX_KEY] = mbox
+
+                # FIXME set recent!
+
+                self._memstore.create_message(
+                    self.mbox, uid_next,
+                    MessageWrapper(
+                        new_fdoc, hdoc.content),
+                    observer=observer,
+                    notify_on_disk=False)
+
+        d = self._get_msg_copy(message)
+        d.addCallback(createCopy)
+        d.addErrback(lambda f: log.msg(f.getTraceback()))
+
+    @deferred_to_thread
+    def _get_msg_copy(self, message):
+        """
+        Get a copy of the fdoc for this message, and check whether
+        it already exists.
+        """
         # XXX  for clarity, this could be delegated to a
         # MessageCollection mixin that implements copy too, and
         # moved out of here.
@@ -822,7 +860,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.warning("Tried to copy a MSG with no fdoc")
             return
         new_fdoc = copy.deepcopy(fdoc.content)
-
         fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
 
         # XXX is this hitting the db??? --- probably.
@@ -830,30 +867,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         dest_fdoc = memstore.get_fdoc_from_chash(
             fdoc_chash, self.mbox)
         exist = dest_fdoc and not empty(dest_fdoc.content)
-
-        if exist:
-            # Should we signal error on the callback?
-            logger.warning("Destination message already exists!")
-
-            # XXX I'm still not clear if we should raise the
-            # errback. This actually rases an ugly warning
-            # in some muas like thunderbird. I guess the user does
-            # not deserve that.
-            observer.callback(True)
-        else:
-            mbox = self.mbox
-            uid_next = memstore.increment_last_soledad_uid(mbox)
-            new_fdoc[self.UID_KEY] = uid_next
-            new_fdoc[self.MBOX_KEY] = mbox
-
-            # FIXME set recent!
-
-            self._memstore.create_message(
-                self.mbox, uid_next,
-                MessageWrapper(
-                    new_fdoc, hdoc.content),
-                observer=observer,
-                notify_on_disk=False)
+        return exist, new_fdoc, hdoc
 
     # convenience fun
 
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index bfa53b6..13f896f 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -216,6 +216,8 @@ class SoledadStore(ContentDedup):
         # TODO could generalize this method into a generic consumer
         # and only implement `process` here
 
+        from twisted.internet import reactor
+
         def docWriteCallBack(doc_wrapper):
             """
             Callback for a successful write of a document wrapper.
@@ -234,10 +236,10 @@ class SoledadStore(ContentDedup):
 
         while not queue.empty():
             doc_wrapper = queue.get()
+
             d = defer.Deferred()
             d.addCallbacks(docWriteCallBack, docWriteErrorBack)
-
-            self._consume_doc(doc_wrapper, d)
+            reactor.callLater(0, self._consume_doc, doc_wrapper, d)
 
     # FIXME this should not run the callback in the deferred thred
     @deferred_to_thread
@@ -254,8 +256,6 @@ class SoledadStore(ContentDedup):
         doc_wrapper.new = False
         doc_wrapper.dirty = False
 
-    # FIXME this should not run the callback in the deferred thred
-    #@deferred_to_thread
     def _consume_doc(self, doc_wrapper, deferred):
         """
         Consume each document wrapper in a separate thread.
@@ -267,33 +267,52 @@ class SoledadStore(ContentDedup):
                          errback depending on whether it succeed.
         :type deferred: Deferred
         """
-        items = self._process(doc_wrapper)
+        def notifyBack(failed, observer, doc_wrapper):
+            if failed:
+                observer.errback(MsgWriteError(
+                    "There was an error writing the mesage"))
+            else:
+                observer.callback(doc_wrapper)
+
+        def doSoledadCalls(items, observer):
+            # we prime the generator, that should return the
+            # message or flags wrapper item in the first place.
+            doc_wrapper = items.next()
+            d_sol = self._soledad_write_document_parts(items)
+            d_sol.addCallback(notifyBack, observer, doc_wrapper)
+            d_sol.addErrback(ebSoledadCalls)
 
-        # we prime the generator, that should return the
-        # message or flags wrapper item in the first place.
-        doc_wrapper = items.next()
+        def ebSoledadCalls(failure):
+            log.msg(failure.getTraceback())
+
+        d = self._iter_wrapper_subparts(doc_wrapper)
+        d.addCallback(doSoledadCalls, deferred)
+        d.addErrback(ebSoledadCalls)
+
+    #
+    # SoledadStore specific methods.
+    #
 
-        # From here, we unpack the subpart items and
-        # the right soledad call.
+    @deferred_to_thread
+    def _soledad_write_document_parts(self, items):
+        """
+        Write the document parts to soledad in a separate thread.
+        :param items: the iterator through the different document wrappers
+                      payloads.
+        :type items: iterator
+        """
         failed = False
         for item, call in items:
             try:
                 self._try_call(call, item)
             except Exception as exc:
                 logger.exception(exc)
-                failed = exc
+                failed = True
                 continue
-        if failed:
-            deferred.errback(MsgWriteError(
-                "There was an error writing the mesage"))
-        else:
-            deferred.callback(doc_wrapper)
+        return failed
 
-    #
-    # SoledadStore specific methods.
-    #
-
-    def _process(self, doc_wrapper):
+    @deferred_to_thread
+    def _iter_wrapper_subparts(self, doc_wrapper):
         """
         Return an iterator that will yield the doc_wrapper in the first place,
         followed by the subparts item and the proper call type for every
-- 
cgit v1.2.3


From bd83f834920709db3350c58dedd3cd2181c1b2cc Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 02:28:54 -0400
Subject: prefetch flag docs

---
 src/leap/mail/imap/mailbox.py     | 20 ++++++++++--
 src/leap/mail/imap/memorystore.py | 53 +++++++++++++++++++++++++++---
 src/leap/mail/imap/messages.py    | 68 +++++++++++++++++++--------------------
 3 files changed, 99 insertions(+), 42 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 84bfa54..f319bf0 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -90,6 +90,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     next_uid_lock = threading.Lock()
 
+    _fdoc_primed = {}
+
     def __init__(self, mbox, soledad, memstore, rw=1):
         """
         SoledadMailbox constructor. Needs to get passed a name, plus a
@@ -129,6 +131,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if self._memstore:
             self.prime_known_uids_to_memstore()
             self.prime_last_uid_to_memstore()
+            self.prime_flag_docs_to_memstore()
 
     @property
     def listeners(self):
@@ -279,6 +282,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         known_uids = self.messages.all_soledad_uid_iter()
         self._memstore.set_known_uids(self.mbox, known_uids)
 
+    def prime_flag_docs_to_memstore(self):
+        """
+        Prime memstore with all the flags documents.
+        """
+        primed = self._fdoc_primed.get(self.mbox, False)
+        if not primed:
+            all_flag_docs = self.messages.get_all_soledad_flag_docs()
+            self._memstore.load_flag_docs(self.mbox, all_flag_docs)
+            self._fdoc_primed[self.mbox] = True
+
     def getUIDValidity(self):
         """
         Return the unique validity identifier for this mailbox.
@@ -606,7 +619,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         messages_asked = self._bound_seq(messages_asked)
         seq_messg = self._filter_msg_seq(messages_asked)
 
-        all_flags = self.messages.all_flags()
+        all_flags = self._memstore.all_flags(self.mbox)
         result = ((msgid, flagsPart(
             msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg)
         return result
@@ -833,7 +846,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 self._memstore.create_message(
                     self.mbox, uid_next,
                     MessageWrapper(
-                        new_fdoc, hdoc.content),
+                        new_fdoc, hdoc),
                     observer=observer,
                     notify_on_disk=False)
 
@@ -860,6 +873,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.warning("Tried to copy a MSG with no fdoc")
             return
         new_fdoc = copy.deepcopy(fdoc.content)
+        copy_hdoc = copy.deepcopy(hdoc.content)
         fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
 
         # XXX is this hitting the db??? --- probably.
@@ -867,7 +881,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         dest_fdoc = memstore.get_fdoc_from_chash(
             fdoc_chash, self.mbox)
         exist = dest_fdoc and not empty(dest_fdoc.content)
-        return exist, new_fdoc, hdoc
+        return exist, new_fdoc, copy_hdoc
 
     # convenience fun
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 8deddda..00cf2cc 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -49,6 +49,11 @@ logger = logging.getLogger(__name__)
 # soledad storage, in seconds.
 SOLEDAD_WRITE_PERIOD = 10
 
+FDOC = MessagePartType.fdoc.key
+HDOC = MessagePartType.hdoc.key
+CDOCS = MessagePartType.cdocs.key
+DOCS_ID = MessagePartType.docs_id.key
+
 
 @contextlib.contextmanager
 def set_bool_flag(obj, att):
@@ -104,6 +109,11 @@ class MemoryStore(object):
         self._write_period = write_period
 
         # Internal Storage: messages
+        # TODO this probably will have better access times if we
+        # use msg_store[mbox][uid] insted of the current key scheme.
+        """
+        key is str(mbox,uid)
+        """
         self._msg_store = {}
 
         # Sizes
@@ -297,11 +307,6 @@ class MemoryStore(object):
         key = mbox, uid
         msg_dict = message.as_dict()
 
-        FDOC = MessagePartType.fdoc.key
-        HDOC = MessagePartType.hdoc.key
-        CDOCS = MessagePartType.cdocs.key
-        DOCS_ID = MessagePartType.docs_id.key
-
         try:
             store = self._msg_store[key]
         except KeyError:
@@ -580,6 +585,44 @@ class MemoryStore(object):
         if self._permanent_store:
             self._permanent_store.write_last_uid(mbox, value)
 
+    def load_flag_docs(self, mbox, flag_docs):
+        """
+        Load the flag documents for the given mbox.
+        Used during initial flag docs prefetch.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param flag_docs: a dict with the content for the flag docs.
+        :type flag_docs: dict
+        """
+        # We can do direct assignments cause we know this will only
+        # be called during initialization of the mailbox.
+        msg_store = self._msg_store
+        for uid in flag_docs:
+            key = mbox, uid
+            msg_store[key] = {}
+            msg_store[key][FDOC] = ReferenciableDict(flag_docs[uid])
+
+    def all_flags(self, mbox):
+        """
+        Return a dictionary with all the flags for a given mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: dict
+        """
+        flags_dict = {}
+        uids = self.get_uids(mbox)
+        store = self._msg_store
+        for uid in uids:
+            key = mbox, uid
+            try:
+                flags = store[key][FDOC][fields.FLAGS_KEY]
+                flags_dict[uid] = flags
+            except KeyError:
+                continue
+        return flags_dict
+
     # Counting sheeps...
 
     def count_new_mbox(self, mbox):
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 89beaaa..3ba9d1b 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -919,7 +919,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         if existing_uid:
             uid = existing_uid
             msg = self.get_msg_by_uid(uid)
-            reactor.callLater(0, msg.setFlags, (fields.DELETED_FLAG,), -1)
+
+            # TODO this cannot be deferred, this has to block.
+            #reactor.callLater(0, msg.setFlags, (fields.DELETED_FLAG,), -1)
+            msg.setFlags((fields.DELETED_FLAG,), -1)
             reactor.callLater(0, observer.callback, uid)
             return
 
@@ -1221,49 +1224,46 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         Return an iterator through the UIDs of all messages, from memory.
         """
-        if self.memstore is not None:
-            mem_uids = self.memstore.get_uids(self.mbox)
-            soledad_known_uids = self.memstore.get_soledad_known_uids(
-                self.mbox)
-            combined = tuple(set(mem_uids).union(soledad_known_uids))
-            return combined
+        mem_uids = self.memstore.get_uids(self.mbox)
+        soledad_known_uids = self.memstore.get_soledad_known_uids(
+            self.mbox)
+        combined = tuple(set(mem_uids).union(soledad_known_uids))
+        return combined
 
-    # XXX MOVE to memstore
-    def all_flags(self):
+    def get_all_soledad_flag_docs(self):
         """
-        Return a dict with all flags documents for this mailbox.
-        """
-        # XXX get all from memstore and cache it there
-        # FIXME should get all uids, get them fro memstore,
-        # and get only the missing ones from disk.
+        Return a dict with the content of all the flag documents
+        in soledad store for the given mbox.
 
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: dict
+        """
+        # XXX we really could return a reduced version with
+        # just {'uid': (flags-tuple,) since the prefetch is
+        # only oriented to get the flag tuples.
         all_flags = dict(((
             doc.content[self.UID_KEY],
-            doc.content[self.FLAGS_KEY]) for doc in
+            dict(doc.content)) for doc in
             self._soledad.get_from_index(
                 fields.TYPE_MBOX_IDX,
                 fields.TYPE_FLAGS_VAL, self.mbox)))
-        if self.memstore is not None:
-            uids = self.memstore.get_uids(self.mbox)
-            docs = ((uid, self.memstore.get_message(self.mbox, uid))
-                    for uid in uids)
-            for uid, doc in docs:
-                all_flags[uid] = doc.fdoc.content[self.FLAGS_KEY]
-
         return all_flags
 
-    def all_flags_chash(self):
-        """
-        Return a dict with the content-hash for all flag documents
-        for this mailbox.
-        """
-        all_flags_chash = dict(((
-            doc.content[self.UID_KEY],
-            doc.content[self.CONTENT_HASH_KEY]) for doc in
-            self._soledad.get_from_index(
-                fields.TYPE_MBOX_IDX,
-                fields.TYPE_FLAGS_VAL, self.mbox)))
-        return all_flags_chash
+    # XXX Move to memstore too. But we don't need it really, since
+    # we can cache the headers docs too.
+    #def all_flags_chash(self):
+        #"""
+        #Return a dict with the content-hash for all flag documents
+        #for this mailbox.
+        #"""
+        #all_flags_chash = dict(((
+            #doc.content[self.UID_KEY],
+            #doc.content[self.CONTENT_HASH_KEY]) for doc in
+            #self._soledad.get_from_index(
+                #fields.TYPE_MBOX_IDX,
+                #fields.TYPE_FLAGS_VAL, self.mbox)))
+        #return all_flags_chash
 
     def all_headers(self):
         """
-- 
cgit v1.2.3


From 3b6ff2133e477441eb8f6956a17be2412fa1ac7c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 10:27:55 -0400
Subject: do not defer fetches to thread

I think this is not a good idea now that all is done in the memstore,
overhead from passing the data to thread and gathering the result
seems to be much higher than just retreiving the data we need from
the memstore.
---
 src/leap/mail/imap/mailbox.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index f319bf0..1fa0554 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -541,7 +541,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         seq_messg = set_asked.intersection(set_exist)
         return seq_messg
 
-    @deferred_to_thread
     def fetch(self, messages_asked, uid):
         """
         Retrieve one or more messages in this mailbox.
@@ -580,7 +579,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
         return result
 
-    @deferred_to_thread
     def fetch_flags(self, messages_asked, uid):
         """
         A fast method to fetch all flags, tricking just the
@@ -624,7 +622,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg)
         return result
 
-    @deferred_to_thread
     def fetch_headers(self, messages_asked, uid):
         """
         A fast method to fetch all headers, tricking just the
-- 
cgit v1.2.3


From 26d6db6210eaca18002c1ec8c5619d7fbd3e4243 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 10:29:18 -0400
Subject: temporarily nuke out the fetch_heders diversion

---
 src/leap/mail/imap/server.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index ba63846..f4b9f71 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -114,14 +114,16 @@ class LeapIMAPServer(imap4.IMAP4Server):
             ).addCallback(
                 cbFetch, tag, query, uid
             ).addErrback(ebFetch, tag)
-        elif len(query) == 1 and str(query[0]) == "rfc822.header":
-            self._oldTimeout = self.setTimeout(None)
+
+        # XXX not implemented yet --- should hit memstore
+        #elif len(query) == 1 and str(query[0]) == "rfc822.header":
+            #self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
-            maybeDeferred(
-                self.mbox.fetch_headers, messages, uid=uid
-            ).addCallback(
-                cbFetch, tag, query, uid
-            ).addErrback(ebFetch, tag)
+            #maybeDeferred(
+                #self.mbox.fetch_headers, messages, uid=uid
+            #).addCallback(
+                #cbFetch, tag, query, uid
+            #).addErrback(ebFetch, tag)
         else:
             self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
-- 
cgit v1.2.3


From b7d28d1ee8208e1361caa73740d826af3b4c572e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 10:29:36 -0400
Subject: defend against empty items

---
 src/leap/mail/imap/soledadstore.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 13f896f..3c0b6f9 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -35,7 +35,7 @@ from leap.mail.imap.messageparts import RecentFlagsDoc
 from leap.mail.imap.fields import fields
 from leap.mail.imap.interfaces import IMessageStore
 from leap.mail.messageflow import IMessageConsumer
-from leap.mail.utils import first
+from leap.mail.utils import first, empty
 
 logger = logging.getLogger(__name__)
 
@@ -303,6 +303,8 @@ class SoledadStore(ContentDedup):
         """
         failed = False
         for item, call in items:
+            if empty(item):
+                continue
             try:
                 self._try_call(call, item)
             except Exception as exc:
-- 
cgit v1.2.3


From ff3a6a640fdb345449a5f9cd3379bbaefa36111e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 15:46:17 -0400
Subject: take recent count from memstore

---
 src/leap/mail/imap/mailbox.py      | 11 ++++++++---
 src/leap/mail/imap/memorystore.py  |  1 -
 src/leap/mail/imap/messages.py     | 25 ++++++++++---------------
 src/leap/mail/imap/service/imap.py |  7 ++++++-
 src/leap/mail/imap/soledadstore.py | 19 +++++++++++--------
 5 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 1fa0554..c188f91 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -559,6 +559,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: A tuple of two-tuples of message sequence numbers and
                 LeapMessage
         """
+        from twisted.internet import reactor
         # For the moment our UID is sequential, so we
         # can treat them all the same.
         # Change this to the flag that twisted expects when we
@@ -577,6 +578,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             raise NotImplementedError
         else:
             result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
+            reactor.callLater(0, self.unset_recent_flags, seq_messg)
         return result
 
     def fetch_flags(self, messages_asked, uid):
@@ -838,6 +840,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 new_fdoc[self.UID_KEY] = uid_next
                 new_fdoc[self.MBOX_KEY] = mbox
 
+                flags = list(new_fdoc[self.FLAGS_KEY])
+                flags.append(fields.RECENT_FLAG)
+                new_fdoc[self.FLAGS_KEY] = flags
+
                 # FIXME set recent!
 
                 self._memstore.create_message(
@@ -890,12 +896,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         for doc in docs:
             self.messages._soledad.delete_doc(doc)
 
-    def unset_recent_flags(self, uids):
+    def unset_recent_flags(self, uid_seq):
         """
         Unset Recent flag for a sequence of UIDs.
         """
-        seq_messg = self._bound_seq(uids)
-        self.messages.unset_recent_flags(seq_messg)
+        self.messages.unset_recent_flags(uid_seq)
 
     def __repr__(self):
         """
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 00cf2cc..bc40a8e 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -827,7 +827,6 @@ class MemoryStore(object):
 
     # Recent Flags
 
-    # TODO --- nice but unused
     def set_recent_flag(self, mbox, uid):
         """
         Set the `Recent` flag for a given mailbox and UID.
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 3ba9d1b..cfad1dc 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1265,6 +1265,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                 #fields.TYPE_FLAGS_VAL, self.mbox)))
         #return all_flags_chash
 
+    # XXX get from memstore
     def all_headers(self):
         """
         Return a dict with all the headers documents for this
@@ -1282,13 +1283,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         :rtype: int
         """
-        # XXX We should cache this in memstore too until next write...
-        count = self._soledad.get_count_from_index(
-            fields.TYPE_MBOX_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox)
-        if self.memstore is not None:
-            count += self.memstore.count_new()
-        return count
+        # XXX get this from a public method in memstore
+        store = self.memstore._msg_store
+        return len([uid for (mbox, uid) in store.keys()
+                    if mbox == self.mbox])
 
     # unseen messages
 
@@ -1300,10 +1298,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :return: iterator through unseen message doc UIDs
         :rtype: iterable
         """
-        return (doc.content[self.UID_KEY] for doc in
-                self._soledad.get_from_index(
-                    fields.TYPE_MBOX_SEEN_IDX,
-                    fields.TYPE_FLAGS_VAL, self.mbox, '0'))
+        # XXX get this from a public method in memstore
+        store = self.memstore._msg_store
+        return (uid for (mbox, uid), d in store.items()
+                if mbox == self.mbox and "\\Seen" not in d["fdoc"]["flags"])
 
     def count_unseen(self):
         """
@@ -1312,10 +1310,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :returns: count
         :rtype: int
         """
-        count = self._soledad.get_count_from_index(
-            fields.TYPE_MBOX_SEEN_IDX,
-            fields.TYPE_FLAGS_VAL, self.mbox, '0')
-        return count
+        return len(list(self.unseen_iter()))
 
     def get_unseen(self):
         """
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 93df51d..726049c 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -115,7 +115,12 @@ class LeapIMAPFactory(ServerFactory):
         # XXX how to pass the store along?
 
     def buildProtocol(self, addr):
-        "Return a protocol suitable for the job."
+        """
+        Return a protocol suitable for the job.
+
+        :param addr: ???
+        :type addr:  ???
+        """
         imapProtocol = LeapIMAPServer(
             uuid=self._uuid,
             userid=self._userid,
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 3c0b6f9..a74b49c 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -86,10 +86,12 @@ class ContentDedup(object):
         if not header_docs:
             return False
 
-        if len(header_docs) != 1:
-            logger.warning("Found more than one copy of chash %s!"
-                           % (chash,))
-        logger.debug("Found header doc with that hash! Skipping save!")
+        # FIXME enable only to debug this problem.
+        #if len(header_docs) != 1:
+            #logger.warning("Found more than one copy of chash %s!"
+                           #% (chash,))
+
+        #logger.debug("Found header doc with that hash! Skipping save!")
         return True
 
     def _content_does_exist(self, doc):
@@ -110,10 +112,11 @@ class ContentDedup(object):
         if not attach_docs:
             return False
 
-        if len(attach_docs) != 1:
-            logger.warning("Found more than one copy of phash %s!"
-                           % (phash,))
-        logger.debug("Found attachment doc with that hash! Skipping save!")
+        # FIXME enable only to debug this problem
+        #if len(attach_docs) != 1:
+            #logger.warning("Found more than one copy of phash %s!"
+                           #% (phash,))
+        #logger.debug("Found attachment doc with that hash! Skipping save!")
         return True
 
 
-- 
cgit v1.2.3


From b849dbb2e79427aabb7c6d2d6364c73778d548d3 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 15:46:52 -0400
Subject: increase writeback period for debug

---
 src/leap/mail/imap/memorystore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index bc40a8e..4a6a3ed 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -47,7 +47,7 @@ logger = logging.getLogger(__name__)
 
 # The default period to do writebacks to the permanent
 # soledad storage, in seconds.
-SOLEDAD_WRITE_PERIOD = 10
+SOLEDAD_WRITE_PERIOD = 30
 
 FDOC = MessagePartType.fdoc.key
 HDOC = MessagePartType.hdoc.key
-- 
cgit v1.2.3


From fec92585f933b6ce9b8c2701a9e28a8b7490d32a Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 19:01:58 -0400
Subject: enable memory-only store

---
 src/leap/mail/imap/memorystore.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 4a6a3ed..04e0af6 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -195,11 +195,17 @@ class MemoryStore(object):
 
             # We can start the write loop right now, why wait?
             self._start_write_loop()
+        else:
+            # We have a memory-only store.
+            self.producer = None
+            self._write_loop = None
 
     def _start_write_loop(self):
         """
         Start loop for writing to disk database.
         """
+        if self._write_loop is None:
+            return
         if not self._write_loop.running:
             self._write_loop.start(self._write_period, now=True)
 
@@ -207,6 +213,8 @@ class MemoryStore(object):
         """
         Stop loop for writing to disk database.
         """
+        if self._write_loop is None:
+            return
         if self._write_loop.running:
             self._write_loop.stop()
 
@@ -961,6 +969,12 @@ class MemoryStore(object):
         :type observer: Deferred
         """
         soledad_store = self._permanent_store
+        if soledad_store is None:
+            # just-in memory store, easy then.
+            self._delete_from_memory(mbox, observer)
+            return
+
+        # We have a soledad storage.
         try:
             # Stop and trigger last write
             self.stop_and_flush()
@@ -973,6 +987,18 @@ class MemoryStore(object):
         except Exception as exc:
             logger.exception(exc)
 
+    def _delete_from_memory(self, mbox, observer):
+        """
+        Remove all messages marked as deleted from soledad and memory.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param observer: a deferred that will be fired when expunge is done
+        :type observer: Deferred
+        """
+        mem_deleted = self.remove_all_deleted(mbox)
+        observer.callback(mem_deleted)
+
     def _delete_from_soledad_and_memory(self, result, mbox, observer):
         """
         Remove all messages marked as deleted from soledad and memory.
@@ -989,12 +1015,7 @@ class MemoryStore(object):
 
         try:
             # 1. Delete all messages marked as deleted in soledad.
-
-            # XXX this could be deferred for faster operation.
-            if soledad_store:
-                sol_deleted = soledad_store.remove_all_deleted(mbox)
-            else:
-                sol_deleted = []
+            sol_deleted = soledad_store.remove_all_deleted(mbox)
 
             try:
                 self._known_uids[mbox].difference_update(set(sol_deleted))
-- 
cgit v1.2.3


From a5c45803dfdc62f22db592d1e542fcbd07170a43 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 19:44:25 -0400
Subject: make last_uid a defaultdict

---
 src/leap/mail/imap/memorystore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 04e0af6..3f3cf83 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -163,7 +163,7 @@ class MemoryStore(object):
         {'mbox-a': 42,
          'mbox-b': 23}
         """
-        self._last_uid = {}
+        self._last_uid = defaultdict(lambda: 0)
 
         """
         known-uids keeps a count of the uids that soledad knows for a given
-- 
cgit v1.2.3


From ee0786c57d72aa8b8da76533f33c3dd65253a878 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 6 Feb 2014 18:11:20 -0400
Subject: long-due update to unittests!

So we're safe under the green lights before further rewriting.
:)
---
 src/leap/mail/imap/account.py         |   6 +
 src/leap/mail/imap/messages.py        |  15 +-
 src/leap/mail/imap/server.py          |   1 +
 src/leap/mail/imap/tests/test_imap.py | 432 ++++++++++++++++------------------
 4 files changed, 218 insertions(+), 236 deletions(-)

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index f985c04..04af3b1 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -36,6 +36,10 @@ from leap.soledad.client import Soledad
 #######################################
 
 
+# TODO change name to LeapIMAPAccount, since we're using
+# the memstore.
+# IndexedDB should also not be here anymore.
+
 class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
     """
     An implementation of IAccount and INamespacePresenteer
@@ -67,6 +71,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         # XXX SHOULD assert too that the name matches the user/uuid with which
         # soledad has been initialized.
 
+        # XXX ??? why is this parsing mailbox name??? it's account...
+        # userid? homogenize.
         self._account_name = self._parse_mailbox_name(account_name)
         self._soledad = soledad
         self._memstore = memstore
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index cfad1dc..3fbe2ad 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -273,11 +273,19 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         Retrieve the date internally associated with this message
 
-        :rtype: C{str}
+        According to the spec, this is NOT the date and time in the
+        RFC-822 header, but rather a date and time that reflects when the
+        message was received.
+
+        * In SMTP, date and time of final delivery.
+        * In COPY, internal date/time of the source message.
+        * In APPEND, date/time specified.
+
         :return: An RFC822-formatted date string.
+        :rtype: str
         """
-        date = self._hdoc.content.get(self.DATE_KEY, '')
-        return str(date)
+        date = self._hdoc.content.get(fields.DATE_KEY, '')
+        return date
 
     #
     # IMessagePart
@@ -882,7 +890,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         leap_assert_type(flags, tuple)
 
         observer = defer.Deferred()
-
         d = self._do_parse(raw)
         d.addCallback(self._do_add_msg, flags, subject, date,
                       notify_on_disk, observer)
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index f4b9f71..89fb46d 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -41,6 +41,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         soledad = kwargs.pop('soledad', None)
         uuid = kwargs.pop('uuid', None)
         userid = kwargs.pop('userid', None)
+
         leap_assert(soledad, "need a soledad instance")
         leap_assert_type(soledad, Soledad)
         leap_assert(uuid, "need a user in the initialization")
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 8c1cf20..fd88440 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -43,6 +43,7 @@ from itertools import chain
 
 from mock import Mock
 from nose.twistedtools import deferred, stop_reactor
+from unittest import skip
 
 
 from twisted.mail import imap4
@@ -64,11 +65,16 @@ import twisted.cred.portal
 from leap.common.testing.basetest import BaseLeapTest
 from leap.mail.imap.account import SoledadBackedAccount
 from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.memorystore import MemoryStore
 from leap.mail.imap.messages import MessageCollection
+from leap.mail.imap.server import LeapIMAPServer
 
 from leap.soledad.client import Soledad
 from leap.soledad.client import SoledadCrypto
 
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
 
 def strip(f):
     return lambda result, f=f: f()
@@ -89,10 +95,10 @@ def initialize_soledad(email, gnupg_home, tempdir):
     """
     Initializes soledad by hand
 
-    @param email: ID for the user
-    @param gnupg_home: path to home used by gnupg
-    @param tempdir: path to temporal dir
-    @rtype: Soledad instance
+    :param email: ID for the user
+    :param gnupg_home: path to home used by gnupg
+    :param tempdir: path to temporal dir
+    :rtype: Soledad instance
     """
 
     uuid = "foobar-uuid"
@@ -125,55 +131,6 @@ def initialize_soledad(email, gnupg_home, tempdir):
     return _soledad
 
 
-#
-# Simple LEAP IMAP4 Server for testing
-#
-
-class SimpleLEAPServer(imap4.IMAP4Server):
-
-    """
-    A Simple IMAP4 Server with mailboxes backed by Soledad.
-
-    This should be pretty close to the real LeapIMAP4Server that we
-    will be instantiating as a service, minus the authentication bits.
-    """
-
-    def __init__(self, *args, **kw):
-
-        soledad = kw.pop('soledad', None)
-
-        imap4.IMAP4Server.__init__(self, *args, **kw)
-        realm = TestRealm()
-
-        # XXX Why I AM PASSING THE ACCOUNT TO
-        # REALM? I AM NOT USING  THAT NOW, AM I???
-        realm.theAccount = SoledadBackedAccount(
-            'testuser',
-            soledad=soledad)
-
-        portal = cred.portal.Portal(realm)
-        c = cred.checkers.InMemoryUsernamePasswordDatabaseDontUse()
-        self.checker = c
-        self.portal = portal
-        portal.registerChecker(c)
-        self.timeoutTest = False
-
-    def lineReceived(self, line):
-        if self.timeoutTest:
-            # Do not send a respones
-            return
-
-        imap4.IMAP4Server.lineReceived(self, line)
-
-    _username = 'testuser'
-    _password = 'password-test'
-
-    def authenticateLogin(self, username, password):
-        if username == self._username and password == self._password:
-            return imap4.IAccount, self.theAccount, lambda: None
-        raise cred.error.UnauthorizedLogin()
-
-
 class TestRealm:
 
     """
@@ -255,13 +212,6 @@ class IMAP4HelperMixin(BaseLeapTest):
         # Soledad: config info
         cls.gnupg_home = "%s/gnupg" % cls.tempdir
         cls.email = 'leap@leap.se'
-        # cls.db1_file = "%s/db1.u1db" % cls.tempdir
-        # cls.db2_file = "%s/db2.u1db" % cls.tempdir
-        # open test dbs
-        # cls._db1 = u1db.open(cls.db1_file, create=True,
-                              # document_factory=SoledadDocument)
-        # cls._db2 = u1db.open(cls.db2_file, create=True,
-                              # document_factory=SoledadDocument)
 
         # initialize soledad by hand so we can control keys
         cls._soledad = initialize_soledad(
@@ -283,8 +233,6 @@ class IMAP4HelperMixin(BaseLeapTest):
         Restores the old path and home environment variables.
         Removes the temporal dir created for tests.
         """
-        # cls._db1.close()
-        # cls._db2.close()
         cls._soledad.close()
 
         os.environ["PATH"] = cls.old_path
@@ -301,8 +249,13 @@ class IMAP4HelperMixin(BaseLeapTest):
         but passing the same Soledad instance (it's costly to initialize),
         so we have to be sure to restore state across tests.
         """
+        UUID = 'deadbeef',
+        USERID = TEST_USER
+        memstore = MemoryStore()
+
         d = defer.Deferred()
-        self.server = SimpleLEAPServer(
+        self.server = LeapIMAPServer(
+            uuid=UUID, userid=USERID,
             contextFactory=self.serverCTX,
             # XXX do we really need this??
             soledad=self._soledad)
@@ -317,9 +270,10 @@ class IMAP4HelperMixin(BaseLeapTest):
         # I THINK we ONLY need to do it at one place now.
 
         theAccount = SoledadBackedAccount(
-            'testuser',
-            soledad=self._soledad)
-        SimpleLEAPServer.theAccount = theAccount
+            USERID,
+            soledad=self._soledad,
+            memstore=memstore)
+        LeapIMAPServer.theAccount = theAccount
 
         # in case we get something from previous tests...
         for mb in self.server.theAccount.mailboxes:
@@ -404,8 +358,9 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         We override mixin method since we are only testing
         MessageCollection interface in this particular TestCase
         """
+        memstore = MemoryStore()
         self.messages = MessageCollection("testmbox%s" % (self.count,),
-                                          self._soledad)
+                                          self._soledad, memstore=memstore)
         MessageCollectionTestCase.count += 1
 
     def tearDown(self):
@@ -414,9 +369,6 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         del self.messages
 
-    def wait(self):
-        time.sleep(2)
-
     def testEmptyMessage(self):
         """
         Test empty message and collection
@@ -425,11 +377,11 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.assertEqual(
             em,
             {
+                "chash": '',
+                "deleted": False,
                 "flags": [],
                 "mbox": "inbox",
-                "recent": True,
                 "seen": False,
-                "deleted": False,
                 "multi": False,
                 "size": 0,
                 "type": "flags",
@@ -441,79 +393,100 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Add multiple messages
         """
-        # TODO really profile addition
         mc = self.messages
-        print "messages", self.messages
         self.assertEqual(self.messages.count(), 0)
-        mc.add_msg('Stuff', uid=1, subject="test1")
-        mc.add_msg('Stuff', uid=2, subject="test2")
-        mc.add_msg('Stuff', uid=3, subject="test3")
-        mc.add_msg('Stuff', uid=4, subject="test4")
-        self.wait()
-        self.assertEqual(self.messages.count(), 4)
-        mc.add_msg('Stuff', uid=5, subject="test5")
-        mc.add_msg('Stuff', uid=6, subject="test6")
-        mc.add_msg('Stuff', uid=7, subject="test7")
-        self.wait()
-        self.assertEqual(self.messages.count(), 7)
-        self.wait()
 
+        def add_first():
+            d = defer.gatherResults([
+                mc.add_msg('Stuff 1', uid=1, subject="test1"),
+                mc.add_msg('Stuff 2', uid=2, subject="test2"),
+                mc.add_msg('Stuff 3', uid=3, subject="test3"),
+                mc.add_msg('Stuff 4', uid=4, subject="test4")])
+            return d
+
+        def add_second(result):
+            d = defer.gatherResults([
+                mc.add_msg('Stuff 5', uid=5, subject="test5"),
+                mc.add_msg('Stuff 6', uid=6, subject="test6"),
+                mc.add_msg('Stuff 7', uid=7, subject="test7")])
+            return d
+
+        def check_second(result):
+            return self.assertEqual(mc.count(), 7)
+
+        d1 = add_first()
+        d1.addCallback(add_second)
+        d1.addCallback(check_second)
+
+    @skip("needs update!")
     def testRecentCount(self):
         """
         Test the recent count
         """
         mc = self.messages
-        self.assertEqual(self.messages.count_recent(), 0)
-        mc.add_msg('Stuff', uid=1, subject="test1")
+        countrecent = mc.count_recent
+        eq = self.assertEqual
+
+        self.assertEqual(countrecent(), 0)
+
+        d = mc.add_msg('Stuff', uid=1, subject="test1")
         # For the semantics defined in the RFC, we auto-add the
         # recent flag by default.
-        self.wait()
-        self.assertEqual(self.messages.count_recent(), 1)
-        mc.add_msg('Stuff', subject="test2", uid=2,
-                   flags=('\\Deleted',))
-        self.wait()
-        self.assertEqual(self.messages.count_recent(), 2)
-        mc.add_msg('Stuff', subject="test3", uid=3,
-                   flags=('\\Recent',))
-        self.wait()
-        self.assertEqual(self.messages.count_recent(), 3)
-        mc.add_msg('Stuff', subject="test4", uid=4,
-                   flags=('\\Deleted', '\\Recent'))
-        self.wait()
-        self.assertEqual(self.messages.count_recent(), 4)
-
-        for msg in mc:
-            msg.removeFlags(('\\Recent',))
-        self.assertEqual(mc.count_recent(), 0)
+
+        def add2(_):
+            return mc.add_msg('Stuff', subject="test2", uid=2,
+                              flags=('\\Deleted',))
+
+        def add3(_):
+            return mc.add_msg('Stuff', subject="test3", uid=3,
+                              flags=('\\Recent',))
+
+        def add4(_):
+            return mc.add_msg('Stuff', subject="test4", uid=4,
+                              flags=('\\Deleted', '\\Recent'))
+
+        d.addCallback(lambda r: eq(countrecent(), 1))
+        d.addCallback(add2)
+        d.addCallback(lambda r: eq(countrecent(), 2))
+        d.addCallback(add3)
+        d.addCallback(lambda r: eq(countrecent(), 3))
+        d.addCallback(add4)
+        d.addCallback(lambda r: eq(countrecent(), 4))
 
     def testFilterByMailbox(self):
         """
         Test that queries filter by selected mailbox
         """
-        def wait():
-            time.sleep(1)
-
         mc = self.messages
         self.assertEqual(self.messages.count(), 0)
-        mc.add_msg('', uid=1, subject="test1")
-        mc.add_msg('', uid=2, subject="test2")
-        mc.add_msg('', uid=3, subject="test3")
-        wait()
-        self.assertEqual(self.messages.count(), 3)
-        newmsg = mc._get_empty_doc()
-        newmsg['mailbox'] = "mailbox/foo"
-        mc._soledad.create_doc(newmsg)
-        self.assertEqual(mc.count(), 3)
-        self.assertEqual(
-            len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
+
+        def add_1():
+            d1 = mc.add_msg('msg 1', uid=1, subject="test1")
+            d2 = mc.add_msg('msg 2', uid=2, subject="test2")
+            d3 = mc.add_msg('msg 3', uid=3, subject="test3")
+            d = defer.gatherResults([d1, d2, d3])
+            return d
+
+        add_1().addCallback(lambda ignored: self.assertEqual(
+                            mc.count(), 3))
+
+        # XXX this has to be redone to fit memstore ------------#
+        #newmsg = mc._get_empty_doc()
+        #newmsg['mailbox'] = "mailbox/foo"
+        #mc._soledad.create_doc(newmsg)
+        #self.assertEqual(mc.count(), 3)
+        #self.assertEqual(
+            #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
 
 
 class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
+    # TODO this currently will use a memory-only store.
+    # create a different one for testing soledad sync.
 
     """
     Tests for the generic behavior of the LeapIMAP4Server
     which, right now, it's just implemented in this test file as
-    SimpleLEAPServer. We will move the implementation, together with
+    LeapIMAPServer. We will move the implementation, together with
     authentication bits, to leap.mail.imap.server so it can be instantiated
     from the tac file.
 
@@ -542,7 +515,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             self.result.append(0)
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def create():
             for name in succeed + fail:
@@ -560,7 +533,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def _cbTestCreate(self, ignored, succeed, fail):
         self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail))
 
-        mbox = SimpleLEAPServer.theAccount.mailboxes
+        mbox = LeapIMAPServer.theAccount.mailboxes
         answers = ['foobox', 'testbox', 'test/box', 'test', 'test/box/box']
         mbox.sort()
         answers.sort()
@@ -571,10 +544,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test whether we can delete mailboxes
         """
-        SimpleLEAPServer.theAccount.addMailbox('delete/me')
+        LeapIMAPServer.theAccount.addMailbox('delete/me')
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def delete():
             return self.client.delete('delete/me')
@@ -586,7 +559,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d = defer.gatherResults([d1, d2])
         d.addCallback(
             lambda _: self.assertEqual(
-                SimpleLEAPServer.theAccount.mailboxes, []))
+                LeapIMAPServer.theAccount.mailboxes, []))
         return d
 
     def testIllegalInboxDelete(self):
@@ -597,7 +570,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.stashed = None
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def delete():
             return self.client.delete('inbox')
@@ -619,10 +592,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def testNonExistentDelete(self):
         """
         Test what happens if we try to delete a non-existent mailbox.
-        We expect an error raised stating 'No such inbox'
+        We expect an error raised stating 'No such mailbox'
         """
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def delete():
             return self.client.delete('delete/me')
@@ -637,8 +610,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
-        d.addCallback(lambda _: self.assertEqual(str(self.failure.value),
-                                                 'No such mailbox'))
+        d.addCallback(lambda _: self.assertTrue(
+            str(self.failure.value).startswith('No such mailbox')))
         return d
 
     @deferred(timeout=None)
@@ -649,14 +622,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         Obs: this test will fail if SoledadMailbox returns hardcoded flags.
         """
-        SimpleLEAPServer.theAccount.addMailbox('delete')
-        to_delete = SimpleLEAPServer.theAccount.getMailbox('delete')
+        LeapIMAPServer.theAccount.addMailbox('delete')
+        to_delete = LeapIMAPServer.theAccount.getMailbox('delete')
         to_delete.setFlags((r'\Noselect',))
         to_delete.getFlags()
-        SimpleLEAPServer.theAccount.addMailbox('delete/me')
+        LeapIMAPServer.theAccount.addMailbox('delete/me')
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def delete():
             return self.client.delete('delete')
@@ -681,10 +654,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test whether we can rename a mailbox
         """
-        SimpleLEAPServer.theAccount.addMailbox('oldmbox')
+        LeapIMAPServer.theAccount.addMailbox('oldmbox')
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def rename():
             return self.client.rename('oldmbox', 'newname')
@@ -696,7 +669,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d = defer.gatherResults([d1, d2])
         d.addCallback(lambda _:
                       self.assertEqual(
-                          SimpleLEAPServer.theAccount.mailboxes,
+                          LeapIMAPServer.theAccount.mailboxes,
                           ['newname']))
         return d
 
@@ -709,7 +682,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.stashed = None
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def rename():
             return self.client.rename('inbox', 'frotz')
@@ -733,11 +706,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Try to rename hierarchical mailboxes
         """
-        SimpleLEAPServer.theAccount.create('oldmbox/m1')
-        SimpleLEAPServer.theAccount.create('oldmbox/m2')
+        LeapIMAPServer.theAccount.create('oldmbox/m1')
+        LeapIMAPServer.theAccount.create('oldmbox/m2')
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def rename():
             return self.client.rename('oldmbox', 'newname')
@@ -750,7 +723,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestHierarchicalRename)
 
     def _cbTestHierarchicalRename(self, ignored):
-        mboxes = SimpleLEAPServer.theAccount.mailboxes
+        mboxes = LeapIMAPServer.theAccount.mailboxes
         expected = ['newname', 'newname/m1', 'newname/m2']
         mboxes.sort()
         self.assertEqual(mboxes, [s for s in expected])
@@ -761,7 +734,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test whether we can mark a mailbox as subscribed to
         """
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def subscribe():
             return self.client.subscribe('this/mbox')
@@ -773,7 +746,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d = defer.gatherResults([d1, d2])
         d.addCallback(lambda _:
                       self.assertEqual(
-                          SimpleLEAPServer.theAccount.subscriptions,
+                          LeapIMAPServer.theAccount.subscriptions,
                           ['this/mbox']))
         return d
 
@@ -782,11 +755,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test whether we can unsubscribe from a set of mailboxes
         """
-        SimpleLEAPServer.theAccount.subscribe('this/mbox')
-        SimpleLEAPServer.theAccount.subscribe('that/mbox')
+        LeapIMAPServer.theAccount.subscribe('this/mbox')
+        LeapIMAPServer.theAccount.subscribe('that/mbox')
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def unsubscribe():
             return self.client.unsubscribe('this/mbox')
@@ -798,7 +771,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         d = defer.gatherResults([d1, d2])
         d.addCallback(lambda _:
                       self.assertEqual(
-                          SimpleLEAPServer.theAccount.subscriptions,
+                          LeapIMAPServer.theAccount.subscriptions,
                           ['that/mbox']))
         return d
 
@@ -811,7 +784,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.selectedArgs = None
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def select():
             def selected(args):
@@ -829,7 +802,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect)
 
     def _cbTestSelect(self, ignored):
-        mbox = SimpleLEAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT')
+        mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT')
         self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox)
         self.assertEqual(self.selectedArgs, {
             'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
@@ -920,7 +893,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test login
         """
         def login():
-            d = self.client.login('testuser', 'password-test')
+            d = self.client.login(TEST_USER, TEST_PASSWD)
             d.addCallback(self._cbStopClient)
         d1 = self.connected.addCallback(
             strip(login)).addErrback(self._ebGeneral)
@@ -928,7 +901,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestLogin)
 
     def _cbTestLogin(self, ignored):
-        self.assertEqual(self.server.account, SimpleLEAPServer.theAccount)
+        self.assertEqual(self.server.account, LeapIMAPServer.theAccount)
         self.assertEqual(self.server.state, 'auth')
 
     @deferred(timeout=None)
@@ -937,7 +910,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test bad login
         """
         def login():
-            d = self.client.login('testuser', 'wrong-password')
+            d = self.client.login("bad_user@leap.se", TEST_PASSWD)
             d.addBoth(self._cbStopClient)
 
         d1 = self.connected.addCallback(
@@ -947,19 +920,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestFailedLogin)
 
     def _cbTestFailedLogin(self, ignored):
-        self.assertEqual(self.server.account, None)
         self.assertEqual(self.server.state, 'unauth')
+        self.assertEqual(self.server.account, None)
 
     @deferred(timeout=None)
     def testLoginRequiringQuoting(self):
         """
         Test login requiring quoting
         """
-        self.server._username = '{test}user'
+        self.server._userid = '{test}user@leap.se'
         self.server._password = '{test}password'
 
         def login():
-            d = self.client.login('{test}user', '{test}password')
+            d = self.client.login('{test}user@leap.se', '{test}password')
             d.addBoth(self._cbStopClient)
 
         d1 = self.connected.addCallback(
@@ -968,7 +941,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         return d.addCallback(self._cbTestLoginRequiringQuoting)
 
     def _cbTestLoginRequiringQuoting(self, ignored):
-        self.assertEqual(self.server.account, SimpleLEAPServer.theAccount)
+        self.assertEqual(self.server.account, LeapIMAPServer.theAccount)
         self.assertEqual(self.server.state, 'auth')
 
     #
@@ -983,7 +956,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.namespaceArgs = None
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def namespace():
             def gotNamespace(args):
@@ -1022,7 +995,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         self.examinedArgs = None
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def examine():
             def examined(args):
@@ -1049,15 +1022,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             'READ-WRITE': False})
 
     def _listSetup(self, f):
-        SimpleLEAPServer.theAccount.addMailbox('root/subthingl',
-                                               creation_ts=42)
-        SimpleLEAPServer.theAccount.addMailbox('root/another-thing',
-                                               creation_ts=42)
-        SimpleLEAPServer.theAccount.addMailbox('non-root/subthing',
-                                               creation_ts=42)
+        LeapIMAPServer.theAccount.addMailbox('root/subthingl',
+                                             creation_ts=42)
+        LeapIMAPServer.theAccount.addMailbox('root/another-thing',
+                                             creation_ts=42)
+        LeapIMAPServer.theAccount.addMailbox('non-root/subthing',
+                                             creation_ts=42)
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def listed(answers):
             self.listed = answers
@@ -1092,7 +1065,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test LSub command
         """
-        SimpleLEAPServer.theAccount.subscribe('root/subthingl2')
+        LeapIMAPServer.theAccount.subscribe('root/subthingl2')
 
         def lsub():
             return self.client.lsub('root', '%')
@@ -1106,12 +1079,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test Status command
         """
-        SimpleLEAPServer.theAccount.addMailbox('root/subthings')
+        LeapIMAPServer.theAccount.addMailbox('root/subthings')
         # XXX FIXME ---- should populate this a little bit,
         # with unseen etc...
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def status():
             return self.client.status(
@@ -1139,7 +1112,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test failed status command with a non-existent mailbox
         """
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def status():
             return self.client.status(
@@ -1180,13 +1153,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         infile = util.sibpath(__file__, 'rfc822.message')
         message = open(infile)
-        SimpleLEAPServer.theAccount.addMailbox('root/subthing')
+        LeapIMAPServer.theAccount.addMailbox('root/subthing')
 
         def login():
-            return self.client.login('testuser', 'password-test')
-
-        def wait():
-            time.sleep(0.5)
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def append():
             return self.client.append(
@@ -1198,21 +1168,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         d1 = self.connected.addCallback(strip(login))
         d1.addCallbacks(strip(append), self._ebGeneral)
-        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
         d = defer.gatherResults([d1, d2])
         return d.addCallback(self._cbTestFullAppend, infile)
 
     def _cbTestFullAppend(self, ignored, infile):
-        mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing')
-        time.sleep(0.5)
+        mb = LeapIMAPServer.theAccount.getMailbox('root/subthing')
         self.assertEqual(1, len(mb.messages))
 
         msg = mb.messages.get_msg_by_uid(1)
         self.assertEqual(
-            ('\\SEEN', '\\DELETED'),
-            msg.getFlags())
+            set(('\\Recent', '\\SEEN', '\\DELETED')),
+            set(msg.getFlags()))
 
         self.assertEqual(
             'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
@@ -1220,14 +1188,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         parsed = self.parser.parse(open(infile))
         body = parsed.get_payload()
-        headers = parsed.items()
+        headers = dict(parsed.items())
         self.assertEqual(
             body,
             msg.getBodyFile().read())
-
-        msg_headers = msg.getHeaders(True, "",)
-        gotheaders = list(chain(
-            *[[(k, item) for item in v] for (k, v) in msg_headers.items()]))
+        gotheaders = msg.getHeaders(True)
 
         self.assertItemsEqual(
             headers, gotheaders)
@@ -1238,13 +1203,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         Test partially appending a message to the mailbox
         """
         infile = util.sibpath(__file__, 'rfc822.message')
-        SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
+        LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
 
         def login():
-            return self.client.login('testuser', 'password-test')
-
-        def wait():
-            time.sleep(1)
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def append():
             message = file(infile)
@@ -1257,7 +1219,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
                 )
             )
         d1 = self.connected.addCallback(strip(login))
-        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(strip(append), self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -1266,16 +1227,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
             self._cbTestPartialAppend, infile)
 
     def _cbTestPartialAppend(self, ignored, infile):
-        mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
-        time.sleep(1)
+        mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
         self.assertEqual(1, len(mb.messages))
         msg = mb.messages.get_msg_by_uid(1)
         self.assertEqual(
-            ('\\SEEN', ),
-            msg.getFlags()
+            set(('\\SEEN', '\\Recent')),
+            set(msg.getFlags())
         )
-        #self.assertEqual(
-            #'Right now', msg.getInternalDate())
         parsed = self.parser.parse(open(infile))
         body = parsed.get_payload()
         self.assertEqual(
@@ -1287,10 +1245,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test check command
         """
-        SimpleLEAPServer.theAccount.addMailbox('root/subthing')
+        LeapIMAPServer.theAccount.addMailbox('root/subthing')
 
         def login():
-            return self.client.login('testuser', 'password-test')
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def select():
             return self.client.select('root/subthing')
@@ -1306,7 +1264,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         # Okay, that was fun
 
-    @deferred(timeout=None)
+    @deferred(timeout=5)
     def testClose(self):
         """
         Test closing the mailbox. We expect to get deleted all messages flagged
@@ -1315,29 +1273,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         name = 'mailbox-close'
         self.server.theAccount.addMailbox(name)
 
-        m = SimpleLEAPServer.theAccount.getMailbox(name)
-        m.messages.add_msg('test 1', uid=1, subject="Message 1",
-                           flags=('\\Deleted', 'AnotherFlag'))
-        m.messages.add_msg('test 2', uid=2, subject="Message 2",
-                           flags=('AnotherFlag',))
-        m.messages.add_msg('test 3', uid=3, subject="Message 3",
-                           flags=('\\Deleted',))
+        m = LeapIMAPServer.theAccount.getMailbox(name)
 
         def login():
-            return self.client.login('testuser', 'password-test')
-
-        def wait():
-            time.sleep(1)
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def select():
             return self.client.select(name)
 
+        def add_messages():
+            d1 = m.messages.add_msg(
+                'test 1', uid=1, subject="Message 1",
+                flags=('\\Deleted', 'AnotherFlag'))
+            d2 = m.messages.add_msg(
+                'test 2', uid=2, subject="Message 2",
+                flags=('AnotherFlag',))
+            d3 = m.messages.add_msg(
+                'test 3', uid=3, subject="Message 3",
+                flags=('\\Deleted',))
+            d = defer.gatherResults([d1, d2, d3])
+            return d
+
         def close():
             return self.client.close()
 
         d = self.connected.addCallback(strip(login))
-        d.addCallbacks(strip(wait), self._ebGeneral)
         d.addCallbacks(strip(select), self._ebGeneral)
+        d.addCallbacks(strip(add_messages), self._ebGeneral)
         d.addCallbacks(strip(close), self._ebGeneral)
         d.addCallbacks(self._cbStopClient, self._ebGeneral)
         d2 = self.loopback()
@@ -1345,37 +1307,42 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
     def _cbTestClose(self, ignored, m):
         self.assertEqual(len(m.messages), 1)
-        messages = [msg for msg in m.messages]
-        self.assertFalse(messages[0] is None)
+
+        msg = m.messages.get_msg_by_uid(2)
+        self.assertFalse(msg is None)
         self.assertEqual(
-            messages[0]._hdoc.content['subject'],
+            msg._hdoc.content['subject'],
             'Message 2')
         self.failUnless(m.closed)
 
-    @deferred(timeout=None)
+    @deferred(timeout=5)
     def testExpunge(self):
         """
         Test expunge command
         """
         name = 'mailbox-expunge'
-        SimpleLEAPServer.theAccount.addMailbox(name)
-        m = SimpleLEAPServer.theAccount.getMailbox(name)
-        m.messages.add_msg('test 1', uid=1, subject="Message 1",
-                           flags=('\\Deleted', 'AnotherFlag'))
-        m.messages.add_msg('test 2', uid=2, subject="Message 2",
-                           flags=('AnotherFlag',))
-        m.messages.add_msg('test 3', uid=3, subject="Message 3",
-                           flags=('\\Deleted',))
+        self.server.theAccount.addMailbox(name)
+        m = LeapIMAPServer.theAccount.getMailbox(name)
 
         def login():
-            return self.client.login('testuser', 'password-test')
-
-        def wait():
-            time.sleep(2)
+            return self.client.login(TEST_USER, TEST_PASSWD)
 
         def select():
             return self.client.select('mailbox-expunge')
 
+        def add_messages():
+            d1 = m.messages.add_msg(
+                'test 1', uid=1, subject="Message 1",
+                flags=('\\Deleted', 'AnotherFlag'))
+            d2 = m.messages.add_msg(
+                'test 2', uid=2, subject="Message 2",
+                flags=('AnotherFlag',))
+            d3 = m.messages.add_msg(
+                'test 3', uid=3, subject="Message 3",
+                flags=('\\Deleted',))
+            d = defer.gatherResults([d1, d2, d3])
+            return d
+
         def expunge():
             return self.client.expunge()
 
@@ -1385,8 +1352,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
 
         self.results = None
         d1 = self.connected.addCallback(strip(login))
-        d1.addCallbacks(strip(wait), self._ebGeneral)
         d1.addCallbacks(strip(select), self._ebGeneral)
+        d1.addCallbacks(strip(add_messages), self._ebGeneral)
         d1.addCallbacks(strip(expunge), self._ebGeneral)
         d1.addCallbacks(expunged, self._ebGeneral)
         d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -1397,9 +1364,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
     def _cbTestExpunge(self, ignored, m):
         # we only left 1 mssage with no deleted flag
         self.assertEqual(len(m.messages), 1)
-        messages = [msg for msg in m.messages]
+
+        msg = m.messages.get_msg_by_uid(2)
         self.assertEqual(
-            messages[0]._hdoc.content['subject'],
+            msg._hdoc.content['subject'],
             'Message 2')
         # the uids of the deleted messages
         self.assertItemsEqual(self.results, [1, 3])
-- 
cgit v1.2.3


From 6586c21a12bfa0d9026068629a9d34203ad577c7 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 7 Feb 2014 02:27:58 -0400
Subject: change internal storage and keying scheme in memstore

---
 src/leap/mail/imap/memorystore.py | 187 +++++++++++++++++++-------------------
 src/leap/mail/imap/messages.py    |  10 +-
 2 files changed, 96 insertions(+), 101 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 3f3cf83..b198e12 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -109,13 +109,14 @@ class MemoryStore(object):
         self._write_period = write_period
 
         # Internal Storage: messages
-        # TODO this probably will have better access times if we
-        # use msg_store[mbox][uid] insted of the current key scheme.
         """
-        key is str(mbox,uid)
+        Flags document store.
+        _fdoc_store[mbox][uid] = { 'content': 'aaa' }
         """
-        self._msg_store = {}
+        self._fdoc_store = defaultdict(lambda: defaultdict(
+            lambda: ReferenciableDict({})))
 
+<<<<<<< HEAD
         # Sizes
         """
         {'mbox, uid': <int>}
@@ -123,10 +124,24 @@ class MemoryStore(object):
         self._sizes = {}
 
         # Internal Storage: payload-hash
+=======
+        # Internal Storage: content-hash:hdoc
+>>>>>>> change internal storage and keying scheme in memstore
         """
-        {'phash': weakreaf.proxy(dict)}
+        hdoc-store keeps references to
+        the header-documents indexed by content-hash.
+
+        {'chash': { dict-stuff }
+        }
+        """
+        self._hdoc_store = defaultdict(lambda: ReferenciableDict({}))
+
+        # Internal Storage: payload-hash:cdoc
+        """
+        content-docs stored by payload-hash
+        {'phash': { dict-stuff } }
         """
-        self._phash_store = {}
+        self._cdoc_store = defaultdict(lambda: ReferenciableDict({}))
 
         # Internal Storage: content-hash:fdoc
         """
@@ -309,26 +324,12 @@ class MemoryStore(object):
         Helper method, called by both create_message and put_message.
         See those for parameter documentation.
         """
-        # XXX have to differentiate between notify_new and notify_dirty
-        # TODO defaultdict the hell outa here...
-
-        key = mbox, uid
         msg_dict = message.as_dict()
 
-        try:
-            store = self._msg_store[key]
-        except KeyError:
-            self._msg_store[key] = {FDOC: {},
-                                    HDOC: {},
-                                    CDOCS: {},
-                                    DOCS_ID: {}}
-            store = self._msg_store[key]
-
         fdoc = msg_dict.get(FDOC, None)
-        if fdoc:
-            if not store.get(FDOC, None):
-                store[FDOC] = ReferenciableDict({})
-            store[FDOC].update(fdoc)
+        if fdoc is not None:
+            fdoc_store = self._fdoc_store[mbox][uid]
+            fdoc_store.update(fdoc)
 
             # content-hash indexing
             chash = fdoc.get(fields.CONTENT_HASH_KEY)
@@ -337,33 +338,21 @@ class MemoryStore(object):
                 chash_fdoc_store[chash] = {}
 
             chash_fdoc_store[chash][mbox] = weakref.proxy(
-                store[FDOC])
+                fdoc_store)
 
         hdoc = msg_dict.get(HDOC, None)
         if hdoc is not None:
-            if not store.get(HDOC, None):
-                store[HDOC] = ReferenciableDict({})
-            store[HDOC].update(hdoc)
-
-        docs_id = msg_dict.get(DOCS_ID, None)
-        if docs_id:
-            if not store.get(DOCS_ID, None):
-                store[DOCS_ID] = {}
-            store[DOCS_ID].update(docs_id)
+            chash = hdoc.get(fields.CONTENT_HASH_KEY)
+            hdoc_store = self._hdoc_store[chash]
+            hdoc_store.update(hdoc)
 
         cdocs = message.cdocs
-        for cdoc_key in cdocs.keys():
-            if not store.get(CDOCS, None):
-                store[CDOCS] = {}
-
-            cdoc = cdocs[cdoc_key]
-            # first we make it weak-referenciable
-            referenciable_cdoc = ReferenciableDict(cdoc)
-            store[CDOCS][cdoc_key] = referenciable_cdoc
+        for cdoc in cdocs.values():
             phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None)
             if not phash:
                 continue
-            self._phash_store[phash] = weakref.proxy(referenciable_cdoc)
+            cdoc_store = self._cdoc_store[phash]
+            cdoc_store.update(cdoc)
 
         # Update memory store size
         # XXX this should use [mbox][uid]
@@ -371,15 +360,13 @@ class MemoryStore(object):
         self._sizes[key] = size.get_size(self._fdoc_store[key])
         # TODO add hdoc and cdocs sizes too
 
-        def prune(seq, store):
-            for key in seq:
-                if key in store and empty(store.get(key)):
-                    store.pop(key)
-
-        prune((FDOC, HDOC, CDOCS, DOCS_ID), store)
+        # XXX what to do with this?
+        #docs_id = msg_dict.get(DOCS_ID, None)
+        #if docs_id is not None:
+            #if not store.get(DOCS_ID, None):
+                #store[DOCS_ID] = {}
+            #store[DOCS_ID].update(docs_id)
 
-        # Update memory store size
-        self._sizes[key] = size(self._msg_store[key])
 
     def get_docid_for_fdoc(self, mbox, uid):
         """
@@ -413,18 +400,20 @@ class MemoryStore(object):
         :return: MessageWrapper or None
         """
         key = mbox, uid
-        FDOC = MessagePartType.fdoc.key
 
-        msg_dict = self._msg_store.get(key, None)
-        if empty(msg_dict):
+        fdoc = self._fdoc_store[mbox][uid]
+        if empty(fdoc):
             return None
+
         new, dirty = self._get_new_dirty_state(key)
         if flags_only:
-            return MessageWrapper(fdoc=msg_dict[FDOC],
+            return MessageWrapper(fdoc=fdoc,
                                   new=new, dirty=dirty,
                                   memstore=weakref.proxy(self))
         else:
-            return MessageWrapper(from_dict=msg_dict,
+            chash = fdoc.get(fields.CONTENT_HASH_KEY)
+            hdoc = self._hdoc_store[chash]
+            return MessageWrapper(fdoc=fdoc, hdoc=hdoc,
                                   new=new, dirty=dirty,
                                   memstore=weakref.proxy(self))
 
@@ -448,10 +437,14 @@ class MemoryStore(object):
             key = mbox, uid
             self._new.discard(key)
             self._dirty.discard(key)
+<<<<<<< HEAD
             self._msg_store.pop(key, None)
             if key in self._sizes:
                 del self._sizes[key]
 
+=======
+            self._fdoc_store[mbox].pop(uid, None)
+>>>>>>> change internal storage and keying scheme in memstore
         except Exception as exc:
             logger.exception(exc)
 
@@ -494,8 +487,7 @@ class MemoryStore(object):
         :type mbox: str or unicode
         :rtype: list
         """
-        all_keys = self._msg_store.keys()
-        return [uid for m, uid in all_keys if m == mbox]
+        return self._fdoc_store[mbox].keys()
 
     def get_soledad_known_uids(self, mbox):
         """
@@ -605,11 +597,9 @@ class MemoryStore(object):
         """
         # We can do direct assignments cause we know this will only
         # be called during initialization of the mailbox.
-        msg_store = self._msg_store
+        fdoc_store = self._fdoc_store[mbox]
         for uid in flag_docs:
-            key = mbox, uid
-            msg_store[key] = {}
-            msg_store[key][FDOC] = ReferenciableDict(flag_docs[uid])
+            fdoc_store[uid] = ReferenciableDict(flag_docs[uid])
 
     def all_flags(self, mbox):
         """
@@ -621,11 +611,10 @@ class MemoryStore(object):
         """
         flags_dict = {}
         uids = self.get_uids(mbox)
-        store = self._msg_store
+        fdoc_store = self._fdoc_store
         for uid in uids:
-            key = mbox, uid
             try:
-                flags = store[key][FDOC][fields.FLAGS_KEY]
+                flags = fdoc_store[uid][fields.FLAGS_KEY]
                 flags_dict[uid] = flags
             except KeyError:
                 continue
@@ -635,7 +624,7 @@ class MemoryStore(object):
 
     def count_new_mbox(self, mbox):
         """
-        Count the new messages by inbox.
+        Count the new messages by mailbox.
 
         :param mbox: the mailbox
         :type mbox: str or unicode
@@ -653,6 +642,32 @@ class MemoryStore(object):
         """
         return len(self._new)
 
+    def count(self, mbox):
+        """
+        Return the count of messages for a given mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: number of messages
+        :rtype: int
+        """
+        return len(self._fdoc_store[mbox])
+
+    def unseen_iter(self, mbox):
+        """
+        Get an iterator for the message UIDs with no `seen` flag
+        for a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :return: iterator through unseen message doc UIDs
+        :rtype: iterable
+        """
+        fdocs = self._fdoc_store[mbox]
+        return [uid for uid, value
+                in fdocs.items()
+                if fields.SEEN_FLAG not in value["flags"]]
+
     def get_cdoc_from_phash(self, phash):
         """
         Return a content-document by its payload-hash.
@@ -661,7 +676,7 @@ class MemoryStore(object):
         :type phash: str or unicode
         :rtype: MessagePartDoc
         """
-        doc = self._phash_store.get(phash, None)
+        doc = self._cdoc_store.get(phash, None)
 
         # XXX return None for consistency?
 
@@ -716,15 +731,15 @@ class MemoryStore(object):
             content=fdoc,
             doc_id=None)
 
-    def all_msg_iter(self):
+    def iter_fdoc_keys(self):
         """
-        Return generator that iterates through all messages in the store.
-
-        :return: generator of MessageWrappers
-        :rtype: generator
+        Return a generator through all the mbox, uid keys in the flags-doc
+        store.
         """
-        return (self.get_message(*key)
-                for key in sorted(self._msg_store.keys()))
+        fdoc_store = self._fdoc_store
+        for mbox in fdoc_store:
+            for uid in fdoc_store[mbox]:
+                yield mbox, uid
 
     def all_new_dirty_msg_iter(self):
         """
@@ -734,23 +749,9 @@ class MemoryStore(object):
         :rtype: generator
         """
         return (self.get_message(*key)
-                for key in sorted(self._msg_store.keys())
+                for key in sorted(self.iter_fdoc_keys())
                 if key in self._new or key in self._dirty)
 
-    def all_msg_dict_for_mbox(self, mbox):
-        """
-        Return all the message dicts for a given mbox.
-
-        :param mbox: the mailbox
-        :type mbox: str or unicode
-        :return: list of dictionaries
-        :rtype: list
-        """
-        # This *needs* to return a fixed sequence. Otherwise the dictionary len
-        # will change during iteration, when we modify it
-        return [self._msg_store[(mb, uid)]
-                for mb, uid in self._msg_store if mb == mbox]
-
     def all_deleted_uid_iter(self, mbox):
         """
         Return a list with the UIDs for all messags
@@ -763,11 +764,10 @@ class MemoryStore(object):
         """
         # This *needs* to return a fixed sequence. Otherwise the dictionary len
         # will change during iteration, when we modify it
-        all_deleted = [
-            msg['fdoc']['uid'] for msg in self.all_msg_dict_for_mbox(mbox)
-            if msg.get('fdoc', None)
-            and fields.DELETED_FLAG in msg['fdoc']['flags']]
-        return all_deleted
+        fdocs = self._fdoc_store[mbox]
+        return [uid for uid, value
+                in fdocs.items()
+                if fields.DELETED_FLAG in value["flags"]]
 
     # new, dirty flags
 
@@ -780,6 +780,7 @@ class MemoryStore(object):
         :return: tuple of bools
         :rtype: tuple
         """
+        # TODO change indexing of sets to [mbox][key] too.
         # XXX should return *first* the news, and *then* the dirty...
         return map(lambda _set: key in _set, (self._new, self._dirty))
 
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 3fbe2ad..3d25598 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1290,10 +1290,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         :rtype: int
         """
-        # XXX get this from a public method in memstore
-        store = self.memstore._msg_store
-        return len([uid for (mbox, uid) in store.keys()
-                    if mbox == self.mbox])
+        return self.memstore.count(self.mbox)
 
     # unseen messages
 
@@ -1305,10 +1302,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :return: iterator through unseen message doc UIDs
         :rtype: iterable
         """
-        # XXX get this from a public method in memstore
-        store = self.memstore._msg_store
-        return (uid for (mbox, uid), d in store.items()
-                if mbox == self.mbox and "\\Seen" not in d["fdoc"]["flags"])
+        return self.memstore.unseen_iter(self.mbox)
 
     def count_unseen(self):
         """
-- 
cgit v1.2.3


From 3149bbe64346d558ef300a3d760732cf499a28d3 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 7 Feb 2014 02:38:00 -0400
Subject: make fdoc, hdoc, chash 'public' properties

---
 src/leap/mail/imap/messages.py | 87 +++++++++++++-----------------------------
 1 file changed, 26 insertions(+), 61 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 3d25598..fbae05f 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -132,7 +132,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
     # XXX make these properties public
 
     @property
-    def _fdoc(self):
+    def fdoc(self):
         """
         An accessor to the flags document.
         """
@@ -149,7 +149,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             return fdoc
 
     @property
-    def _hdoc(self):
+    def hdoc(self):
         """
         An accessor to the headers document.
         """
@@ -161,23 +161,23 @@ class LeapMessage(fields, MailParser, MBoxParser):
         return self._get_headers_doc()
 
     @property
-    def _chash(self):
+    def chash(self):
         """
         An accessor to the content hash for this message.
         """
-        if not self._fdoc:
+        if not self.fdoc:
             return None
-        if not self.__chash and self._fdoc:
-            self.__chash = self._fdoc.content.get(
+        if not self.__chash and self.fdoc:
+            self.__chash = self.fdoc.content.get(
                 fields.CONTENT_HASH_KEY, None)
         return self.__chash
 
     @property
-    def _bdoc(self):
+    def bdoc(self):
         """
         An accessor to the body document.
         """
-        if not self._hdoc:
+        if not self.hdoc:
             return None
         if not self.__bdoc:
             self.__bdoc = self._get_body_doc()
@@ -204,7 +204,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         uid = self._uid
 
         flags = set([])
-        fdoc = self._fdoc
+        fdoc = self.fdoc
         if fdoc:
             flags = set(fdoc.content.get(self.FLAGS_KEY, None))
 
@@ -232,7 +232,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
         log.msg('setting flags: %s (%s)' % (self._uid, flags))
 
-        doc = self._fdoc
+        doc = self.fdoc
         if not doc:
             logger.warning(
                 "Could not find FDOC for %s:%s while setting flags!" %
@@ -284,7 +284,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: An RFC822-formatted date string.
         :rtype: str
         """
-        date = self._hdoc.content.get(fields.DATE_KEY, '')
+        date = self.hdoc.content.get(fields.DATE_KEY, '')
         return date
 
     #
@@ -310,8 +310,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
         fd = StringIO.StringIO()
 
-        if self._bdoc is not None:
-            bdoc_content = self._bdoc.content
+        if self.bdoc is not None:
+            bdoc_content = self.bdoc.content
             if empty(bdoc_content):
                 logger.warning("No BDOC content found for message!!!")
                 return write_fd("")
@@ -360,8 +360,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :rtype: int
         """
         size = None
-        if self._fdoc:
-            fdoc_content = self._fdoc.content
+        if self.fdoc is not None:
+            fdoc_content = self.fdoc.content
             size = fdoc_content.get(self.SIZE_KEY, False)
         else:
             logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
@@ -430,8 +430,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         Return the headers dict for this message.
         """
-        if self._hdoc is not None:
-            hdoc_content = self._hdoc.content
+        if self.hdoc is not None:
+            hdoc_content = self.hdoc.content
             headers = hdoc_content.get(self.HEADERS_KEY, {})
             return headers
 
@@ -445,8 +445,8 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         Return True if this message is multipart.
         """
-        if self._fdoc:
-            fdoc_content = self._fdoc.content
+        if self.fdoc:
+            fdoc_content = self.fdoc.content
             is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
             return is_multipart
         else:
@@ -485,11 +485,11 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :raises: KeyError if key does not exist
         :rtype: dict
         """
-        if not self._hdoc:
+        if not self.hdoc:
             logger.warning("Tried to get part but no HDOC found!")
             return None
 
-        hdoc_content = self._hdoc.content
+        hdoc_content = self.hdoc.content
         pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
 
         # remember, lads, soledad is using strings in its keys,
@@ -523,7 +523,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         head_docs = self._soledad.get_from_index(
             fields.TYPE_C_HASH_IDX,
-            fields.TYPE_HEADERS_VAL, str(self._chash))
+            fields.TYPE_HEADERS_VAL, str(self.chash))
         return first(head_docs)
 
     def _get_body_doc(self):
@@ -531,7 +531,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         Return the document that keeps the body for this
         message.
         """
-        hdoc_content = self._hdoc.content
+        hdoc_content = self.hdoc.content
         body_phash = hdoc_content.get(
             fields.BODY_KEY, None)
         if not body_phash:
@@ -568,14 +568,14 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :return: The content value indexed by C{key} or None
         :rtype: str
         """
-        return self._fdoc.content.get(key, None)
+        return self.fdoc.content.get(key, None)
 
     def does_exist(self):
         """
         Return True if there is actually a flags document for this
         UID and mbox.
         """
-        return not empty(self._fdoc)
+        return not empty(self.fdoc)
 
 
 class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
@@ -680,8 +680,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     _rdoc_lock = threading.Lock()
     _rdoc_property_lock = threading.Lock()
-    _hdocset_lock = threading.Lock()
-    _hdocset_property_lock = threading.Lock()
 
     def __init__(self, mbox=None, soledad=None, memstore=None):
         """
@@ -722,7 +720,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         self.memstore = memstore
 
         self.__rflags = None
-        self.__hdocset = None
         self.initialize_db()
 
         # ensure that we have a recent-flags and a hdocs-sec doc
@@ -751,18 +748,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                 rdoc[fields.MBOX_KEY] = self.mbox
             self._soledad.create_doc(rdoc)
 
-    def _get_or_create_hdocset(self):
-        """
-        Try to retrieve the hdocs-set doc for this MessageCollection,
-        and create one if not found.
-        """
-        hdocset = self._get_hdocset_doc()
-        if not hdocset:
-            hdocset = self._get_empty_doc(self.HDOCS_SET_DOC)
-            if self.mbox != fields.INBOX_VAL:
-                hdocset[fields.MBOX_KEY] = self.mbox
-            self._soledad.create_doc(hdocset)
-
     @deferred_to_thread
     def _do_parse(self, raw):
         """
@@ -1257,32 +1242,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                 fields.TYPE_FLAGS_VAL, self.mbox)))
         return all_flags
 
-    # XXX Move to memstore too. But we don't need it really, since
-    # we can cache the headers docs too.
-    #def all_flags_chash(self):
-        #"""
-        #Return a dict with the content-hash for all flag documents
-        #for this mailbox.
-        #"""
-        #all_flags_chash = dict(((
-            #doc.content[self.UID_KEY],
-            #doc.content[self.CONTENT_HASH_KEY]) for doc in
-            #self._soledad.get_from_index(
-                #fields.TYPE_MBOX_IDX,
-                #fields.TYPE_FLAGS_VAL, self.mbox)))
-        #return all_flags_chash
-
-    # XXX get from memstore
+    # TODO get from memstore
     def all_headers(self):
         """
         Return a dict with all the headers documents for this
         mailbox.
         """
-        all_headers = dict(((
-            doc.content[self.CONTENT_HASH_KEY],
-            doc.content[self.HEADERS_KEY]) for doc in
-            self._soledad.get_docs(self._hdocset)))
-        return all_headers
 
     def count(self):
         """
-- 
cgit v1.2.3


From 813db4a356141592337f39f9c801203367c63193 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 7 Feb 2014 02:54:52 -0400
Subject: remove hdoc copy since it's in its own structure now

---
 src/leap/mail/imap/mailbox.py     | 22 +++++++++-------------
 src/leap/mail/imap/memorystore.py | 17 ++++++++++++++++-
 src/leap/mail/imap/messages.py    | 14 +++++++++++---
 3 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index c188f91..6e472ee 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -824,12 +824,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         memstore = self._memstore
 
         def createCopy(result):
-            exist, new_fdoc, hdoc = result
+            exist, new_fdoc = result
             if exist:
                 # Should we signal error on the callback?
                 logger.warning("Destination message already exists!")
 
-                # XXX I'm still not clear if we should raise the
+                # XXX I'm not sure if we should raise the
                 # errback. This actually rases an ugly warning
                 # in some muas like thunderbird. I guess the user does
                 # not deserve that.
@@ -848,8 +848,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
                 self._memstore.create_message(
                     self.mbox, uid_next,
-                    MessageWrapper(
-                        new_fdoc, hdoc),
+                    MessageWrapper(new_fdoc),
                     observer=observer,
                     notify_on_disk=False)
 
@@ -862,6 +861,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Get a copy of the fdoc for this message, and check whether
         it already exists.
+
+        :return: exist, new_fdoc
+        :rtype: tuple
         """
         # XXX  for clarity, this could be delegated to a
         # MessageCollection mixin that implements copy too, and
@@ -869,22 +871,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         msg = message
         memstore = self._memstore
 
-        # XXX should use a public api instead
-        fdoc = msg._fdoc
-        hdoc = msg._hdoc
-        if not fdoc:
+        if empty(msg.fdoc):
             logger.warning("Tried to copy a MSG with no fdoc")
             return
-        new_fdoc = copy.deepcopy(fdoc.content)
-        copy_hdoc = copy.deepcopy(hdoc.content)
+        new_fdoc = copy.deepcopy(msg.fdoc.content)
         fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
 
-        # XXX is this hitting the db??? --- probably.
-        # We should profile after the pre-fetch.
         dest_fdoc = memstore.get_fdoc_from_chash(
             fdoc_chash, self.mbox)
         exist = dest_fdoc and not empty(dest_fdoc.content)
-        return exist, new_fdoc, copy_hdoc
+        return exist, new_fdoc
 
     # convenience fun
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index b198e12..4156c0b 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -592,7 +592,8 @@ class MemoryStore(object):
 
         :param mbox: the mailbox
         :type mbox: str or unicode
-        :param flag_docs: a dict with the content for the flag docs.
+        :param flag_docs: a dict with the content for the flag docs, indexed
+                          by uid.
         :type flag_docs: dict
         """
         # We can do direct assignments cause we know this will only
@@ -601,6 +602,20 @@ class MemoryStore(object):
         for uid in flag_docs:
             fdoc_store[uid] = ReferenciableDict(flag_docs[uid])
 
+    def load_header_docs(self, header_docs):
+        """
+        Load the flag documents for the given mbox.
+        Used during header docs prefetch, and during cache after
+        a read from soledad if the hdoc property in message did not
+        find its value in here.
+
+        :param flag_docs: a dict with the content for the flag docs.
+        :type flag_docs: dict
+        """
+        hdoc_store = self._hdoc_store
+        for chash in header_docs:
+            hdoc_store[chash] = ReferenciableDict(header_docs[chash])
+
     def all_flags(self, mbox):
         """
         Return a dictionary with all the flags for a given mbox.
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index fbae05f..4b95689 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -153,12 +153,20 @@ class LeapMessage(fields, MailParser, MBoxParser):
         """
         An accessor to the headers document.
         """
-        if self._container is not None:
+        container = self._container
+        if container is not None:
             hdoc = self._container.hdoc
             if hdoc and not empty(hdoc.content):
                 return hdoc
-        # XXX cache this into the memory store !!!
-        return self._get_headers_doc()
+        hdoc = self._get_headers_doc()
+
+        if container and not empty(hdoc.content):
+            # mem-cache it
+            hdoc_content = hdoc.content
+            chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
+            hdocs = {chash: hdoc_content}
+            container.memstore.load_header_docs(hdocs)
+        return hdoc
 
     @property
     def chash(self):
-- 
cgit v1.2.3


From b92e63c316c1cf9f8b6481dbfa70737acfb3eee9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 7 Feb 2014 05:50:55 -0400
Subject: separate better dirty/new flags; add cdocs

---
 src/leap/mail/imap/memorystore.py  | 88 +++++++++++++++++++++++++-------------
 src/leap/mail/imap/messages.py     | 21 ++++-----
 src/leap/mail/imap/soledadstore.py | 22 ++++++++--
 src/leap/mail/utils.py             | 19 ++++++++
 4 files changed, 106 insertions(+), 44 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 4156c0b..ee3ee92 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -24,6 +24,7 @@ import weakref
 
 from collections import defaultdict
 from copy import copy
+from itertools import chain
 
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
@@ -33,7 +34,7 @@ from zope.interface import implements
 from leap.common.check import leap_assert_type
 from leap.mail import size
 from leap.mail.decorators import deferred_to_thread
-from leap.mail.utils import empty
+from leap.mail.utils import empty, phash_iter
 from leap.mail.messageflow import MessageProducer
 from leap.mail.imap import interfaces
 from leap.mail.imap.fields import fields
@@ -110,13 +111,12 @@ class MemoryStore(object):
 
         # Internal Storage: messages
         """
-        Flags document store.
+        flags document store.
         _fdoc_store[mbox][uid] = { 'content': 'aaa' }
         """
         self._fdoc_store = defaultdict(lambda: defaultdict(
             lambda: ReferenciableDict({})))
 
-<<<<<<< HEAD
         # Sizes
         """
         {'mbox, uid': <int>}
@@ -124,9 +124,14 @@ class MemoryStore(object):
         self._sizes = {}
 
         # Internal Storage: payload-hash
-=======
+        """
+        fdocs:doc-id store, stores document IDs for putting
+        the dirty flags-docs.
+        """
+        self._fdoc_id_store = defaultdict(lambda: defaultdict(
+            lambda: ''))
+
         # Internal Storage: content-hash:hdoc
->>>>>>> change internal storage and keying scheme in memstore
         """
         hdoc-store keeps references to
         the header-documents indexed by content-hash.
@@ -360,14 +365,6 @@ class MemoryStore(object):
         self._sizes[key] = size.get_size(self._fdoc_store[key])
         # TODO add hdoc and cdocs sizes too
 
-        # XXX what to do with this?
-        #docs_id = msg_dict.get(DOCS_ID, None)
-        #if docs_id is not None:
-            #if not store.get(DOCS_ID, None):
-                #store[DOCS_ID] = {}
-            #store[DOCS_ID].update(docs_id)
-
-
     def get_docid_for_fdoc(self, mbox, uid):
         """
         Return Soledad document id for the flags-doc for a given mbox and uid,
@@ -379,13 +376,18 @@ class MemoryStore(object):
         :type uid: int
         :rtype: unicode or None
         """
-        fdoc = self._permanent_store.get_flags_doc(mbox, uid)
-        if empty(fdoc):
-            return None
-        doc_id = fdoc.doc_id
+        doc_id = self._fdoc_id_store[mbox][uid]
+
+        if empty(doc_id):
+            fdoc = self._permanent_store.get_flags_doc(mbox, uid)
+            if empty(fdoc.content):
+                return None
+            doc_id = fdoc.doc_id
+            self._fdoc_id_store[mbox][uid] = doc_id
+
         return doc_id
 
-    def get_message(self, mbox, uid, flags_only=False):
+    def get_message(self, mbox, uid, dirtystate="none", flags_only=False):
         """
         Get a MessageWrapper for the given mbox and uid combination.
 
@@ -393,19 +395,32 @@ class MemoryStore(object):
         :type mbox: str or unicode
         :param uid: the message UID
         :type uid: int
+        :param dirtystate: one of `dirty`, `new` or `none` (default)
+        :type dirtystate: str
         :param flags_only: whether the message should carry only a reference
                            to the flags document.
         :type flags_only: bool
+        :
 
         :return: MessageWrapper or None
         """
+        if dirtystate == "dirty":
+            flags_only = True
+
         key = mbox, uid
 
         fdoc = self._fdoc_store[mbox][uid]
         if empty(fdoc):
             return None
 
-        new, dirty = self._get_new_dirty_state(key)
+        new, dirty = False, False
+        if dirtystate == "none":
+            new, dirty = self._get_new_dirty_state(key)
+        if dirtystate == "dirty":
+            new, dirty = False, True
+        if dirtystate == "new":
+            new, dirty = True, False
+
         if flags_only:
             return MessageWrapper(fdoc=fdoc,
                                   new=new, dirty=dirty,
@@ -413,7 +428,22 @@ class MemoryStore(object):
         else:
             chash = fdoc.get(fields.CONTENT_HASH_KEY)
             hdoc = self._hdoc_store[chash]
-            return MessageWrapper(fdoc=fdoc, hdoc=hdoc,
+            if empty(hdoc):
+                hdoc = self._permanent_store.get_headers_doc(chash)
+                if not empty(hdoc.content):
+                    self._hdoc_store[chash] = hdoc.content
+                    hdoc = hdoc.content
+            cdocs = None
+
+            pmap = hdoc.get(fields.PARTS_MAP_KEY, None)
+            if new and pmap is not None:
+                # take the different cdocs for write...
+                cdoc_store = self._cdoc_store
+                cdocs_list = phash_iter(hdoc)
+                cdocs = dict(enumerate(
+                    [cdoc_store[phash] for phash in cdocs_list], 1))
+
+            return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs,
                                   new=new, dirty=dirty,
                                   memstore=weakref.proxy(self))
 
@@ -437,14 +467,9 @@ class MemoryStore(object):
             key = mbox, uid
             self._new.discard(key)
             self._dirty.discard(key)
-<<<<<<< HEAD
-            self._msg_store.pop(key, None)
             if key in self._sizes:
                 del self._sizes[key]
-
-=======
             self._fdoc_store[mbox].pop(uid, None)
->>>>>>> change internal storage and keying scheme in memstore
         except Exception as exc:
             logger.exception(exc)
 
@@ -464,7 +489,7 @@ class MemoryStore(object):
         # XXX this could return the deferred for all the enqueued operations
 
         if not self.producer.is_queue_empty():
-            return
+            return False
 
         if any(map(lambda i: not empty(i), (self._new, self._dirty))):
             logger.info("Writing messages to Soledad...")
@@ -598,6 +623,7 @@ class MemoryStore(object):
         """
         # We can do direct assignments cause we know this will only
         # be called during initialization of the mailbox.
+
         fdoc_store = self._fdoc_store[mbox]
         for uid in flag_docs:
             fdoc_store[uid] = ReferenciableDict(flag_docs[uid])
@@ -626,7 +652,8 @@ class MemoryStore(object):
         """
         flags_dict = {}
         uids = self.get_uids(mbox)
-        fdoc_store = self._fdoc_store
+        fdoc_store = self._fdoc_store[mbox]
+
         for uid in uids:
             try:
                 flags = fdoc_store[uid][fields.FLAGS_KEY]
@@ -763,9 +790,10 @@ class MemoryStore(object):
         :return: generator of MessageWrappers
         :rtype: generator
         """
-        return (self.get_message(*key)
-                for key in sorted(self.iter_fdoc_keys())
-                if key in self._new or key in self._dirty)
+        gm = self.get_message
+        new = (gm(*key) for key in self._new)
+        dirty = (gm(*key, flags_only=True) for key in self._dirty)
+        return chain(new, dirty)
 
     def all_deleted_uid_iter(self, mbox):
         """
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 4b95689..8b6d3f3 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -264,17 +264,15 @@ class LeapMessage(fields, MailParser, MBoxParser):
             # to put it under the lock...
             doc.content[self.FLAGS_KEY] = newflags
             doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+
+            # XXX check if this is working ok.
             doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
 
-            if self._collection.memstore is not None:
-                log.msg("putting message in collection")
-                self._collection.memstore.put_message(
-                    self._mbox, self._uid,
-                    MessageWrapper(fdoc=doc.content, new=False, dirty=True,
-                                   docs_id={'fdoc': doc.doc_id}))
-            else:
-                # fallback for non-memstore initializations.
-                self._soledad.put_doc(doc)
+            log.msg("putting message in collection")
+            self._collection.memstore.put_message(
+                self._mbox, self._uid,
+                MessageWrapper(fdoc=doc.content, new=False, dirty=True,
+                               docs_id={'fdoc': doc.doc_id}))
         return map(str, newflags)
 
     def getInternalDate(self):
@@ -524,6 +522,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         finally:
             return result
 
+    # TODO move to soledadstore instead of accessing soledad directly
     def _get_headers_doc(self):
         """
         Return the document that keeps the headers for this
@@ -534,6 +533,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             fields.TYPE_HEADERS_VAL, str(self.chash))
         return first(head_docs)
 
+    # TODO move to soledadstore instead of accessing soledad directly
     def _get_body_doc(self):
         """
         Return the document that keeps the body for this
@@ -1165,7 +1165,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                  or None if not found.
         :rtype: LeapMessage
         """
-        msg_container = self.memstore.get_message(self.mbox, uid, flags_only)
+        msg_container = self.memstore.get_message(
+            self.mbox, uid, flags_only=flags_only)
         if msg_container is not None:
             if mem_only:
                 msg = LeapMessage(None, uid, self.mbox, collection=self,
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index a74b49c..6cd3749 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -212,10 +212,8 @@ class SoledadStore(ContentDedup):
                       to be inserted.
         :type queue: Queue
         """
-        # TODO should delete the original message from incoming only after
-        # the writes are done.
         # TODO should handle the delete case
-        # TODO should handle errors
+        # TODO should handle errors better
         # TODO could generalize this method into a generic consumer
         # and only implement `process` here
 
@@ -235,7 +233,7 @@ class SoledadStore(ContentDedup):
             Errorback for write operations.
             """
             log.msg("ERROR: Error while processing item.")
-            log.msg(failure.getTraceBack())
+            log.msg(failure.getTraceback())
 
         while not queue.empty():
             doc_wrapper = queue.get()
@@ -354,6 +352,7 @@ class SoledadStore(ContentDedup):
                 doc = self._GET_DOC_FUN(doc_id)
                 doc.content = dict(item.content)
                 item = doc
+
             try:
                 call(item)
             except u1db_errors.RevisionConflict as exc:
@@ -451,6 +450,7 @@ class SoledadStore(ContentDedup):
         :type mbox: str or unicode
         :param uid: the UID for the message
         :type uid: int
+        :rtype: SoledadDocument or None
         """
         result = None
         try:
@@ -465,6 +465,20 @@ class SoledadStore(ContentDedup):
         finally:
             return result
 
+    def get_headers_doc(self, chash):
+        """
+        Return the document that keeps the headers for a message
+        indexed by its content-hash.
+
+        :param chash: the content-hash to retrieve the document from.
+        :type chash: str or unicode
+        :rtype: SoledadDocument or None
+        """
+        head_docs = self._soledad.get_from_index(
+            fields.TYPE_C_HASH_IDX,
+            fields.TYPE_HEADERS_VAL, str(chash))
+        return first(head_docs)
+
     def write_last_uid(self, mbox, value):
         """
         Write the `last_uid` integer to the proper mailbox document
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 942acfb..8b75cfc 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -94,6 +94,7 @@ def lowerdict(_dict):
 
 
 PART_MAP = "part_map"
+PHASH = "phash"
 
 
 def _str_dict(d, k):
@@ -130,6 +131,24 @@ def stringify_parts_map(d):
     return d
 
 
+def phash_iter(d):
+    """
+    A recursive generator that extracts all the payload-hashes
+    from an arbitrary nested parts-map dictionary.
+
+    :param d: the dictionary to walk
+    :type d: dictionary
+    :return: a list of all the phashes found
+    :rtype: list
+    """
+    if PHASH in d:
+        yield d[PHASH]
+    if PART_MAP in d:
+        for key in d[PART_MAP]:
+            for phash in phash_iter(d[PART_MAP][key]):
+                yield phash
+
+
 class CustomJsonScanner(object):
     """
     This class is a context manager definition used to monkey patch the default
-- 
cgit v1.2.3


From ce55f761a55f78cb122296e91686fa6fde8959b8 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 7 Feb 2014 07:00:47 -0400
Subject: two versions of accumulator util

---
 src/leap/mail/utils.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 80 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 8b75cfc..3ba4291 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -17,10 +17,10 @@
 """
 Mail utilities.
 """
-import copy
 import json
 import re
 import traceback
+import Queue
 
 from leap.soledad.common.document import SoledadDocument
 
@@ -149,6 +149,85 @@ def phash_iter(d):
                 yield phash
 
 
+def accumulator(fun, lim):
+    """
+    A simple accumulator that uses a closure and a mutable
+    object to collect items.
+    When the count of items is greater than `lim`, the
+    collection is flushed after invoking a map of the function `fun`
+    over it.
+
+    The returned accumulator can also be flushed at any moment
+    by passing a boolean as a second parameter.
+
+    :param fun: the function to call over the collection
+                when its size is greater than `lim`
+    :type fun: callable
+    :param lim: the turning point for the collection
+    :type lim: int
+    :rtype: function
+
+    >>> from pprint import pprint
+    >>> acc = accumulator(pprint, 2)
+    >>> acc(1)
+    >>> acc(2)
+    [1, 2]
+    >>> acc(3)
+    >>> acc(4)
+    [3, 4]
+    >>> acc = accumulator(pprint, 5)
+    >>> acc(1)
+    >>> acc(2)
+    >>> acc(3)
+    >>> acc(None, flush=True)
+    [1,2,3]
+    """
+    KEY = "items"
+    _o = {KEY: []}
+
+    def _accumulator(item, flush=False):
+        collection = _o[KEY]
+        collection.append(item)
+        if len(collection) >= lim or flush:
+            map(fun, filter(None, collection))
+            _o[KEY] = []
+
+    return _accumulator
+
+
+def accumulator_queue(fun, lim):
+    """
+    A version of the accumulator that uses a queue.
+
+    When the count of items is greater than `lim`, the
+    queue is flushed after invoking the function `fun`
+    over its items.
+
+    The returned accumulator can also be flushed at any moment
+    by passing a boolean as a second parameter.
+
+    :param fun: the function to call over the collection
+                when its size is greater than `lim`
+    :type fun: callable
+    :param lim: the turning point for the collection
+    :type lim: int
+    :rtype: function
+    """
+    _q = Queue.Queue()
+
+    def _accumulator(item, flush=False):
+        _q.put(item)
+        if _q.qsize() >= lim or flush:
+            collection = [_q.get() for i in range(_q.qsize())]
+            map(fun, filter(None, collection))
+
+    return _accumulator
+
+
+#
+# String manipulation
+#
+
 class CustomJsonScanner(object):
     """
     This class is a context manager definition used to monkey patch the default
-- 
cgit v1.2.3


From 9ffcaa09c2d6a57f3f34350298eff8412b540bc9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 7 Feb 2014 07:27:38 -0400
Subject: defer_to_thread the bulk of write operations

and batch the notifications back to the memorystore,
within the reactor thread.
---
 src/leap/mail/imap/memorystore.py  |  9 ++--
 src/leap/mail/imap/soledadstore.py | 88 +++++++++++++++-----------------------
 2 files changed, 39 insertions(+), 58 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index ee3ee92..786a9c4 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -380,7 +380,7 @@ class MemoryStore(object):
 
         if empty(doc_id):
             fdoc = self._permanent_store.get_flags_doc(mbox, uid)
-            if empty(fdoc.content):
+            if empty(fdoc) or empty(fdoc.content):
                 return None
             doc_id = fdoc.doc_id
             self._fdoc_id_store[mbox][uid] = doc_id
@@ -706,9 +706,10 @@ class MemoryStore(object):
         :rtype: iterable
         """
         fdocs = self._fdoc_store[mbox]
+
         return [uid for uid, value
                 in fdocs.items()
-                if fields.SEEN_FLAG not in value["flags"]]
+                if fields.SEEN_FLAG not in value.get(fields.FLAGS_KEY, [])]
 
     def get_cdoc_from_phash(self, phash):
         """
@@ -760,7 +761,7 @@ class MemoryStore(object):
         # We want to create a new one in this case.
         # Hmmm what if the deletion is un-done?? We would end with a
         # duplicate...
-        if fdoc and fields.DELETED_FLAG in fdoc[fields.FLAGS_KEY]:
+        if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []):
             return None
 
         uid = fdoc[fields.UID_KEY]
@@ -810,7 +811,7 @@ class MemoryStore(object):
         fdocs = self._fdoc_store[mbox]
         return [uid for uid, value
                 in fdocs.items()
-                if fields.DELETED_FLAG in value["flags"]]
+                if fields.DELETED_FLAG in value.get(fields.FLAGS_KEY, [])]
 
     # new, dirty flags
 
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 6cd3749..e7c6b29 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -23,7 +23,6 @@ import threading
 from itertools import chain
 
 from u1db import errors as u1db_errors
-from twisted.internet import defer
 from twisted.python import log
 from zope.interface import implements
 
@@ -35,7 +34,7 @@ from leap.mail.imap.messageparts import RecentFlagsDoc
 from leap.mail.imap.fields import fields
 from leap.mail.imap.interfaces import IMessageStore
 from leap.mail.messageflow import IMessageConsumer
-from leap.mail.utils import first, empty
+from leap.mail.utils import first, empty, accumulator_queue
 
 logger = logging.getLogger(__name__)
 
@@ -142,12 +141,18 @@ class SoledadStore(ContentDedup):
         :param soledad: the soledad instance
         :type soledad: Soledad
         """
+        from twisted.internet import reactor
         self._soledad = soledad
 
         self._CREATE_DOC_FUN = self._soledad.create_doc
         self._PUT_DOC_FUN = self._soledad.put_doc
         self._GET_DOC_FUN = self._soledad.get_doc
 
+        # we instantiate an accumulator to batch the notifications
+        self.docs_notify_queue = accumulator_queue(
+            lambda item: reactor.callFromThread(self._unset_new_dirty, item),
+            20)
+
     # IMessageStore
 
     # -------------------------------------------------------------------
@@ -202,7 +207,10 @@ class SoledadStore(ContentDedup):
 
     # IMessageConsumer
 
-    # It's not thread-safe to defer this to a different thread
+    # TODO should handle the delete case
+    # TODO should handle errors better
+    # TODO could generalize this method into a generic consumer
+    # and only implement `process` here
 
     def consume(self, queue):
         """
@@ -212,38 +220,16 @@ class SoledadStore(ContentDedup):
                       to be inserted.
         :type queue: Queue
         """
-        # TODO should handle the delete case
-        # TODO should handle errors better
-        # TODO could generalize this method into a generic consumer
-        # and only implement `process` here
-
         from twisted.internet import reactor
 
-        def docWriteCallBack(doc_wrapper):
-            """
-            Callback for a successful write of a document wrapper.
-            """
-            if isinstance(doc_wrapper, MessageWrapper):
-                # If everything went well, we can unset the new flag
-                # in the source store (memory store)
-                self._unset_new_dirty(doc_wrapper)
-
-        def docWriteErrorBack(failure):
-            """
-            Errorback for write operations.
-            """
-            log.msg("ERROR: Error while processing item.")
-            log.msg(failure.getTraceback())
-
         while not queue.empty():
             doc_wrapper = queue.get()
+            reactor.callInThread(self._consume_doc, doc_wrapper,
+                                 self.docs_notify_queue)
 
-            d = defer.Deferred()
-            d.addCallbacks(docWriteCallBack, docWriteErrorBack)
-            reactor.callLater(0, self._consume_doc, doc_wrapper, d)
+        # Queue empty, flush the notifications queue.
+        self.docs_notify_queue(None, flush=True)
 
-    # FIXME this should not run the callback in the deferred thred
-    @deferred_to_thread
     def _unset_new_dirty(self, doc_wrapper):
         """
         Unset the `new` and `dirty` flags for this document wrapper in the
@@ -252,49 +238,38 @@ class SoledadStore(ContentDedup):
         :param doc_wrapper: a MessageWrapper instance
         :type doc_wrapper: MessageWrapper
         """
-        # XXX debug msg id/mbox?
-        logger.info("unsetting new flag!")
-        doc_wrapper.new = False
-        doc_wrapper.dirty = False
+        if isinstance(doc_wrapper, MessageWrapper):
+            logger.info("unsetting new flag!")
+            doc_wrapper.new = False
+            doc_wrapper.dirty = False
 
-    def _consume_doc(self, doc_wrapper, deferred):
+    @deferred_to_thread
+    def _consume_doc(self, doc_wrapper, notify_queue):
         """
         Consume each document wrapper in a separate thread.
 
         :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance
         :type doc_wrapper: MessageWrapper or RecentFlagsDoc
-        :param deferred: a deferred that will be fired when the write operation
-                         has finished, either calling its callback or its
-                         errback depending on whether it succeed.
-        :type deferred: Deferred
         """
-        def notifyBack(failed, observer, doc_wrapper):
+        def queueNotifyBack(failed, doc_wrapper):
             if failed:
-                observer.errback(MsgWriteError(
-                    "There was an error writing the mesage"))
+                log.msg("There was an error writing the mesage...")
             else:
-                observer.callback(doc_wrapper)
+                notify_queue(doc_wrapper)
 
-        def doSoledadCalls(items, observer):
+        def doSoledadCalls(items):
             # we prime the generator, that should return the
             # message or flags wrapper item in the first place.
             doc_wrapper = items.next()
-            d_sol = self._soledad_write_document_parts(items)
-            d_sol.addCallback(notifyBack, observer, doc_wrapper)
-            d_sol.addErrback(ebSoledadCalls)
-
-        def ebSoledadCalls(failure):
-            log.msg(failure.getTraceback())
+            failed = self._soledad_write_document_parts(items)
+            queueNotifyBack(failed, doc_wrapper)
 
-        d = self._iter_wrapper_subparts(doc_wrapper)
-        d.addCallback(doSoledadCalls, deferred)
-        d.addErrback(ebSoledadCalls)
+        doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper))
 
     #
     # SoledadStore specific methods.
     #
 
-    @deferred_to_thread
     def _soledad_write_document_parts(self, items):
         """
         Write the document parts to soledad in a separate thread.
@@ -314,7 +289,6 @@ class SoledadStore(ContentDedup):
                 continue
         return failed
 
-    @deferred_to_thread
     def _iter_wrapper_subparts(self, doc_wrapper):
         """
         Return an iterator that will yield the doc_wrapper in the first place,
@@ -350,6 +324,12 @@ class SoledadStore(ContentDedup):
             if call == self._PUT_DOC_FUN:
                 doc_id = item.doc_id
                 doc = self._GET_DOC_FUN(doc_id)
+
+                if doc is None:
+                    logger.warning("BUG! Dirty doc but could not "
+                                   "find document %s" % (doc_id,))
+                    return
+
                 doc.content = dict(item.content)
                 item = doc
 
-- 
cgit v1.2.3


From 0d61ed62ed1a2b3bead50b16324d50acfe71727d Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:35:35 -0400
Subject: add profile-command utility

---
 src/leap/mail/imap/mailbox.py | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 6e472ee..122875b 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -50,6 +50,25 @@ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
 notifying clients of new messages. Use during stress tests.
 """
 NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False)
+PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
+
+if PROFILE_CMD:
+    import time
+
+    def _debugProfiling(result, cmdname, start):
+        took = (time.time() - start) * 1000
+        log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
+        return result
+
+    def do_profile_cmd(d, name):
+        """
+        Add the profiling debug to the passed callback.
+        :param d: deferred
+        :param name: name of the command
+        :type name: str
+        """
+        d.addCallback(_debugProfiling, name, time.time())
+        d.addErrback(lambda f: log.msg(f.getTraceback()))
 
 
 class SoledadMailbox(WithMsgFields, MBoxParser):
@@ -133,6 +152,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             self.prime_last_uid_to_memstore()
             self.prime_flag_docs_to_memstore()
 
+        from twisted.internet import reactor
+        self.reactor = reactor
+
     @property
     def listeners(self):
         """
@@ -711,14 +733,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :raise ReadOnlyMailbox: Raised if this mailbox is not open for
                                 read-write.
         """
-        from twisted.internet import reactor
         if not self.isWriteable():
             log.msg('read only mailbox!')
             raise imap4.ReadOnlyMailbox
 
         d = defer.Deferred()
-        deferLater(reactor, 0, self._do_store, messages_asked, flags,
-                   mode, uid, d)
+        self.reactor.callLater(0, self._do_store, messages_asked, flags,
+                               mode, uid, d)
+        if PROFILE_CMD:
+            do_profile_cmd(d, "STORE")
         return d
 
     def _do_store(self, messages_asked, flags, mode, uid, observer):
@@ -797,15 +820,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                  uid when the copy succeed.
         :rtype: Deferred
         """
-        from twisted.internet import reactor
-
         d = defer.Deferred()
-        # XXX this should not happen ... track it down,
-        # probably to FETCH...
-        if message is None:
-            log.msg("BUG: COPY found a None in passed message")
-            d.callback(None)
-        deferLater(reactor, 0, self._do_copy, message, d)
+        if PROFILE_CMD:
+            do_profile_cmd(d, "COPY")
+        d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
+        deferLater(self.reactor, 0, self._do_copy, message, d)
         return d
 
     def _do_copy(self, message, observer):
-- 
cgit v1.2.3


From 912873a939214bc805fb398bc5a2fe1949fe34d6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:37:23 -0400
Subject: do not get last_uid from the set of soledad messages

but always from the counter instead. once assigned, the uid must
never be reused, unless the uidvalidity mailbox value changes.
doing otherwise will cause messages not to be shown until next session.

Also, renamed get_mbox method for clarity.
---
 src/leap/mail/imap/mailbox.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 122875b..018f88e 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -108,6 +108,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     _listeners = defaultdict(set)
 
     next_uid_lock = threading.Lock()
+    last_uid_lock = threading.Lock()
 
     _fdoc_primed = {}
 
@@ -196,7 +197,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self.listeners.remove(listener)
 
     # TODO move completely to soledadstore, under memstore reponsibility.
-    def _get_mbox(self):
+    def _get_mbox_doc(self):
         """
         Return mailbox document.
 
@@ -220,7 +221,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :returns: tuple of flags for this mailbox
         :rtype: tuple of str
         """
-        mbox = self._get_mbox()
+        mbox = self._get_mbox_doc()
         if not mbox:
             return None
         flags = mbox.content.get(self.FLAGS_KEY, [])
@@ -235,7 +236,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         leap_assert(isinstance(flags, tuple),
                     "flags expected to be a tuple")
-        mbox = self._get_mbox()
+        mbox = self._get_mbox_doc()
         if not mbox:
             return None
         mbox.content[self.FLAGS_KEY] = map(str, flags)
@@ -250,7 +251,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :return: True if the mailbox is closed
         :rtype: bool
         """
-        mbox = self._get_mbox()
+        mbox = self._get_mbox_doc()
         return mbox.content.get(self.CLOSED_KEY, False)
 
     def _set_closed(self, closed):
@@ -261,7 +262,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :type closed: bool
         """
         leap_assert(isinstance(closed, bool), "closed needs to be boolean")
-        mbox = self._get_mbox()
+        mbox = self._get_mbox_doc()
         mbox.content[self.CLOSED_KEY] = closed
         self._soledad.put_doc(mbox)
 
@@ -290,8 +291,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Prime memstore with last_uid value
         """
-        set_exist = set(self.messages.all_uid_iter())
-        last = max(set_exist) if set_exist else 0
+        mbox = self._get_mbox_doc()
+        last = mbox.content.get('lastuid', 0)
         logger.info("Priming Soledad last_uid to %s" % (last,))
         self._memstore.set_last_soledad_uid(self.mbox, last)
 
@@ -321,7 +322,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :return: unique validity identifier
         :rtype: int
         """
-        mbox = self._get_mbox()
+        mbox = self._get_mbox_doc()
         return mbox.content.get(self.CREATED_KEY, 1)
 
     def getUID(self, message):
@@ -483,12 +484,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         exists = self.getMessageCount()
         recent = self.getRecentCount()
         logger.debug("NOTIFY (%r): there are %s messages, %s recent" % (
-            self.mbox,
-            exists,
-            recent))
+            self.mbox, exists, recent))
 
         for l in self.listeners:
-            logger.debug('notifying...')
             l.newMessages(exists, recent)
 
     # commands, do not rename methods
@@ -507,7 +505,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         # we should postpone the removal
 
         # XXX move to memory store??
-        self._soledad.delete_doc(self._get_mbox())
+        self._soledad.delete_doc(self._get_mbox_doc())
 
     def _close_cb(self, result):
         self.closed = True
@@ -756,7 +754,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :type observer: deferred
         """
         # XXX implement also sequence (uid = 0)
-        # XXX we should prevent cclient from setting Recent flag?
+        # XXX we should prevent client from setting Recent flag?
         leap_assert(not isinstance(flags, basestring),
                     "flags cannot be a string")
         flags = tuple(flags)
-- 
cgit v1.2.3


From 5e96a249ab541cedcc79e4e60f46cd4a187e47fb Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:39:43 -0400
Subject: fix repeated recent flag

---
 src/leap/mail/imap/mailbox.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 018f88e..fa97512 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -854,12 +854,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             else:
                 mbox = self.mbox
                 uid_next = memstore.increment_last_soledad_uid(mbox)
+
                 new_fdoc[self.UID_KEY] = uid_next
                 new_fdoc[self.MBOX_KEY] = mbox
 
                 flags = list(new_fdoc[self.FLAGS_KEY])
                 flags.append(fields.RECENT_FLAG)
-                new_fdoc[self.FLAGS_KEY] = flags
+                new_fdoc[self.FLAGS_KEY] = tuple(set(flags))
 
                 # FIXME set recent!
 
@@ -896,7 +897,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         dest_fdoc = memstore.get_fdoc_from_chash(
             fdoc_chash, self.mbox)
-        exist = dest_fdoc and not empty(dest_fdoc.content)
+
+        exist = not empty(dest_fdoc)
         return exist, new_fdoc
 
     # convenience fun
-- 
cgit v1.2.3


From a05a00a78d750904456a649334bba619b07a3380 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:40:30 -0400
Subject: call notify in reactor

---
 src/leap/mail/imap/server.py | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 89fb46d..3497a8b 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -56,6 +56,9 @@ class LeapIMAPServer(imap4.IMAP4Server):
         # populate the test account properly (and only once
         # per session)
 
+        from twisted.internet import reactor
+        self.reactor = reactor
+
     def lineReceived(self, line):
         """
         Attempt to parse a single line from the server.
@@ -141,30 +144,23 @@ class LeapIMAPServer(imap4.IMAP4Server):
                     imap4.IMAP4Server.arg_fetchatt)
 
     def on_fetch_finished(self, _, messages):
-        from twisted.internet import reactor
-
-        print "FETCH FINISHED -- NOTIFY NEW"
-        deferLater(reactor, 0, self.notifyNew)
-        deferLater(reactor, 0, self.mbox.unset_recent_flags, messages)
-        deferLater(reactor, 0, self.mbox.signal_unread_to_ui)
+        deferLater(self.reactor, 0, self.notifyNew)
+        deferLater(self.reactor, 0, self.mbox.unset_recent_flags, messages)
+        deferLater(self.reactor, 0, self.mbox.signal_unread_to_ui)
 
     def on_copy_finished(self, defers):
         d = defer.gatherResults(filter(None, defers))
 
         def when_finished(result):
-            log.msg("COPY FINISHED")
             self.notifyNew()
             self.mbox.signal_unread_to_ui()
         d.addCallback(when_finished)
-        #d.addCallback(self.notifyNew)
-        #d.addCallback(self.mbox.signal_unread_to_ui)
 
     def do_COPY(self, tag, messages, mailbox, uid=0):
-        from twisted.internet import reactor
         defers = []
         d = imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
         defers.append(d)
-        deferLater(reactor, 0, self.on_copy_finished, defers)
+        deferLater(self.reactor, 0, self.on_copy_finished, defers)
 
     select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
                    imap4.IMAP4Server.arg_astring)
@@ -173,8 +169,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
         """
         Notify new messages to listeners.
         """
-        print "TRYING TO NOTIFY NEW"
-        self.mbox.notify_new()
+        self.reactor.callFromThread(self.mbox.notify_new)
 
     def _cbSelectWork(self, mbox, cmdName, tag):
         """
-- 
cgit v1.2.3


From 080c9207c70572a02f33d50697f39878713ab6ac Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:41:05 -0400
Subject: make the condition optional

---
 src/leap/mail/imap/service/imap.py | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 726049c..6041961 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -129,7 +129,7 @@ class LeapIMAPFactory(ServerFactory):
         imapProtocol.factory = self
         return imapProtocol
 
-    def doStop(self, cv):
+    def doStop(self, cv=None):
         """
         Stops imap service (fetcher, factory and port).
 
@@ -142,21 +142,23 @@ class LeapIMAPFactory(ServerFactory):
         """
         ServerFactory.doStop(self)
 
-        def _stop_imap_cb():
-            logger.debug('Stopping in memory store.')
-            self._memstore.stop_and_flush()
-            while not self._memstore.producer.is_queue_empty():
-                logger.debug('Waiting for queue to be empty.')
-                # TODO use a gatherResults over the new/dirty deferred list,
-                # as in memorystore's expunge() method.
-                time.sleep(1)
-            # notify that service has stopped
-            logger.debug('Notifying that service has stopped.')
-            cv.acquire()
-            cv.notify()
-            cv.release()
-
-        return threads.deferToThread(_stop_imap_cb)
+        if cv is not None:
+            def _stop_imap_cb():
+                logger.debug('Stopping in memory store.')
+                self._memstore.stop_and_flush()
+                while not self._memstore.producer.is_queue_empty():
+                    logger.debug('Waiting for queue to be empty.')
+                    # TODO use a gatherResults over the new/dirty
+                    # deferred list,
+                    # as in memorystore's expunge() method.
+                    time.sleep(1)
+                # notify that service has stopped
+                logger.debug('Notifying that service has stopped.')
+                cv.acquire()
+                cv.notify()
+                cv.release()
+
+            return threads.deferToThread(_stop_imap_cb)
 
 
 def run_service(*args, **kwargs):
-- 
cgit v1.2.3


From de762b5c6e529f4e668bee1ec848eb1f6380369b Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:41:51 -0400
Subject: catch typeerror too in empty definition

---
 src/leap/mail/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index 3ba4291..fed24b3 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -49,7 +49,7 @@ def empty(thing):
         thing = thing.content
     try:
         return len(thing) == 0
-    except ReferenceError:
+    except (ReferenceError, TypeError):
         return True
 
 
@@ -267,6 +267,8 @@ class CustomJsonScanner(object):
         if not monkey_patched:
             return self._orig_scanstring(s, idx, *args, **kwargs)
 
+        # TODO profile to see if a compiled regex can get us some
+        # benefit here.
         found = False
         end = s.find("\"", idx)
         while not found:
-- 
cgit v1.2.3


From 4338368aa2ba0efaee742e9000e21b81af34d3db Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:43:14 -0400
Subject: separate new and dirty queues

---
 src/leap/mail/imap/memorystore.py  | 80 ++++++++++++++++++++++++++------------
 src/leap/mail/imap/messageparts.py | 25 ++++++------
 src/leap/mail/imap/soledadstore.py | 20 ++++++----
 src/leap/mail/messageflow.py       | 26 ++++++++++---
 4 files changed, 102 insertions(+), 49 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 786a9c4..a053f3f 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -24,7 +24,6 @@ import weakref
 
 from collections import defaultdict
 from copy import copy
-from itertools import chain
 
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
@@ -33,7 +32,6 @@ from zope.interface import implements
 
 from leap.common.check import leap_assert_type
 from leap.mail import size
-from leap.mail.decorators import deferred_to_thread
 from leap.mail.utils import empty, phash_iter
 from leap.mail.messageflow import MessageProducer
 from leap.mail.imap import interfaces
@@ -48,7 +46,7 @@ logger = logging.getLogger(__name__)
 
 # The default period to do writebacks to the permanent
 # soledad storage, in seconds.
-SOLEDAD_WRITE_PERIOD = 30
+SOLEDAD_WRITE_PERIOD = 15
 
 FDOC = MessagePartType.fdoc.key
 HDOC = MessagePartType.hdoc.key
@@ -106,6 +104,9 @@ class MemoryStore(object):
         :param write_period: the interval to dump messages to disk, in seconds.
         :type write_period: int
         """
+        from twisted.internet import reactor
+        self.reactor = reactor
+
         self._permanent_store = permanent_store
         self._write_period = write_period
 
@@ -195,11 +196,15 @@ class MemoryStore(object):
 
         # New and dirty flags, to set MessageWrapper State.
         self._new = set([])
+        self._new_queue = set([])
         self._new_deferreds = {}
+
         self._dirty = set([])
-        self._rflags_dirty = set([])
+        self._dirty_queue = set([])
         self._dirty_deferreds = {}
 
+        self._rflags_dirty = set([])
+
         # Flag for signaling we're busy writing to the disk storage.
         setattr(self, self.WRITING_FLAG, False)
 
@@ -297,7 +302,7 @@ class MemoryStore(object):
         """
         Put an existing message.
 
-        This will set the dirty flag on the MemoryStore.
+        This will also set the dirty flag on the MemoryStore.
 
         :param mbox: the mailbox
         :type mbox: str or unicode
@@ -498,9 +503,14 @@ class MemoryStore(object):
         # is accquired
         with set_bool_flag(self, self.WRITING_FLAG):
             for rflags_doc_wrapper in self.all_rdocs_iter():
-                self.producer.push(rflags_doc_wrapper)
-            for msg_wrapper in self.all_new_dirty_msg_iter():
-                self.producer.push(msg_wrapper)
+                self.producer.push(rflags_doc_wrapper,
+                                   state=self.producer.STATE_DIRTY)
+            for msg_wrapper in self.all_new_msg_iter():
+                self.producer.push(msg_wrapper,
+                                   state=self.producer.STATE_NEW)
+            for msg_wrapper in self.all_dirty_msg_iter():
+                self.producer.push(msg_wrapper,
+                                   state=self.producer.STATE_DIRTY)
 
     # MemoryStore specific methods.
 
@@ -784,17 +794,34 @@ class MemoryStore(object):
             for uid in fdoc_store[mbox]:
                 yield mbox, uid
 
-    def all_new_dirty_msg_iter(self):
+    def all_new_msg_iter(self):
         """
-        Return generator that iterates through all new and dirty messages.
+        Return generator that iterates through all new messages.
 
         :return: generator of MessageWrappers
         :rtype: generator
         """
         gm = self.get_message
-        new = (gm(*key) for key in self._new)
-        dirty = (gm(*key, flags_only=True) for key in self._dirty)
-        return chain(new, dirty)
+        new = [gm(*key) for key in self._new]
+        # move content from new set to the queue
+        self._new_queue.update(self._new)
+        self._new.difference_update(self._new)
+        return new
+
+    def all_dirty_msg_iter(self):
+        """
+        Return generator that iterates through all dirty messages.
+
+        :return: generator of MessageWrappers
+        :rtype: generator
+        """
+        gm = self.get_message
+        dirty = [gm(*key, flags_only=True) for key in self._dirty]
+        # move content from new and dirty sets to the queue
+
+        self._dirty_queue.update(self._dirty)
+        self._dirty.difference_update(self._dirty)
+        return dirty
 
     def all_deleted_uid_iter(self, mbox):
         """
@@ -826,25 +853,28 @@ class MemoryStore(object):
         """
         # TODO change indexing of sets to [mbox][key] too.
         # XXX should return *first* the news, and *then* the dirty...
+
+        # TODO should query in queues too , true?
+        #
         return map(lambda _set: key in _set, (self._new, self._dirty))
 
-    def set_new(self, key):
+    def set_new_queued(self, key):
         """
-        Add the key value to the `new` set.
+        Add the key value to the `new-queue` set.
 
         :param key: the key for the message, in the form mbox, uid
         :type key: tuple
         """
-        self._new.add(key)
+        self._new_queue.add(key)
 
-    def unset_new(self, key):
+    def unset_new_queued(self, key):
         """
-        Remove the key value from the `new` set.
+        Remove the key value from the `new-queue` set.
 
         :param key: the key for the message, in the form mbox, uid
         :type key: tuple
         """
-        self._new.discard(key)
+        self._new_queue.discard(key)
         deferreds = self._new_deferreds
         d = deferreds.get(key, None)
         if d:
@@ -853,23 +883,23 @@ class MemoryStore(object):
             d.callback('%s, ok' % str(key))
             deferreds.pop(key)
 
-    def set_dirty(self, key):
+    def set_dirty_queued(self, key):
         """
-        Add the key value to the `dirty` set.
+        Add the key value to the `dirty-queue` set.
 
         :param key: the key for the message, in the form mbox, uid
         :type key: tuple
         """
-        self._dirty.add(key)
+        self._dirty_queue.add(key)
 
-    def unset_dirty(self, key):
+    def unset_dirty_queued(self, key):
         """
-        Remove the key value from the `dirty` set.
+        Remove the key value from the `dirty-queue` set.
 
         :param key: the key for the message, in the form mbox, uid
         :type key: tuple
         """
-        self._dirty.discard(key)
+        self._dirty_queue.discard(key)
         deferreds = self._dirty_deferreds
         d = deferreds.get(key, None)
         if d:
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index b1f333a..9b7de86 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -98,7 +98,7 @@ class MessageWrapper(object):
     CDOCS = "cdocs"
     DOCS_ID = "docs_id"
 
-    # Using slots to limit some the memory footprint,
+    # Using slots to limit some the memory use,
     # Add your attribute here.
 
     __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"]
@@ -148,7 +148,7 @@ class MessageWrapper(object):
         """
         return self._new
 
-    def _set_new(self, value=True):
+    def _set_new(self, value=False):
         """
         Set the value for the `new` flag, and propagate it
         to the memory store if any.
@@ -161,8 +161,8 @@ class MessageWrapper(object):
             mbox = self.fdoc.content['mbox']
             uid = self.fdoc.content['uid']
             key = mbox, uid
-            fun = [self.memstore.unset_new,
-                   self.memstore.set_new][int(value)]
+            fun = [self.memstore.unset_new_queued,
+                   self.memstore.set_new_queued][int(value)]
             fun(key)
         else:
             logger.warning("Could not find a memstore referenced from this "
@@ -193,8 +193,8 @@ class MessageWrapper(object):
             mbox = self.fdoc.content['mbox']
             uid = self.fdoc.content['uid']
             key = mbox, uid
-            fun = [self.memstore.unset_dirty,
-                   self.memstore.set_dirty][int(value)]
+            fun = [self.memstore.unset_dirty_queued,
+                   self.memstore.set_dirty_queued][int(value)]
             fun(key)
         else:
             logger.warning("Could not find a memstore referenced from this "
@@ -271,11 +271,14 @@ class MessageWrapper(object):
         :rtype: generator
         """
         if self._dirty:
-            mbox = self.fdoc.content[fields.MBOX_KEY]
-            uid = self.fdoc.content[fields.UID_KEY]
-            docid_dict = self._dict[self.DOCS_ID]
-            docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc(
-                mbox, uid)
+            try:
+                mbox = self.fdoc.content[fields.MBOX_KEY]
+                uid = self.fdoc.content[fields.UID_KEY]
+                docid_dict = self._dict[self.DOCS_ID]
+                docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc(
+                    mbox, uid)
+            except Exception as exc:
+                logger.exception(exc)
 
         if not empty(self.fdoc.content):
             yield self.fdoc
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index e7c6b29..667e64d 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -220,12 +220,15 @@ class SoledadStore(ContentDedup):
                       to be inserted.
         :type queue: Queue
         """
-        from twisted.internet import reactor
-
-        while not queue.empty():
-            doc_wrapper = queue.get()
-            reactor.callInThread(self._consume_doc, doc_wrapper,
-                                 self.docs_notify_queue)
+        new, dirty = queue
+        while not new.empty():
+            doc_wrapper = new.get()
+            self.reactor.callInThread(self._consume_doc, doc_wrapper,
+                                      self.docs_notify_queue)
+        while not dirty.empty():
+            doc_wrapper = dirty.get()
+            self.reactor.callInThread(self._consume_doc, doc_wrapper,
+                                      self.docs_notify_queue)
 
         # Queue empty, flush the notifications queue.
         self.docs_notify_queue(None, flush=True)
@@ -239,7 +242,8 @@ class SoledadStore(ContentDedup):
         :type doc_wrapper: MessageWrapper
         """
         if isinstance(doc_wrapper, MessageWrapper):
-            logger.info("unsetting new flag!")
+            # XXX still needed for debug quite often
+            #logger.info("unsetting new flag!")
             doc_wrapper.new = False
             doc_wrapper.dirty = False
 
@@ -284,6 +288,8 @@ class SoledadStore(ContentDedup):
             try:
                 self._try_call(call, item)
             except Exception as exc:
+                logger.debug("ITEM WAS: %s" % str(item))
+                logger.debug("ITEM CONTENT WAS: %s" % str(item.content))
                 logger.exception(exc)
                 failed = True
                 continue
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index 80121c8..c8f224c 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -49,7 +49,7 @@ class IMessageProducer(Interface):
     entities.
     """
 
-    def push(self, item):
+    def push(self, item, state=None):
         """
         Push a new item in the queue.
         """
@@ -101,6 +101,10 @@ class MessageProducer(object):
     # and consumption is not likely (?) to consume huge amounts of memory in
     # our current settings, so the need to pause the stream is not urgent now.
 
+    # TODO use enum
+    STATE_NEW = 1
+    STATE_DIRTY = 2
+
     def __init__(self, consumer, queue=Queue.Queue, period=1):
         """
         Initializes the MessageProducer
@@ -115,7 +119,8 @@ class MessageProducer(object):
         # it should implement a `consume` method
         self._consumer = consumer
 
-        self._queue = queue()
+        self._queue_new = queue()
+        self._queue_dirty = queue()
         self._period = period
 
         self._loop = LoopingCall(self._check_for_new)
@@ -130,7 +135,7 @@ class MessageProducer(object):
         If the queue is found empty, the loop is stopped. It will be started
         again after the addition of new items.
         """
-        self._consumer.consume(self._queue)
+        self._consumer.consume((self._queue_new, self._queue_dirty))
         if self.is_queue_empty():
             self.stop()
 
@@ -138,11 +143,13 @@ class MessageProducer(object):
         """
         Return True if queue is empty, False otherwise.
         """
-        return self._queue.empty()
+        new = self._queue_new
+        dirty = self._queue_dirty
+        return new.empty() and dirty.empty()
 
     # public methods: IMessageProducer
 
-    def push(self, item):
+    def push(self, item, state=None):
         """
         Push a new item in the queue.
 
@@ -150,7 +157,14 @@ class MessageProducer(object):
         """
         # XXX this might raise if the queue does not accept any new
         # items. what to do then?
-        self._queue.put(item)
+        queue = self._queue_new
+
+        if state == self.STATE_NEW:
+            queue = self._queue_new
+        if state == self.STATE_DIRTY:
+            queue = self._queue_dirty
+
+        queue.put(item)
         self.start()
 
     def start(self):
-- 
cgit v1.2.3


From f869b7eecab67d07a23dfb8b2931b3844f7523e3 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:45:20 -0400
Subject: fine grained locks for puts

---
 src/leap/mail/imap/messages.py     | 35 +++++++++++++++++++++------------
 src/leap/mail/imap/soledadstore.py | 40 +++++++++++++++++++++++++++++++++-----
 2 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 8b6d3f3..de5dd1f 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -88,6 +88,13 @@ def try_unique_query(curried):
         logger.exception("Unhandled error %r" % exc)
 
 
+"""
+A dictionary that keeps one lock per mbox and uid.
+"""
+# XXX too much overhead?
+fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+
+
 class LeapMessage(fields, MailParser, MBoxParser):
     """
     The main representation of a message.
@@ -102,8 +109,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     implements(imap4.IMessage)
 
-    flags_lock = threading.Lock()
-
     def __init__(self, soledad, uid, mbox, collection=None, container=None):
         """
         Initializes a LeapMessage.
@@ -129,6 +134,9 @@ class LeapMessage(fields, MailParser, MBoxParser):
         self.__chash = None
         self.__bdoc = None
 
+        from twisted.internet import reactor
+        self.reactor = reactor
+
     # XXX make these properties public
 
     @property
@@ -238,20 +246,21 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :type mode: int
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        log.msg('setting flags: %s (%s)' % (self._uid, flags))
+        #log.msg('setting flags: %s (%s)' % (self._uid, flags))
 
-        doc = self.fdoc
-        if not doc:
-            logger.warning(
-                "Could not find FDOC for %s:%s while setting flags!" %
-                (self._mbox, self._uid))
-            return
+        mbox, uid = self._mbox, self._uid
 
         APPEND = 1
         REMOVE = -1
         SET = 0
 
-        with self.flags_lock:
+        with fdoc_locks[mbox][uid]:
+            doc = self.fdoc
+            if not doc:
+                logger.warning(
+                    "Could not find FDOC for %r:%s while setting flags!" %
+                    (mbox, uid))
+                return
             current = doc.content[self.FLAGS_KEY]
             if mode == APPEND:
                 newflags = tuple(set(tuple(current) + flags))
@@ -733,6 +742,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # ensure that we have a recent-flags and a hdocs-sec doc
         self._get_or_create_rdoc()
 
+        from twisted.internet import reactor
+        self.reactor = reactor
+
     def _get_empty_doc(self, _type=FLAGS_DOC):
         """
         Returns an empty doc for storing different message parts.
@@ -877,7 +889,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                  uid when the adding succeed.
         :rtype: deferred
         """
-        logger.debug('adding message')
+        logger.debug('Adding message')
         if flags is None:
             flags = tuple()
         leap_assert_type(flags, tuple)
@@ -921,7 +933,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             msg = self.get_msg_by_uid(uid)
 
             # TODO this cannot be deferred, this has to block.
-            #reactor.callLater(0, msg.setFlags, (fields.DELETED_FLAG,), -1)
             msg.setFlags((fields.DELETED_FLAG,), -1)
             reactor.callLater(0, observer.callback, uid)
             return
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 667e64d..9d19857 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -20,6 +20,7 @@ A MessageStore that writes to Soledad.
 import logging
 import threading
 
+from collections import defaultdict
 from itertools import chain
 
 from u1db import errors as u1db_errors
@@ -123,6 +124,17 @@ class MsgWriteError(Exception):
     """
     Raised if any exception is found while saving message parts.
     """
+    pass
+
+
+"""
+A lock per document.
+"""
+# TODO should bound the space of this!!!
+# http://stackoverflow.com/a/2437645/1157664
+# Setting this to twice the number of threads in the threadpool
+# should be safe.
+put_locks = defaultdict(lambda: threading.Lock())
 
 
 class SoledadStore(ContentDedup):
@@ -142,6 +154,8 @@ class SoledadStore(ContentDedup):
         :type soledad: Soledad
         """
         from twisted.internet import reactor
+        self.reactor = reactor
+
         self._soledad = soledad
 
         self._CREATE_DOC_FUN = self._soledad.create_doc
@@ -326,9 +340,9 @@ class SoledadStore(ContentDedup):
         if call is None:
             return
 
-        with self._soledad_rw_lock:
-            if call == self._PUT_DOC_FUN:
-                doc_id = item.doc_id
+        if call == self._PUT_DOC_FUN:
+            doc_id = item.doc_id
+            with put_locks[doc_id]:
                 doc = self._GET_DOC_FUN(doc_id)
 
                 if doc is None:
@@ -337,13 +351,26 @@ class SoledadStore(ContentDedup):
                     return
 
                 doc.content = dict(item.content)
+
                 item = doc
+                try:
+                    call(item)
+                except u1db_errors.RevisionConflict as exc:
+                    logger.exception("Error: %r" % (exc,))
+                    raise exc
+                except Exception as exc:
+                    logger.exception("Error: %r" % (exc,))
+                    raise exc
 
+        else:
             try:
                 call(item)
             except u1db_errors.RevisionConflict as exc:
                 logger.exception("Error: %r" % (exc,))
                 raise exc
+            except Exception as exc:
+                logger.exception("Error: %r" % (exc,))
+                raise exc
 
     def _get_calls_for_msg_parts(self, msg_wrapper):
         """
@@ -383,10 +410,11 @@ class SoledadStore(ContentDedup):
                 # XXX FIXME Give error if dirty and not doc_id !!!
                 doc_id = item.doc_id  # defend!
                 if not doc_id:
+                    logger.warning("Dirty item but no doc_id!")
                     continue
 
                 if item.part == MessagePartType.fdoc:
-                    logger.debug("PUT dirty fdoc")
+                    #logger.debug("PUT dirty fdoc")
                     yield item, call
 
                 # XXX also for linkage-doc !!!
@@ -443,6 +471,9 @@ class SoledadStore(ContentDedup):
             flag_docs = self._soledad.get_from_index(
                 fields.TYPE_MBOX_UID_IDX,
                 fields.TYPE_FLAGS_VAL, mbox, str(uid))
+            if len(flag_docs) != 1:
+                logger.warning("More than one flag doc for %r:%s" %
+                               (mbox, uid))
             result = first(flag_docs)
         except Exception as exc:
             # ugh! Something's broken down there!
@@ -506,7 +537,6 @@ class SoledadStore(ContentDedup):
                 fields.TYPE_MBOX_DEL_IDX,
                 fields.TYPE_FLAGS_VAL, mbox, '1'))
 
-    # TODO can deferToThread this?
     def remove_all_deleted(self, mbox):
         """
         Remove from Soledad all messages flagged as deleted for a given
-- 
cgit v1.2.3


From b83b94bf84ffb6d3bcabb192d64cf91fcfef96b4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:46:00 -0400
Subject: fix last_uid write to avoid updates to lesser values

---
 src/leap/mail/imap/soledadstore.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 9d19857..657f21f 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -515,11 +515,12 @@ class SoledadStore(ContentDedup):
         with self._last_uid_lock:
             mbox_doc = self._get_mbox_document(mbox)
             old_val = mbox_doc.content[key]
-            if value < old_val:
+            if value > old_val:
+                mbox_doc.content[key] = value
+                self._soledad.put_doc(mbox_doc)
+            else:
                 logger.error("%r:%s Tried to write a UID lesser than what's "
                              "stored!" % (mbox, value))
-            mbox_doc.content[key] = value
-            self._soledad.put_doc(mbox_doc)
 
     # deleted messages
 
-- 
cgit v1.2.3


From df9cd2e0d59600840acb6aa00f36b7eb43e48297 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:49:01 -0400
Subject: fix several bugs in copy/store

---
 src/leap/mail/imap/messages.py | 64 +++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 39 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index de5dd1f..bbc9deb 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -268,20 +268,12 @@ class LeapMessage(fields, MailParser, MBoxParser):
                 newflags = tuple(set(current).difference(set(flags)))
             elif mode == SET:
                 newflags = flags
+            new_fdoc = {
+                self.FLAGS_KEY: newflags,
+                self.SEEN_KEY: self.SEEN_FLAG in newflags,
+                self.DEL_KEY: self.DELETED_FLAG in newflags}
+            self._collection.memstore.update_flags(mbox, uid, new_fdoc)
 
-            # We could defer this, but I think it's better
-            # to put it under the lock...
-            doc.content[self.FLAGS_KEY] = newflags
-            doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-
-            # XXX check if this is working ok.
-            doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
-
-            log.msg("putting message in collection")
-            self._collection.memstore.put_message(
-                self._mbox, self._uid,
-                MessageWrapper(fdoc=doc.content, new=False, dirty=True,
-                               docs_id={'fdoc': doc.doc_id}))
         return map(str, newflags)
 
     def getInternalDate(self):
@@ -334,7 +326,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
             body = bdoc_content.get(self.RAW_KEY, "")
             content_type = bdoc_content.get('content-type', "")
             charset = find_charset(content_type)
-            logger.debug('got charset from content-type: %s' % charset)
+            #logger.debug('got charset from content-type: %s' % charset)
             if charset is None:
                 charset = self._get_charset(body)
             try:
@@ -855,8 +847,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :return: False, if it does not exist, or UID.
         """
         exist = False
-        if self.memstore is not None:
-            exist = self.memstore.get_fdoc_from_chash(chash, self.mbox)
+        exist = self.memstore.get_fdoc_from_chash(chash, self.mbox)
 
         if not exist:
             exist = self._get_fdoc_from_chash(chash)
@@ -1115,6 +1106,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
+    @deferred_to_thread
     def set_flags(self, mbox, messages, flags, mode, observer):
         """
         Set flags for a sequence of messages.
@@ -1132,28 +1124,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                          done.
         :type observer: deferred
         """
-        # XXX we could defer *this* to thread pool, and gather results...
-        # XXX use deferredList
+        reactor = self.reactor
+        getmsg = self.get_msg_by_uid
 
-        deferreds = []
-        for msg_id in messages:
-            deferreds.append(
-                self._set_flag_for_uid(msg_id, flags, mode))
+        def set_flags(uid, flags, mode):
+            msg = getmsg(uid, mem_only=True, flags_only=True)
+            if msg is not None:
+                return uid, msg.setFlags(flags, mode)
 
-        def notify(result):
-            observer.callback(dict(result))
-        d1 = defer.gatherResults(deferreds, consumeErrors=True)
-        d1.addCallback(notify)
+        result = dict(
+            set_flags(uid, tuple(flags), mode) for uid in messages)
 
-    @deferred_to_thread
-    def _set_flag_for_uid(self, msg_id, flags, mode):
-        """
-        Run the set_flag operation in the thread pool.
-        """
-        log.msg("MSG ID = %s" % msg_id)
-        msg = self.get_msg_by_uid(msg_id, mem_only=True, flags_only=True)
-        if msg is not None:
-            return msg_id, msg.setFlags(flags, mode)
+        reactor.callFromThread(observer.callback, result)
 
     # getters: generic for a mailbox
 
@@ -1229,7 +1211,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         db_uids = set([doc.content[self.UID_KEY] for doc in
                        self._soledad.get_from_index(
                            fields.TYPE_MBOX_IDX,
-                           fields.TYPE_FLAGS_VAL, self.mbox)])
+                           fields.TYPE_FLAGS_VAL, self.mbox)
+                       if not empty(doc)])
         return db_uids
 
     def all_uid_iter(self):
@@ -1254,12 +1237,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX we really could return a reduced version with
         # just {'uid': (flags-tuple,) since the prefetch is
         # only oriented to get the flag tuples.
-        all_flags = dict(((
+        all_docs = [(
             doc.content[self.UID_KEY],
-            dict(doc.content)) for doc in
+            dict(doc.content))
+            for doc in
             self._soledad.get_from_index(
                 fields.TYPE_MBOX_IDX,
-                fields.TYPE_FLAGS_VAL, self.mbox)))
+                fields.TYPE_FLAGS_VAL, self.mbox)
+            if not empty(doc.content)]
+        all_flags = dict(all_docs)
         return all_flags
 
     # TODO get from memstore
-- 
cgit v1.2.3


From 04dfa3afdbb2080c717bfd32d6e47641615967fc Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 01:52:21 -0400
Subject: improve flag-docs relative internal storage

---
 src/leap/mail/imap/memorystore.py | 58 +++++++++++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index a053f3f..2835826 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -92,6 +92,7 @@ class MemoryStore(object):
 
     WRITING_FLAG = "_writing"
     _last_uid_lock = threading.Lock()
+    _fdoc_docid_lock = threading.Lock()
 
     def __init__(self, permanent_store=None,
                  write_period=SOLEDAD_WRITE_PERIOD):
@@ -158,7 +159,7 @@ class MemoryStore(object):
                    'mbox-b': weakref.proxy(dict)}
         }
         """
-        self._chash_fdoc_store = {}
+        self._chash_fdoc_store = defaultdict(lambda: defaultdict(lambda: None))
 
         # Internal Storage: recent-flags store
         """
@@ -275,7 +276,7 @@ class MemoryStore(object):
         """
         from twisted.internet import reactor
 
-        log.msg("adding new doc to memstore %r (%r)" % (mbox, uid))
+        log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid))
         key = mbox, uid
 
         self._add_message(mbox, uid, message, notify_on_disk)
@@ -340,15 +341,12 @@ class MemoryStore(object):
         if fdoc is not None:
             fdoc_store = self._fdoc_store[mbox][uid]
             fdoc_store.update(fdoc)
+            chash_fdoc_store = self._chash_fdoc_store
 
             # content-hash indexing
             chash = fdoc.get(fields.CONTENT_HASH_KEY)
-            chash_fdoc_store = self._chash_fdoc_store
-            if not chash in chash_fdoc_store:
-                chash_fdoc_store[chash] = {}
-
             chash_fdoc_store[chash][mbox] = weakref.proxy(
-                fdoc_store)
+                self._fdoc_store[mbox][uid])
 
         hdoc = msg_dict.get(HDOC, None)
         if hdoc is not None:
@@ -381,7 +379,8 @@ class MemoryStore(object):
         :type uid: int
         :rtype: unicode or None
         """
-        doc_id = self._fdoc_id_store[mbox][uid]
+        with self._fdoc_docid_lock:
+            doc_id = self._fdoc_id_store[mbox][uid]
 
         if empty(doc_id):
             fdoc = self._permanent_store.get_flags_doc(mbox, uid)
@@ -475,6 +474,8 @@ class MemoryStore(object):
             if key in self._sizes:
                 del self._sizes[key]
             self._fdoc_store[mbox].pop(uid, None)
+            with self._fdoc_docid_lock:
+                self._fdoc_id_store[mbox].pop(uid, None)
         except Exception as exc:
             logger.exception(exc)
 
@@ -571,7 +572,8 @@ class MemoryStore(object):
         :param value: the value to set
         :type value: int
         """
-        leap_assert_type(value, int)
+        # can be long???
+        #leap_assert_type(value, int)
         logger.info("setting last soledad uid for %s to %s" %
                     (mbox, value))
         # if we already have a value here, don't do anything
@@ -603,10 +605,9 @@ class MemoryStore(object):
         with self._last_uid_lock:
             self._last_uid[mbox] += 1
             value = self._last_uid[mbox]
-            self.write_last_uid(mbox, value)
+            self.reactor.callInThread(self.write_last_uid, mbox, value)
             return value
 
-    @deferred_to_thread
     def write_last_uid(self, mbox, value):
         """
         Increment the soledad integer cache for the highest uid value.
@@ -633,10 +634,36 @@ class MemoryStore(object):
         """
         # We can do direct assignments cause we know this will only
         # be called during initialization of the mailbox.
+        # TODO could hook here a sanity-check
+        # for duplicates
 
         fdoc_store = self._fdoc_store[mbox]
+        chash_fdoc_store = self._chash_fdoc_store
         for uid in flag_docs:
-            fdoc_store[uid] = ReferenciableDict(flag_docs[uid])
+            rdict = ReferenciableDict(flag_docs[uid])
+            fdoc_store[uid] = rdict
+            # populate chash dict too, to avoid fdoc duplication
+            chash = flag_docs[uid]["chash"]
+            chash_fdoc_store[chash][mbox] = weakref.proxy(
+                self._fdoc_store[mbox][uid])
+
+    def update_flags(self, mbox, uid, fdoc):
+        """
+        Update the flag document for a given mbox and uid combination,
+        and set the dirty flag.
+        We could use put_message, but this is faster.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param uid: the uid of the message
+        :type uid: int
+
+        :param fdoc: a dict with the content for the flag docs
+        :type fdoc: dict
+        """
+        key = mbox, uid
+        self._fdoc_store[mbox][uid].update(fdoc)
+        self._dirty.add(key)
 
     def load_header_docs(self, header_docs):
         """
@@ -759,8 +786,7 @@ class MemoryStore(object):
         :return: MessagePartDoc. It will return None if the flags document
                  has empty content or it is flagged as \\Deleted.
         """
-        docs_dict = self._chash_fdoc_store.get(chash, None)
-        fdoc = docs_dict.get(mbox, None) if docs_dict else None
+        fdoc = self._chash_fdoc_store[chash][mbox]
 
         # a couple of special cases.
         # 1. We might have a doc with empty content...
@@ -778,6 +804,7 @@ class MemoryStore(object):
         key = mbox, uid
         new = key in self._new
         dirty = key in self._dirty
+
         return MessagePartDoc(
             new=new, dirty=dirty, store="mem",
             part=MessagePartType.fdoc,
@@ -1027,6 +1054,8 @@ class MemoryStore(object):
         """
         self._stop_write_loop()
         if self._permanent_store is not None:
+            # XXX we should check if we did get a True value on this
+            # operation. If we got False we should retry! (queue was not empty)
             self.write_messages(self._permanent_store)
             self.producer.flush()
 
@@ -1090,6 +1119,7 @@ class MemoryStore(object):
 
         try:
             # 1. Delete all messages marked as deleted in soledad.
+            logger.debug("DELETING FROM SOLEDAD ALL FOR %r" % (mbox,))
             sol_deleted = soledad_store.remove_all_deleted(mbox)
 
             try:
-- 
cgit v1.2.3


From fd9c8c2e3c88476b90805b689f6914fe5eac16df Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 02:53:28 -0400
Subject: defer fetch to thread

also, dispatch query for all headers to its own method.
---
 src/leap/mail/imap/mailbox.py     | 38 ++++++++++++++++++++++++++++++++------
 src/leap/mail/imap/memorystore.py | 27 +++++++++++++++++++++++++++
 src/leap/mail/imap/messages.py    |  7 ++++---
 src/leap/mail/imap/server.py      | 15 +++++++--------
 4 files changed, 70 insertions(+), 17 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index fa97512..21f0554 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -211,6 +211,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 fields.TYPE_MBOX_VAL, self.mbox)
             if query:
                 return query.pop()
+            else:
+                logger.error("Could not find mbox document for %r" %
+                             (self.mbox,))
         except Exception as exc:
             logger.exception("Unhandled error %r" % exc)
 
@@ -576,10 +579,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                     otherwise.
         :type uid: bool
 
+        :rtype: deferred
+        """
+        d = defer.Deferred()
+        self.reactor.callInThread(self._do_fetch, messages_asked, uid, d)
+        if PROFILE_CMD:
+            do_profile_cmd(d, "FETCH")
+        return d
+
+    # called in thread
+    def _do_fetch(self, messages_asked, uid, d):
+        """
+        :param messages_asked: IDs of the messages to retrieve information
+                               about
+        :type messages_asked: MessageSet
+
+        :param uid: If true, the IDs are UIDs. They are message sequence IDs
+                    otherwise.
+        :type uid: bool
+        :param d: deferred whose callback will be called with result.
+        :type d: Deferred
+
         :rtype: A tuple of two-tuples of message sequence numbers and
                 LeapMessage
         """
-        from twisted.internet import reactor
         # For the moment our UID is sequential, so we
         # can treat them all the same.
         # Change this to the flag that twisted expects when we
@@ -597,9 +620,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.debug("Getting msg by index: INEFFICIENT call!")
             raise NotImplementedError
         else:
-            result = ((msgid, getmsg(msgid)) for msgid in seq_messg)
-            reactor.callLater(0, self.unset_recent_flags, seq_messg)
-        return result
+            got_msg = [(msgid, getmsg(msgid)) for msgid in seq_messg]
+            result = ((msgid, msg) for msgid, msg in got_msg
+                      if msg is not None)
+            self.reactor.callLater(0, self.unset_recent_flags, seq_messg)
+            self.reactor.callFromThread(d.callback, result)
 
     def fetch_flags(self, messages_asked, uid):
         """
@@ -668,6 +693,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 MessagePart.
         :rtype: tuple
         """
+        # TODO how often is thunderbird doing this?
+
         class headersPart(object):
             def __init__(self, uid, headers):
                 self.uid = uid
@@ -685,10 +712,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         messages_asked = self._bound_seq(messages_asked)
         seq_messg = self._filter_msg_seq(messages_asked)
 
-        all_chash = self.messages.all_flags_chash()
         all_headers = self.messages.all_headers()
         result = ((msgid, headersPart(
-            msgid, all_headers.get(all_chash.get(msgid, 'nil'), {})))
+            msgid, all_headers.get(msgid, {})))
             for msgid in seq_messg)
         return result
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 2835826..e8e8152 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -434,6 +434,8 @@ class MemoryStore(object):
             hdoc = self._hdoc_store[chash]
             if empty(hdoc):
                 hdoc = self._permanent_store.get_headers_doc(chash)
+                if empty(hdoc):
+                    return None
                 if not empty(hdoc.content):
                     self._hdoc_store[chash] = hdoc.content
                     hdoc = hdoc.content
@@ -699,6 +701,31 @@ class MemoryStore(object):
                 continue
         return flags_dict
 
+    def all_headers(self, mbox):
+        """
+        Return a dictionary with all the header docs for a given mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: dict
+        """
+        headers_dict = {}
+        uids = self.get_uids(mbox)
+        fdoc_store = self._fdoc_store[mbox]
+        hdoc_store = self._hdoc_store
+
+        for uid in uids:
+            try:
+                chash = fdoc_store[uid][fields.CONTENT_HASH_KEY]
+                hdoc = hdoc_store[chash]
+                if not empty(hdoc):
+                    headers_dict[uid] = hdoc
+            except KeyError:
+                continue
+
+        import pprint; pprint.pprint(headers_dict)
+        return headers_dict
+
     # Counting sheeps...
 
     def count_new_mbox(self, mbox):
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index bbc9deb..7884fb0 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -28,7 +28,6 @@ from functools import partial
 
 from twisted.mail import imap4
 from twisted.internet import defer
-from twisted.python import log
 from zope.interface import implements
 from zope.proxy import sameProxiedObjects
 
@@ -1248,12 +1247,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         all_flags = dict(all_docs)
         return all_flags
 
-    # TODO get from memstore
     def all_headers(self):
         """
-        Return a dict with all the headers documents for this
+        Return a dict with all the header documents for this
         mailbox.
+
+        :rtype: dict
         """
+        return self.memstore.all_headers(self.mbox)
 
     def count(self):
         """
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 3497a8b..7c09784 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -119,15 +119,14 @@ class LeapIMAPServer(imap4.IMAP4Server):
                 cbFetch, tag, query, uid
             ).addErrback(ebFetch, tag)
 
-        # XXX not implemented yet --- should hit memstore
-        #elif len(query) == 1 and str(query[0]) == "rfc822.header":
-            #self._oldTimeout = self.setTimeout(None)
+        elif len(query) == 1 and str(query[0]) == "rfc822.header":
+            self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
-            #maybeDeferred(
-                #self.mbox.fetch_headers, messages, uid=uid
-            #).addCallback(
-                #cbFetch, tag, query, uid
-            #).addErrback(ebFetch, tag)
+            maybeDeferred(
+                self.mbox.fetch_headers, messages, uid=uid
+            ).addCallback(
+                cbFetch, tag, query, uid
+            ).addErrback(ebFetch, tag)
         else:
             self._oldTimeout = self.setTimeout(None)
             # no need to call iter, we get a generator
-- 
cgit v1.2.3


From 484c5fc316c0f95ebccc4a2c2a04c1cda96a34f8 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 03:04:04 -0400
Subject: defend against malformed fdocs during unset dirty/new

---
 src/leap/mail/imap/messageparts.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 9b7de86..6f1376a 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -158,8 +158,11 @@ class MessageWrapper(object):
         """
         self._new = value
         if self.memstore:
-            mbox = self.fdoc.content['mbox']
-            uid = self.fdoc.content['uid']
+            mbox = self.fdoc.content.get('mbox', None)
+            uid = self.fdoc.content.get('uid', None)
+            if not mbox or not uid:
+                logger.warning("Malformed fdoc")
+                return
             key = mbox, uid
             fun = [self.memstore.unset_new_queued,
                    self.memstore.set_new_queued][int(value)]
@@ -190,8 +193,11 @@ class MessageWrapper(object):
         """
         self._dirty = value
         if self.memstore:
-            mbox = self.fdoc.content['mbox']
-            uid = self.fdoc.content['uid']
+            mbox = self.fdoc.content.get('mbox', None)
+            uid = self.fdoc.content.get('uid', None)
+            if not mbox or not uid:
+                logger.warning("Malformed fdoc")
+                return
             key = mbox, uid
             fun = [self.memstore.unset_dirty_queued,
                    self.memstore.set_dirty_queued][int(value)]
@@ -278,6 +284,7 @@ class MessageWrapper(object):
                 docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc(
                     mbox, uid)
             except Exception as exc:
+                logger.debug("Error while walking message...")
                 logger.exception(exc)
 
         if not empty(self.fdoc.content):
-- 
cgit v1.2.3


From f6566fe83c93625b918664526e8858f7be667354 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Feb 2014 16:20:26 -0400
Subject: defer appends too

and cut some more time by firing the callback as soon as we've got
an UID.
---
 src/leap/mail/imap/mailbox.py     | 22 +++++++++++-----------
 src/leap/mail/imap/memorystore.py | 32 ++++++++++++--------------------
 src/leap/mail/imap/messages.py    | 19 +++++++++----------
 3 files changed, 32 insertions(+), 41 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 21f0554..7083316 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -111,6 +111,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     last_uid_lock = threading.Lock()
 
     _fdoc_primed = {}
+    _last_uid_primed = {}
 
     def __init__(self, mbox, soledad, memstore, rw=1):
         """
@@ -294,10 +295,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         Prime memstore with last_uid value
         """
-        mbox = self._get_mbox_doc()
-        last = mbox.content.get('lastuid', 0)
-        logger.info("Priming Soledad last_uid to %s" % (last,))
-        self._memstore.set_last_soledad_uid(self.mbox, last)
+        primed = self._last_uid_primed.get(self.mbox, False)
+        if not primed:
+            mbox = self._get_mbox_doc()
+            last = mbox.content.get('lastuid', 0)
+            logger.info("Priming Soledad last_uid to %s" % (last,))
+            self._memstore.set_last_soledad_uid(self.mbox, last)
+            self._last_uid_primed[self.mbox] = True
 
     def prime_known_uids_to_memstore(self):
         """
@@ -459,6 +463,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             flags = tuple(str(flag) for flag in flags)
 
         d = self._do_add_message(message, flags=flags, date=date)
+        if PROFILE_CMD:
+            do_profile_cmd(d, "APPEND")
+        # XXX should notify here probably
         return d
 
     def _do_add_message(self, message, flags, date):
@@ -467,13 +474,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Invoked from addMessage.
         """
         d = self.messages.add_msg(message, flags=flags, date=date)
-        # XXX Removing notify temporarily.
-        # This is interfering with imaptest results. I'm not clear if it's
-        # because we clutter the logging or because the set of listeners is
-        # ever-growing. We should come up with some smart way of dealing with
-        # it, or maybe just disabling it using an environmental variable since
-        # we will only have just a few listeners in the regular desktop case.
-        #d.addCallback(self.notify_new)
         return d
 
     def notify_new(self, *args):
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index e8e8152..423b891 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -274,30 +274,24 @@ class MemoryStore(object):
                                be fired.
         :type notify_on_disk: bool
         """
-        from twisted.internet import reactor
-
         log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid))
         key = mbox, uid
 
         self._add_message(mbox, uid, message, notify_on_disk)
         self._new.add(key)
 
-        # XXX use this while debugging the callback firing,
-        # remove after unittesting this.
-        #def log_add(result):
-            #return result
-        #observer.addCallback(log_add)
-
-        if notify_on_disk:
-            # We store this deferred so we can keep track of the pending
-            # operations internally.
-            # TODO this should fire with the UID !!! -- change that in
-            # the soledad store code.
-            self._new_deferreds[key] = observer
-        if not notify_on_disk:
-            # Caller does not care, just fired and forgot, so we pass
-            # a defer that will inmediately have its callback triggered.
-            reactor.callLater(0, observer.callback, uid)
+        if observer is not None:
+            if notify_on_disk:
+                # We store this deferred so we can keep track of the pending
+                # operations internally.
+                # TODO this should fire with the UID !!! -- change that in
+                # the soledad store code.
+                self._new_deferreds[key] = observer
+
+            else:
+                # Caller does not care, just fired and forgot, so we pass
+                # a defer that will inmediately have its callback triggered.
+                self.reactor.callFromThread(observer.callback, uid)
 
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
@@ -722,8 +716,6 @@ class MemoryStore(object):
                     headers_dict[uid] = hdoc
             except KeyError:
                 continue
-
-        import pprint; pprint.pprint(headers_dict)
         return headers_dict
 
     # Counting sheeps...
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 7884fb0..c133a6d 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -879,19 +879,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                  uid when the adding succeed.
         :rtype: deferred
         """
-        logger.debug('Adding message')
         if flags is None:
             flags = tuple()
         leap_assert_type(flags, tuple)
 
         observer = defer.Deferred()
         d = self._do_parse(raw)
-        d.addCallback(self._do_add_msg, flags, subject, date,
-                      notify_on_disk, observer)
+        d.addCallback(lambda result: self.reactor.callInThread(
+            self._do_add_msg, result, flags, subject, date,
+            notify_on_disk, observer))
         return observer
 
-    # We SHOULD defer the heavy load here) to the thread pool,
-    # but it gives troubles with the QSocketNotifier used by Qt...
+    # Called in thread
     def _do_add_msg(self, parse_result, flags, subject,
                     date, notify_on_disk, observer):
         """
@@ -912,7 +911,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # TODO add the linked-from info !
         # TODO add reference to the original message
 
-        from twisted.internet import reactor
         msg, parts, chash, size, multi = parse_result
 
         # check for uniqueness --------------------------------
@@ -922,13 +920,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             uid = existing_uid
             msg = self.get_msg_by_uid(uid)
 
-            # TODO this cannot be deferred, this has to block.
+            # We can say the observer that we're done
+            self.reactor.callFromThread(observer.callback, uid)
             msg.setFlags((fields.DELETED_FLAG,), -1)
-            reactor.callLater(0, observer.callback, uid)
             return
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
-        logger.info("ADDING MSG WITH UID: %s" % uid)
+        # We can say the observer that we're done
+        self.reactor.callFromThread(observer.callback, uid)
 
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
@@ -953,7 +952,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         msg_container = MessageWrapper(fd, hd, cdocs)
         self.memstore.create_message(
             self.mbox, uid, msg_container,
-            observer=observer, notify_on_disk=notify_on_disk)
+            observer=None, notify_on_disk=notify_on_disk)
 
     #
     # getters: specific queries
-- 
cgit v1.2.3


From f5fe0fdefa61f4989735800980dc2a3241c2fdf3 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 01:35:48 -0400
Subject: avoid revision conflict during deletion

---
 src/leap/mail/imap/soledadstore.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 657f21f..3415fa8 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -143,6 +143,7 @@ class SoledadStore(ContentDedup):
     """
     _last_uid_lock = threading.Lock()
     _soledad_rw_lock = threading.Lock()
+    _remove_lock = threading.Lock()
 
     implements(IMessageConsumer, IMessageStore)
 
@@ -526,7 +527,7 @@ class SoledadStore(ContentDedup):
 
     def deleted_iter(self, mbox):
         """
-        Get an iterator for the SoledadDocuments for messages
+        Get an iterator for the the doc_id for SoledadDocuments for messages
         with \\Deleted flag for a given mailbox.
 
         :param mbox: the mailbox
@@ -534,9 +535,9 @@ class SoledadStore(ContentDedup):
         :return: iterator through deleted message docs
         :rtype: iterable
         """
-        return (doc for doc in self._soledad.get_from_index(
+        return [doc.doc_id for doc in self._soledad.get_from_index(
                 fields.TYPE_MBOX_DEL_IDX,
-                fields.TYPE_FLAGS_VAL, mbox, '1'))
+                fields.TYPE_FLAGS_VAL, mbox, '1')]
 
     def remove_all_deleted(self, mbox):
         """
@@ -547,7 +548,13 @@ class SoledadStore(ContentDedup):
         :type mbox: str or unicode
         """
         deleted = []
-        for doc in self.deleted_iter(mbox):
-            deleted.append(doc.content[fields.UID_KEY])
-            self._soledad.delete_doc(doc)
+        for doc_id in self.deleted_iter(mbox):
+            with self._remove_lock:
+                doc = self._soledad.get_doc(doc_id)
+                self._soledad.delete_doc(doc)
+            try:
+                deleted.append(doc.content[fields.UID_KEY])
+            except TypeError:
+                # empty content
+                pass
         return deleted
-- 
cgit v1.2.3


From 54114126d0b8e16784b67ee972e549e5c152c9d0 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 12:37:31 -0400
Subject: purge empty fdocs on select

---
 src/leap/mail/imap/mailbox.py     |  3 +++
 src/leap/mail/imap/memorystore.py | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 7083316..087780f 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -157,6 +157,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         from twisted.internet import reactor
         self.reactor = reactor
 
+        # purge memstore from empty fdocs.
+        self._memstore.purge_fdoc_store(mbox)
+
     @property
     def listeners(self):
         """
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 423b891..4aaee75 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -362,6 +362,27 @@ class MemoryStore(object):
         self._sizes[key] = size.get_size(self._fdoc_store[key])
         # TODO add hdoc and cdocs sizes too
 
+    def purge_fdoc_store(self, mbox):
+        """
+        Purge the empty documents from a fdoc store.
+        Called during initialization of the SoledadMailbox
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        """
+        # XXX This is really a workaround until I find the conditions
+        # that are making the empty items remain there.
+        # This happens, for instance, after running several times
+        # the regression test, that issues a store deleted + expunge + select
+        # The items are being correclty deleted, but in succesive appends
+        # the empty items with previously deleted uids reappear as empty
+        # documents. I suspect it's a timing condition with a previously
+        # evaluated sequence being used after the items has been removed.
+
+        for uid, value in self._fdoc_store[mbox].items():
+            if empty(value):
+                del self._fdoc_store[mbox][uid]
+
     def get_docid_for_fdoc(self, mbox, uid):
         """
         Return Soledad document id for the flags-doc for a given mbox and uid,
-- 
cgit v1.2.3


From b520a60d0e48f36dcebe03d19b65839afc460fe9 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 12:39:33 -0400
Subject: move mbox-doc handling to soledadstore, and lock it

---
 src/leap/mail/imap/mailbox.py      |  22 ++-----
 src/leap/mail/imap/memorystore.py  |  36 ++++++++++++
 src/leap/mail/imap/soledadstore.py | 115 ++++++++++++++++++++++++++-----------
 3 files changed, 120 insertions(+), 53 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 087780f..d18bc9a 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -200,7 +200,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         self.listeners.remove(listener)
 
-    # TODO move completely to soledadstore, under memstore reponsibility.
     def _get_mbox_doc(self):
         """
         Return mailbox document.
@@ -209,17 +208,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                  the query failed.
         :rtype: SoledadDocument or None.
         """
-        try:
-            query = self._soledad.get_from_index(
-                fields.TYPE_MBOX_IDX,
-                fields.TYPE_MBOX_VAL, self.mbox)
-            if query:
-                return query.pop()
-            else:
-                logger.error("Could not find mbox document for %r" %
-                             (self.mbox,))
-        except Exception as exc:
-            logger.exception("Unhandled error %r" % exc)
+        return self._memstore.get_mbox_doc(self.mbox)
 
     def getFlags(self):
         """
@@ -234,6 +223,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         flags = mbox.content.get(self.FLAGS_KEY, [])
         return map(str, flags)
 
+    # XXX move to memstore->soledadstore
     def setFlags(self, flags):
         """
         Sets flags for this mailbox.
@@ -258,8 +248,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :return: True if the mailbox is closed
         :rtype: bool
         """
-        mbox = self._get_mbox_doc()
-        return mbox.content.get(self.CLOSED_KEY, False)
+        return self._memstore.get_mbox_closed(self.mbox)
 
     def _set_closed(self, closed):
         """
@@ -268,10 +257,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param closed: the state to be set
         :type closed: bool
         """
-        leap_assert(isinstance(closed, bool), "closed needs to be boolean")
-        mbox = self._get_mbox_doc()
-        mbox.content[self.CLOSED_KEY] = closed
-        self._soledad.put_doc(mbox)
+        self._memstore.set_mbox_closed(self.mbox, closed)
 
     closed = property(
         _get_closed, _set_closed, doc="Closed attribute.")
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 4aaee75..ba444b0 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -293,6 +293,7 @@ class MemoryStore(object):
                 # a defer that will inmediately have its callback triggered.
                 self.reactor.callFromThread(observer.callback, uid)
 
+
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
         Put an existing message.
@@ -1176,8 +1177,43 @@ class MemoryStore(object):
             logger.exception(exc)
         finally:
             self._start_write_loop()
+
         observer.callback(all_deleted)
 
+    # Mailbox documents and attributes
+
+    # This could be also be cached in memstore, but proxying directly
+    # to soledad since it's not too performance-critical.
+
+    def get_mbox_doc(self, mbox):
+        """
+        Return the soledad document for a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: SoledadDocument or None.
+        """
+        return self.permanent_store.get_mbox_document(mbox)
+
+    def get_mbox_closed(self, mbox):
+        """
+        Return the closed attribute for a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: bool
+        """
+        return self.permanent_store.get_mbox_closed(mbox)
+
+    def set_mbox_closed(self, mbox, closed):
+        """
+        Set the closed attribute for a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        """
+        self.permanent_store.set_mbox_closed(mbox, closed)
+
     # Dump-to-disk controls.
 
     @property
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 3415fa8..f415894 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -27,7 +27,7 @@ from u1db import errors as u1db_errors
 from twisted.python import log
 from zope.interface import implements
 
-from leap.common.check import leap_assert_type
+from leap.common.check import leap_assert_type, leap_assert
 from leap.mail.decorators import deferred_to_thread
 from leap.mail.imap.messageparts import MessagePartType
 from leap.mail.imap.messageparts import MessageWrapper
@@ -141,9 +141,9 @@ class SoledadStore(ContentDedup):
     """
     This will create docs in the local Soledad database.
     """
-    _last_uid_lock = threading.Lock()
     _soledad_rw_lock = threading.Lock()
     _remove_lock = threading.Lock()
+    _mbox_doc_locks = defaultdict(lambda: threading.Lock())
 
     implements(IMessageConsumer, IMessageStore)
 
@@ -438,7 +438,9 @@ class SoledadStore(ContentDedup):
             logger.debug("Saving RFLAGS to Soledad...")
             yield payload, call
 
-    def _get_mbox_document(self, mbox):
+    # Mbox documents and attributes
+
+    def get_mbox_document(self, mbox):
         """
         Return mailbox document.
 
@@ -448,15 +450,83 @@ class SoledadStore(ContentDedup):
                  the query failed.
         :rtype: SoledadDocument or None.
         """
+        with self._mbox_doc_locks[mbox]:
+            return self._get_mbox_document(mbox)
+
+    def _get_mbox_document(self, mbox):
+        """
+        Helper for returning the mailbox document.
+        """
         try:
             query = self._soledad.get_from_index(
                 fields.TYPE_MBOX_IDX,
                 fields.TYPE_MBOX_VAL, mbox)
             if query:
                 return query.pop()
+            else:
+                logger.error("Could not find mbox document for %r" %
+                            (self.mbox,))
         except Exception as exc:
             logger.exception("Unhandled error %r" % exc)
 
+    def get_mbox_closed(self, mbox):
+        """
+        Return the closed attribute for a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: bool
+        """
+        mbox_doc = self.get_mbox_document()
+        return mbox_doc.content.get(fields.CLOSED_KEY, False)
+
+    def set_mbox_closed(self, mbox, closed):
+        """
+        Set the closed attribute for a given mailbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param closed:  the value to be set
+        :type closed: bool
+        """
+        leap_assert(isinstance(closed, bool), "closed needs to be boolean")
+        with self._mbox_doc_locks[mbox]:
+            mbox_doc = self._get_mbox_document(mbox)
+            if mbox_doc is None:
+                logger.error(
+                    "Could not find mbox document for %r" % (mbox,))
+                return
+            mbox_doc.content[fields.CLOSED_KEY] = closed
+            self._soledad.put_doc(mbox_doc)
+
+    def write_last_uid(self, mbox, value):
+        """
+        Write the `last_uid` integer to the proper mailbox document
+        in Soledad.
+        This is called from the deferred triggered by
+        memorystore.increment_last_soledad_uid, which is expected to
+        run in a separate thread.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: the value to set
+        :type value: int
+        """
+        leap_assert_type(value, int)
+        key = fields.LAST_UID_KEY
+
+        # XXX change for a lock related to the mbox document
+        # itself.
+        with self._mbox_doc_locks[mbox]:
+            mbox_doc = self._get_mbox_document(mbox)
+            old_val = mbox_doc.content[key]
+            if value > old_val:
+                mbox_doc.content[key] = value
+                self._soledad.put_doc(mbox_doc)
+            else:
+                logger.error("%r:%s Tried to write a UID lesser than what's "
+                             "stored!" % (mbox, value))
+
     def get_flags_doc(self, mbox, uid):
         """
         Return the SoledadDocument for the given mbox and uid.
@@ -497,32 +567,6 @@ class SoledadStore(ContentDedup):
             fields.TYPE_HEADERS_VAL, str(chash))
         return first(head_docs)
 
-    def write_last_uid(self, mbox, value):
-        """
-        Write the `last_uid` integer to the proper mailbox document
-        in Soledad.
-        This is called from the deferred triggered by
-        memorystore.increment_last_soledad_uid, which is expected to
-        run in a separate thread.
-
-        :param mbox: the mailbox
-        :type mbox: str or unicode
-        :param value: the value to set
-        :type value: int
-        """
-        leap_assert_type(value, int)
-        key = fields.LAST_UID_KEY
-
-        with self._last_uid_lock:
-            mbox_doc = self._get_mbox_document(mbox)
-            old_val = mbox_doc.content[key]
-            if value > old_val:
-                mbox_doc.content[key] = value
-                self._soledad.put_doc(mbox_doc)
-            else:
-                logger.error("%r:%s Tried to write a UID lesser than what's "
-                             "stored!" % (mbox, value))
-
     # deleted messages
 
     def deleted_iter(self, mbox):
@@ -551,10 +595,11 @@ class SoledadStore(ContentDedup):
         for doc_id in self.deleted_iter(mbox):
             with self._remove_lock:
                 doc = self._soledad.get_doc(doc_id)
-                self._soledad.delete_doc(doc)
-            try:
-                deleted.append(doc.content[fields.UID_KEY])
-            except TypeError:
-                # empty content
-                pass
+                if doc is not None:
+                    self._soledad.delete_doc(doc)
+                    try:
+                        deleted.append(doc.content[fields.UID_KEY])
+                    except TypeError:
+                        # empty content
+                        pass
         return deleted
-- 
cgit v1.2.3


From ac4c70f0be36c985e16e3f4ec0a38ef6f8d48166 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 12:42:02 -0400
Subject: remove all refs during removal, and protect from empty docs

---
 src/leap/mail/imap/mailbox.py      |  2 +-
 src/leap/mail/imap/memorystore.py  | 17 +++++++++++++++--
 src/leap/mail/imap/messageparts.py |  4 +---
 src/leap/mail/imap/messages.py     | 17 ++++++++++++-----
 4 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index d18bc9a..045de82 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -609,7 +609,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.debug("Getting msg by index: INEFFICIENT call!")
             raise NotImplementedError
         else:
-            got_msg = [(msgid, getmsg(msgid)) for msgid in seq_messg]
+            got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg)
             result = ((msgid, msg) for msgid, msg in got_msg
                       if msg is not None)
             self.reactor.callLater(0, self.unset_recent_flags, seq_messg)
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index ba444b0..1e4262a 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -485,16 +485,26 @@ class MemoryStore(object):
         # XXX implement elijah's idea of using a PUT document as a
         # token to ensure consistency in the removal.
 
+        try:
+            del self._fdoc_store[mbox][uid]
+        except KeyError:
+            pass
+
         try:
             key = mbox, uid
             self._new.discard(key)
             self._dirty.discard(key)
             if key in self._sizes:
                 del self._sizes[key]
-            self._fdoc_store[mbox].pop(uid, None)
+            self._known_uids[mbox].discard(uid)
+        except Exception as exc:
+            logger.error("error while removing message!")
+            logger.exception(exc)
+        try:
             with self._fdoc_docid_lock:
-                self._fdoc_id_store[mbox].pop(uid, None)
+                del self._fdoc_id_store[mbox][uid]
         except Exception as exc:
+            logger.error("error while removing message!")
             logger.exception(exc)
 
     # IMessageStoreWriter
@@ -1124,6 +1134,8 @@ class MemoryStore(object):
             # Stop and trigger last write
             self.stop_and_flush()
             # Wait on the writebacks to finish
+
+            # XXX what if pending deferreds is empty?
             pending_deferreds = (self._new_deferreds.get(mbox, []) +
                                  self._dirty_deferreds.get(mbox, []))
             d1 = defer.gatherResults(pending_deferreds, consumeErrors=True)
@@ -1169,6 +1181,7 @@ class MemoryStore(object):
                 logger.exception(exc)
 
             # 2. Delete all messages marked as deleted in memory.
+            logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,))
             mem_deleted = self.remove_all_deleted(mbox)
 
             all_deleted = set(mem_deleted).union(set(sol_deleted))
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index 6f1376a..257721c 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -287,7 +287,7 @@ class MessageWrapper(object):
                 logger.debug("Error while walking message...")
                 logger.exception(exc)
 
-        if not empty(self.fdoc.content):
+        if not empty(self.fdoc.content) and 'uid' in self.fdoc.content:
             yield self.fdoc
         if not empty(self.hdoc.content):
             yield self.hdoc
@@ -418,10 +418,8 @@ class MessagePart(object):
         if payload:
             content_type = self._get_ctype_from_document(phash)
             charset = find_charset(content_type)
-            logger.debug("Got charset from header: %s" % (charset,))
             if charset is None:
                 charset = self._get_charset(payload)
-                logger.debug("Got charset: %s" % (charset,))
             try:
                 if isinstance(payload, unicode):
                     payload = payload.encode(charset)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index c133a6d..0aa40f1 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -850,7 +850,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         if not exist:
             exist = self._get_fdoc_from_chash(chash)
-        if exist:
+        if exist and exist.content is not None:
             return exist.content.get(fields.UID_KEY, "unknown-uid")
         else:
             return False
@@ -926,8 +926,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             return
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
-        # We can say the observer that we're done
+        # We can say the observer that we're done at this point.
+        # Make sure it has no serious consequences if we're issued
+        # a fetch command right after...
         self.reactor.callFromThread(observer.callback, uid)
+        # if we did the notify, we need to invalidate the deferred
+        # so not to try to fire it twice.
+        observer = None
 
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
@@ -952,7 +957,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         msg_container = MessageWrapper(fd, hd, cdocs)
         self.memstore.create_message(
             self.mbox, uid, msg_container,
-            observer=None, notify_on_disk=notify_on_disk)
+            observer=observer, notify_on_disk=notify_on_disk)
 
     #
     # getters: specific queries
@@ -1130,8 +1135,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             if msg is not None:
                 return uid, msg.setFlags(flags, mode)
 
-        result = dict(
-            set_flags(uid, tuple(flags), mode) for uid in messages)
+        setted_flags = [set_flags(uid, flags, mode) for uid in messages]
+        result = dict(filter(None, setted_flags))
 
         reactor.callFromThread(observer.callback, result)
 
@@ -1158,6 +1163,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         msg_container = self.memstore.get_message(
             self.mbox, uid, flags_only=flags_only)
+
         if msg_container is not None:
             if mem_only:
                 msg = LeapMessage(None, uid, self.mbox, collection=self,
@@ -1170,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                                   collection=self, container=msg_container)
         else:
             msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
+
         if not msg.does_exist():
             return None
         return msg
-- 
cgit v1.2.3


From 3490773898c0b1af76ff18b088b099b7cde677e7 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 12:44:18 -0400
Subject: select instead of examine

---
 src/leap/mail/imap/tests/regressions | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions
index 0a43398..efe3f46 100755
--- a/src/leap/mail/imap/tests/regressions
+++ b/src/leap/mail/imap/tests/regressions
@@ -101,7 +101,6 @@ def compare_msg_parts(a, b):
         pprint(b[index])
         print
 
-
     return all_match
 
 
@@ -328,7 +327,7 @@ def cbAppendNextMessage(proto):
     return proto.append(
         REGRESSIONS_FOLDER, msg
     ).addCallback(
-        lambda r: proto.examine(REGRESSIONS_FOLDER)
+        lambda r: proto.select(REGRESSIONS_FOLDER)
     ).addCallback(
         cbAppend, proto, raw
     ).addErrback(
@@ -379,6 +378,9 @@ def cbCompareMessage(result, proto, raw):
     if result:
         keys = result.keys()
         keys.sort()
+    else:
+        print "[-] GOT NO RESULT"
+        return proto.logout()
 
     latest = max(keys)
 
-- 
cgit v1.2.3


From 1ff1663fb2968adff87208134c374c4084670ace Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 13:05:08 -0400
Subject: docstring fixes

---
 src/leap/mail/imap/messages.py     |  3 ---
 src/leap/mail/imap/soledadstore.py | 19 +++++++++++--------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 0aa40f1..a49ea90 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -245,8 +245,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :type mode: int
         """
         leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
-        #log.msg('setting flags: %s (%s)' % (self._uid, flags))
-
         mbox, uid = self._mbox, self._uid
 
         APPEND = 1
@@ -325,7 +323,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
             body = bdoc_content.get(self.RAW_KEY, "")
             content_type = bdoc_content.get('content-type', "")
             charset = find_charset(content_type)
-            #logger.debug('got charset from content-type: %s' % charset)
             if charset is None:
                 charset = self._get_charset(body)
             try:
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index f415894..6d6d382 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -41,9 +41,7 @@ logger = logging.getLogger(__name__)
 
 
 # TODO
-# [ ] Delete original message from the incoming queue after all successful
-#     writes.
-# [ ] Implement a retry queue.
+# [ ] Implement a retry queue?
 # [ ] Consider journaling of operations.
 
 
@@ -231,9 +229,9 @@ class SoledadStore(ContentDedup):
         """
         Creates a new document in soledad db.
 
-        :param queue: queue to get item from, with content of the document
-                      to be inserted.
-        :type queue: Queue
+        :param queue: a tuple of queues to get item from, with content of the
+                      document to be inserted.
+        :type queue: tuple of Queues
         """
         new, dirty = queue
         while not new.empty():
@@ -266,9 +264,14 @@ class SoledadStore(ContentDedup):
     def _consume_doc(self, doc_wrapper, notify_queue):
         """
         Consume each document wrapper in a separate thread.
+        We pass an instance of an accumulator that handles the notifications
+        to the memorystore when the write has been done.
 
         :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance
         :type doc_wrapper: MessageWrapper or RecentFlagsDoc
+        :param notify_queue: a callable that handles the writeback
+                             notifications to the memstore.
+        :type notify_queue: callable
         """
         def queueNotifyBack(failed, doc_wrapper):
             if failed:
@@ -316,8 +319,8 @@ class SoledadStore(ContentDedup):
         followed by the subparts item and the proper call type for every
         item in the queue, if any.
 
-        :param queue: the queue from where we'll pick item.
-        :type queue: Queue
+        :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance
+        :type doc_wrapper: MessageWrapper or RecentFlagsDoc
         """
         if isinstance(doc_wrapper, MessageWrapper):
             return chain((doc_wrapper,),
-- 
cgit v1.2.3


From 1217be6c792d87134f6801591c7bfa9536c9a3d1 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 12:40:04 -0400
Subject: suggest bigger threadpool to reactors that honor it

---
 src/leap/mail/imap/service/imap.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 6041961..a7799ca 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -171,6 +171,9 @@ def run_service(*args, **kwargs):
               the protocol.
     """
     from twisted.internet import reactor
+    # it looks like qtreactor does not honor this,
+    # but other reactors should.
+    reactor.suggestThreadPoolSize(20)
 
     leap_assert(len(args) == 2)
     soledad, keymanager = args
-- 
cgit v1.2.3


From 1baafbaa8e3dd7d62580ba4ad3a829ceaf16f583 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 12 Feb 2014 13:13:36 -0400
Subject: remove early notification on append for now

this can be done to save some msec, but additional measures
have to be taken to avoid inconsistencies with reads right after
this is done. we could make those wait until a second deferred
is done, for example.
---
 src/leap/mail/imap/messages.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index a49ea90..fc1ec55 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -923,13 +923,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             return
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
-        # We can say the observer that we're done at this point.
-        # Make sure it has no serious consequences if we're issued
-        # a fetch command right after...
-        self.reactor.callFromThread(observer.callback, uid)
+
+        # We can say the observer that we're done at this point, but
+        # before that we should make sure it has no serious consequences
+        # if we're issued, for instance, a fetch command right after...
+        #self.reactor.callFromThread(observer.callback, uid)
         # if we did the notify, we need to invalidate the deferred
         # so not to try to fire it twice.
-        observer = None
+        #observer = None
 
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
-- 
cgit v1.2.3


From 7eba9d5badcd3ebbdb746e6598ce452a8d9b7649 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 13 Feb 2014 11:42:06 -0400
Subject: avoid hitting db on every select

---
 src/leap/mail/imap/account.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 04af3b1..fd35698 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -18,6 +18,7 @@
 Soledad Backed Account.
 """
 import copy
+import logging
 import time
 
 from twisted.mail import imap4
@@ -30,6 +31,8 @@ from leap.mail.imap.parser import MBoxParser
 from leap.mail.imap.mailbox import SoledadMailbox
 from leap.soledad.client import Soledad
 
+logger = logging.getLogger(__name__)
+
 
 #######################################
 # Soledad Account
@@ -77,10 +80,13 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         self._soledad = soledad
         self._memstore = memstore
 
+        self.__mailboxes = set([])
+
         self.initialize_db()
 
         # every user should have the right to an inbox folder
         # at least, so let's make one!
+        self._load_mailboxes()
 
         if not self.mailboxes:
             self.addMailbox(self.INBOX_NAME)
@@ -112,9 +118,13 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         """
         A list of the current mailboxes for this account.
         """
-        return [doc.content[self.MBOX_KEY]
-                for doc in self._soledad.get_from_index(
-                    self.TYPE_IDX, self.MBOX_KEY)]
+        return self.__mailboxes
+
+    def _load_mailboxes(self):
+        self.__mailboxes.update(
+            [doc.content[self.MBOX_KEY]
+             for doc in self._soledad.get_from_index(
+                 self.TYPE_IDX, self.MBOX_KEY)])
 
     @property
     def subscriptions(self):
@@ -179,6 +189,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         mbox[self.CREATED_KEY] = creation_ts
 
         doc = self._soledad.create_doc(mbox)
+        self._load_mailboxes()
         return bool(doc)
 
     def create(self, pathspec):
@@ -209,6 +220,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         except imap4.MailboxCollision:
             if not pathspec.endswith('/'):
                 return False
+        self._load_mailboxes()
         return True
 
     def select(self, name, readwrite=1):
@@ -221,13 +233,13 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
         :param readwrite: 1 for readwrite permissions.
         :type readwrite: int
 
-        :rtype: bool
+        :rtype: SoledadMailbox
         """
         name = self._parse_mailbox_name(name)
 
         if name not in self.mailboxes:
+            logger.warning("No such mailbox!")
             return None
-
         self.selected = name
 
         return SoledadMailbox(
@@ -266,6 +278,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
                             "Hierarchically inferior mailboxes "
                             "exist and \\Noselect is set")
         mbox.destroy()
+        self._load_mailboxes()
 
         # XXX FIXME --- not honoring the inferior names...
 
@@ -303,6 +316,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
             mbox.content[self.MBOX_KEY] = new
             self._soledad.put_doc(mbox)
 
+        self._load_mailboxes()
+
         # XXX ---- FIXME!!!! ------------------------------------
         # until here we just renamed the index...
         # We have to rename also the occurrence of this
-- 
cgit v1.2.3


From 45733a231128cc06e123f352b4eb9886d6820878 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 14 Feb 2014 12:41:58 -0400
Subject: docstring fixes

---
 src/leap/mail/imap/mailbox.py      |  2 ++
 src/leap/mail/imap/memorystore.py  | 27 +++++++++++++++++----------
 src/leap/mail/imap/service/imap.py |  4 ++--
 src/leap/mail/imap/soledadstore.py |  3 +++
 4 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 045de82..d55cae6 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -895,6 +895,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Get a copy of the fdoc for this message, and check whether
         it already exists.
 
+        :param message: an IMessage implementor
+        :type message: LeapMessage
         :return: exist, new_fdoc
         :rtype: tuple
         """
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 1e4262a..53b8d99 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -25,6 +25,7 @@ import weakref
 from collections import defaultdict
 from copy import copy
 
+from enum import Enum
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
 from twisted.python import log
@@ -69,6 +70,9 @@ def set_bool_flag(obj, att):
         setattr(obj, att, False)
 
 
+DirtyState = Enum("none", "dirty", "new")
+
+
 class MemoryStore(object):
     """
     An in-memory store to where we can write the different parts that
@@ -293,7 +297,6 @@ class MemoryStore(object):
                 # a defer that will inmediately have its callback triggered.
                 self.reactor.callFromThread(observer.callback, uid)
 
-
     def put_message(self, mbox, uid, message, notify_on_disk=True):
         """
         Put an existing message.
@@ -407,7 +410,8 @@ class MemoryStore(object):
 
         return doc_id
 
-    def get_message(self, mbox, uid, dirtystate="none", flags_only=False):
+    def get_message(self, mbox, uid, dirtystate=DirtyState.none,
+                    flags_only=False):
         """
         Get a MessageWrapper for the given mbox and uid combination.
 
@@ -415,8 +419,9 @@ class MemoryStore(object):
         :type mbox: str or unicode
         :param uid: the message UID
         :type uid: int
-        :param dirtystate: one of `dirty`, `new` or `none` (default)
-        :type dirtystate: str
+        :param dirtystate: DirtyState enum: one of `dirty`, `new`
+                           or `none` (default)
+        :type dirtystate: enum
         :param flags_only: whether the message should carry only a reference
                            to the flags document.
         :type flags_only: bool
@@ -424,7 +429,7 @@ class MemoryStore(object):
 
         :return: MessageWrapper or None
         """
-        if dirtystate == "dirty":
+        if dirtystate == DirtyState.dirty:
             flags_only = True
 
         key = mbox, uid
@@ -434,11 +439,11 @@ class MemoryStore(object):
             return None
 
         new, dirty = False, False
-        if dirtystate == "none":
+        if dirtystate == DirtyState.none:
             new, dirty = self._get_new_dirty_state(key)
-        if dirtystate == "dirty":
+        if dirtystate == DirtyState.dirty:
             new, dirty = False, True
-        if dirtystate == "new":
+        if dirtystate == DirtyState.new:
             new, dirty = True, False
 
         if flags_only:
@@ -514,6 +519,7 @@ class MemoryStore(object):
         Write the message documents in this MemoryStore to a different store.
 
         :param store: the IMessageStore to write to
+        :rtype: False if queue is not empty, None otherwise.
         """
         # For now, we pass if the queue is not empty, to avoid duplicate
         # queuing.
@@ -880,7 +886,7 @@ class MemoryStore(object):
         :rtype: generator
         """
         gm = self.get_message
-        new = [gm(*key) for key in self._new]
+        new = [gm(*key, dirtystate=DirtyState.new) for key in self._new]
         # move content from new set to the queue
         self._new_queue.update(self._new)
         self._new.difference_update(self._new)
@@ -894,7 +900,8 @@ class MemoryStore(object):
         :rtype: generator
         """
         gm = self.get_message
-        dirty = [gm(*key, flags_only=True) for key in self._dirty]
+        dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty)
+                 for key in self._dirty]
         # move content from new and dirty sets to the queue
 
         self._dirty_queue.update(self._dirty)
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index a7799ca..b79d42d 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -118,8 +118,8 @@ class LeapIMAPFactory(ServerFactory):
         """
         Return a protocol suitable for the job.
 
-        :param addr: ???
-        :type addr:  ???
+        :param addr: remote ip address
+        :type addr:  str
         """
         imapProtocol = LeapIMAPServer(
             uuid=self._uuid,
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 6d6d382..e1a278a 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -295,9 +295,12 @@ class SoledadStore(ContentDedup):
     def _soledad_write_document_parts(self, items):
         """
         Write the document parts to soledad in a separate thread.
+
         :param items: the iterator through the different document wrappers
                       payloads.
         :type items: iterator
+        :return: whether the write was successful or not
+        :rtype: bool
         """
         failed = False
         for item, call in items:
-- 
cgit v1.2.3


From f67aabeb382592f3d7d597acb0389b39a353d8b8 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 14 Feb 2014 12:42:58 -0400
Subject: add cProfiler instrumentation

---
 src/leap/mail/imap/service/imap.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index b79d42d..1175cdc 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -25,6 +25,7 @@ from twisted.internet import defer, threads
 from twisted.internet.protocol import ServerFactory
 from twisted.internet.error import CannotListenError
 from twisted.mail import imap4
+from twisted.python import log
 
 logger = logging.getLogger(__name__)
 
@@ -71,6 +72,15 @@ DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
 if DO_MANHOLE:
     from leap.mail.imap.service import manhole
 
+DO_PROFILE = os.environ.get("LEAP_PROFILE", None)
+if DO_PROFILE:
+    import cProfile
+    log.msg("Starting PROFILING...")
+
+    PROFILE_DAT = "/tmp/leap_mail_profile.pstats"
+    pr = cProfile.Profile()
+    pr.enable()
+
 
 class IMAPAuthRealm(object):
     """
@@ -140,6 +150,11 @@ class LeapIMAPFactory(ServerFactory):
                  disk in another thread.
         :rtype: Deferred
         """
+        if DO_PROFILE:
+            log.msg("Stopping PROFILING")
+            pr.disable()
+            pr.dump_stats(PROFILE_DAT)
+
         ServerFactory.doStop(self)
 
         if cv is not None:
-- 
cgit v1.2.3


From ed270dcbdca5f3887953121aa0750f99b618dd3a Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 10:45:39 -0400
Subject: profile select

---
 src/leap/mail/imap/account.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index fd35698..1b5d4a0 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -19,9 +19,11 @@ Soledad Backed Account.
 """
 import copy
 import logging
+import os
 import time
 
 from twisted.mail import imap4
+from twisted.python import log
 from zope.interface import implements
 
 from leap.common.check import leap_assert, leap_assert_type
@@ -33,6 +35,15 @@ from leap.soledad.client import Soledad
 
 logger = logging.getLogger(__name__)
 
+PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
+
+if PROFILE_CMD:
+
+    def _debugProfiling(result, cmdname, start):
+        took = (time.time() - start) * 1000
+        log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
+        return result
+
 
 #######################################
 # Soledad Account
@@ -235,15 +246,20 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         :rtype: SoledadMailbox
         """
-        name = self._parse_mailbox_name(name)
+        if PROFILE_CMD:
+            start = time.time()
 
+        name = self._parse_mailbox_name(name)
         if name not in self.mailboxes:
             logger.warning("No such mailbox!")
             return None
         self.selected = name
 
-        return SoledadMailbox(
+        sm = SoledadMailbox(
             name, self._soledad, self._memstore, readwrite)
+        if PROFILE_CMD:
+            _debugProfiling(None, "SELECT", start)
+        return sm
 
     def delete(self, name, force=False):
         """
-- 
cgit v1.2.3


From f72fd69e637aa252f64c0e0ec8f7e3ebeb2290bb Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 10:50:31 -0400
Subject: speedup mailbox select

---
 src/leap/mail/imap/mailbox.py | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index d55cae6..57505f0 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -110,8 +110,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
     next_uid_lock = threading.Lock()
     last_uid_lock = threading.Lock()
 
+    # TODO unify all the `primed` dicts
     _fdoc_primed = {}
     _last_uid_primed = {}
+    _known_uids_primed = {}
 
     def __init__(self, mbox, soledad, memstore, rw=1):
         """
@@ -130,6 +132,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param rw: read-and-write flag for this mailbox
         :type rw: int
         """
+        logger.debug("Initializing mailbox %r" % (mbox,))
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
@@ -146,6 +149,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self.messages = MessageCollection(
             mbox=mbox, soledad=self._soledad, memstore=self._memstore)
 
+        # XXX careful with this get/set (it would be
+        # hitting db unconditionally, move to memstore too)
+        # Now it's returning a fixed amount of flags from mem
+        # as a workaround.
         if not self.getFlags():
             self.setFlags(self.INIT_FLAGS)
 
@@ -159,6 +166,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         # purge memstore from empty fdocs.
         self._memstore.purge_fdoc_store(mbox)
+        logger.debug("DONE initializing mailbox %r" % (mbox,))
 
     @property
     def listeners(self):
@@ -217,10 +225,18 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :returns: tuple of flags for this mailbox
         :rtype: tuple of str
         """
-        mbox = self._get_mbox_doc()
-        if not mbox:
-            return None
-        flags = mbox.content.get(self.FLAGS_KEY, [])
+        flags = self.INIT_FLAGS
+
+        # XXX returning fixed flags always
+        # Since I have not found a case where the client
+        # wants to modify this, as a way of speeding up
+        # selects. To do it right, we probably should keep
+        # track of the set of all flags used by msgs
+        # in this mailbox. Does it matter?
+        #mbox = self._get_mbox_doc()
+        #if not mbox:
+            #return None
+        #flags = mbox.content.get(self.FLAGS_KEY, [])
         return map(str, flags)
 
     # XXX move to memstore->soledadstore
@@ -237,6 +253,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not mbox:
             return None
         mbox.content[self.FLAGS_KEY] = map(str, flags)
+        logger.debug("Writing mbox document for %r to Soledad"
+                     % (self.mbox,))
         self._soledad.put_doc(mbox)
 
     # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
@@ -298,8 +316,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         We do this to be able to filter the requests efficiently.
         """
-        known_uids = self.messages.all_soledad_uid_iter()
-        self._memstore.set_known_uids(self.mbox, known_uids)
+        primed = self._known_uids_primed.get(self.mbox, False)
+        if not primed:
+            known_uids = self.messages.all_soledad_uid_iter()
+            self._memstore.set_known_uids(self.mbox, known_uids)
+            self._known_uids_primed[self.mbox] = True
 
     def prime_flag_docs_to_memstore(self):
         """
@@ -465,6 +486,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = self.messages.add_msg(message, flags=flags, date=date)
         return d
 
+    @deferred_to_thread
     def notify_new(self, *args):
         """
         Notify of new messages to all the listeners.
@@ -836,7 +858,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = defer.Deferred()
         if PROFILE_CMD:
             do_profile_cmd(d, "COPY")
-        d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
         deferLater(self.reactor, 0, self._do_copy, message, d)
         return d
 
@@ -863,9 +884,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
                 # XXX I'm not sure if we should raise the
                 # errback. This actually rases an ugly warning
-                # in some muas like thunderbird. I guess the user does
-                # not deserve that.
-                observer.callback(True)
+                # in some muas like thunderbird.
+                # UID 0 seems a good convention for no uid.
+                observer.callback(0)
             else:
                 mbox = self.mbox
                 uid_next = memstore.increment_last_soledad_uid(mbox)
-- 
cgit v1.2.3


From 5af059a237833f52869a632e490ff932315a4939 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 10:52:17 -0400
Subject: defer message push to thread

---
 src/leap/mail/imap/memorystore.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 53b8d99..2d1f95b 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -42,6 +42,8 @@ from leap.mail.imap.messageparts import RecentFlagsDoc
 from leap.mail.imap.messageparts import MessageWrapper
 from leap.mail.imap.messageparts import ReferenciableDict
 
+from leap.mail.decorators import deferred_to_thread
+
 logger = logging.getLogger(__name__)
 
 
@@ -514,6 +516,7 @@ class MemoryStore(object):
 
     # IMessageStoreWriter
 
+    @deferred_to_thread
     def write_messages(self, store):
         """
         Write the message documents in this MemoryStore to a different store.
-- 
cgit v1.2.3


From e9488bf377f07f6f05d3fdd2eb316843cf561605 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 10:52:48 -0400
Subject: freeze dirty/new sets to avoid changes during iteration

---
 src/leap/mail/imap/memorystore.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 2d1f95b..f23a234 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -889,7 +889,8 @@ class MemoryStore(object):
         :rtype: generator
         """
         gm = self.get_message
-        new = [gm(*key, dirtystate=DirtyState.new) for key in self._new]
+        # need to freeze, set can change during iteration
+        new = [gm(*key, dirtystate=DirtyState.new) for key in tuple(self._new)]
         # move content from new set to the queue
         self._new_queue.update(self._new)
         self._new.difference_update(self._new)
@@ -903,8 +904,9 @@ class MemoryStore(object):
         :rtype: generator
         """
         gm = self.get_message
+        # need to freeze, set can change during iteration
         dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty)
-                 for key in self._dirty]
+                 for key in tuple(self._dirty)]
         # move content from new and dirty sets to the queue
 
         self._dirty_queue.update(self._dirty)
-- 
cgit v1.2.3


From 2f114c5a85775f4044aa12cd421231ed92544549 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 10:53:30 -0400
Subject: remove floody log

---
 src/leap/mail/imap/soledadstore.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index e1a278a..732fe03 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -529,9 +529,6 @@ class SoledadStore(ContentDedup):
             if value > old_val:
                 mbox_doc.content[key] = value
                 self._soledad.put_doc(mbox_doc)
-            else:
-                logger.error("%r:%s Tried to write a UID lesser than what's "
-                             "stored!" % (mbox, value))
 
     def get_flags_doc(self, mbox, uid):
         """
-- 
cgit v1.2.3


From 985ff0a78a8df0eafb7789383f711b9e5ceb1cb6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 11:08:37 -0400
Subject: Remove notify_new callbacks from fetch and copy.

This fixes a bug with qtreactor that was making the 'OK foo copied'
not being delivered.

This or something similar will probably have to be re-added, because
on the current state the destination folder will not receive the
notification if it's selected *before* the copy operation has finished.

But in this way we have a clean slate that is working properly.
The bottleneck in the copy/append operations seems to have moved to
the select operation now.
---
 src/leap/mail/imap/server.py | 28 +---------------------------
 1 file changed, 1 insertion(+), 27 deletions(-)

diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 7c09784..5da9bfd 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -20,9 +20,7 @@ Leap IMAP4 Server Implementation.
 from copy import copy
 
 from twisted import cred
-from twisted.internet import defer
 from twisted.internet.defer import maybeDeferred
-from twisted.internet.task import deferLater
 from twisted.mail import imap4
 from twisted.python import log
 
@@ -135,35 +133,11 @@ class LeapIMAPServer(imap4.IMAP4Server):
             ).addCallback(
                 cbFetch, tag, query, uid
             ).addErrback(
-                ebFetch, tag
-            ).addCallback(
-                self.on_fetch_finished, messages)
+                ebFetch, tag)
 
     select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
                     imap4.IMAP4Server.arg_fetchatt)
 
-    def on_fetch_finished(self, _, messages):
-        deferLater(self.reactor, 0, self.notifyNew)
-        deferLater(self.reactor, 0, self.mbox.unset_recent_flags, messages)
-        deferLater(self.reactor, 0, self.mbox.signal_unread_to_ui)
-
-    def on_copy_finished(self, defers):
-        d = defer.gatherResults(filter(None, defers))
-
-        def when_finished(result):
-            self.notifyNew()
-            self.mbox.signal_unread_to_ui()
-        d.addCallback(when_finished)
-
-    def do_COPY(self, tag, messages, mailbox, uid=0):
-        defers = []
-        d = imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid)
-        defers.append(d)
-        deferLater(self.reactor, 0, self.on_copy_finished, defers)
-
-    select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset,
-                   imap4.IMAP4Server.arg_astring)
-
     def notifyNew(self, ignored=None):
         """
         Notify new messages to listeners.
-- 
cgit v1.2.3


From ce9185be85d82d95799dfb0644d4f363feeeeeae Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Mon, 17 Feb 2014 18:46:12 -0300
Subject: Update keymanager kwargs, related to #5120.

---
 src/leap/mail/imap/service/imap-server.tac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index b65bb17..feeca06 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -132,7 +132,7 @@ tempdir = "/tmp/"
 soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir)
 km_args = (userid, "https://localhost", soledad)
 km_kwargs = { 
-	"session_id":  "", 
+	"token":  "",
 	"ca_cert_path":  "", 
 	"api_uri":  "", 
 	"api_version":  "", 
-- 
cgit v1.2.3


From c8b2163c8d8063f5e9a5cc7a2a721e933f002be0 Mon Sep 17 00:00:00 2001
From: Ivan Alejandro <ivanalejandro0@gmail.com>
Date: Mon, 17 Feb 2014 18:50:53 -0300
Subject: pep8 fixes.

---
 src/leap/mail/imap/service/imap-server.tac | 33 ++++++++++++++++--------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index feeca06..651f71b 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# imap-server.tac 
+# imap-server.tac
 # Copyright (C) 2013,2014 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
@@ -97,8 +97,10 @@ print "[+] Running LEAP IMAP Service"
 
 bmconf = os.environ.get("LEAP_MAIL_CONF", "")
 if not bmconf:
-	print "[-] Please set LEAP_MAIL_CONF environment variable pointing to your config."
-	sys.exit(1)
+    print ("[-] Please set LEAP_MAIL_CONF environment variable "
+           "pointing to your config.")
+    sys.exit(1)
+
 SECTION = "leap_mail"
 cp = ConfigParser.ConfigParser()
 cp.read(bmconf)
@@ -111,11 +113,11 @@ passwd = unicode(cp.get(SECTION, "passwd"))
 port = 1984
 
 if not userid or not uuid:
-	print "[-] Config file missing userid or uuid field"
-	sys.exit(1)
+    print "[-] Config file missing userid or uuid field"
+    sys.exit(1)
 
 if not passwd:
-	passwd = unicode(getpass.getpass("Soledad passphrase: "))
+    passwd = unicode(getpass.getpass("Soledad passphrase: "))
 
 
 secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,))
@@ -129,16 +131,17 @@ tempdir = "/tmp/"
 
 # Ad-hoc soledad/keymanager initialization.
 
-soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir)
+soledad = initialize_soledad(uuid, userid, passwd, secrets,
+                             localdb, gnupg_home, tempdir)
 km_args = (userid, "https://localhost", soledad)
-km_kwargs = { 
-	"token":  "",
-	"ca_cert_path":  "", 
-	"api_uri":  "", 
-	"api_version":  "", 
-	"uid": uuid,
-	"gpgbinary": "/usr/bin/gpg"
-}   
+km_kwargs = {
+    "token": "",
+    "ca_cert_path": "",
+    "api_uri":  "",
+    "api_version": "",
+    "uid": uuid,
+    "gpgbinary": "/usr/bin/gpg"
+}
 keymanager = KeyManager(*km_args, **km_kwargs)
 
 ##################################################
-- 
cgit v1.2.3


From daa0c9fd588c61f2e440453be83af137779ce207 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 12:18:41 -0400
Subject: cache uidvalidity

---
 src/leap/mail/imap/mailbox.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 57505f0..6513db9 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -149,6 +149,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self.messages = MessageCollection(
             mbox=mbox, soledad=self._soledad, memstore=self._memstore)
 
+        self._uidvalidity = None
+
         # XXX careful with this get/set (it would be
         # hitting db unconditionally, move to memstore too)
         # Now it's returning a fixed amount of flags from mem
@@ -339,8 +341,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :return: unique validity identifier
         :rtype: int
         """
-        mbox = self._get_mbox_doc()
-        return mbox.content.get(self.CREATED_KEY, 1)
+        if self._uidvalidity is None:
+            mbox = self._get_mbox_doc()
+            self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1)
+        return self._uidvalidity
 
     def getUID(self, message):
         """
-- 
cgit v1.2.3


From e87dba3288345ba28dce5a844a7faf9f5a5a0b9c Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 12:19:02 -0400
Subject: remove size calculation until we defer it to thread properly

---
 src/leap/mail/imap/memorystore.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index f23a234..56cd000 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -364,9 +364,11 @@ class MemoryStore(object):
 
         # Update memory store size
         # XXX this should use [mbox][uid]
-        key = mbox, uid
-        self._sizes[key] = size.get_size(self._fdoc_store[key])
+        # TODO --- this has to be deferred to thread,
         # TODO add hdoc and cdocs sizes too
+        # it's slowing things down here.
+        #key = mbox, uid
+        #self._sizes[key] = size.get_size(self._fdoc_store[key])
 
     def purge_fdoc_store(self, mbox):
         """
-- 
cgit v1.2.3


From 0f2f53c8819133e36780e521fecbfadda331255a Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 13:00:41 -0400
Subject: defer fetch-all-flags too

---
 src/leap/mail/imap/mailbox.py     | 28 +++++++++++++++++++++++++---
 src/leap/mail/imap/memorystore.py |  9 ++++-----
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 6513db9..be8b429 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -656,15 +656,37 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                                about
         :type messages_asked: MessageSet
 
-        :param uid: If true, the IDs are UIDs. They are message sequence IDs
+        :param uid: If 1, the IDs are UIDs. They are message sequence IDs
                     otherwise.
-        :type uid: bool
+        :type uid: int
 
         :return: A tuple of two-tuples of message sequence numbers and
                 flagsPart, which is a only a partial implementation of
                 MessagePart.
         :rtype: tuple
         """
+        d = defer.Deferred()
+        self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d)
+        if PROFILE_CMD:
+            do_profile_cmd(d, "FETCH-ALL-FLAGS")
+        return d
+
+    # called in thread
+    def _do_fetch_flags(self, messages_asked, uid, d):
+        """
+        :param messages_asked: IDs of the messages to retrieve information
+                               about
+        :type messages_asked: MessageSet
+
+        :param uid: If 1, the IDs are UIDs. They are message sequence IDs
+                    otherwise.
+        :type uid: int
+        :param d: deferred whose callback will be called with result.
+        :type d: Deferred
+
+        :rtype: A tuple of two-tuples of message sequence numbers and
+                flagsPart
+        """
         class flagsPart(object):
             def __init__(self, uid, flags):
                 self.uid = uid
@@ -682,7 +704,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         all_flags = self._memstore.all_flags(self.mbox)
         result = ((msgid, flagsPart(
             msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg)
-        return result
+        self.reactor.callFromThread(d.callback, result)
 
     def fetch_headers(self, messages_asked, uid):
         """
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 56cd000..875b1b8 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -726,17 +726,16 @@ class MemoryStore(object):
         :type mbox: str or unicode
         :rtype: dict
         """
-        flags_dict = {}
+        fdict = {}
         uids = self.get_uids(mbox)
-        fdoc_store = self._fdoc_store[mbox]
+        fstore = self._fdoc_store[mbox]
 
         for uid in uids:
             try:
-                flags = fdoc_store[uid][fields.FLAGS_KEY]
-                flags_dict[uid] = flags
+                fdict[uid] = fstore[uid][fields.FLAGS_KEY]
             except KeyError:
                 continue
-        return flags_dict
+        return fdict
 
     def all_headers(self, mbox):
         """
-- 
cgit v1.2.3


From 99ec94f08fb2d062eb2c350b64971ea9ad8d87dd Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 17 Feb 2014 13:59:06 -0400
Subject: avoid unneeded db index updates and rdoc creation

---
 src/leap/mail/imap/mailbox.py  |  6 ------
 src/leap/mail/imap/messages.py | 10 +++++++---
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index be8b429..d7be662 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -132,14 +132,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :param rw: read-and-write flag for this mailbox
         :type rw: int
         """
-        logger.debug("Initializing mailbox %r" % (mbox,))
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
-        # XXX should move to wrapper
-        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    #"soledad._db must be an instance of SQLCipherDatabase")
-
         self.mbox = self._parse_mailbox_name(mbox)
         self.rw = rw
 
@@ -168,7 +163,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
         # purge memstore from empty fdocs.
         self._memstore.purge_fdoc_store(mbox)
-        logger.debug("DONE initializing mailbox %r" % (mbox,))
 
     @property
     def listeners(self):
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index fc1ec55..9bd64fc 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -686,6 +686,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
     _rdoc_lock = threading.Lock()
     _rdoc_property_lock = threading.Lock()
 
+    _initialized = {}
+
     def __init__(self, mbox=None, soledad=None, memstore=None):
         """
         Constructor for MessageCollection.
@@ -725,10 +727,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         self.memstore = memstore
 
         self.__rflags = None
-        self.initialize_db()
 
-        # ensure that we have a recent-flags and a hdocs-sec doc
-        self._get_or_create_rdoc()
+        if not self._initialized.get(mbox, False):
+            self.initialize_db()
+            # ensure that we have a recent-flags and a hdocs-sec doc
+            self._get_or_create_rdoc()
+            self._initialized[mbox] = True
 
         from twisted.internet import reactor
         self.reactor = reactor
-- 
cgit v1.2.3


From d6ee618534596e76733d3c6b6cf0a75bdb4aa905 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 18 Feb 2014 09:58:54 -0400
Subject: catch soledad error while updating mbox doc

---
 src/leap/mail/imap/account.py      |  1 +
 src/leap/mail/imap/soledadstore.py | 21 ++++++++++++---------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 1b5d4a0..ede63d3 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -119,6 +119,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
 
         :rtype: SoledadDocument
         """
+        # XXX use soledadstore instead ...;
         doc = self._soledad.get_from_index(
             self.TYPE_MBOX_IDX, self.MBOX_KEY,
             self._parse_mailbox_name(name))
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 732fe03..919f834 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -133,15 +133,14 @@ A lock per document.
 # Setting this to twice the number of threads in the threadpool
 # should be safe.
 put_locks = defaultdict(lambda: threading.Lock())
+mbox_doc_locks = defaultdict(lambda: threading.Lock())
 
 
 class SoledadStore(ContentDedup):
     """
     This will create docs in the local Soledad database.
     """
-    _soledad_rw_lock = threading.Lock()
     _remove_lock = threading.Lock()
-    _mbox_doc_locks = defaultdict(lambda: threading.Lock())
 
     implements(IMessageConsumer, IMessageStore)
 
@@ -456,7 +455,7 @@ class SoledadStore(ContentDedup):
                  the query failed.
         :rtype: SoledadDocument or None.
         """
-        with self._mbox_doc_locks[mbox]:
+        with mbox_doc_locks[mbox]:
             return self._get_mbox_document(mbox)
 
     def _get_mbox_document(self, mbox):
@@ -471,7 +470,7 @@ class SoledadStore(ContentDedup):
                 return query.pop()
             else:
                 logger.error("Could not find mbox document for %r" %
-                            (self.mbox,))
+                            (mbox,))
         except Exception as exc:
             logger.exception("Unhandled error %r" % exc)
 
@@ -496,7 +495,7 @@ class SoledadStore(ContentDedup):
         :type closed: bool
         """
         leap_assert(isinstance(closed, bool), "closed needs to be boolean")
-        with self._mbox_doc_locks[mbox]:
+        with mbox_doc_locks[mbox]:
             mbox_doc = self._get_mbox_document(mbox)
             if mbox_doc is None:
                 logger.error(
@@ -521,14 +520,18 @@ class SoledadStore(ContentDedup):
         leap_assert_type(value, int)
         key = fields.LAST_UID_KEY
 
-        # XXX change for a lock related to the mbox document
-        # itself.
-        with self._mbox_doc_locks[mbox]:
+        # XXX use accumulator to reduce number of hits
+        with mbox_doc_locks[mbox]:
             mbox_doc = self._get_mbox_document(mbox)
             old_val = mbox_doc.content[key]
             if value > old_val:
                 mbox_doc.content[key] = value
-                self._soledad.put_doc(mbox_doc)
+                try:
+                    self._soledad.put_doc(mbox_doc)
+                except Exception as exc:
+                    logger.error("Error while setting last_uid for %r"
+                                 % (mbox,))
+                    logger.exception(exc)
 
     def get_flags_doc(self, mbox, uid):
         """
-- 
cgit v1.2.3


From 95e00da239bd119ae161f249a74af229cb6c7759 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 00:00:24 -0400
Subject: fix attribute error on debug line

---
 src/leap/mail/imap/soledadstore.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 919f834..ed5259a 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -308,8 +308,10 @@ class SoledadStore(ContentDedup):
             try:
                 self._try_call(call, item)
             except Exception as exc:
-                logger.debug("ITEM WAS: %s" % str(item))
-                logger.debug("ITEM CONTENT WAS: %s" % str(item.content))
+                logger.debug("ITEM WAS: %s" % repr(item))
+                if hasattr(item, 'content'):
+                        logger.debug("ITEM CONTENT WAS: %s" %
+                                     repr(item.content))
                 logger.exception(exc)
                 failed = True
                 continue
-- 
cgit v1.2.3


From 4bcb32639bff9a5aab076dba2bdc7667cea60c7f Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 01:11:26 -0400
Subject: fix rdoc duplication

---
 src/leap/mail/imap/mailbox.py      |  6 ++---
 src/leap/mail/imap/memorystore.py  |  2 ++
 src/leap/mail/imap/messages.py     | 51 ++++++++++++++++++++++----------------
 src/leap/mail/imap/soledadstore.py |  7 ++++--
 4 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index d7be662..59b2b40 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -135,6 +135,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
+        from twisted.internet import reactor
+        self.reactor = reactor
+
         self.mbox = self._parse_mailbox_name(mbox)
         self.rw = rw
 
@@ -158,9 +161,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             self.prime_last_uid_to_memstore()
             self.prime_flag_docs_to_memstore()
 
-        from twisted.internet import reactor
-        self.reactor = reactor
-
         # purge memstore from empty fdocs.
         self._memstore.purge_fdoc_store(mbox)
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 875b1b8..aa7da3d 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -506,6 +506,8 @@ class MemoryStore(object):
             if key in self._sizes:
                 del self._sizes[key]
             self._known_uids[mbox].discard(uid)
+        except KeyError:
+            pass
         except Exception as exc:
             logger.error("error while removing message!")
             logger.exception(exc)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 9bd64fc..8c777f5 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -77,7 +77,7 @@ def try_unique_query(curried):
                 # TODO we could take action, like trigger a background
                 # process to kill dupes.
                 name = getattr(curried, 'expected', 'doc')
-                logger.debug(
+                logger.warning(
                     "More than one %s found for this mbox, "
                     "we got a duplicate!!" % (name,))
             return query.pop()
@@ -683,8 +683,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # TODO we would abstract this to a SoledadProperty class
 
-    _rdoc_lock = threading.Lock()
-    _rdoc_property_lock = threading.Lock()
+    _rdoc_lock = defaultdict(lambda: threading.Lock())
+    _rdoc_write_lock = defaultdict(lambda: threading.Lock())
+    _rdoc_read_lock = defaultdict(lambda: threading.Lock())
+    _rdoc_property_lock = defaultdict(lambda: threading.Lock())
 
     _initialized = {}
 
@@ -729,10 +731,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         self.__rflags = None
 
         if not self._initialized.get(mbox, False):
-            self.initialize_db()
-            # ensure that we have a recent-flags and a hdocs-sec doc
-            self._get_or_create_rdoc()
-            self._initialized[mbox] = True
+            try:
+                self.initialize_db()
+                # ensure that we have a recent-flags doc
+                self._get_or_create_rdoc()
+            except Exception:
+                logger.debug("Error initializing %r" % (mbox,))
+            else:
+                self._initialized[mbox] = True
 
         from twisted.internet import reactor
         self.reactor = reactor
@@ -753,12 +759,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         Try to retrieve the recent-flags doc for this MessageCollection,
         and create one if not found.
         """
-        rdoc = self._get_recent_doc()
-        if not rdoc:
-            rdoc = self._get_empty_doc(self.RECENT_DOC)
-            if self.mbox != fields.INBOX_VAL:
-                rdoc[fields.MBOX_KEY] = self.mbox
-            self._soledad.create_doc(rdoc)
+        # XXX should move this to memstore too
+        with self._rdoc_write_lock[self.mbox]:
+            rdoc = self._get_recent_doc_from_soledad()
+            if rdoc is None:
+                rdoc = self._get_empty_doc(self.RECENT_DOC)
+                if self.mbox != fields.INBOX_VAL:
+                    rdoc[fields.MBOX_KEY] = self.mbox
+                self._soledad.create_doc(rdoc)
 
     @deferred_to_thread
     def _do_parse(self, raw):
@@ -976,12 +984,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             return self.__rflags
 
         if self.memstore is not None:
-            with self._rdoc_lock:
+            with self._rdoc_lock[self.mbox]:
                 rflags = self.memstore.get_recent_flags(self.mbox)
                 if not rflags:
                     # not loaded in the memory store yet.
                     # let's fetch them from soledad...
-                    rdoc = self._get_recent_doc()
+                    rdoc = self._get_recent_doc_from_soledad()
                     rflags = set(rdoc.content.get(
                         fields.RECENTFLAGS_KEY, []))
                     # ...and cache them now.
@@ -1001,8 +1009,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
 
-    # XXX change naming, indicate soledad query.
-    def _get_recent_doc(self):
+    def _get_recent_doc_from_soledad(self):
         """
         Get recent-flags document from Soledad for this mailbox.
         :rtype: SoledadDocument or None
@@ -1012,8 +1019,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             fields.TYPE_MBOX_IDX,
             fields.TYPE_RECENT_VAL, self.mbox)
         curried.expected = "rdoc"
-        rdoc = try_unique_query(curried)
-        return rdoc
+        with self._rdoc_read_lock[self.mbox]:
+            return try_unique_query(curried)
 
     # Property-set modification (protected by a different
     # lock to give atomicity to the read/write operation)
@@ -1025,7 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param uids: the uids to unset
         :type uid: sequence
         """
-        with self._rdoc_property_lock:
+        with self._rdoc_property_lock[self.mbox]:
             self.recent_flags.difference_update(
                 set(uids))
 
@@ -1038,7 +1045,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param uid: the uid to unset
         :type uid: int
         """
-        with self._rdoc_property_lock:
+        with self._rdoc_property_lock[self.mbox]:
             self.recent_flags.difference_update(
                 set([uid]))
 
@@ -1050,7 +1057,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param uid: the uid to set
         :type uid: int
         """
-        with self._rdoc_property_lock:
+        with self._rdoc_property_lock[self.mbox]:
             self.recent_flags = self.recent_flags.union(
                 set([uid]))
 
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index ed5259a..e047e2e 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -350,6 +350,9 @@ class SoledadStore(ContentDedup):
 
         if call == self._PUT_DOC_FUN:
             doc_id = item.doc_id
+            if doc_id is None:
+                logger.warning("BUG! Dirty doc but has no doc_id!")
+                return
             with put_locks[doc_id]:
                 doc = self._GET_DOC_FUN(doc_id)
 
@@ -438,12 +441,12 @@ class SoledadStore(ContentDedup):
         :return: a tuple with recent-flags doc payload and callable
         :rtype: tuple
         """
-        call = self._CREATE_DOC_FUN
+        call = self._PUT_DOC_FUN
 
         payload = rflags_wrapper.content
         if payload:
             logger.debug("Saving RFLAGS to Soledad...")
-            yield payload, call
+            yield rflags_wrapper, call
 
     # Mbox documents and attributes
 
-- 
cgit v1.2.3


From 52868a4c67170abbfa19deda9bd20931c21554b7 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 01:21:46 -0400
Subject: catch stopiteration

---
 src/leap/mail/imap/soledadstore.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index e047e2e..25f00bb 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -281,9 +281,13 @@ class SoledadStore(ContentDedup):
         def doSoledadCalls(items):
             # we prime the generator, that should return the
             # message or flags wrapper item in the first place.
-            doc_wrapper = items.next()
-            failed = self._soledad_write_document_parts(items)
-            queueNotifyBack(failed, doc_wrapper)
+            try:
+                doc_wrapper = items.next()
+            except StopIteration:
+                pass
+            else:
+                failed = self._soledad_write_document_parts(items)
+                queueNotifyBack(failed, doc_wrapper)
 
         doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper))
 
-- 
cgit v1.2.3


From c9ada6da8f3c94efd0739abd8be46c6356854a49 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 01:22:01 -0400
Subject: catch empty rdoc

---
 src/leap/mail/imap/messages.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 8c777f5..9f7f6e2 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -990,6 +990,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                     # not loaded in the memory store yet.
                     # let's fetch them from soledad...
                     rdoc = self._get_recent_doc_from_soledad()
+                    if rdoc is None:
+                        return set([])
                     rflags = set(rdoc.content.get(
                         fields.RECENTFLAGS_KEY, []))
                     # ...and cache them now.
-- 
cgit v1.2.3


From 976ec85451bef3fd380f69c64e803d7740d7dae4 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 01:27:17 -0400
Subject: ignore keyerror on deletion

---
 src/leap/mail/imap/memorystore.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index aa7da3d..6206468 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -514,6 +514,8 @@ class MemoryStore(object):
         try:
             with self._fdoc_docid_lock:
                 del self._fdoc_id_store[mbox][uid]
+        except KeyError:
+            pass
         except Exception as exc:
             logger.error("error while removing message!")
             logger.exception(exc)
-- 
cgit v1.2.3


From b2d97c9faef6037a065e2903afe5b0ab2624917e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 02:52:17 -0400
Subject: mail parsing performance improvements

Although the do_parse function is deferred to threads, we were actually
waiting till its return to fire the callback of the deferred, and hence
the "append ok" was being delayed. During massive appends, this was a
tight loop contributing as much as 35 msec, of a total of 100 msec
average.

Several ineficiencies are addressed here:

* use pycryptopp hash functions.
* avoiding function calling overhead.
* avoid duplicate call to message.as_string
* make use of the string size caching capabilities.
* avoiding the mail Parser initialization/method call completely,
  in favor of the module helper to get the object from string.

Overall, these changes cut parsing to 50% of the initial timing by my
measurements with line_profiler, YMMV.
---
 src/leap/mail/imap/messages.py       | 25 ++++++------
 src/leap/mail/imap/parser.py         | 75 +-----------------------------------
 src/leap/mail/imap/soledadstore.py   |  4 +-
 src/leap/mail/imap/tests/walktree.py |  4 +-
 src/leap/mail/walk.py                |  7 ++--
 5 files changed, 21 insertions(+), 94 deletions(-)

diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 9f7f6e2..9a001b3 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -24,8 +24,10 @@ import threading
 import StringIO
 
 from collections import defaultdict
+from email import message_from_string
 from functools import partial
 
+from pycryptopp.hash import sha256
 from twisted.mail import imap4
 from twisted.internet import defer
 from zope.interface import implements
@@ -42,7 +44,7 @@ from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.memorystore import MessageWrapper
 from leap.mail.imap.messageparts import MessagePart
-from leap.mail.imap.parser import MailParser, MBoxParser
+from leap.mail.imap.parser import MBoxParser
 
 logger = logging.getLogger(__name__)
 
@@ -94,7 +96,7 @@ A dictionary that keeps one lock per mbox and uid.
 fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
 
 
-class LeapMessage(fields, MailParser, MBoxParser):
+class LeapMessage(fields, MBoxParser):
     """
     The main representation of a message.
 
@@ -123,7 +125,6 @@ class LeapMessage(fields, MailParser, MBoxParser):
         :param container: a IMessageContainer implementor instance
         :type container: IMessageContainer
         """
-        MailParser.__init__(self)
         self._soledad = soledad
         self._uid = int(uid)
         self._mbox = self._parse_mailbox_name(mbox)
@@ -583,7 +584,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
         return not empty(self.fdoc)
 
 
-class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
+class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
     """
     A collection of messages, surprisingly.
 
@@ -713,7 +714,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param memstore: a MemoryStore instance
         :type memstore: MemoryStore
         """
-        MailParser.__init__(self)
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(mbox.strip() != "", "mbox cannot be blank space")
         leap_assert(isinstance(mbox, (str, unicode)),
@@ -782,11 +782,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :return: msg, parts, chash, size, multi
         :rtype: tuple
         """
-        msg = self._get_parsed_msg(raw)
-        chash = self._get_hash(msg)
-        size = len(msg.as_string())
-        multi = msg.is_multipart()
+        msg = message_from_string(raw)
         parts = walk.get_parts(msg)
+        size = len(raw)
+        chash = sha256.SHA256(raw).hexdigest()
+        multi = msg.is_multipart()
         return msg, parts, chash, size, multi
 
     def _populate_flags(self, flags, uid, chash, size, multi):
@@ -803,7 +803,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         fd[self.SIZE_KEY] = size
         fd[self.MULTIPART_KEY] = multi
         if flags:
-            fd[self.FLAGS_KEY] = map(self._stringify, flags)
+            fd[self.FLAGS_KEY] = flags
             fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
             fd[self.DEL_KEY] = self.DELETED_FLAG in flags
             fd[self.RECENT_KEY] = True  # set always by default
@@ -926,11 +926,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # Watch out! We're reserving a UID right after this!
         existing_uid = self._fdoc_already_exists(chash)
         if existing_uid:
-            uid = existing_uid
-            msg = self.get_msg_by_uid(uid)
+            msg = self.get_msg_by_uid(existing_uid)
 
             # We can say the observer that we're done
-            self.reactor.callFromThread(observer.callback, uid)
+            self.reactor.callFromThread(observer.callback, existing_uid)
             msg.setFlags((fields.DELETED_FLAG,), -1)
             return
 
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
index 6a9ace9..4a801b0 100644
--- a/src/leap/mail/imap/parser.py
+++ b/src/leap/mail/imap/parser.py
@@ -15,83 +15,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-Mail parser mixins.
+Mail parser mixin.
 """
-import cStringIO
-import StringIO
-import hashlib
 import re
 
-from email.message import Message
-from email.parser import Parser
-
-from leap.common.check import leap_assert_type
-
-
-class MailParser(object):
-    """
-    Mixin with utility methods to parse raw messages.
-    """
-    def __init__(self):
-        """
-        Initializes the mail parser.
-        """
-        self._parser = Parser()
-
-    def _get_parsed_msg(self, raw, headersonly=False):
-        """
-        Return a parsed Message.
-
-        :param raw: the raw string to parse
-        :type raw: basestring, or StringIO object
-
-        :param headersonly: True for parsing only the headers.
-        :type headersonly: bool
-        """
-        msg = self._get_parser_fun(raw)(raw, headersonly=headersonly)
-        return msg
-
-    def _get_hash(self, msg):
-        """
-        Returns a hash of the string representation of the raw message,
-        suitable for indexing the inmutable pieces.
-
-        :param msg: a Message object
-        :type msg: Message
-        """
-        leap_assert_type(msg, Message)
-        return hashlib.sha256(msg.as_string()).hexdigest()
-
-    def _get_parser_fun(self, o):
-        """
-        Retunn the proper parser function for an object.
-
-        :param o: object
-        :type o: object
-        :param parser: an instance of email.parser.Parser
-        :type parser: email.parser.Parser
-        """
-        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-            return self._parser.parse
-        if isinstance(o, basestring):
-            return self._parser.parsestr
-        # fallback
-        return self._parser.parsestr
-
-    def _stringify(self, o):
-        """
-        Return a string object.
-
-        :param o: object
-        :type o: object
-        """
-        # XXX Maybe we don't need no more, we're using
-        # msg.as_string()
-        if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
-            return o.getvalue()
-        else:
-            return o
-
 
 class MBoxParser(object):
     """
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 25f00bb..f3de8eb 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -314,8 +314,8 @@ class SoledadStore(ContentDedup):
             except Exception as exc:
                 logger.debug("ITEM WAS: %s" % repr(item))
                 if hasattr(item, 'content'):
-                        logger.debug("ITEM CONTENT WAS: %s" %
-                                     repr(item.content))
+                    logger.debug("ITEM CONTENT WAS: %s" %
+                                 repr(item.content))
                 logger.exception(exc)
                 failed = True
                 continue
diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py
index f3cbcb0..695f487 100644
--- a/src/leap/mail/imap/tests/walktree.py
+++ b/src/leap/mail/imap/tests/walktree.py
@@ -36,11 +36,11 @@ p = parser.Parser()
 if len(sys.argv) > 1:
     FILENAME = sys.argv[1]
 else:
-    FILENAME = "rfc822.multi-minimal.message"
+    FILENAME = "rfc822.multi-signed.message"
 
 """
-FILENAME = "rfc822.multi-signed.message"
 FILENAME = "rfc822.plain.message"
+FILENAME = "rfc822.multi-minimal.message"
 """
 
 msg = p.parse(open(FILENAME))
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 49f2c22..f747377 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -17,17 +17,18 @@
 """
 Utilities for walking along a message tree.
 """
-import hashlib
 import os
 
+from pycryptopp.hash import sha256
+
 from leap.mail.utils import first
 
 DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
 
 if DEBUG:
-    get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10]
+    get_hash = lambda s: sha256.SHA256(s).hexdigest()[:10]
 else:
-    get_hash = lambda s: hashlib.sha256(s).hexdigest()
+    get_hash = lambda s: sha256.SHA256(s).hexdigest()
 
 
 """
-- 
cgit v1.2.3


From bd476d7ba97a479db14a9b72b8b52ef5997d98f6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 20 Feb 2014 17:07:58 -0400
Subject: Fix regression on "duplicate drafts" issue.

Not a permanent solution, but it looks for fdoc matching a given
msgid to avoid duplication of drafts in thunderbird folders.
---
 src/leap/mail/imap/mailbox.py  |  4 +++-
 src/leap/mail/imap/messages.py | 39 ++++++++++++++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 59b2b40..947cf1b 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -354,7 +354,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: int
         """
         msg = self.messages.get_msg_by_uid(message)
-        return msg.getUID()
+        if msg is not None:
+            return msg.getUID()
 
     def getUIDNext(self):
         """
@@ -854,6 +855,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if len(query) > 2:
             if query[1] == 'HEADER' and query[2].lower() == "message-id":
                 msgid = str(query[3]).strip()
+                logger.debug("Searching for %s" % (msgid,))
                 d = self.messages._get_uid_from_msgid(str(msgid))
                 d1 = defer.gatherResults([d])
                 # we want a list, so return it all the same
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 9a001b3..b0b2f95 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -43,7 +43,7 @@ from leap.mail.decorators import deferred_to_thread
 from leap.mail.imap.index import IndexedDB
 from leap.mail.imap.fields import fields, WithMsgFields
 from leap.mail.imap.memorystore import MessageWrapper
-from leap.mail.imap.messageparts import MessagePart
+from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
 from leap.mail.imap.parser import MBoxParser
 
 logger = logging.getLogger(__name__)
@@ -126,7 +126,7 @@ class LeapMessage(fields, MBoxParser):
         :type container: IMessageContainer
         """
         self._soledad = soledad
-        self._uid = int(uid)
+        self._uid = int(uid) if uid is not None else None
         self._mbox = self._parse_mailbox_name(mbox)
         self._collection = collection
         self._container = container
@@ -1077,7 +1077,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
             fields.TYPE_MBOX_C_HASH_IDX,
             fields.TYPE_FLAGS_VAL, self.mbox, chash)
         curried.expected = "fdoc"
-        return try_unique_query(curried)
+        fdoc = try_unique_query(curried)
+        if fdoc is not None:
+            return fdoc
+        else:
+            # probably this should be the other way round,
+            # ie, try fist on memstore...
+            cf = self.memstore._chash_fdoc_store
+            fdoc = cf[chash][self.mbox]
+            # hey, I just needed to wrap fdoc thing into
+            # a "content" attribute, look a better way...
+            if not empty(fdoc):
+                return MessagePartDoc(
+                    new=None, dirty=None, part=None,
+                    store=None, doc_id=None,
+                    content=fdoc)
 
     def _get_uid_from_msgidCb(self, msgid):
         hdoc = None
@@ -1088,11 +1102,26 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
         curried.expected = "hdoc"
         hdoc = try_unique_query(curried)
 
-        if hdoc is None:
+        # XXX this is only a quick hack to avoid regression
+        # on the "multiple copies of the draft" issue, but
+        # this is currently broken since  it's not efficient to
+        # look for this. Should lookup better.
+        # FIXME!
+
+        if hdoc is not None:
+            hdoc_dict = hdoc.content
+
+        else:
+            hdocstore = self.memstore._hdoc_store
+            match = [x for _, x in hdocstore.items() if x['msgid'] == msgid]
+            hdoc_dict = first(match)
+
+        if hdoc_dict is None:
             logger.warning("Could not find hdoc for msgid %s"
                            % (msgid,))
             return None
-        msg_chash = hdoc.content.get(fields.CONTENT_HASH_KEY)
+        msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY)
+
         fdoc = self._get_fdoc_from_chash(msg_chash)
         if not fdoc:
             logger.warning("Could not find fdoc for msgid %s"
-- 
cgit v1.2.3


From cffce1a7dfbca91278862e7a173e661e6644e6ec Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 25 Feb 2014 12:19:21 -0400
Subject: Workaround for broken notify-after-copy

---
 src/leap/mail/imap/mailbox.py | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 947cf1b..9b1f4e5 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -474,7 +474,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = self._do_add_message(message, flags=flags, date=date)
         if PROFILE_CMD:
             do_profile_cmd(d, "APPEND")
-        # XXX should notify here probably
+
+        # A better place for this would be  the COPY/APPEND dispatcher
+        # in server.py, but qtreactor hangs when I do that, so this seems
+        # to work fine for now.
+        d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
         return d
 
     def _do_add_message(self, message, flags, date):
@@ -485,7 +489,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = self.messages.add_msg(message, flags=flags, date=date)
         return d
 
-    @deferred_to_thread
     def notify_new(self, *args):
         """
         Notify of new messages to all the listeners.
@@ -494,13 +497,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         if not NOTIFY_NEW:
             return
+
+        def cbNotifyNew(result):
+            exists, recent = result
+            for l in self.listeners:
+                l.newMessages(exists, recent)
+        d = self._get_notify_count()
+        d.addCallback(cbNotifyNew)
+
+    @deferred_to_thread
+    def _get_notify_count(self):
+        """
+        Get message count and recent count for this mailbox
+        Executed in a separate thread. Called from notify_new.
+
+        :return: number of messages and number of recent messages.
+        :rtype: tuple
+        """
         exists = self.getMessageCount()
         recent = self.getRecentCount()
         logger.debug("NOTIFY (%r): there are %s messages, %s recent" % (
             self.mbox, exists, recent))
-
-        for l in self.listeners:
-            l.newMessages(exists, recent)
+        return exists, recent
 
     # commands, do not rename methods
 
@@ -880,6 +898,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = defer.Deferred()
         if PROFILE_CMD:
             do_profile_cmd(d, "COPY")
+
+        # A better place for this would be  the COPY/APPEND dispatcher
+        # in server.py, but qtreactor hangs when I do that, so this seems
+        # to work fine for now.
+        d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
         deferLater(self.reactor, 0, self._do_copy, message, d)
         return d
 
-- 
cgit v1.2.3


From 127ee651a09284524b185419e61914fe6911cf71 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 25 Feb 2014 12:21:57 -0400
Subject: changes file

---
 changes/bug_5167_fix-notify-after-copy | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 changes/bug_5167_fix-notify-after-copy

diff --git a/changes/bug_5167_fix-notify-after-copy b/changes/bug_5167_fix-notify-after-copy
new file mode 100644
index 0000000..36ecd0b
--- /dev/null
+++ b/changes/bug_5167_fix-notify-after-copy
@@ -0,0 +1,2 @@
+  o Fix bug in which destination folder sometimes was not showing messages after copy/append.
+    Closes: #5167
-- 
cgit v1.2.3


From 6480fe087e764ace849f552bef3339e1fcd85eff Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 25 Feb 2014 21:41:53 -0400
Subject: fix unread notification to UI

---
 changes/bug_5177_fix_unread_signal_to_ui |  1 +
 src/leap/mail/imap/mailbox.py            | 37 ++++++++++++++++++++------------
 2 files changed, 24 insertions(+), 14 deletions(-)
 create mode 100644 changes/bug_5177_fix_unread_signal_to_ui

diff --git a/changes/bug_5177_fix_unread_signal_to_ui b/changes/bug_5177_fix_unread_signal_to_ui
new file mode 100644
index 0000000..eac79f2
--- /dev/null
+++ b/changes/bug_5177_fix_unread_signal_to_ui
@@ -0,0 +1 @@
+  o Fix unread notifications to client UI. Only INBOX is notified. Closes: #5177
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 9b1f4e5..d8e6cb1 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -371,15 +371,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         :rtype: int
         """
         with self.next_uid_lock:
-            if self._memstore:
-                return self.last_uid + 1
-            else:
-                # XXX after lock, it should be safe to
-                # return just the increment here, and
-                # have a different method that actually increments
-                # the counter when really adding.
-                self.last_uid += 1
-                return self.last_uid
+            return self.last_uid + 1
 
     def getMessageCount(self):
         """
@@ -474,11 +466,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d = self._do_add_message(message, flags=flags, date=date)
         if PROFILE_CMD:
             do_profile_cmd(d, "APPEND")
-
         # A better place for this would be  the COPY/APPEND dispatcher
         # in server.py, but qtreactor hangs when I do that, so this seems
         # to work fine for now.
         d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
+        d.addCallback(self.cb_signal_unread_to_ui)
+        d.addErrback(lambda f: log.msg(f.getTraceback()))
         return d
 
     def _do_add_message(self, message, flags, date):
@@ -613,6 +606,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         self.reactor.callInThread(self._do_fetch, messages_asked, uid, d)
         if PROFILE_CMD:
             do_profile_cmd(d, "FETCH")
+        d.addCallback(self.cb_signal_unread_to_ui)
         return d
 
     # called in thread
@@ -768,14 +762,27 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             for msgid in seq_messg)
         return result
 
-    def signal_unread_to_ui(self, *args, **kwargs):
+    def cb_signal_unread_to_ui(self, result):
         """
         Sends unread event to ui.
+        Used as a callback in several commands.
+
+        :param result: ignored
+        """
+        d = self._get_unseen_deferred()
+        d.addCallback(self.__cb_signal_unread_to_ui)
+        return result
+
+    @deferred_to_thread
+    def _get_unseen_deferred(self):
+        return self.getUnseenCount()
 
-        :param args: ignored
-        :param kwargs: ignored
+    def __cb_signal_unread_to_ui(self, unseen):
+        """
+        Send the unread signal to UI.
+        :param unseen: number of unseen messages.
+        :type unseen: int
         """
-        unseen = self.getUnseenCount()
         leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
 
     def store(self, messages_asked, flags, mode, uid):
@@ -816,6 +823,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                                mode, uid, d)
         if PROFILE_CMD:
             do_profile_cmd(d, "STORE")
+        d.addCallback(self.cb_signal_unread_to_ui)
+        d.addErrback(lambda f: log.msg(f.getTraceback()))
         return d
 
     def _do_store(self, messages_asked, flags, mode, uid, observer):
-- 
cgit v1.2.3


From 3f69d4216c731c2985bb5d09127c58e196a29006 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 26 Feb 2014 10:46:44 -0400
Subject: Implement non-synchronizing literals (rfc2088) Closes: #5190

This also paves the way to MULTIAPPEND IMAP Extension (rfc3502)
Related to: Feature #5182
---
 changes/feature_literal-plus |   2 +
 src/leap/mail/imap/server.py | 187 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 189 insertions(+)
 create mode 100644 changes/feature_literal-plus

diff --git a/changes/feature_literal-plus b/changes/feature_literal-plus
new file mode 100644
index 0000000..39192b9
--- /dev/null
+++ b/changes/feature_literal-plus
@@ -0,0 +1,2 @@
+  o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs can be made
+    in a single round-trip. Closes: #5190
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 5da9bfd..fe56ea6 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -29,6 +29,11 @@ from leap.common.check import leap_assert, leap_assert_type
 from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN
 from leap.soledad.client import Soledad
 
+# imports for LITERAL+ patch
+from twisted.internet import defer, interfaces
+from twisted.mail.imap4 import IllegalClientResponse
+from twisted.mail.imap4 import LiteralString, LiteralFile
+
 
 class LeapIMAPServer(imap4.IMAP4Server):
     """
@@ -186,3 +191,185 @@ class LeapIMAPServer(imap4.IMAP4Server):
         # TODO return the output of _memstore.is_writing
         # XXX and that should return a deferred!
         return None
+
+    #############################################################
+    #
+    # Twisted imap4 patch to support LITERAL+ extension
+    # TODO send this patch upstream asap!
+    #
+    #############################################################
+
+    def capabilities(self):
+        cap = {'AUTH': self.challengers.keys()}
+        if self.ctx and self.canStartTLS:
+            t = self.transport
+            ti = interfaces.ISSLTransport
+            if not self.startedTLS and ti(t, None) is None:
+                cap['LOGINDISABLED'] = None
+                cap['STARTTLS'] = None
+        cap['NAMESPACE'] = None
+        cap['IDLE'] = None
+        # patched ############
+        cap['LITERAL+'] = None
+        ######################
+        return cap
+
+    def _stringLiteral(self, size, literal_plus=False):
+        if size > self._literalStringLimit:
+            raise IllegalClientResponse(
+                "Literal too long! I accept at most %d octets" %
+                (self._literalStringLimit,))
+        d = defer.Deferred()
+        self.parseState = 'pending'
+        self._pendingLiteral = LiteralString(size, d)
+        # Patched ###########################################################
+        if not literal_plus:
+            self.sendContinuationRequest('Ready for %d octets of text' % size)
+        #####################################################################
+        self.setRawMode()
+        return d
+
+    def _fileLiteral(self, size, literal_plus=False):
+        d = defer.Deferred()
+        self.parseState = 'pending'
+        self._pendingLiteral = LiteralFile(size, d)
+        if not literal_plus:
+            self.sendContinuationRequest('Ready for %d octets of data' % size)
+        self.setRawMode()
+        return d
+
+    def arg_astring(self, line):
+        """
+        Parse an astring from the line, return (arg, rest), possibly
+        via a deferred (to handle literals)
+        """
+        line = line.strip()
+        if not line:
+            raise IllegalClientResponse("Missing argument")
+        d = None
+        arg, rest = None, None
+        if line[0] == '"':
+            try:
+                spam, arg, rest = line.split('"', 2)
+                rest = rest[1:]  # Strip space
+            except ValueError:
+                raise IllegalClientResponse("Unmatched quotes")
+        elif line[0] == '{':
+            # literal
+            if line[-1] != '}':
+                raise IllegalClientResponse("Malformed literal")
+
+            # Patched ################
+            if line[-2] == "+":
+                literalPlus = True
+                size_end = -2
+            else:
+                literalPlus = False
+                size_end = -1
+
+            try:
+                size = int(line[1:size_end])
+            except ValueError:
+                raise IllegalClientResponse(
+                    "Bad literal size: " + line[1:size_end])
+            d = self._stringLiteral(size, literalPlus)
+            ##########################
+        else:
+            arg = line.split(' ', 1)
+            if len(arg) == 1:
+                arg.append('')
+            arg, rest = arg
+        return d or (arg, rest)
+
+    def arg_literal(self, line):
+        """
+        Parse a literal from the line
+        """
+        if not line:
+            raise IllegalClientResponse("Missing argument")
+
+        if line[0] != '{':
+            raise IllegalClientResponse("Missing literal")
+
+        if line[-1] != '}':
+            raise IllegalClientResponse("Malformed literal")
+
+        # Patched ##################
+        if line[-2] == "+":
+            literalPlus = True
+            size_end = -2
+        else:
+            literalPlus = False
+            size_end = -1
+
+        try:
+            size = int(line[1:size_end])
+        except ValueError:
+            raise IllegalClientResponse(
+                "Bad literal size: " + line[1:size_end])
+
+        return self._fileLiteral(size, literalPlus)
+        #############################
+
+    # Need to override the command table after patching
+    # arg_astring and arg_literal
+
+    do_LOGIN = imap4.IMAP4Server.do_LOGIN
+    do_CREATE = imap4.IMAP4Server.do_CREATE
+    do_DELETE = imap4.IMAP4Server.do_DELETE
+    do_RENAME = imap4.IMAP4Server.do_RENAME
+    do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
+    do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
+    do_STATUS = imap4.IMAP4Server.do_STATUS
+    do_APPEND = imap4.IMAP4Server.do_APPEND
+    do_COPY = imap4.IMAP4Server.do_COPY
+
+    _selectWork = imap4.IMAP4Server._selectWork
+    _listWork = imap4.IMAP4Server._listWork
+    arg_plist = imap4.IMAP4Server.arg_plist
+    arg_seqset = imap4.IMAP4Server.arg_seqset
+    opt_plist = imap4.IMAP4Server.opt_plist
+    opt_datetime = imap4.IMAP4Server.opt_datetime
+
+    unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring)
+
+    auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT')
+    select_SELECT = auth_SELECT
+
+    auth_CREATE = (do_CREATE, arg_astring)
+    select_CREATE = auth_CREATE
+
+    auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE')
+    select_EXAMINE = auth_EXAMINE
+
+    auth_DELETE = (do_DELETE, arg_astring)
+    select_DELETE = auth_DELETE
+
+    auth_RENAME = (do_RENAME, arg_astring, arg_astring)
+    select_RENAME = auth_RENAME
+
+    auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
+    select_SUBSCRIBE = auth_SUBSCRIBE
+
+    auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
+    select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+    auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST')
+    select_LIST = auth_LIST
+
+    auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB')
+    select_LSUB = auth_LSUB
+
+    auth_STATUS = (do_STATUS, arg_astring, arg_plist)
+    select_STATUS = auth_STATUS
+
+    auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime,
+                   arg_literal)
+    select_APPEND = auth_APPEND
+
+    select_COPY = (do_COPY, arg_seqset, arg_astring)
+
+
+    #############################################################
+    # END of Twisted imap4 patch to support LITERAL+ extension
+    #############################################################
-- 
cgit v1.2.3


From 733994d68b9f3ce528b552f67e9cbec005e57e9f Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 25 Feb 2014 22:38:29 -0400
Subject: rename all fdocs when folder is renamed

---
 changes/bug_5179_delete_folder    |  1 +
 src/leap/mail/imap/account.py     |  9 +--------
 src/leap/mail/imap/mailbox.py     |  2 ++
 src/leap/mail/imap/memorystore.py | 21 +++++++++++++++++++++
 4 files changed, 25 insertions(+), 8 deletions(-)
 create mode 100644 changes/bug_5179_delete_folder

diff --git a/changes/bug_5179_delete_folder b/changes/bug_5179_delete_folder
new file mode 100644
index 0000000..3de52cc
--- /dev/null
+++ b/changes/bug_5179_delete_folder
@@ -0,0 +1 @@
+  o Fix bug in which deleted folder wouldn't show its messages inside. Closes: #5179
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index ede63d3..199a2a4 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -329,20 +329,13 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
                 raise imap4.MailboxCollision(repr(new))
 
         for (old, new) in inferiors:
+            self._memstore.rename_fdocs_mailbox(old, new)
             mbox = self._get_mailbox_by_name(old)
             mbox.content[self.MBOX_KEY] = new
             self._soledad.put_doc(mbox)
 
         self._load_mailboxes()
 
-        # XXX ---- FIXME!!!! ------------------------------------
-        # until here we just renamed the index...
-        # We have to rename also the occurrence of this
-        # mailbox on ALL the messages that are contained in it!!!
-        # ... we maybe could use a reference to the doc_id
-        # in each msg, instead of the "mbox" field in msgs
-        # -------------------------------------------------------
-
     def _inferiorNames(self, name):
         """
         Return hierarchically inferior mailboxes.
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index d8e6cb1..503e38b 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -337,6 +337,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         """
         if self._uidvalidity is None:
             mbox = self._get_mbox_doc()
+            if mbox is None:
+                return 0
             self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1)
         return self._uidvalidity
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 6206468..d383b79 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -1244,6 +1244,27 @@ class MemoryStore(object):
         """
         self.permanent_store.set_mbox_closed(mbox, closed)
 
+    # Rename flag-documents
+
+    def rename_fdocs_mailbox(self, old_mbox, new_mbox):
+        """
+        Change the mailbox name for all flag documents in a given mailbox.
+        Used from account.rename
+
+        :param old_mbox: name for the old mbox
+        :type old_mbox: str or unicode
+        :param new_mbox: name for the new mbox
+        :type new_mbox: str or unicode
+        """
+        fs = self._fdoc_store
+        keys = fs[old_mbox].keys()
+        for k in keys:
+            fdoc = fs[old_mbox][k]
+            fdoc['mbox'] = new_mbox
+            fs[new_mbox][k] = fdoc
+            fs[old_mbox].pop(k)
+            self._dirty.add((new_mbox, k))
+
     # Dump-to-disk controls.
 
     @property
-- 
cgit v1.2.3


From 38863f978437fcd33254b830e7d4906f4fbeccdc Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 5 Mar 2014 12:13:43 -0400
Subject: workaround attempt for the recursionlimit bug with qtreactor.

Increasing the recursion limit by an order of magnitude here seems to
allow a fetch of a mailbox with 500 mails.

See #5196 for discussion of alternatives.
---
 src/leap/mail/imap/service/imap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 1175cdc..10ba32a 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -57,7 +57,7 @@ import resource
 import sys
 
 try:
-    sys.setrecursionlimit(10**6)
+    sys.setrecursionlimit(10**7)
 except Exception:
     print "Error setting recursion limit"
 try:
-- 
cgit v1.2.3


From 40197f87b86c20ecc3f9dfd38687f25a4158d6e7 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 31 Jan 2014 14:50:16 -0400
Subject: keep processing after decoding errors during fetch

---
 src/leap/mail/imap/fetch.py  | 186 ++++++++++++++++++++++++++++++-------------
 src/leap/mail/imap/fields.py |  15 ++++
 2 files changed, 145 insertions(+), 56 deletions(-)

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 40dadb3..6e12b3f 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -23,6 +23,7 @@ import threading
 import time
 import sys
 import traceback
+import warnings
 
 from email.parser import Parser
 from email.generator import Generator
@@ -32,6 +33,8 @@ from StringIO import StringIO
 from twisted.python import log
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
+from twisted.internet.task import deferLater
+from u1db import errors as u1db_errors
 from zope.proxy import sameProxiedObjects
 
 from leap.common import events as leap_events
@@ -46,7 +49,8 @@ from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.mail.decorators import deferred_to_thread
-from leap.mail.utils import json_loads
+from leap.mail.imap.fields import fields
+from leap.mail.utils import json_loads, empty, first
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
 
@@ -80,8 +84,6 @@ class LeapIncomingMail(object):
     """
 
     RECENT_FLAG = "\\Recent"
-
-    INCOMING_KEY = "incoming"
     CONTENT_KEY = "content"
 
     LEAP_SIGNATURE_HEADER = 'X-Leap-Signature'
@@ -130,17 +132,9 @@ class LeapIncomingMail(object):
         self._loop = None
         self._check_period = check_period
 
-        self._create_soledad_indexes()
-
         # initialize a mail parser only once
         self._parser = Parser()
 
-    def _create_soledad_indexes(self):
-        """
-        Create needed indexes on soledad.
-        """
-        self._soledad.create_index("just-mail", "incoming")
-
     @property
     def _pkey(self):
         if sameProxiedObjects(self._keymanager, None):
@@ -159,13 +153,29 @@ class LeapIncomingMail(object):
         Calls a deferred that will execute the fetch callback
         in a separate thread
         """
+        def syncSoledadCallback(result):
+            # FIXME this needs a matching change in mx!!!
+            # --> need to add ERROR_DECRYPTING_KEY = False
+            # as default.
+            try:
+                doclist = self._soledad.get_from_index(
+                    fields.JUST_MAIL_IDX, "*", "0")
+            except u1db_errors.InvalidGlobbing:
+                # It looks like we are a dealing with an outdated
+                # mx. Fallback to the version of the index
+                warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
+                              DeprecationWarning)
+                doclist = self._soledad.get_from_index(
+                    fields.JUST_MAIL_COMPAT_IDX, "*")
+            self._process_doclist(doclist)
+
         logger.debug("fetching mail for: %s %s" % (
             self._soledad.uuid, self._userid))
         if not self.fetching_lock.locked():
             d1 = self._sync_soledad()
             d = defer.gatherResults([d1], consumeErrors=True)
+            d.addCallbacks(syncSoledadCallback, self._errback)
             d.addCallbacks(self._signal_fetch_to_ui, self._errback)
-            d.addCallbacks(self._signal_unread_to_ui, self._errback)
             return d
         else:
             logger.debug("Already fetching mail.")
@@ -202,46 +212,44 @@ class LeapIncomingMail(object):
     @deferred_to_thread
     def _sync_soledad(self):
         """
-        Synchronizes with remote soledad.
+        Synchronize with remote soledad.
 
         :returns: a list of LeapDocuments, or None.
         :rtype: iterable or None
         """
         with self.fetching_lock:
-            log.msg('syncing soledad...')
+            log.msg('FETCH: syncing soledad...')
             self._soledad.sync()
-            log.msg('soledad synced.')
-            doclist = self._soledad.get_from_index("just-mail", "*")
-        self._process_doclist(doclist)
-
-    def _signal_unread_to_ui(self, *args):
-        """
-        Sends unread event to ui.
-        """
-        leap_events.signal(
-            IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount()))
+            log.msg('FETCH soledad SYNCED.')
 
     def _signal_fetch_to_ui(self, doclist):
         """
-        Sends leap events to ui.
+        Send leap events to ui.
 
         :param doclist: iterable with msg documents.
         :type doclist: iterable.
         :returns: doclist
         :rtype: iterable
         """
-        doclist = doclist[0]  # gatherResults pass us a list
-        fetched_ts = time.mktime(time.gmtime())
-        num_mails = len(doclist) if doclist is not None else 0
-        if num_mails != 0:
-            log.msg("there are %s mails" % (num_mails,))
+        doclist = first(doclist)  # gatherResults pass us a list
+        if doclist:
+            fetched_ts = time.mktime(time.gmtime())
+            num_mails = len(doclist) if doclist is not None else 0
+            if num_mails != 0:
+                log.msg("there are %s mails" % (num_mails,))
+            leap_events.signal(
+                IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts))
+            return doclist
+
+    def _signal_unread_to_ui(self, *args):
+        """
+        Sends unread event to ui.
+        """
         leap_events.signal(
-            IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts))
-        return doclist
+            IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount()))
 
     # process incoming mail.
 
-    @defer.inlineCallbacks
     def _process_doclist(self, doclist):
         """
         Iterates through the doclist, checks if each doc
@@ -262,22 +270,36 @@ class LeapIncomingMail(object):
             logger.debug("processing doc %d of %d" % (index + 1, num_mails))
             leap_events.signal(
                 IMAP_MSG_PROCESSING, str(index), str(num_mails))
+
             keys = doc.content.keys()
-            if self._is_msg(keys):
-                # Ok, this looks like a legit msg.
+
+            # TODO Compatibility check with the index in pre-0.6 mx
+            # that does not write the ERROR_DECRYPTING_KEY
+            # This should be removed in 0.7
+
+            has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None)
+            if has_errors is None:
+                warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
+                              DeprecationWarning)
+            if has_errors:
+                logger.debug("skipping msg with decrypting errors...")
+
+            if self._is_msg(keys) and not has_errors:
+                # Evaluating to bool of has_errors is intentional here.
+                # We don't mind at this point if it's None or False.
+
+                # Ok, this looks like a legit msg, and with no errors.
                 # Let's process it!
-                decrypted = list(self._decrypt_doc(doc))[0]
-                res = self._add_message_locally(decrypted)
-                yield res
 
-            else:
-                # Ooops, this does not.
-                logger.debug('This does not look like a proper msg.')
+                d1 = self._decrypt_doc(doc)
+                d = defer.gatherResults([d1], consumeErrors=True)
+                d.addCallbacks(self._add_message_locally, self._errback)
 
     #
     # operations on individual messages
     #
 
+    @deferred_to_thread
     def _decrypt_doc(self, doc):
         """
         Decrypt the contents of a document.
@@ -302,8 +324,8 @@ class LeapIncomingMail(object):
             decrdata = ""
         leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0")
 
-        data = list(self._process_decrypted_doc((doc, decrdata)))
-        yield (doc, data)
+        data = self._process_decrypted_doc((doc, decrdata))
+        return (doc, data)
 
     def _process_decrypted_doc(self, msgtuple):
         """
@@ -319,11 +341,35 @@ class LeapIncomingMail(object):
         """
         log.msg('processing decrypted doc')
         doc, data = msgtuple
-        msg = json_loads(data)
+
+        from twisted.internet import reactor
+
+        # XXX turn this into an errBack for each one of
+        # the deferreds that would process an individual document
+        try:
+            msg = json_loads(data)
+        except UnicodeError as exc:
+            logger.error("Error while decrypting %s" % (doc.doc_id,))
+            logger.exception(exc)
+
+            # we flag the message as "with decrypting errors",
+            # to avoid further decryption attempts during sync
+            # cycles until we're prepared to deal with that.
+            # What is the same, when Ivan deals with it...
+            # A new decrypting attempt event could be triggered by a
+            # future a library upgrade, or a cli flag to the client,
+            # we just `defer` that for now... :)
+            doc.content[fields.ERROR_DECRYPTING_KEY] = True
+            deferLater(reactor, 0, self._update_incoming_message, doc)
+
+            # FIXME this is just a dirty hack to delay the proper
+            # deferred organization here...
+            # and remember, boys, do not do this at home.
+            return []
 
         if not isinstance(msg, dict):
             defer.returnValue(False)
-        if not msg.get(self.INCOMING_KEY, False):
+        if not msg.get(fields.INCOMING_KEY, False):
             defer.returnValue(False)
 
         # ok, this is an incoming message
@@ -332,6 +378,27 @@ class LeapIncomingMail(object):
             return False
         return self._maybe_decrypt_msg(rawmsg)
 
+    @deferred_to_thread
+    def _update_incoming_message(self, doc):
+        """
+        Do a put for a soledad document. This probably has been called only
+        in the case that we've needed to update the ERROR_DECRYPTING_KEY
+        flag in an incoming message, to get it out of the decrypting queue.
+
+        :param doc: the SoledadDocument to update
+        :type doc: SoledadDocument
+        """
+        log.msg("Updating SoledadDoc %s" % (doc.doc_id))
+        self._soledad.put_doc(doc)
+
+    @deferred_to_thread
+    def _delete_incoming_message(self, doc):
+        """
+        Delete document.
+        """
+        log.msg("Deleting SoledadDoc %s" % (doc.doc_id))
+        self._soledad.delete_doc(doc)
+
     def _maybe_decrypt_msg(self, data):
         """
         Tries to decrypt a gpg message if data looks like one.
@@ -384,7 +451,7 @@ class LeapIncomingMail(object):
                 self.LEAP_SIGNATURE_INVALID,
                 pubkey=senderPubkey.key_id)
 
-        yield decrmsg.as_string()
+        return decrmsg.as_string()
 
     def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey):
         """
@@ -505,32 +572,39 @@ class LeapIncomingMail(object):
                 data, self._pkey)
         return (decrdata, valid_sig)
 
-    def _add_message_locally(self, msgtuple):
+    def _add_message_locally(self, result):
         """
         Adds a message to local inbox and delete it from the incoming db
         in soledad.
 
+        # XXX this comes from a gatherresult...
         :param msgtuple: a tuple consisting of a SoledadDocument
                          instance containing the incoming message
                          and data, the json-encoded, decrypted content of the
                          incoming message
         :type msgtuple: (SoledadDocument, str)
         """
-        log.msg('adding message to local db')
+        from twisted.internet import reactor
+        msgtuple = first(result)
+
         doc, data = msgtuple
+        log.msg('adding message %s to local db' % (doc.doc_id,))
 
         if isinstance(data, list):
+            if empty(data):
+                return False
             data = data[0]
 
-        self._inbox.addMessage(data, flags=(self.RECENT_FLAG,))
+        def msgSavedCallback(result):
+            if not empty(result):
+                leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
+                deferLater(reactor, 0, self._delete_incoming_message, result)
+                leap_events.signal(IMAP_MSG_DELETED_INCOMING)
+                deferLater(reactor, 1, self._signal_unread_to_ui)
 
-        leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
-        doc_id = doc.doc_id
-        self._soledad.delete_doc(doc)
-        log.msg("deleted doc %s from incoming" % doc_id)
-        leap_events.signal(IMAP_MSG_DELETED_INCOMING)
-        self._signal_unread_to_ui()
-        return True
+        # XXX should pass a notify_on_disk=True along...
+        d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,))
+        d.addCallbacks(msgSavedCallback, self._errback)
 
     #
     # helpers
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 886ee63..4576939 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -108,6 +108,14 @@ class WithMsgFields(object):
     # correct since the recent flag is volatile.
     TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
 
+    # Soledad index for incoming mail, without decrypting errors.
+    JUST_MAIL_IDX = "just-mail"
+    # XXX the backward-compatible index, will be deprecated at 0.7
+    JUST_MAIL_COMPAT_IDX = "just-mail-compat"
+
+    INCOMING_KEY = "incoming"
+    ERROR_DECRYPTING_KEY = "errdecr"
+
     KTYPE = TYPE_KEY
     MBOX_VAL = TYPE_MBOX_VAL
     CHASH_VAL = CONTENT_HASH_KEY
@@ -140,6 +148,13 @@ class WithMsgFields(object):
         TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
         TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
                                   'bool(recent)', 'bool(seen)'],
+
+        # incoming queue
+        JUST_MAIL_IDX: [INCOMING_KEY,
+                        "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
+
+        # the backward-compatible index, will be deprecated at 0.7
+        JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
     }
 
     MBOX_KEY = MBOX_VAL
-- 
cgit v1.2.3


From f6891cbaf171b5264af031e4af2649236194e137 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 11 Mar 2014 10:38:59 -0400
Subject: changes file

---
 changes/bug_5307_keep-processing | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/bug_5307_keep-processing

diff --git a/changes/bug_5307_keep-processing b/changes/bug_5307_keep-processing
new file mode 100644
index 0000000..7194adf
--- /dev/null
+++ b/changes/bug_5307_keep-processing
@@ -0,0 +1 @@
+  o Keep processing after a decryption error. Closes: #5307
-- 
cgit v1.2.3


From 8981ff7de49401fcc9c3031a386ae0402021a6e6 Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Mon, 17 Mar 2014 17:45:49 -0300
Subject: Signal the UI in case the soledad token is invalid when syncing
 (#5191).

---
 changes/feature_5191_signal-invalid-auth-token |  1 +
 src/leap/mail/decorators.py                    |  1 +
 src/leap/mail/imap/fetch.py                    | 13 ++++++++++---
 3 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 changes/feature_5191_signal-invalid-auth-token

diff --git a/changes/feature_5191_signal-invalid-auth-token b/changes/feature_5191_signal-invalid-auth-token
new file mode 100644
index 0000000..f833a3e
--- /dev/null
+++ b/changes/feature_5191_signal-invalid-auth-token
@@ -0,0 +1 @@
+  o Signal the client when auth token is invalid for syncing Soledad (#5191).
diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py
index ae115f8..5105de9 100644
--- a/src/leap/mail/decorators.py
+++ b/src/leap/mail/decorators.py
@@ -24,6 +24,7 @@ from functools import wraps
 
 from twisted.internet.threads import deferToThread
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 6e12b3f..5f951c3 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -45,6 +45,7 @@ from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED
 from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY
 from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING
 from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
+from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN
 from leap.common.mail import get_email_charset
 from leap.keymanager import errors as keymanager_errors
 from leap.keymanager.openpgp import OpenPGPKey
@@ -53,6 +54,7 @@ from leap.mail.imap.fields import fields
 from leap.mail.utils import json_loads, empty, first
 from leap.soledad.client import Soledad
 from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
+from leap.soledad.common.errors import InvalidAuthTokenError
 
 
 logger = logging.getLogger(__name__)
@@ -218,9 +220,14 @@ class LeapIncomingMail(object):
         :rtype: iterable or None
         """
         with self.fetching_lock:
-            log.msg('FETCH: syncing soledad...')
-            self._soledad.sync()
-            log.msg('FETCH soledad SYNCED.')
+            try:
+                log.msg('FETCH: syncing soledad...')
+                self._soledad.sync()
+                log.msg('FETCH soledad SYNCED.')
+            except InvalidAuthTokenError:
+                # if the token is invalid, send an event so the GUI can
+                # disable mail and show an error message.
+                leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN)
 
     def _signal_fetch_to_ui(self, doclist):
         """
-- 
cgit v1.2.3


From eef78aae4164a740f5673c38202f0a32b3615c1e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 26 Mar 2014 12:06:26 -0400
Subject: fix wrong object being passed in the messageSaved callback

this was the result of a bad merge during the last fetch refactor.
---
 src/leap/mail/imap/fetch.py   | 12 +++++++-----
 src/leap/mail/imap/mailbox.py | 19 +++++++++++++------
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 6e12b3f..8e94051 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -395,8 +395,11 @@ class LeapIncomingMail(object):
     def _delete_incoming_message(self, doc):
         """
         Delete document.
+
+        :param doc: the SoledadDocument to delete
+        :type doc: SoledadDocument
         """
-        log.msg("Deleting SoledadDoc %s" % (doc.doc_id))
+        log.msg("Deleting Incoming message: %s" % (doc.doc_id,))
         self._soledad.delete_doc(doc)
 
     def _maybe_decrypt_msg(self, data):
@@ -598,12 +601,11 @@ class LeapIncomingMail(object):
         def msgSavedCallback(result):
             if not empty(result):
                 leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
-                deferLater(reactor, 0, self._delete_incoming_message, result)
+                deferLater(reactor, 0, self._delete_incoming_message, doc)
                 leap_events.signal(IMAP_MSG_DELETED_INCOMING)
-                deferLater(reactor, 1, self._signal_unread_to_ui)
 
-        # XXX should pass a notify_on_disk=True along...
-        d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,))
+        d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,),
+                                   notify_on_disk=True)
         d.addCallbacks(msgSavedCallback, self._errback)
 
     #
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 503e38b..47c7ff1 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -439,7 +439,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             r[self.CMD_UNSEEN] = self.getUnseenCount()
         return defer.succeed(r)
 
-    def addMessage(self, message, flags, date=None):
+    def addMessage(self, message, flags, date=None, notify_on_disk=False):
         """
         Adds a message to this mailbox.
 
@@ -465,23 +465,29 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         else:
             flags = tuple(str(flag) for flag in flags)
 
-        d = self._do_add_message(message, flags=flags, date=date)
+        d = self._do_add_message(message, flags=flags, date=date,
+                                 notify_on_disk=notify_on_disk)
         if PROFILE_CMD:
             do_profile_cmd(d, "APPEND")
         # A better place for this would be  the COPY/APPEND dispatcher
         # in server.py, but qtreactor hangs when I do that, so this seems
         # to work fine for now.
-        d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
-        d.addCallback(self.cb_signal_unread_to_ui)
+
+        def notifyCallback(x):
+            self.reactor.callLater(0, self.notify_new)
+            return x
+
+        d.addCallback(notifyCallback)
         d.addErrback(lambda f: log.msg(f.getTraceback()))
         return d
 
-    def _do_add_message(self, message, flags, date):
+    def _do_add_message(self, message, flags, date, notify_on_disk=False):
         """
         Calls to the messageCollection add_msg method.
         Invoked from addMessage.
         """
-        d = self.messages.add_msg(message, flags=flags, date=date)
+        d = self.messages.add_msg(message, flags=flags, date=date,
+                                  notify_on_disk=notify_on_disk)
         return d
 
     def notify_new(self, *args):
@@ -499,6 +505,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
                 l.newMessages(exists, recent)
         d = self._get_notify_count()
         d.addCallback(cbNotifyNew)
+        d.addCallback(self.cb_signal_unread_to_ui)
 
     @deferred_to_thread
     def _get_notify_count(self):
-- 
cgit v1.2.3


From 0f10cbbc48927f89d2a1ec76490905a3c386f168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= <chiiph@leap.se>
Date: Fri, 4 Apr 2014 16:38:28 -0300
Subject: Update requirements

---
 changes/VERSION_COMPAT | 3 ---
 pkg/requirements.pip   | 6 +++---
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
index 03caa3e..cc00ecf 100644
--- a/changes/VERSION_COMPAT
+++ b/changes/VERSION_COMPAT
@@ -8,6 +8,3 @@
 #
 # BEGIN DEPENDENCY LIST -------------------------
 # leap.foo.bar>=x.y.z
-leap.soledad.client 0.5.0 # get_count_by_index
-leap.common 0.3.7 # get_email_charset
-leap.keymanager 0.3.8 # openpgp.decrypt
diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index 603eaf6..17ceba6 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -1,7 +1,7 @@
 zope.interface
-leap.soledad.client>=0.3.0
-leap.common>=0.3.5
-leap.keymanager>=0.3.7
+leap.soledad.client>=0.4.5
+leap.common>=0.3.7
+leap.keymanager>=0.3.8
 twisted  # >= 12.0.3 ??
 zope.proxy
 enum
-- 
cgit v1.2.3


From 3258025ea15e3f20c8054fd67ec617b7a87eb309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= <chiiph@leap.se>
Date: Fri, 4 Apr 2014 16:43:07 -0300
Subject: Fold in changes

---
 CHANGELOG                                          | 54 ++++++++++++++++++++++
 changes/bug-4791_url-should-not-end-in-period      |  1 -
 changes/bug-5021_handle-non-ascii-headers          |  1 -
 changes/bug_4715_fix_message_adding                |  1 -
 .../bug_4830_convert-unicode-to-str-when-raising   |  1 -
 changes/bug_4830_handle-unicode-in-folder-names    |  2 -
 changes/bug_4925_close_session                     |  1 -
 changes/bug_4933_check_for_none                    |  1 -
 changes/bug_4949-check-fdoc-uniqueness             |  2 -
 ...bug_5014_fix-attachment-processing-when-signing |  1 -
 changes/bug_5167_fix-notify-after-copy             |  2 -
 changes/bug_5177_fix_unread_signal_to_ui           |  1 -
 changes/bug_5179_delete_folder                     |  1 -
 changes/bug_5307_keep-processing                   |  1 -
 changes/bug_enqueue-unset-recent                   |  2 -
 changes/bug_fetch_size                             |  4 --
 changes/bug_properly_parse_apple_mails             |  1 -
 ...t-adding-outgoing-footer-to-text-plain-messages |  1 -
 changes/bug_safety-check-for-last-uid              |  1 -
 .../feature_4335_stop-providing-hostname-for-helo  |  1 -
 ...to-fetch-keys-for-multipart-signed-or-encrypted |  1 -
 changes/feature_4943-offline-flag                  |  1 -
 .../feature_5095_flush-data-to-disk-when-stopping  |  1 -
 changes/feature_5191_signal-invalid-auth-token     |  1 -
 changes/feature_enable-search-by-msg-id            |  3 --
 changes/feature_in-memory-store                    |  1 -
 changes/feature_literal-plus                       |  2 -
 changes/feature_split_message_docs                 |  7 ---
 changes/feaure_4616_fix_mail_indexing              |  1 -
 changes/handle-unicode-characters                  |  1 -
 30 files changed, 54 insertions(+), 45 deletions(-)
 delete mode 100644 changes/bug-4791_url-should-not-end-in-period
 delete mode 100644 changes/bug-5021_handle-non-ascii-headers
 delete mode 100644 changes/bug_4715_fix_message_adding
 delete mode 100644 changes/bug_4830_convert-unicode-to-str-when-raising
 delete mode 100644 changes/bug_4830_handle-unicode-in-folder-names
 delete mode 100644 changes/bug_4925_close_session
 delete mode 100644 changes/bug_4933_check_for_none
 delete mode 100644 changes/bug_4949-check-fdoc-uniqueness
 delete mode 100644 changes/bug_5014_fix-attachment-processing-when-signing
 delete mode 100644 changes/bug_5167_fix-notify-after-copy
 delete mode 100644 changes/bug_5177_fix_unread_signal_to_ui
 delete mode 100644 changes/bug_5179_delete_folder
 delete mode 100644 changes/bug_5307_keep-processing
 delete mode 100644 changes/bug_enqueue-unset-recent
 delete mode 100644 changes/bug_fetch_size
 delete mode 100644 changes/bug_properly_parse_apple_mails
 delete mode 100644 changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages
 delete mode 100644 changes/bug_safety-check-for-last-uid
 delete mode 100644 changes/feature_4335_stop-providing-hostname-for-helo
 delete mode 100644 changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
 delete mode 100644 changes/feature_4943-offline-flag
 delete mode 100644 changes/feature_5095_flush-data-to-disk-when-stopping
 delete mode 100644 changes/feature_5191_signal-invalid-auth-token
 delete mode 100644 changes/feature_enable-search-by-msg-id
 delete mode 100644 changes/feature_in-memory-store
 delete mode 100644 changes/feature_literal-plus
 delete mode 100644 changes/feature_split_message_docs
 delete mode 100644 changes/feaure_4616_fix_mail_indexing
 delete mode 100644 changes/handle-unicode-characters

diff --git a/CHANGELOG b/CHANGELOG
index fea58c8..08d20cc 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,57 @@
+0.3.9 Apr 4:
+  o Footer url shouldn't end in period. Closes #4791.
+  o Handle non-ascii headers. Closes #5021.
+  o Soledad writer consumes messages eagerly. Fixes failing
+    tests. Closes #4715.
+  o Convert unicode to str when raising exceptions in IMAP server.
+    Fixes #4830.
+  o Remove conversion of IMAP folder names to string. This makes the
+    IMAP server use twisted's transparent 7bit conversion. Fixes
+    #4830.
+  o Add a flag to be able to reset the session. Closes #4925.
+  o Check for none in payload detection. Closes #4933.
+  o Check for flags doc uniqueness before adding a message. Avoids
+    duplicates of a single message in the same mailbox while copying
+    or moving. Closes #4949.
+  o Correctly process attachments when signing. Fixes #5014.
+  o Fix bug in which destination folder sometimes was not showing
+    messages after copy/append. Closes #5167.
+  o Fix unread notifications to client UI. Only INBOX is
+    notified. Closes #5177.
+  o Fix bug in which deleted folder wouldn't show its messages
+    inside. Closes #5179.
+  o Keep processing after a decryption error. Closes #5307.
+  o Enqueue unsetting of recent flag. this was holding the new mails
+    from being displayed soonish.
+  o Properly parse emails crafted by Mail.app. Fixes #5013.
+  o Restrict adding outgoing footer to text/plain messages.
+  o Sanity check on last_uid setter. Avoids incomplete fetches.
+  o Stop providing hostname for helo in smtp gateway. Fixes #4335.
+  o Only try to fetch keys for multipart signed or encrypted emails.
+    Fixes #4671.
+  o Add a flag for offline mode in imap. Related to #4943.
+  o Flush IMAP data to disk when stopping. Closes #5095.
+  o Signal the client when auth token is invalid for syncing Soledad.
+    Fixes #5191.
+  o Ability to support SEARCH Commands, limited to HEADER Message-ID.
+    This is a quick workaround for avoiding duplicate saves in Drafts
+    Folder. Closes #4209.
+  o Use a memory store as write-buffer and read-cache.
+  o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs
+    can be made in a single round-trip. Closes #5190.
+  o Defer costly operations to a pool of threads.
+  o Split the internal representation of messages into three distinct
+    documents: 1) Flags 2) Headers 3) Content.
+  o Make use of the Twisted MIME interface.
+  o Add deduplication ability to the save operation, for body and
+    attachments.
+  o Add IMessageCopier interface to mailbox implementation, so bulk
+    moves are costless. Closes #4654.
+  o Makes efficient use of indexes and count method. Closes #4616.
+  o Handle correctly unicode characters in emails. Closes #4838.
+
+-- 2014 --
+
 0.3.8 Dec 6:
   o Fail gracefully when failing to decrypt incoming messages. Closes
     #4589.
diff --git a/changes/bug-4791_url-should-not-end-in-period b/changes/bug-4791_url-should-not-end-in-period
deleted file mode 100644
index d4ff29c..0000000
--- a/changes/bug-4791_url-should-not-end-in-period
+++ /dev/null
@@ -1 +0,0 @@
-  o Footer url shouldn't end in period. Closes #4791.
diff --git a/changes/bug-5021_handle-non-ascii-headers b/changes/bug-5021_handle-non-ascii-headers
deleted file mode 100644
index 098cfa0..0000000
--- a/changes/bug-5021_handle-non-ascii-headers
+++ /dev/null
@@ -1 +0,0 @@
-  o Handle non-ascii headers. Closes #5021.
diff --git a/changes/bug_4715_fix_message_adding b/changes/bug_4715_fix_message_adding
deleted file mode 100644
index 53b875c..0000000
--- a/changes/bug_4715_fix_message_adding
+++ /dev/null
@@ -1 +0,0 @@
-  o Soledad writer consumes messages eagerly. Fixes failing tests. Closes: #4715
diff --git a/changes/bug_4830_convert-unicode-to-str-when-raising b/changes/bug_4830_convert-unicode-to-str-when-raising
deleted file mode 100644
index 86d9b1c..0000000
--- a/changes/bug_4830_convert-unicode-to-str-when-raising
+++ /dev/null
@@ -1 +0,0 @@
-  o Convert unicode to str when raising exceptions in IMAP server (#4830).
diff --git a/changes/bug_4830_handle-unicode-in-folder-names b/changes/bug_4830_handle-unicode-in-folder-names
deleted file mode 100644
index 6824745..0000000
--- a/changes/bug_4830_handle-unicode-in-folder-names
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Remove conversion of IMAP folder names to string. This makes the IMAP
-    server use twisted's transparent 7bit conversion (#4830).
diff --git a/changes/bug_4925_close_session b/changes/bug_4925_close_session
deleted file mode 100644
index 93dab55..0000000
--- a/changes/bug_4925_close_session
+++ /dev/null
@@ -1 +0,0 @@
-  o Add a flag to be able to reset the session. Closes: #4925
diff --git a/changes/bug_4933_check_for_none b/changes/bug_4933_check_for_none
deleted file mode 100644
index 33f3bd5..0000000
--- a/changes/bug_4933_check_for_none
+++ /dev/null
@@ -1 +0,0 @@
-  o Check for none in payload detection. Closes: #4933
diff --git a/changes/bug_4949-check-fdoc-uniqueness b/changes/bug_4949-check-fdoc-uniqueness
deleted file mode 100644
index bf49d1f..0000000
--- a/changes/bug_4949-check-fdoc-uniqueness
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Check for flags doc uniqueness before adding a message. Avoids duplicates of
-    a single message in the same mailbox while copying or moving. Closes: #4949
diff --git a/changes/bug_5014_fix-attachment-processing-when-signing b/changes/bug_5014_fix-attachment-processing-when-signing
deleted file mode 100644
index c12e35e..0000000
--- a/changes/bug_5014_fix-attachment-processing-when-signing
+++ /dev/null
@@ -1 +0,0 @@
-  o Correctly process attachments when signing. Fixes #5014.
diff --git a/changes/bug_5167_fix-notify-after-copy b/changes/bug_5167_fix-notify-after-copy
deleted file mode 100644
index 36ecd0b..0000000
--- a/changes/bug_5167_fix-notify-after-copy
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Fix bug in which destination folder sometimes was not showing messages after copy/append.
-    Closes: #5167
diff --git a/changes/bug_5177_fix_unread_signal_to_ui b/changes/bug_5177_fix_unread_signal_to_ui
deleted file mode 100644
index eac79f2..0000000
--- a/changes/bug_5177_fix_unread_signal_to_ui
+++ /dev/null
@@ -1 +0,0 @@
-  o Fix unread notifications to client UI. Only INBOX is notified. Closes: #5177
diff --git a/changes/bug_5179_delete_folder b/changes/bug_5179_delete_folder
deleted file mode 100644
index 3de52cc..0000000
--- a/changes/bug_5179_delete_folder
+++ /dev/null
@@ -1 +0,0 @@
-  o Fix bug in which deleted folder wouldn't show its messages inside. Closes: #5179
diff --git a/changes/bug_5307_keep-processing b/changes/bug_5307_keep-processing
deleted file mode 100644
index 7194adf..0000000
--- a/changes/bug_5307_keep-processing
+++ /dev/null
@@ -1 +0,0 @@
-  o Keep processing after a decryption error. Closes: #5307
diff --git a/changes/bug_enqueue-unset-recent b/changes/bug_enqueue-unset-recent
deleted file mode 100644
index 8903804..0000000
--- a/changes/bug_enqueue-unset-recent
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Enqueue unsetting of recent flag. this was holding the new
-    mails from being displayed soonish. 
diff --git a/changes/bug_fetch_size b/changes/bug_fetch_size
deleted file mode 100644
index e9e97b9..0000000
--- a/changes/bug_fetch_size
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Limit the size of the messages returned to the IMAP client to 100,
-    since Thunderbird hangs with numbers bigger than those. This is a
-    quick fix until we figure out how does Thunderbird want to receive
-    more than 100 mails at a time.
\ No newline at end of file
diff --git a/changes/bug_properly_parse_apple_mails b/changes/bug_properly_parse_apple_mails
deleted file mode 100644
index 1bf42ae..0000000
--- a/changes/bug_properly_parse_apple_mails
+++ /dev/null
@@ -1 +0,0 @@
-  o Properly parse emails crafted by Mail.app. Fixes #5013.
\ No newline at end of file
diff --git a/changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages b/changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages
deleted file mode 100644
index 9983404..0000000
--- a/changes/bug_restrict-adding-outgoing-footer-to-text-plain-messages
+++ /dev/null
@@ -1 +0,0 @@
-  o Restrict adding outgoing footer to text/plain messages.
diff --git a/changes/bug_safety-check-for-last-uid b/changes/bug_safety-check-for-last-uid
deleted file mode 100644
index bb0229f..0000000
--- a/changes/bug_safety-check-for-last-uid
+++ /dev/null
@@ -1 +0,0 @@
-  o Sanity check on last_uid setter. Avoids incomplete fetches.
diff --git a/changes/feature_4335_stop-providing-hostname-for-helo b/changes/feature_4335_stop-providing-hostname-for-helo
deleted file mode 100644
index f4b6c29..0000000
--- a/changes/feature_4335_stop-providing-hostname-for-helo
+++ /dev/null
@@ -1 +0,0 @@
-  o Stop providing hostname for helo in smtp gateway (#4335).
diff --git a/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted b/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
deleted file mode 100644
index de3bb86..0000000
--- a/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
+++ /dev/null
@@ -1 +0,0 @@
-  o Only try to fetch keys for multipart signed or encrypted emails (#4671).
diff --git a/changes/feature_4943-offline-flag b/changes/feature_4943-offline-flag
deleted file mode 100644
index 6edfd4d..0000000
--- a/changes/feature_4943-offline-flag
+++ /dev/null
@@ -1 +0,0 @@
-  o Add a flag for offline mode in imap. Related to #4943
diff --git a/changes/feature_5095_flush-data-to-disk-when-stopping b/changes/feature_5095_flush-data-to-disk-when-stopping
deleted file mode 100644
index d7c1ce7..0000000
--- a/changes/feature_5095_flush-data-to-disk-when-stopping
+++ /dev/null
@@ -1 +0,0 @@
-  o Flush IMAP data to disk when stopping. Closes #5095.
diff --git a/changes/feature_5191_signal-invalid-auth-token b/changes/feature_5191_signal-invalid-auth-token
deleted file mode 100644
index f833a3e..0000000
--- a/changes/feature_5191_signal-invalid-auth-token
+++ /dev/null
@@ -1 +0,0 @@
-  o Signal the client when auth token is invalid for syncing Soledad (#5191).
diff --git a/changes/feature_enable-search-by-msg-id b/changes/feature_enable-search-by-msg-id
deleted file mode 100644
index accc12f..0000000
--- a/changes/feature_enable-search-by-msg-id
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Ability to support SEARCH Commands, limited to HEADER Message-ID.
-    This is a quick workaround for avoiding duplicate saves in Drafts Folder.
-    Closes: #4209
diff --git a/changes/feature_in-memory-store b/changes/feature_in-memory-store
deleted file mode 100644
index a7a4d7a..0000000
--- a/changes/feature_in-memory-store
+++ /dev/null
@@ -1 +0,0 @@
-  o Use a memory store as write-buffer and read-cache.
diff --git a/changes/feature_literal-plus b/changes/feature_literal-plus
deleted file mode 100644
index 39192b9..0000000
--- a/changes/feature_literal-plus
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs can be made
-    in a single round-trip. Closes: #5190
diff --git a/changes/feature_split_message_docs b/changes/feature_split_message_docs
deleted file mode 100644
index 0109501..0000000
--- a/changes/feature_split_message_docs
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Defer costly operations to a pool of threads.
-  o Split the internal representation of messages into three distinct documents:
-    1) Flags 2) Headers 3) Content.
-  o Make use of the Twisted MIME interface.
-  o Add deduplication ability to the save operation, for body and attachments.
-  o Add IMessageCopier interface to mailbox implementation, so bulk moves
-    are costless. Closes: #4654
diff --git a/changes/feaure_4616_fix_mail_indexing b/changes/feaure_4616_fix_mail_indexing
deleted file mode 100644
index 6e94100..0000000
--- a/changes/feaure_4616_fix_mail_indexing
+++ /dev/null
@@ -1 +0,0 @@
-  o Makes efficient use of indexes and count method. Closes: #4616
diff --git a/changes/handle-unicode-characters b/changes/handle-unicode-characters
deleted file mode 100644
index 052c543..0000000
--- a/changes/handle-unicode-characters
+++ /dev/null
@@ -1 +0,0 @@
-  o Handle correctly unicode characters in emails. Closes #4838.
-- 
cgit v1.2.3