[feature] post-sync mail processing hooks

using the new soledad plugin capablity, mail hooks to the post-sync
event by subscribing to the Meta-Doc type of documents. In this way, we
can create the uid tables and the uid entries needed to keep local
indexes for mail that has been processed in another instance.

however, this won't prevent a conflict if a given mail is received and
processed in two different instances. that is a problem that we still
have to deal with.

Resolves: #6996
Releases: 0.4.0

6 files changed, 153 insertions, 2 deletions

diff --git a/mail/.gitignore b/mail/.gitignore
index 7ac82894..aafbdd13 100644
--- a/mail/.gitignore
+++ b/mail/.gitignore
@@ -1,4 +1,5 @@
 *.pyc
+dropin.cache
 build/
 dist/
 *.egg
diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py
index 370c5132..e401283e 100644
--- a/mail/src/leap/mail/imap/service/imap.py
+++ b/mail/src/leap/mail/imap/service/imap.py
@@ -35,6 +35,7 @@ from leap.common.events import emit, catalog
 from leap.common.check import leap_assert_type, leap_check
 from leap.mail.imap.account import IMAPAccount
 from leap.mail.imap.server import LEAPIMAPServer
+from leap.mail.plugins import soledad_sync_hooks
 from leap.soledad.client import Soledad
 
 
@@ -91,10 +92,17 @@ class LeapIMAPFactory(ServerFactory):
 
         theAccount = IMAPAccount(uuid, soledad)
         self.theAccount = theAccount
+        self._initialize_sync_hooks()
 
         self._connections = defaultdict()
         # XXX how to pass the store along?
 
+    def _initialize_sync_hooks(self):
+        soledad_sync_hooks.post_sync_uid_reindexer.set_account(self.theAccount)
+
+    def _teardown_sync_hooks(self):
+        soledad_sync_hooks.post_sync_uid_reindexer.set_account(None)
+
     def buildProtocol(self, addr):
         """
         Return a protocol suitable for the job.
@@ -128,6 +136,7 @@ class LeapIMAPFactory(ServerFactory):
         # mark account as unusable, so any imap command will fail
         # with unauth state.
         self.theAccount.end_session()
+        self._teardown_sync_hooks()
 
         # TODO should wait for all the pending deferreds,
         # the twisted way!
diff --git a/mail/src/leap/mail/mail.py b/mail/src/leap/mail/mail.py
index 1649d4ad..bab73cb6 100644
--- a/mail/src/leap/mail/mail.py
+++ b/mail/src/leap/mail/mail.py
@@ -42,8 +42,6 @@ logger = logging.getLogger(name=__name__)
 # TODO LIST
 # [ ] Probably change the name of this module to "api" or "account", mail is
 #     too generic (there's also IncomingMail, and OutgoingMail
-# [ ] Change the doc_ids scheme for part-docs: use mailbox UID validity
-#     identifier, instead of name! (renames are broken!)
 # [ ] Profile add_msg.
 
 def _get_mdoc_id(mbox, chash):
diff --git a/mail/src/leap/mail/plugins/__init__.py b/mail/src/leap/mail/plugins/__init__.py
new file mode 100644
index 00000000..ddb86917
--- /dev/null
+++ b/mail/src/leap/mail/plugins/__init__.py
@@ -0,0 +1,3 @@
+from twisted.plugin import pluginPackagePaths
+__path__.extend(pluginPackagePaths(__name__))
+__all__ = []
diff --git a/mail/src/leap/mail/plugins/soledad_sync_hooks.py b/mail/src/leap/mail/plugins/soledad_sync_hooks.py
new file mode 100644
index 00000000..9d48126e
--- /dev/null
+++ b/mail/src/leap/mail/plugins/soledad_sync_hooks.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# soledad_sync_hooks.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from leap.mail.sync_hooks import MailProcessingPostSyncHook
+post_sync_uid_reindexer = MailProcessingPostSyncHook()
diff --git a/mail/src/leap/mail/sync_hooks.py b/mail/src/leap/mail/sync_hooks.py
new file mode 100644
index 00000000..b5bded5b
--- /dev/null
+++ b/mail/src/leap/mail/sync_hooks.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# sync_hooks.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad PostSync Hooks.
+
+Process every new document of interest after every soledad synchronization,
+using the hooks that soledad exposes via plugins.
+"""
+import logging
+
+from re import compile as regex_compile
+
+from zope.interface import implements
+from twisted.internet import defer
+from twisted.plugin import IPlugin
+from twisted.python import log
+
+from leap.soledad.client.interfaces import ISoledadPostSyncPlugin
+from leap.mail import constants
+
+
+logger = logging.getLogger(__name__)
+
+_get_doc_type_preffix = lambda s: s[:2]
+
+
+class MailProcessingPostSyncHook(object):
+    implements(IPlugin, ISoledadPostSyncPlugin)
+
+    META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID)
+    watched_doc_types = (META_DOC_PREFFIX, )
+
+    _account = None
+    _pending_docs = []
+    _processing_deferreds = []
+
+    def process_received_docs(self, doc_id_list):
+        if self._has_configured_account():
+            process_fun = self._make_uid_index
+        else:
+            self._processing_deferreds = []
+            process_fun = self._queue_doc_id
+
+        for doc_id in doc_id_list:
+            if _get_doc_type_preffix(doc_id) in self.watched_doc_types:
+                log.msg("Mail post-sync hook: processing %s" % doc_id)
+                process_fun(doc_id)
+
+        if self._processing_deferreds:
+            return defer.gatherResults(self._processing_deferreds)
+
+    def set_account(self, account):
+        self._account = account
+        if account:
+            self._process_queued_docs()
+
+    def _has_configured_account(self):
+        return self._account is not None
+
+    def _queue_doc_id(self, doc_id):
+        self._pending_docs.append(doc_id)
+
+    def _make_uid_index(self, mdoc_id):
+        indexer = self._account.account.mbox_indexer
+        mbox_uuid = _get_mbox_uuid(mdoc_id)
+        if mbox_uuid:
+            chash = _get_chash_from_mdoc(mdoc_id)
+            logger.debug("Making index table for %s:%s" % (mbox_uuid, chash))
+            index_docid = constants.METAMSGID.format(
+                mbox_uuid=mbox_uuid.replace('-', '_'),
+                chash=chash)
+            # XXX could avoid creating table if I track which ones I already
+            # have seen -- but make sure *it's already created* before
+            # inserting the index entry!.
+            d = indexer.create_table(mbox_uuid)
+            d.addCallback(lambda _: indexer.insert_doc(mbox_uuid, index_docid))
+            self._processing_deferreds.append(d)
+
+    def _process_queued_docs(self):
+        assert(self._has_configured_account())
+        pending = self._pending_docs
+        log.msg("Mail post-sync hook: processing queued docs")
+
+        def remove_pending_docs(res):
+            self._pending_docs = []
+            return res
+
+        d = self.process_received_docs(pending)
+        if d:
+            d.addCallback(remove_pending_docs)
+            return d
+
+
+_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE)
+_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE)
+
+
+def _get_mbox_uuid(doc_id):
+    matches = _mbox_uuid_regex.findall(doc_id)
+    if matches:
+        return matches[0].replace('_', '-')
+
+
+def _get_chash_from_mdoc(doc_id):
+    matches = _mdoc_chash_regex.findall(doc_id)
+    if matches:
+        return matches[0]