From ece1f50d25dd9810fedd7498557dd2048fba2540 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 21 May 2015 12:47:27 -0400 Subject: [feature] post-sync mail processing hooks using the new soledad plugin capablity, mail hooks to the post-sync event by subscribing to the Meta-Doc type of documents. In this way, we can create the uid tables and the uid entries needed to keep local indexes for mail that has been processed in another instance. however, this won't prevent a conflict if a given mail is received and processed in two different instances. that is a problem that we still have to deal with. Resolves: #6996 Releases: 0.4.0 --- mail/.gitignore | 1 + mail/src/leap/mail/imap/service/imap.py | 9 ++ mail/src/leap/mail/mail.py | 2 - mail/src/leap/mail/plugins/__init__.py | 3 + mail/src/leap/mail/plugins/soledad_sync_hooks.py | 19 ++++ mail/src/leap/mail/sync_hooks.py | 121 +++++++++++++++++++++++ 6 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 mail/src/leap/mail/plugins/__init__.py create mode 100644 mail/src/leap/mail/plugins/soledad_sync_hooks.py create mode 100644 mail/src/leap/mail/sync_hooks.py (limited to 'mail') diff --git a/mail/.gitignore b/mail/.gitignore index 7ac82894..aafbdd13 100644 --- a/mail/.gitignore +++ b/mail/.gitignore @@ -1,4 +1,5 @@ *.pyc +dropin.cache build/ dist/ *.egg diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py index 370c5132..e401283e 100644 --- a/mail/src/leap/mail/imap/service/imap.py +++ b/mail/src/leap/mail/imap/service/imap.py @@ -35,6 +35,7 @@ from leap.common.events import emit, catalog from leap.common.check import leap_assert_type, leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.plugins import soledad_sync_hooks from leap.soledad.client import Soledad @@ -91,10 +92,17 @@ class LeapIMAPFactory(ServerFactory): theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount + self._initialize_sync_hooks() self._connections = defaultdict() # XXX how to pass the store along? + def _initialize_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(self.theAccount) + + def _teardown_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) + def buildProtocol(self, addr): """ Return a protocol suitable for the job. @@ -128,6 +136,7 @@ class LeapIMAPFactory(ServerFactory): # mark account as unusable, so any imap command will fail # with unauth state. self.theAccount.end_session() + self._teardown_sync_hooks() # TODO should wait for all the pending deferreds, # the twisted way! diff --git a/mail/src/leap/mail/mail.py b/mail/src/leap/mail/mail.py index 1649d4ad..bab73cb6 100644 --- a/mail/src/leap/mail/mail.py +++ b/mail/src/leap/mail/mail.py @@ -42,8 +42,6 @@ logger = logging.getLogger(name=__name__) # TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail -# [ ] Change the doc_ids scheme for part-docs: use mailbox UID validity -# identifier, instead of name! (renames are broken!) # [ ] Profile add_msg. def _get_mdoc_id(mbox, chash): diff --git a/mail/src/leap/mail/plugins/__init__.py b/mail/src/leap/mail/plugins/__init__.py new file mode 100644 index 00000000..ddb86917 --- /dev/null +++ b/mail/src/leap/mail/plugins/__init__.py @@ -0,0 +1,3 @@ +from twisted.plugin import pluginPackagePaths +__path__.extend(pluginPackagePaths(__name__)) +__all__ = [] diff --git a/mail/src/leap/mail/plugins/soledad_sync_hooks.py b/mail/src/leap/mail/plugins/soledad_sync_hooks.py new file mode 100644 index 00000000..9d48126e --- /dev/null +++ b/mail/src/leap/mail/plugins/soledad_sync_hooks.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# soledad_sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from leap.mail.sync_hooks import MailProcessingPostSyncHook +post_sync_uid_reindexer = MailProcessingPostSyncHook() diff --git a/mail/src/leap/mail/sync_hooks.py b/mail/src/leap/mail/sync_hooks.py new file mode 100644 index 00000000..b5bded5b --- /dev/null +++ b/mail/src/leap/mail/sync_hooks.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad PostSync Hooks. + +Process every new document of interest after every soledad synchronization, +using the hooks that soledad exposes via plugins. +""" +import logging + +from re import compile as regex_compile + +from zope.interface import implements +from twisted.internet import defer +from twisted.plugin import IPlugin +from twisted.python import log + +from leap.soledad.client.interfaces import ISoledadPostSyncPlugin +from leap.mail import constants + + +logger = logging.getLogger(__name__) + +_get_doc_type_preffix = lambda s: s[:2] + + +class MailProcessingPostSyncHook(object): + implements(IPlugin, ISoledadPostSyncPlugin) + + META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID) + watched_doc_types = (META_DOC_PREFFIX, ) + + _account = None + _pending_docs = [] + _processing_deferreds = [] + + def process_received_docs(self, doc_id_list): + if self._has_configured_account(): + process_fun = self._make_uid_index + else: + self._processing_deferreds = [] + process_fun = self._queue_doc_id + + for doc_id in doc_id_list: + if _get_doc_type_preffix(doc_id) in self.watched_doc_types: + log.msg("Mail post-sync hook: processing %s" % doc_id) + process_fun(doc_id) + + if self._processing_deferreds: + return defer.gatherResults(self._processing_deferreds) + + def set_account(self, account): + self._account = account + if account: + self._process_queued_docs() + + def _has_configured_account(self): + return self._account is not None + + def _queue_doc_id(self, doc_id): + self._pending_docs.append(doc_id) + + def _make_uid_index(self, mdoc_id): + indexer = self._account.account.mbox_indexer + mbox_uuid = _get_mbox_uuid(mdoc_id) + if mbox_uuid: + chash = _get_chash_from_mdoc(mdoc_id) + logger.debug("Making index table for %s:%s" % (mbox_uuid, chash)) + index_docid = constants.METAMSGID.format( + mbox_uuid=mbox_uuid.replace('-', '_'), + chash=chash) + # XXX could avoid creating table if I track which ones I already + # have seen -- but make sure *it's already created* before + # inserting the index entry!. + d = indexer.create_table(mbox_uuid) + d.addCallback(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) + self._processing_deferreds.append(d) + + def _process_queued_docs(self): + assert(self._has_configured_account()) + pending = self._pending_docs + log.msg("Mail post-sync hook: processing queued docs") + + def remove_pending_docs(res): + self._pending_docs = [] + return res + + d = self.process_received_docs(pending) + if d: + d.addCallback(remove_pending_docs) + return d + + +_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE) +_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE) + + +def _get_mbox_uuid(doc_id): + matches = _mbox_uuid_regex.findall(doc_id) + if matches: + return matches[0].replace('_', '-') + + +def _get_chash_from_mdoc(doc_id): + matches = _mdoc_chash_regex.findall(doc_id) + if matches: + return matches[0] -- cgit v1.2.3