From 87433821ad3ade5440814ffe9950e04c211a7a4c Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 2 Jan 2015 21:05:22 -0600 Subject: Port `enum` to `enum34` --- changes/bug-6601_port_enum34 | 1 + pkg/requirements.pip | 2 +- src/leap/mail/imap/memorystore.py | 10 +++++----- src/leap/mail/imap/messageparts.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changes/bug-6601_port_enum34 diff --git a/changes/bug-6601_port_enum34 b/changes/bug-6601_port_enum34 new file mode 100644 index 0000000..2ca551d --- /dev/null +++ b/changes/bug-6601_port_enum34 @@ -0,0 +1 @@ +- Port `enum` to `enum34` (Closes #6601) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 17ceba6..5bd4972 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -4,4 +4,4 @@ leap.common>=0.3.7 leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy -enum +enum34 diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py index 5eea4ef..e075394 100644 --- a/src/leap/mail/imap/memorystore.py +++ b/src/leap/mail/imap/memorystore.py @@ -52,10 +52,10 @@ logger = logging.getLogger(__name__) # soledad storage, in seconds. SOLEDAD_WRITE_PERIOD = 15 -FDOC = MessagePartType.fdoc.key -HDOC = MessagePartType.hdoc.key -CDOCS = MessagePartType.cdocs.key -DOCS_ID = MessagePartType.docs_id.key +FDOC = MessagePartType.fdoc.name +HDOC = MessagePartType.hdoc.name +CDOCS = MessagePartType.cdocs.name +DOCS_ID = MessagePartType.docs_id.name @contextlib.contextmanager @@ -73,7 +73,7 @@ def set_bool_flag(obj, att): setattr(obj, att, False) -DirtyState = Enum("none", "dirty", "new") +DirtyState = Enum("DirtyState", "none dirty new") class MemoryStore(object): diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py index 257721c..fb1d75a 100644 --- a/src/leap/mail/imap/messageparts.py +++ b/src/leap/mail/imap/messageparts.py @@ -32,7 +32,7 @@ from leap.mail.imap import interfaces from leap.mail.imap.fields import fields from leap.mail.utils import empty, first, find_charset -MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id") +MessagePartType = Enum("MessagePartType", "hdoc fdoc cdoc cdocs docs_id") logger = logging.getLogger(__name__) -- cgit v1.2.3 From 4eaf845479487d8cb245eeecf8bb82e10ed33664 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 15 Jan 2015 16:31:22 -0400 Subject: remove enum dep --- pkg/requirements.pip | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 5bd4972..64ff28c 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -4,4 +4,3 @@ leap.common>=0.3.7 leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy -enum34 -- cgit v1.2.3 From 82069cdd10d388bc3559e9ab882d514179374571 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:16:38 -0400 Subject: bump version compat for soledad client --- changes/VERSION_COMPAT | 1 + 1 file changed, 1 insertion(+) diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index 1eadcbe..12822ac 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -9,3 +9,4 @@ # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z leap.keymanager>=0.4.0 +leap.soledad.client>=0.7.0 -- cgit v1.2.3 From 2deb9547cbc61b42d0de7c5126a1cae5fc843939 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:19:53 -0400 Subject: add service-identity as a dependency for leap.mail --- pkg/requirements.pip | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 64ff28c..20f93a6 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -4,3 +4,4 @@ leap.common>=0.3.7 leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy +service-identity -- cgit v1.2.3 From 15ece6aeacb08f34a76d66747929e70cda334437 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Oct 2014 14:41:42 +0200 Subject: specify not syncable shared db --- src/leap/mail/imap/tests/utils.py | 5 +++-- src/leap/mail/tests/__init__.py | 20 ++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 0932bd4..5339acf 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -65,7 +65,7 @@ def initialize_soledad(email, gnupg_home, tempdir): passphrase = u"verysecretpassphrase" secret_path = os.path.join(tempdir, "secret.gpg") local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "http://provider" + server_url = "https://provider" cert_file = "" class MockSharedDB(object): @@ -86,7 +86,8 @@ def initialize_soledad(email, gnupg_home, tempdir): secret_path, local_db_path, server_url, - cert_file) + cert_file, + syncable=False) return _soledad diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index dc24293..10bc5fe 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -14,12 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ -Base classes and keys for SMTP gateway tests. +Base classes and keys for leap.mail tests. """ - import os import distutils.spawn import shutil @@ -27,9 +24,6 @@ import tempfile from mock import Mock -from twisted.trial import unittest - - from leap.soledad.client import Soledad from leap.keymanager import ( KeyManager, @@ -43,7 +37,7 @@ from leap.common.testing.basetest import BaseLeapTest def _find_gpg(): gpg_path = distutils.spawn.find_executable('gpg') return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" - + class TestCaseWithKeyManager(BaseLeapTest): @@ -100,23 +94,25 @@ class TestCaseWithKeyManager(BaseLeapTest): def __call__(self): return self - Soledad._shared_db = MockSharedDB() - - return Soledad( + soledad = Soledad( uuid, passphrase, secrets_path=secrets_path, local_db_path=local_db_path, server_url=server_url, cert_file=cert_file, + syncable=False ) + soledad._shared_db = MockSharedDB() + return soledad + def _keymanager_instance(self, address): """ Return a Key Manager instance for tests. """ self._config = { - 'host': 'http://provider/', + 'host': 'https://provider/', 'port': 25, 'username': address, 'password': '', -- cgit v1.2.3 From 3357b71f88a0d3e46c7a6ad2471917666b26e55a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 23 Dec 2014 10:21:06 -0400 Subject: fix typo in docs --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 4801833..d8634ea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ Welcome to leap.mail's documentation! ===================================== -This is the documentation for the ``leap.imap`` module. It is a twisted package +This is the documentation for the ``leap.mail`` module. It is a twisted package that exposes two services, ``smtp`` and ``imap``, that run local proxies and interact with a remote ``LEAP`` provider that offers *a soledad syncronization endpoint* and receive the outgoing email. -- cgit v1.2.3 From 1f6687d1375ff97f1ad0746e45f91f922866f32d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Oct 2014 14:51:53 +0200 Subject: adapt to soledad 0.7 async API --- src/leap/mail/imap/account.py | 253 +++++++++++++++++++++++++--------- src/leap/mail/imap/index.py | 51 +++++-- src/leap/mail/imap/mailbox.py | 57 ++++++-- src/leap/mail/imap/memorystore.py | 39 +++--- src/leap/mail/imap/messages.py | 136 ++++++++++-------- src/leap/mail/imap/server.py | 204 +++++++++++++++++++++++++-- src/leap/mail/imap/soledadstore.py | 11 +- src/leap/mail/imap/tests/test_imap.py | 185 ++++++++++++++++--------- src/leap/mail/imap/tests/utils.py | 25 ++-- 9 files changed, 698 insertions(+), 263 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 70ed13b..fe466cb 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -22,6 +22,7 @@ import logging import os import time +from twisted.internet import defer from twisted.mail import imap4 from twisted.python import log from zope.interface import implements @@ -65,6 +66,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): _soledad = None selected = None closed = False + _initialized = False def __init__(self, account_name, soledad, memstore=None): """ @@ -93,14 +95,39 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): self.__mailboxes = set([]) - self.initialize_db() + self._deferred_initialization = defer.Deferred() + self._initialize_storage() - # every user should have the right to an inbox folder - # at least, so let's make one! - self._load_mailboxes() + def _initialize_storage(self): - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.addMailbox(self.INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.initialize_db() + + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization def _get_empty_mailbox(self): """ @@ -120,10 +147,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ # XXX use soledadstore instead ...; - doc = self._soledad.get_from_index( + def get_first_if_any(docs): + return docs[0] if docs else None + + d = self._soledad.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, self._parse_mailbox_name(name)) - return doc[0] if doc else None + d.addCallback(get_first_if_any) + return d @property def mailboxes(self): @@ -134,19 +165,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): return sorted(self.__mailboxes) def _load_mailboxes(self): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)]) - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + def update_mailboxes(db_indexes): + self.__mailboxes.update( + [doc.content[self.MBOX_KEY] for doc in db_indexes]) + d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) + d.addCallback(update_mailboxes) + return d def getMailbox(self, name): """ @@ -182,7 +206,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): one is provided. :type creation_ts: int - :returns: True if successful + :returns: a Deferred that will contain the document if successful. :rtype: bool """ name = self._parse_mailbox_name(name) @@ -203,21 +227,29 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): mbox[self.MBOX_KEY] = name mbox[self.CREATED_KEY] = creation_ts - doc = self._soledad.create_doc(mbox) - self._load_mailboxes() - return bool(doc) + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self._soledad.create_doc(mbox) + d.addCallback(load_mbox_cache) + return d def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. + :param pathspec: + The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. :type pathspec: str - :return: A true value if the creation succeeds. - :rtype: bool + :return: + A deferred that will fire with a true value if the creation + succeeds. + :rtype: Deferred :raise MailboxException: Raised if this mailbox cannot be added. """ @@ -225,18 +257,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): paths = filter( None, self._parse_mailbox_name(pathspec).split('/')) + + subs = [] + sep = '/' + for accum in range(1, len(paths)): try: - self.addMailbox('/'.join(paths[:accum])) + partial = sep.join(paths[:accum]) + d = self.addMailbox(partial) + subs.append(d) except imap4.MailboxCollision: pass try: - self.addMailbox('/'.join(paths)) + df = self.addMailbox(sep.join(paths)) except imap4.MailboxCollision: if not pathspec.endswith('/'): - return False - self._load_mailboxes() - return True + df = defer.succeed(False) + else: + df = defer.succeed(True) + finally: + subs.append(df) + + def all_good(result): + return all(result) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + if subs: + d1 = defer.gatherResults(subs, consumeErrors=True) + d1.addCallback(load_mbox_cache) + d1.addCallback(all_good) + else: + d1 = defer.succeed(False) + d1.addCallback(load_mbox_cache) + return d1 def select(self, name, readwrite=1): """ @@ -275,17 +332,20 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: the mailbox to be deleted :type name: str - :param force: if True, it will not check for noselect flag or inferior - names. use with care. + :param force: + if True, it will not check for noselect flag or inferior + names. use with care. :type force: bool + :rtype: Deferred """ name = self._parse_mailbox_name(name) if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) + err = imap4.MailboxException("No such mailbox: %r" % name) + return defer.fail(err) mbox = self.getMailbox(name) - if force is False: + if not force: # See if this box is flagged \Noselect # XXX use mbox.flags instead? mbox_flags = mbox.getFlags() @@ -294,11 +354,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # as part of their root. for others in self.mailboxes: if others != name and others.startswith(name): - raise imap4.MailboxException, ( + err = imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") + return defer.fail(err) self.__mailboxes.discard(name) - mbox.destroy() + return mbox.destroy() # XXX FIXME --- not honoring the inferior names... @@ -331,14 +392,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): if new in self.mailboxes: raise imap4.MailboxCollision(repr(new)) + rename_deferreds = [] + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + def update_mbox_doc_name(mbox, oldname, newname, update_deferred): + mbox.content[self.MBOX_KEY] = newname + d = self._soledad.put_doc(mbox) + d.addCallback(lambda r: update_deferred.callback(True)) + for (old, new) in inferiors: - self._memstore.rename_fdocs_mailbox(old, new) - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new self.__mailboxes.discard(old) - self._soledad.put_doc(mbox) + self._memstore.rename_fdocs_mailbox(old, new) + + d0 = defer.Deferred() + d = self._get_mailbox_by_name(old) + d.addCallback(update_mbox_doc_name, old, new, d0) + rename_deferreds.append(d0) - self._load_mailboxes() + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + d1.addCallback(load_mbox_cache) + return d1 def _inferiorNames(self, name): """ @@ -354,6 +431,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO ------------------ can we preserve the attr? + # maybe add to memory store. def isSubscribed(self, name): """ Returns True if user is subscribed to this mailbox. @@ -361,10 +440,35 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: the mailbox to be checked. :type name: str - :rtype: bool + :rtype: Deferred (will fire with bool) + """ + subscribed = self.SUBSCRIBED_KEY + + def is_subscribed(mbox): + subs_bool = bool(mbox.content.get(subscribed, False)) + return subs_bool + + d = self._get_mailbox_by_name(name) + d.addCallback(is_subscribed) + return d + + # TODO ------------------ can we preserve the property? + # maybe add to memory store. + + def _get_subscriptions(self): """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) + Return a list of the current subscriptions for this account. + + :returns: A deferred that will fire with the subscriptions. + :rtype: Deferred + """ + def get_docs_content(docs): + return [doc.content[self.MBOX_KEY] for doc in docs] + + d = self._soledad.get_from_index( + self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') + d.addCallback(get_docs_content) + return d def _set_subscription(self, name, value): """ @@ -376,26 +480,42 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param value: the boolean value :type value: bool """ + # XXX Note that this kind of operation has + # no guarantees of atomicity. We should not be accessing mbox + # documents concurrently. + + subscribed = self.SUBSCRIBED_KEY + + def update_subscribed_value(mbox): + mbox.content[subscribed] = value + return self._soledad.put_doc(mbox) + # maybe we should store subscriptions in another # document... if name not in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) + d = self.addMailbox(name) + d.addCallback(lambda v: self._get_mailbox_by_name(name)) + else: + d = self._get_mailbox_by_name(name) + d.addCallback(update_subscribed_value) + return d def subscribe(self, name): """ - Subscribe to this mailbox + Subscribe to this mailbox if not already subscribed. :param name: name of the mailbox :type name: str + :rtype: Deferred """ name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) + + def check_and_subscribe(subscriptions): + if name not in subscriptions: + return self._set_subscription(name, True) + d = self._get_subscriptions() + d.addCallback(check_and_subscribe) + return d def unsubscribe(self, name): """ @@ -403,12 +523,21 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: name of the mailbox :type name: str + :rtype: Deferred """ name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException( - "Not currently subscribed to %r" % name) - self._set_subscription(name, False) + + def check_and_unsubscribe(subscriptions): + if name not in subscriptions: + raise imap4.MailboxException( + "Not currently subscribed to %r" % name) + return self._set_subscription(name, False) + d = self._get_subscriptions() + d.addCallback(check_and_unsubscribe) + return d + + def getSubscriptions(self): + return self._get_subscriptions() def listMailboxes(self, ref, wildcard): """ diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py index 5f0919a..ea35fff 100644 --- a/src/leap/mail/imap/index.py +++ b/src/leap/mail/imap/index.py @@ -19,6 +19,8 @@ Index for SoledadBackedAccount, Mailbox and Messages. """ import logging +from twisted.internet import defer + from leap.common.check import leap_assert, leap_assert_type from leap.mail.imap.fields import fields @@ -39,6 +41,9 @@ class IndexedDB(object): """ # TODO we might want to move this to soledad itself, check + _index_creation_deferreds = [] + index_ready = False + def initialize_db(self): """ Initialize the database. @@ -46,24 +51,40 @@ class IndexedDB(object): leap_assert(self._soledad, "Need a soledad attribute accesible in the instance") leap_assert_type(self.INDEXES, dict) + self._index_creation_deferreds = [] + + def _on_indexes_created(ignored): + self.index_ready = True + + def _create_index(name, expression): + d = self._soledad.create_index(name, *expression) + self._index_creation_deferreds.append(d) + + def _create_indexes(db_indexes): + db_indexes = dict(db_indexes) + for name, expression in fields.INDEXES.items(): + if name not in db_indexes: + # The index does not yet exist. + _create_index(name, expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so + # we delete it and add the proper index expression. + d1 = self._soledad.delete_index(name) + d1.addCallback(lambda _: _create_index(name, expression)) + + all_created = defer.gatherResults(self._index_creation_deferreds) + all_created.addCallback(_on_indexes_created) + return all_created # Ask the database for currently existing indexes. if not self._soledad: logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") return - db_indexes = dict() if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) + d = self._soledad.list_indexes() + d.addCallback(_create_indexes) + return d diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 34cf535..3c1769a 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -303,21 +303,32 @@ class SoledadMailbox(WithMsgFields, MBoxParser): We do this to be able to filter the requests efficiently. """ primed = self._known_uids_primed.get(self.mbox, False) - if not primed: - known_uids = self.messages.all_soledad_uid_iter() + # XXX handle the maybeDeferred + + def set_primed(known_uids): self._memstore.set_known_uids(self.mbox, known_uids) self._known_uids_primed[self.mbox] = True + if not primed: + d = self.messages.all_soledad_uid_iter() + d.addCallback(set_primed) + return d + def prime_flag_docs_to_memstore(self): """ Prime memstore with all the flags documents. """ primed = self._fdoc_primed.get(self.mbox, False) - if not primed: - all_flag_docs = self.messages.get_all_soledad_flag_docs() - self._memstore.load_flag_docs(self.mbox, all_flag_docs) + + def set_flag_docs(flag_docs): + self._memstore.load_flag_docs(self.mbox, flag_docs) self._fdoc_primed[self.mbox] = True + if not primed: + d = self.messages.get_all_soledad_flag_docs() + d.addCallback(set_flag_docs) + return d + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -522,21 +533,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Should cleanup resources, and set the \\Noselect flag on the mailbox. + """ # XXX this will overwrite all the existing flags! # should better simply addFlag self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() # XXX removing the mailbox in situ for now, # we should postpone the removal - # XXX move to memory store?? - mbox_doc = self._get_mbox_doc() - if mbox_doc is None: - # memory-only store! - return - self._soledad.delete_doc(self._get_mbox_doc()) + def remove_mbox_doc(ignored): + # XXX move to memory store?? + + def _remove_mbox_doc(doc): + if doc is None: + # memory-only store! + return defer.succeed(True) + return self._soledad.delete_doc(doc) + + doc = self._get_mbox_doc() + return _remove_mbox_doc(doc) + + d = self.deleteAllDocs() + d.addCallback(remove_mbox_doc) + return d def _close_cb(self, result): self.closed = True @@ -1006,9 +1026,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Delete all docs in this mailbox """ - docs = self.messages.get_all_docs() - for doc in docs: - self.messages._soledad.delete_doc(doc) + def del_all_docs(docs): + todelete = [] + for doc in docs: + d = self.messages._soledad.delete_doc(doc) + todelete.append(d) + return defer.gatherResults(todelete) + + d = self.messages.get_all_docs() + d.addCallback(del_all_docs) + return d def unset_recent_flags(self, uid_seq): """ diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py index e075394..eda5b96 100644 --- a/src/leap/mail/imap/memorystore.py +++ b/src/leap/mail/imap/memorystore.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # memorystore.py # Copyright (C) 2014 LEAP # @@ -112,8 +112,6 @@ class MemoryStore(object): :param write_period: the interval to dump messages to disk, in seconds. :type write_period: int """ - self.reactor = reactor - self._permanent_store = permanent_store self._write_period = write_period @@ -241,6 +239,7 @@ class MemoryStore(object): self.producer = None self._write_loop = None + # TODO -- remove def _start_write_loop(self): """ Start loop for writing to disk database. @@ -250,6 +249,7 @@ class MemoryStore(object): if not self._write_loop.running: self._write_loop.start(self._write_period, now=True) + # TODO -- remove def _stop_write_loop(self): """ Stop loop for writing to disk database. @@ -278,17 +278,18 @@ class MemoryStore(object): :type uid: int :param message: a message to be added :type message: MessageWrapper - :param observer: the deferred that will fire with the - UID of the message. If notify_on_disk is True, - this will happen when the message is written to - Soledad. Otherwise it will fire as soon as we've - added the message to the memory store. + :param observer: + the deferred that will fire with the UID of the message. If + notify_on_disk is True, this will happen when the message is + written to Soledad. Otherwise it will fire as soon as we've added + the message to the memory store. :type observer: Deferred - :param notify_on_disk: whether the `observer` deferred should - wait until the message is written to disk to - be fired. + :param notify_on_disk: + whether the `observer` deferred should wait until the message is + written to disk to be fired. :type notify_on_disk: bool """ + # TODO -- return a deferred log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) key = mbox, uid @@ -306,7 +307,7 @@ class MemoryStore(object): else: # Caller does not care, just fired and forgot, so we pass # a defer that will inmediately have its callback triggered. - self.reactor.callFromThread(observer.callback, uid) + reactor.callFromThread(observer.callback, uid) def put_message(self, mbox, uid, message, notify_on_disk=True): """ @@ -442,6 +443,7 @@ class MemoryStore(object): :return: MessageWrapper or None """ + # TODO -- return deferred if dirtystate == DirtyState.dirty: flags_only = True @@ -467,6 +469,7 @@ class MemoryStore(object): chash = fdoc.get(fields.CONTENT_HASH_KEY) hdoc = self._hdoc_store[chash] if empty(hdoc): + # XXX this will be a deferred hdoc = self._permanent_store.get_headers_doc(chash) if empty(hdoc): return None @@ -531,7 +534,8 @@ class MemoryStore(object): # IMessageStoreWriter - @deferred_to_thread + # TODO -- I think we don't need this anymore. + # instead, we can have def write_messages(self, store): """ Write the message documents in this MemoryStore to a different store. @@ -657,7 +661,7 @@ class MemoryStore(object): with self._last_uid_lock: self._last_uid[mbox] += 1 value = self._last_uid[mbox] - self.reactor.callInThread(self.write_last_uid, mbox, value) + reactor.callInThread(self.write_last_uid, mbox, value) return value def write_last_uid(self, mbox, value): @@ -1077,6 +1081,7 @@ class MemoryStore(object): return None return self._rflags_store[mbox]['set'] + # XXX -- remove def all_rdocs_iter(self): """ Return an iterator through all in-memory recent flag dicts, wrapped @@ -1125,6 +1130,7 @@ class MemoryStore(object): self.remove_message(mbox, uid) return mem_deleted + # TODO -- remove def stop_and_flush(self): """ Stop the write loop and trigger a write to the producer. @@ -1180,6 +1186,7 @@ class MemoryStore(object): :type observer: Deferred """ mem_deleted = self.remove_all_deleted(mbox) + # TODO return a DeferredList observer.callback(mem_deleted) def _delete_from_soledad_and_memory(self, result, mbox, observer): @@ -1313,8 +1320,8 @@ class MemoryStore(object): :rtype: bool """ # FIXME this should return a deferred !!! - # XXX ----- can fire when all new + dirty deferreds - # are done (gatherResults) + # TODO this should be moved to soledadStore instead + # (all pending deferreds) return getattr(self, self.WRITING_FLAG) @property diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index e8d64d1..c761091 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -71,6 +71,7 @@ def try_unique_query(curried): :param curried: a curried function :type curried: callable """ + # XXX FIXME ---------- convert to deferreds leap_assert(callable(curried), "A callable is expected") try: query = curried() @@ -134,10 +135,11 @@ class LeapMessage(fields, MBoxParser): self.__chash = None self.__bdoc = None - self.reactor = reactor - # XXX make these properties public + # XXX FIXME ------ the documents can be + # deferreds too.... niice. + @property def fdoc(self): """ @@ -506,18 +508,15 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the flags for this message. """ - result = {} - try: - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % self._uid) - logger.exception(exc) - finally: - return result + def get_first_if_any(docs): + result = first(docs) + return result if result else {} + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + d.addCallback(get_first_if_any) + return d # TODO move to soledadstore instead of accessing soledad directly def _get_headers_doc(self): @@ -525,10 +524,11 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the headers for this message. """ - head_docs = self._soledad.get_from_index( + d = self._soledad.get_from_index( fields.TYPE_C_HASH_IDX, fields.TYPE_HEADERS_VAL, str(self.chash)) - return first(head_docs) + d.addCallback(lambda docs: first(docs)) + return d # TODO move to soledadstore instead of accessing soledad directly def _get_body_doc(self): @@ -536,6 +536,8 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the body for this message. """ + # XXX FIXME --- this might need a maybedeferred + # on the receiving side... hdoc_content = self.hdoc.content body_phash = hdoc_content.get( fields.BODY_KEY, None) @@ -554,13 +556,11 @@ class LeapMessage(fields, MBoxParser): return bdoc # no memstore, or no body doc found there - if self._soledad: - body_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(body_phash)) - return first(body_docs) - else: - logger.error("No phash in container, and no soledad found!") + d = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(body_phash)) + d.addCallback(lambda docs: first(docs)) + return d def __getitem__(self, key): """ @@ -739,8 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): else: self._initialized[mbox] = True - self.reactor = reactor - def _get_empty_doc(self, _type=FLAGS_DOC): """ Returns an empty doc for storing different message parts. @@ -887,9 +885,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) + # TODO return soledad deferred instead observer = defer.Deferred() d = self._do_parse(raw) - d.addCallback(lambda result: self.reactor.callInThread( + d.addCallback(lambda result: reactor.callInThread( self._do_add_msg, result, flags, subject, date, notify_on_disk, observer)) return observer @@ -924,17 +923,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): msg = self.get_msg_by_uid(existing_uid) # We can say the observer that we're done - self.reactor.callFromThread(observer.callback, existing_uid) + # TODO return soledad deferred instead + reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return - # XXX get FUCKING UID from autoincremental table + # TODO S2 -- get FUCKING UID from autoincremental table uid = self.memstore.increment_last_soledad_uid(self.mbox) # We can say the observer that we're done at this point, but # before that we should make sure it has no serious consequences # if we're issued, for instance, a fetch command right after... - # self.reactor.callFromThread(observer.callback, uid) + # reactor.callFromThread(observer.callback, uid) # if we did the notify, we need to invalidate the deferred # so not to try to fire it twice. # observer = None @@ -960,6 +960,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): self.set_recent_flag(uid) msg_container = MessageWrapper(fd, hd, cdocs) + + # TODO S1 -- just pass this to memstore and return that deferred. self.memstore.create_message( self.mbox, uid, msg_container, observer=observer, notify_on_disk=notify_on_disk) @@ -1011,6 +1013,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): Get recent-flags document from Soledad for this mailbox. :rtype: SoledadDocument or None """ + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_IDX, @@ -1029,6 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uids: the uids to unset :type uid: sequence """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags.difference_update( set(uids)) @@ -1042,11 +1046,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uid: the uid to unset :type uid: int """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags.difference_update( set([uid])) - @deferred_to_thread def set_recent_flag(self, uid): """ Set Recent flag for a given uid. @@ -1054,6 +1058,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uid: the uid to set :type uid: int """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags = self.recent_flags.union( set([uid])) @@ -1068,6 +1073,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_C_HASH_IDX, @@ -1125,7 +1131,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): return None return fdoc.content.get(fields.UID_KEY, None) - @deferred_to_thread def _get_uid_from_msgid(self, msgid): """ Return a UID for a given message-id. @@ -1144,7 +1149,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # XXX is this working? return self._get_uid_from_msgidCb(msgid) - @deferred_to_thread def set_flags(self, mbox, messages, flags, mode, observer): """ Set flags for a sequence of messages. @@ -1162,7 +1166,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): done. :type observer: deferred """ - reactor = self.reactor getmsg = self.get_msg_by_uid def set_flags(uid, flags, mode): @@ -1173,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): setted_flags = [set_flags(uid, flags, mode) for uid in messages] result = dict(filter(None, setted_flags)) + # TODO -- remove reactor.callFromThread(observer.callback, result) # getters: generic for a mailbox @@ -1223,37 +1227,45 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): If you want acess to the content, use __iter__ instead - :return: a list of u1db documents - :rtype: list of SoledadDocument + :return: a Deferred, that will fire with a list of u1db documents + :rtype: Deferred (promise of list of SoledadDocument) """ if _type not in fields.__dict__.values(): raise TypeError("Wrong type passed to get_all_docs") + # FIXME ----- either raise or return a deferred wrapper. if sameProxiedObjects(self._soledad, None): logger.warning('Tried to get messages but soledad is None!') return [] - all_docs = [doc for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - _type, self.mbox)] + def get_sorted_docs(docs): + all_docs = [doc for doc in docs] + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad + # FIXME ---------------------------------------------- + return sorted(all_docs, key=lambda item: item.content['uid']) - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, _type, self.mbox) + d.addCallback(get_sorted_docs) + return d def all_soledad_uid_iter(self): """ Return an iterator through the UIDs of all messages, sorted in ascending order. """ - db_uids = set([doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc)]) - return db_uids + # XXX FIXME ------ sorted??? + + def get_uids(docs): + return set([ + doc.content[self.UID_KEY] for doc in docs if not empty(doc)]) + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) + d.addCallback(get_uids) + return d def all_uid_iter(self): """ @@ -1277,16 +1289,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # XXX we really could return a reduced version with # just {'uid': (flags-tuple,) since the prefetch is # only oriented to get the flag tuples. - all_docs = [( - doc.content[self.UID_KEY], - dict(doc.content)) - for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc.content)] - all_flags = dict(all_docs) - return all_flags + + def get_content(docs): + all_docs = [( + doc.content[self.UID_KEY], + dict(doc.content)) + for doc in docs + if not empty(doc.content)] + all_flags = dict(all_docs) + return all_flags + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + d.addCallback(get_content) + return d def all_headers(self): """ @@ -1339,6 +1356,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # recent messages # XXX take it from memstore + # XXX Used somewhere? def count_recent(self): """ Count all messages with the `Recent` flag. diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index fe56ea6..cf0ba74 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation. from copy import copy from twisted import cred +from twisted.internet import reactor from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log @@ -50,6 +51,7 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid + self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -59,9 +61,6 @@ class LeapIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) - from twisted.internet import reactor - self.reactor = reactor - def lineReceived(self, line): """ Attempt to parse a single line from the server. @@ -311,21 +310,203 @@ class LeapIMAPServer(imap4.IMAP4Server): return self._fileLiteral(size, literalPlus) ############################# + # --------------------------------- isSubscribed patch + # TODO -- send patch upstream. + # There is a bug in twisted implementation: + # in cbListWork, it's assumed that account.isSubscribed IS a callable, + # although in the interface documentation it's stated that it can be + # a deferred. + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = self._parseMbox(mbox) + mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) + mailboxes.addCallback(self._cbSubscribed) + mailboxes.addCallback( + self._cbListWork, tag, sub, cmdName, + ).addErrback(self._ebListWork, tag) + + def _cbSubscribed(self, mailboxes): + subscribed = [ + maybeDeferred(self.account.isSubscribed, name) + for (name, box) in mailboxes] + + def get_mailboxes_and_subs(result): + subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] + return mailboxes, subscribed + + d = defer.gatherResults(subscribed) + d.addCallback(get_mailboxes_and_subs) + return d + + def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): + mailboxes, subscribed = mailboxes_subscribed + + for (name, box) in mailboxes: + if not sub or name in subscribed: + flags = box.getFlags() + delim = box.getHierarchicalDelimiter() + resp = (imap4.DontQuoteMe(cmdName), + map(imap4.DontQuoteMe, flags), + delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse( + imap4.collapseNestedLists(resp)) + self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) + # -------------------- end isSubscribed patch ----------- + + # TODO ---- + # subscribe method had also to be changed to accomodate + # deferred + # Revert to regular methods as soon as we implement non-deferred memory + # cache. + def do_SUBSCRIBE(self, tag, name): + name = self._parseMbox(name) + + def _subscribeCb(_): + self.sendPositiveResponse(tag, 'Subscribed') + + def _subscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while subscribing to mailbox") + + d = self.account.subscribe(name) + d.addCallbacks(_subscribeCb, _subscribeEb) + return d + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + def do_UNSUBSCRIBE(self, tag, name): + # unsubscribe method had also to be changed to accomodate + # deferred + name = self._parseMbox(name) + + def _unsubscribeCb(_): + self.sendPositiveResponse(tag, 'Unsubscribed') + + def _unsubscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while unsubscribing " + "from mailbox") + + d = self.account.unsubscribe(name) + d.addCallbacks(_unsubscribeCb, _unsubscribeEb) + return d + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [self._parseMbox(n) for n in oldname, newname] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse( + tag, + 'You cannot rename the inbox, or ' + 'rename another mailbox to inbox.') + return + + def _renameCb(_): + self.sendPositiveResponse(tag, 'Mailbox renamed') + + def _renameEb(failure): + m = failure.value + print "rename failure!" + if failure.check(TypeError): + self.sendBadResponse(tag, 'Invalid command syntax') + elif failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + log.err() + self.sendBadResponse( + tag, + "Server error encountered while " + "renaming mailbox") + + d = self.account.rename(oldname, newname) + d.addCallbacks(_renameCb, _renameEb) + return d + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + def do_CREATE(self, tag, name): + name = self._parseMbox(name) + + def _createCb(result): + if result: + self.sendPositiveResponse(tag, 'Mailbox created') + else: + self.sendNegativeResponse(tag, 'Mailbox not created') + + def _createEb(failure): + c = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(c)) + else: + log.err() + self.sendBadResponse( + tag, "Server error encountered while creating mailbox") + + d = self.account.create(name) + d.addCallbacks(_createCb, _createEb) + return d + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + def do_DELETE(self, tag, name): + name = self._parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, 'You cannot delete the inbox') + return + + def _deleteCb(result): + self.sendPositiveResponse(tag, 'Mailbox deleted') + + def _deleteEb(failure): + m = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + print "other error" + log.err() + self.sendBadResponse( + tag, + "Server error encountered while deleting mailbox") + + d = self.account.delete(name) + d.addCallbacks(_deleteCb, _deleteEb) + return d + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + # Need to override the command table after patching # arg_astring and arg_literal + # do_DELETE = imap4.IMAP4Server.do_DELETE + # do_CREATE = imap4.IMAP4Server.do_CREATE + # do_RENAME = imap4.IMAP4Server.do_RENAME + # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_LOGIN = imap4.IMAP4Server.do_LOGIN - do_CREATE = imap4.IMAP4Server.do_CREATE - do_DELETE = imap4.IMAP4Server.do_DELETE - do_RENAME = imap4.IMAP4Server.do_RENAME - do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE - do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_STATUS = imap4.IMAP4Server.do_STATUS do_APPEND = imap4.IMAP4Server.do_APPEND do_COPY = imap4.IMAP4Server.do_COPY _selectWork = imap4.IMAP4Server._selectWork - _listWork = imap4.IMAP4Server._listWork + arg_plist = imap4.IMAP4Server.arg_plist arg_seqset = imap4.IMAP4Server.arg_seqset opt_plist = imap4.IMAP4Server.opt_plist @@ -342,8 +523,8 @@ class LeapIMAPServer(imap4.IMAP4Server): auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') select_EXAMINE = auth_EXAMINE - auth_DELETE = (do_DELETE, arg_astring) - select_DELETE = auth_DELETE + # auth_DELETE = (do_DELETE, arg_astring) + # select_DELETE = auth_DELETE auth_RENAME = (do_RENAME, arg_astring, arg_astring) select_RENAME = auth_RENAME @@ -369,7 +550,6 @@ class LeapIMAPServer(imap4.IMAP4Server): select_COPY = (do_COPY, arg_seqset, arg_astring) - ############################################################# # END of Twisted imap4 patch to support LITERAL+ extension ############################################################# diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py index f3de8eb..fc8ea55 100644 --- a/src/leap/mail/imap/soledadstore.py +++ b/src/leap/mail/imap/soledadstore.py @@ -40,11 +40,6 @@ from leap.mail.utils import first, empty, accumulator_queue logger = logging.getLogger(__name__) -# TODO -# [ ] Implement a retry queue? -# [ ] Consider journaling of operations. - - class ContentDedup(object): """ Message deduplication. @@ -132,6 +127,7 @@ A lock per document. # http://stackoverflow.com/a/2437645/1157664 # Setting this to twice the number of threads in the threadpool # should be safe. + put_locks = defaultdict(lambda: threading.Lock()) mbox_doc_locks = defaultdict(lambda: threading.Lock()) @@ -429,7 +425,6 @@ class SoledadStore(ContentDedup): continue if item.part == MessagePartType.fdoc: - #logger.debug("PUT dirty fdoc") yield item, call # XXX also for linkage-doc !!! @@ -479,7 +474,7 @@ class SoledadStore(ContentDedup): return query.pop() else: logger.error("Could not find mbox document for %r" % - (mbox,)) + (mbox,)) except Exception as exc: logger.exception("Unhandled error %r" % exc) @@ -552,8 +547,10 @@ class SoledadStore(ContentDedup): :type uid: int :rtype: SoledadDocument or None """ + # TODO -- inlineCallbacks result = None try: + # TODO -- yield flag_docs = self._soledad.get_from_index( fields.TYPE_MBOX_UID_IDX, fields.TYPE_FLAGS_VAL, mbox, str(uid)) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 7837aaa..dd4294c 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -68,7 +68,6 @@ def sortNest(l): class TestRealm: - """ A minimal auth realm for testing purposes only """ @@ -83,7 +82,6 @@ class TestRealm: # class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): - """ Tests for the MessageCollection class """ @@ -254,14 +252,18 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.login(TEST_USER, TEST_PASSWD) def create(): + create_deferreds = [] for name in succeed + fail: d = self.client.create(name) d.addCallback(strip(cb)).addErrback(eb) - d.addCallbacks(self._cbStopClient, self._ebGeneral) + create_deferreds.append(d) + dd = defer.gatherResults(create_deferreds) + dd.addCallbacks(self._cbStopClient, self._ebGeneral) + return dd self.result = [] - d1 = self.connected.addCallback(strip(login)).addCallback( - strip(create)) + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create)) d2 = self.loopback() d = defer.gatherResults([d1, d2]) return d.addCallback(self._cbTestCreate, succeed, fail) @@ -269,24 +271,27 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestCreate(self, ignored, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = list(LeapIMAPServer.theAccount.mailboxes) - answers = ([u'INBOX', u'foobox', 'test', u'test/box', - u'test/box/box', 'testbox']) - self.assertEqual(mboxes, [a for a in answers]) + mboxes = LeapIMAPServer.theAccount.mailboxes + + answers = ([u'INBOX', u'testbox', u'test/box', u'test', + u'test/box/box', 'foobox']) + self.assertEqual(sorted(mboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = LeapIMAPServer.theAccount + d0 = lambda: acc.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) def delete(): - return self.client.delete('delete/me') + return self.client.delete('test-delete/me') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -352,11 +357,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - LeapIMAPServer.theAccount.addMailbox('delete') - to_delete = LeapIMAPServer.theAccount.getMailbox('delete') - to_delete.setFlags((r'\Noselect',)) - to_delete.getFlags() - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = LeapIMAPServer.theAccount + d_del0 = lambda: acc.addMailbox('delete') + d_del1 = lambda: acc.addMailbox('delete/me') + + def set_noselect_flag(): + mbox = acc.getMailbox('delete') + mbox.setFlags((r'\Noselect',)) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -369,6 +376,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.failure = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d_del0)) + d1.addCallback(strip(d_del1)) + d1.addCallback(strip(set_noselect_flag)) d1.addCallback(strip(delete)).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -385,7 +395,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can rename a mailbox """ - LeapIMAPServer.theAccount.addMailbox('oldmbox') + d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -394,6 +404,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.rename('oldmbox', 'newname') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -435,8 +446,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - LeapIMAPServer.theAccount.create('oldmbox/m1') - LeapIMAPServer.theAccount.create('oldmbox/m2') + acc = LeapIMAPServer.theAccount + dc1 = lambda: acc.create('oldmbox/m1') + dc2 = lambda: acc.create('oldmbox/m2') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -445,6 +457,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.rename('oldmbox', 'newname') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -454,7 +468,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestHierarchicalRename(self, ignored): mboxes = LeapIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] - self.assertEqual(mboxes, [s for s in expected]) + self.assertEqual(sorted(mboxes), sorted([s for s in expected])) def testSubscribe(self): """ @@ -466,23 +480,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def subscribe(): return self.client.subscribe('this/mbox') + def get_subscriptions(ignored): + return LeapIMAPServer.theAccount.getSubscriptions() + d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['this/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['this/mbox'])) return d def testUnsubscribe(self): """ Test whether we can unsubscribe from a set of mailboxes """ - LeapIMAPServer.theAccount.subscribe('this/mbox') - LeapIMAPServer.theAccount.subscribe('that/mbox') + acc = LeapIMAPServer.theAccount + + dc1 = lambda: acc.subscribe('this/mbox') + dc2 = lambda: acc.subscribe('that/mbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -490,22 +509,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def unsubscribe(): return self.client.unsubscribe('this/mbox') + def get_subscriptions(ignored): + return LeapIMAPServer.theAccount.getSubscriptions() + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['that/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['that/mbox'])) return d def testSelect(self): """ Try to select a mailbox """ - self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + acc = self.server.theAccount + d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) self.selectedArgs = None def login(): @@ -520,6 +545,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallback(strip(select)) d1.addErrback(self._ebGeneral) @@ -754,13 +780,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) - def _listSetup(self, f): - LeapIMAPServer.theAccount.addMailbox('root/subthingl', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('root/another-thing', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('non-root/subthing', - creation_ts=42) + def _listSetup(self, f, f2=None): + acc = LeapIMAPServer.theAccount + + dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -770,6 +795,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.listed = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallback(strip(dc3)) + + if f2 is not None: + d1.addCallback(f2) + d1.addCallbacks(strip(f), self._ebGeneral) d1.addCallbacks(listed, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -786,7 +818,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"), + (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") ]) )) @@ -796,20 +828,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - LeapIMAPServer.theAccount.subscribe('root/subthingl2') + acc = LeapIMAPServer.theAccount + + def subs_mailbox(): + # why not client.subscribe instead? + return acc.subscribe('root/subthing') def lsub(): return self.client.lsub('root', '%') - d = self._listSetup(lsub) + + d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")]) + [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) return d def testStatus(self): """ Test Status command """ - LeapIMAPServer.theAccount.addMailbox('root/subthings') + acc = LeapIMAPServer.theAccount + + def add_mailbox(): + return acc.addMailbox('root/subthings') + # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -824,7 +865,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.statused = result self.statused = None - d1 = self.connected.addCallback(strip(login)) + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(status), self._ebGeneral) d1.addCallbacks(statused, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -930,7 +973,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -946,6 +989,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ) ) d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -995,10 +1039,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test closing the mailbox. We expect to get deleted all messages flagged as such. """ + acc = self.server.theAccount name = 'mailbox-close' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + d0 = lambda: acc.addMailbox(name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1006,14 +1050,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def select(): return self.client.select(name) + def get_mailbox(): + self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + def add_messages(): - d1 = m.messages.add_msg( + d1 = self.mailbox.messages.add_msg( 'test 1', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( + d2 = self.mailbox.messages.add_msg( 'test 2', subject="Message 2", flags=('AnotherFlag',)) - d3 = m.messages.add_msg( + d3 = self.mailbox.messages.add_msg( 'test 3', subject="Message 3", flags=('\\Deleted',)) d = defer.gatherResults([d1, d2, d3]) @@ -1023,30 +1070,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.close() d = self.connected.addCallback(strip(login)) + d.addCallback(strip(d0)) d.addCallbacks(strip(select), self._ebGeneral) + d.addCallback(strip(get_mailbox)) d.addCallbacks(strip(add_messages), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m) + return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - def _cbTestClose(self, ignored, m): - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) + def _cbTestClose(self, ignored): + self.assertEqual(len(self.mailbox.messages), 1) + msg = self.mailbox.messages.get_msg_by_uid(2) self.assertTrue(msg is not None) self.assertEqual( dict(msg.hdoc.content)['subject'], 'Message 2') - self.failUnless(m.closed) + self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ + acc = self.server.theAccount name = 'mailbox-expunge' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + + d0 = lambda: acc.addMailbox(name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1054,14 +1104,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def select(): return self.client.select('mailbox-expunge') + def get_mailbox(): + self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + def add_messages(): - d1 = m.messages.add_msg( + d1 = self.mailbox.messages.add_msg( 'test 1', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( + d2 = self.mailbox.messages.add_msg( 'test 2', subject="Message 2", flags=('AnotherFlag',)) - d3 = m.messages.add_msg( + d3 = self.mailbox.messages.add_msg( 'test 3', subject="Message 3", flags=('\\Deleted',)) d = defer.gatherResults([d1, d2, d3]) @@ -1076,21 +1129,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestExpunge, m) + return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored, m): + def _cbTestExpunge(self, ignored): # we only left 1 mssage with no deleted flag - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) + self.assertEqual(len(self.mailbox.messages), 1) + msg = self.mailbox.messages.get_msg_by_uid(2) - msg = list(m.messages)[0] + msg = list(self.mailbox.messages)[0] self.assertTrue(msg is not None) self.assertEqual( diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 5339acf..9a3868c 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -139,31 +139,32 @@ class IMAP4HelperMixin(BaseLeapTest): ########### - d = defer.Deferred() + d_server_ready = defer.Deferred() + self.server = LeapIMAPServer( uuid=UUID, userid=USERID, contextFactory=self.serverCTX, - # XXX do we really need this?? soledad=self._soledad) - self.client = SimpleClient(d, contextFactory=self.clientCTX) - self.connected = d - - # XXX REVIEW-ME. - # We're adding theAccount here to server - # but it was also passed to initialization - # as it was passed to realm. - # I THINK we ONLY need to do it at one place now. + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) theAccount = SoledadBackedAccount( USERID, soledad=self._soledad, memstore=memstore) + d_account_ready = theAccount.callWhenReady(lambda r: None) LeapIMAPServer.theAccount = theAccount + self.connected = defer.gatherResults( + [d_server_ready, d_account_ready]) + + # XXX FIXME -------------------------------------------- + # XXX this needs to be done differently, + # have to be hooked on initialization callback instead. # in case we get something from previous tests... - for mb in self.server.theAccount.mailboxes: - self.server.theAccount.delete(mb) + #for mb in self.server.theAccount.mailboxes: + #self.server.theAccount.delete(mb) # email parser self.parser = parser.Parser() -- cgit v1.2.3 From 02a88688344070120ef09287c5d7cb654bc28e6e Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Tue, 9 Dec 2014 12:18:40 -0600 Subject: New keymanager async API --- src/leap/mail/imap/fetch.py | 312 +++++++++++++------------ src/leap/mail/imap/tests/test_incoming_mail.py | 38 +-- src/leap/mail/service.py | 201 +++++++++------- src/leap/mail/smtp/gateway.py | 38 +-- src/leap/mail/smtp/tests/test_gateway.py | 25 +- src/leap/mail/tests/__init__.py | 88 ++----- src/leap/mail/tests/test_service.py | 216 ++++++++--------- 7 files changed, 474 insertions(+), 444 deletions(-) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 01373be..dbc726a 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -36,7 +36,6 @@ from twisted.internet import defer, reactor from twisted.internet.task import LoopingCall from twisted.internet.task import deferLater from u1db import errors as u1db_errors -from zope.proxy import sameProxiedObjects from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type @@ -138,13 +137,6 @@ class LeapIncomingMail(object): # initialize a mail parser only once self._parser = Parser() - @property - def _pkey(self): - if sameProxiedObjects(self._keymanager, None): - logger.warning('tried to get key, but null keymanager found') - return None - return self._keymanager.get_key(self._userid, OpenPGPKey, private=True) - # # Public API: fetch, start_loop, stop. # @@ -312,40 +304,46 @@ class LeapIncomingMail(object): :param doc: A document containing an encrypted message. :type doc: SoledadDocument - :return: A tuple containing the document and the decrypted message. - :rtype: (SoledadDocument, str) + :return: A Deferred that will be fired with the document and the + decrypted message. + :rtype: SoledadDocument, str """ log.msg('decrypting msg') - success = False - try: - decrdata = self._keymanager.decrypt( - doc.content[ENC_JSON_KEY], - self._pkey) - success = True - except Exception as exc: - # XXX move this to errback !!! - logger.error("Error while decrypting msg: %r" % (exc,)) - decrdata = "" - leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") + def process_decrypted(res): + if isinstance(res, tuple): + decrdata, _ = res + success = True + else: + decrdata = "" + success = False - data = self._process_decrypted_doc((doc, decrdata)) - return (doc, data) + leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - def _process_decrypted_doc(self, msgtuple): + data = self._process_decrypted_doc(doc, decrdata) + return doc, data + + d = self._keymanager.decrypt( + doc.content[ENC_JSON_KEY], + self._userid, OpenPGPKey) + d.addErrback(self._errback) + d.addCallback(process_decrypted) + return d + + def _process_decrypted_doc(self, doc, data): """ Process a document containing a succesfully decrypted message. - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) + :param doc: the incoming message + :type doc: SoledadDocument + :param data: the json-encoded, decrypted content of the incoming + message + :type data: str + :return: the processed data. :rtype: str """ log.msg('processing decrypted doc') - doc, data = msgtuple # XXX turn this into an errBack for each one of # the deferreds that would process an individual document @@ -421,45 +419,40 @@ class LeapIncomingMail(object): encoding = get_email_charset(data) msg = self._parser.parsestr(data) - # try to obtain sender public key - senderPubkey = None fromHeader = msg.get('from', None) + senderAddress = None if (fromHeader is not None and (msg.get_content_type() == MULTIPART_ENCRYPTED or msg.get_content_type() == MULTIPART_SIGNED)): - _, senderAddress = parseaddr(fromHeader) - try: - senderPubkey = self._keymanager.get_key( - senderAddress, OpenPGPKey) - except keymanager_errors.KeyNotFound: - pass - - valid_sig = False # we will add a header saying if sig is valid - decrypt_multi = self._decrypt_multipart_encrypted_msg - decrypt_inline = self._maybe_decrypt_inline_encrypted_msg + senderAddress = parseaddr(fromHeader) + + def add_leap_header(decrmsg, signkey): + if (senderAddress is None or + isinstance(signkey, keymanager_errors.KeyNotFound)): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_COULD_NOT_VERIFY) + elif isinstance(signkey, keymanager_errors.InvalidSignature): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_INVALID) + else: + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_VALID, + pubkey=signkey.key_id) + return decrmsg.as_string() if msg.get_content_type() == MULTIPART_ENCRYPTED: - decrmsg, valid_sig = decrypt_multi( - msg, encoding, senderPubkey) + d = self._decrypt_multipart_encrypted_msg( + msg, encoding, senderAddress) else: - decrmsg, valid_sig = decrypt_inline( - msg, encoding, senderPubkey) - - # add x-leap-signature header - if senderPubkey is None: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_COULD_NOT_VERIFY) - else: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_VALID if valid_sig else - self.LEAP_SIGNATURE_INVALID, - pubkey=senderPubkey.key_id) - - return decrmsg.as_string() + d = self._maybe_decrypt_inline_encrypted_msg( + msg, encoding, senderAddress) + d.addCallback(add_leap_header) + return d - def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey): + def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): """ Decrypt a message with content-type 'multipart/encrypted'. @@ -467,12 +460,13 @@ class LeapIncomingMail(object): :type msg: Message :param encoding: The encoding of the email message. :type encoding: str - :param senderPubkey: The key of the sender of the message. - :type senderPubkey: OpenPGPKey + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str - :return: A tuple containing a decrypted message and - a bool indicating whether the signature is valid. - :rtype: (Message, bool) + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred """ log.msg('decrypting multipart encrypted msg') msg = copy.deepcopy(msg) @@ -483,33 +477,33 @@ class LeapIncomingMail(object): encdata = pgpencmsg.get_payload() # decrypt or fail gracefully - try: - decrdata, valid_sig = self._decrypt_and_verify_data( - encdata, senderPubkey) - except keymanager_errors.DecryptError as e: - logger.warning('Failed to decrypt encrypted message (%s). ' - 'Storing message without modifications.' % str(e)) - # Bailing out! - return (msg, False) + def build_msg(res): + decrdata, signkey = res - decrmsg = self._parser.parsestr(decrdata) - # remove original message's multipart/encrypted content-type - del(msg['content-type']) + decrmsg = self._parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(msg['content-type']) - # replace headers back in original message - for hkey, hval in decrmsg.items(): - try: - # this will raise KeyError if header is not present - msg.replace_header(hkey, hval) - except KeyError: - msg[hkey] = hval + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + msg.replace_header(hkey, hval) + except KeyError: + msg[hkey] = hval + + # all ok, replace payload by unencrypted payload + msg.set_payload(decrmsg.get_payload()) + return (msg, signkey) - # all ok, replace payload by unencrypted payload - msg.set_payload(decrmsg.get_payload()) - return (msg, valid_sig) + d = self._keymanager.decrypt( + encdata, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) + return d def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, - senderPubkey): + senderAddress): """ Possibly decrypt an inline OpenPGP encrypted message. @@ -517,12 +511,13 @@ class LeapIncomingMail(object): :type origmsg: Message :param encoding: The encoding of the email message. :type encoding: str - :param senderPubkey: The key of the sender of the message. - :type senderPubkey: OpenPGPKey + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str - :return: A tuple containing a decrypted message and - a bool indicating whether the signature is valid. - :rtype: (Message, bool) + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred """ log.msg('maybe decrypting inline encrypted msg') # serialize the original message @@ -530,54 +525,48 @@ class LeapIncomingMail(object): g = Generator(buf) g.flatten(origmsg) data = buf.getvalue() + + def decrypted_data(res): + decrdata, signkey = res + return data.replace(pgp_message, decrdata), signkey + + def encode_and_return(res): + data, signkey = res + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return (self._parser.parsestr(data), signkey) + # handle exactly one inline PGP message - valid_sig = False if PGP_BEGIN in data: begin = data.find(PGP_BEGIN) end = data.find(PGP_END) pgp_message = data[begin:end + len(PGP_END)] - try: - decrdata, valid_sig = self._decrypt_and_verify_data( - pgp_message, senderPubkey) - # replace encrypted by decrypted content - data = data.replace(pgp_message, decrdata) - except keymanager_errors.DecryptError: - logger.warning('Failed to decrypt potential inline encrypted ' - 'message. Storing message as is...') - - # if message is not encrypted, return raw data - if isinstance(data, unicode): - data = data.encode(encoding, 'replace') - return (self._parser.parsestr(data), valid_sig) + d = self._keymanager.decrypt( + pgp_message, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(decrypted_data, self._decryption_error, + errbackArgs=(data,)) + else: + d = defer.succeed((data, None)) + d.addCallback(encode_and_return) + return d - def _decrypt_and_verify_data(self, data, senderPubkey): + def _decryption_error(self, failure, msg): """ - Decrypt C{data} using our private key and attempt to verify a - signature using C{senderPubkey}. - - :param data: The text to be decrypted. - :type data: unicode - :param senderPubkey: The public key of the sender of the message. - :type senderPubkey: OpenPGPKey - - :return: The decrypted data and a boolean stating whether the - signature could be verified. - :rtype: (str, bool) - - :raise DecryptError: Raised if failed to decrypt. + Check for known decryption errors """ - log.msg('decrypting and verifying data') - valid_sig = False - try: - decrdata = self._keymanager.decrypt( - data, self._pkey, - verify=senderPubkey) - if senderPubkey is not None: - valid_sig = True - except keymanager_errors.InvalidSignature: - decrdata = self._keymanager.decrypt( - data, self._pkey) - return (decrdata, valid_sig) + if failure.check(keymanager_errors.DecryptError): + logger.warning('Failed to decrypt encrypted message (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + elif failure.check(keymanager_errors.KeyNotFound): + logger.error('Failed to find private key for decryption (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + else: + return failure def _extract_keys(self, msgtuple): """ @@ -592,6 +581,10 @@ class LeapIncomingMail(object): and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired with msgtuple when key + extraction finishes + :rtype: Deferred """ OpenPGP_HEADER = 'OpenPGP' doc, data = msgtuple @@ -603,13 +596,17 @@ class LeapIncomingMail(object): _, fromAddress = parseaddr(msg['from']) header = msg.get(OpenPGP_HEADER, None) + dh = defer.success() if header is not None: - self._extract_openpgp_header(header, fromAddress) + dh = self._extract_openpgp_header(header, fromAddress) + da = defer.success() if msg.is_multipart(): - self._extract_attached_key(msg.get_payload(), fromAddress) + da = self._extract_attached_key(msg.get_payload(), fromAddress) - return msgtuple + d = defer.gatherResults([dh, da]) + d.addCallback(lambda _: msgtuple) + return d def _extract_openpgp_header(self, header, address): """ @@ -619,7 +616,11 @@ class LeapIncomingMail(object): :type header: str :param address: email address in the from header :type address: str + + :return: A Deferred that will be fired when header extraction is done + :rtype: Deferred """ + d = defer.success() fields = dict([f.strip(' ').split('=') for f in header.split(';')]) if 'url' in fields: url = shlex.split(fields['url'])[0] # remove quotations @@ -627,21 +628,28 @@ class LeapIncomingMail(object): addressHostname = address.split('@')[1] if (urlparts.scheme == 'https' and urlparts.hostname == addressHostname): - try: - self._keymanager.fetch_key(address, url, OpenPGPKey) - logger.info("Imported key from header %s" % (url,)) - except keymanager_errors.KeyNotFound: - logger.warning("Url from OpenPGP header %s failed" - % (url,)) - except keymanager_errors.KeyAttributesDiffer: - logger.warning("Key from OpenPGP header url %s didn't " - "match the from address %s" - % (url, address)) + def fetch_error(failure): + if failure.check(keymanager_errors.KeyNotFound): + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + elif failure.check(keymanager_errors.KeyAttributesDiffer): + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, address)) + else: + return failure + + d = self._keymanager.fetch_key(address, url, OpenPGPKey) + d.addCallback( + lambda _: + logger.info("Imported key from header %s" % (url,))) + d.addErrback(fetch_error) else: logger.debug("No valid url on OpenPGP header %s" % (url,)) else: logger.debug("There is no url on the OpenPGP header: %s" % (header,)) + return d def _extract_attached_key(self, attachments, address): """ @@ -651,16 +659,22 @@ class LeapIncomingMail(object): :type attachments: list(email.Message) :param address: email address in the from header :type address: str + + :return: A Deferred that will be fired when all the keys are stored + :rtype: Deferred """ MIME_KEY = "application/pgp-keys" + deferreds = [] for attachment in attachments: if MIME_KEY == attachment.get_content_type(): logger.debug("Add key from attachment") - self._keymanager.put_raw_key( + d = self._keymanager.put_raw_key( attachment.get_payload(), OpenPGPKey, address=address) + deferreds.append(d) + return defer.gatherResults(deferreds) def _add_message_locally(self, msgtuple): """ @@ -672,6 +686,9 @@ class LeapIncomingMail(object): and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired when the messages is stored + :rtype: Defferred """ doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) @@ -690,6 +707,7 @@ class LeapIncomingMail(object): d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), notify_on_disk=True) d.addCallbacks(msgSavedCallback, self._errback) + return d # # helpers diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py index ce6d56a..03c0164 100644 --- a/src/leap/mail/imap/tests/test_incoming_mail.py +++ b/src/leap/mail/imap/tests/test_incoming_mail.py @@ -28,7 +28,6 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.parser import Parser from mock import Mock -from twisted.trial import unittest from leap.keymanager.openpgp import OpenPGPKey from leap.mail.imap.account import SoledadBackedAccount @@ -48,7 +47,7 @@ from leap.soledad.common.crypto import ( ) -class LeapIncomingMailTestCase(TestCaseWithKeyManager, unittest.TestCase): +class LeapIncomingMailTestCase(TestCaseWithKeyManager): """ Tests for the incoming mail parser """ @@ -147,31 +146,42 @@ subject: independence of cyberspace key = MIMEApplication("", "pgp-keys") key.set_payload(KEY) message.attach(key) - email = self._create_incoming_email(message.as_string()) - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) - self.fetcher._keymanager.put_raw_key = Mock() def put_raw_key_called(ret): self.fetcher._keymanager.put_raw_key.assert_called_once_with( KEY, OpenPGPKey, address=self.FROM_ADDRESS) - d = self.fetcher.fetch() + d = self.mock_fetch(message.as_string()) d.addCallback(put_raw_key_called) return d + def _mock_fetch(self, message): + self.fetcher._keymanager.fetch_key = Mock() + d = self._create_incoming_email(message) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + return d + def _create_incoming_email(self, email_str): email = SoledadDocument() - pubkey = self._km.get_key(ADDRESS, OpenPGPKey) data = json.dumps( {"incoming": True, "content": email_str}, ensure_ascii=False) - email.content = { - fields.INCOMING_KEY: True, - fields.ERROR_DECRYPTING_KEY: False, - ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, - ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) - } - return email + + def set_email_content(pubkey): + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) + } + return email + + d = self._km.get_key(ADDRESS, OpenPGPKey) + d.addCallback(set_email_content) + return d def _mock_soledad_get_from_index(self, index_name, value): get_from_index = self._soledad.get_from_index diff --git a/src/leap/mail/service.py b/src/leap/mail/service.py index f6e4d11..a99f13a 100644 --- a/src/leap/mail/service.py +++ b/src/leap/mail/service.py @@ -24,7 +24,6 @@ from OpenSSL import SSL from twisted.mail import smtp from twisted.internet import reactor from twisted.internet import defer -from twisted.internet.threads import deferToThread from twisted.protocols.amp import ssl from twisted.python import log @@ -111,17 +110,17 @@ class OutgoingMail: :type recipient: smtp.User :return: a deferred which delivers the message when fired """ - d = deferToThread(lambda: self._maybe_encrypt_and_sign(raw, recipient)) + d = self._maybe_encrypt_and_sign(raw, recipient) d.addCallback(self._route_msg) d.addErrback(self.sendError) - return d def sendSuccess(self, smtp_sender_result): """ Callback for a successful send. - :param smtp_sender_result: The result from the ESMTPSender from _route_msg + :param smtp_sender_result: The result from the ESMTPSender from + _route_msg :type smtp_sender_result: tuple(int, list(tuple)) """ dest_addrstr = smtp_sender_result[1][0][0] @@ -145,7 +144,8 @@ class OutgoingMail: """ Sends the msg using the ESMTPSenderFactory. - :param encrypt_and_sign_result: A tuple containing the 'maybe' encrypted message and the recipient + :param encrypt_and_sign_result: A tuple containing the 'maybe' + encrypted message and the recipient :type encrypt_and_sign_result: tuple """ message, recipient = encrypt_and_sign_result @@ -173,7 +173,6 @@ class OutgoingMail: self._host, self._port, factory, contextFactory=SSLContextFactory(self._cert, self._key)) - def _maybe_encrypt_and_sign(self, raw, recipient): """ Attempt to encrypt and sign the outgoing message. @@ -209,16 +208,20 @@ class OutgoingMail: :param recipient: The recipient for the message :type: recipient: smtp.User + :return: A Deferred that will be fired with a MIMEMultipart message + and the original recipient Message + :rtype: Deferred """ # pass if the original message's content-type is "multipart/encrypted" lines = raw.split('\r\n') origmsg = Parser().parsestr(raw) if origmsg.get_content_type() == 'multipart/encrypted': - return origmsg + return defer.success((origmsg, recipient)) from_address = validate_address(self._from_address) username, domain = from_address.split('@') + to_address = validate_address(recipient.dest.addrstr) # add a nice footer to the outgoing message # XXX: footer will eventually optional or be removed @@ -230,80 +233,93 @@ class OutgoingMail: origmsg = Parser().parsestr('\r\n'.join(lines)) - # get sender and recipient data - signkey = self._keymanager.get_key(from_address, OpenPGPKey, private=True) - log.msg("Will sign the message with %s." % signkey.fingerprint) - to_address = validate_address(recipient.dest.addrstr) - try: - # try to get the recipient pubkey - pubkey = self._keymanager.get_key(to_address, OpenPGPKey) - log.msg("Will encrypt the message to %s." % pubkey.fingerprint) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) - newmsg = self._encrypt_and_sign(origmsg, pubkey, signkey) - + def signal_encrypt_sign(newmsg): signal(proto.SMTP_END_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) - except KeyNotFound: - # at this point we _can_ send unencrypted mail, because if the - # configuration said the opposite the address would have been - # rejected in SMTPDelivery.validateTo(). - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._from_address) - newmsg = self._sign(origmsg, signkey) - signal(proto.SMTP_END_SIGN, self._from_address) - return newmsg, recipient + return newmsg, recipient + def signal_sign(newmsg): + signal(proto.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + def if_key_not_found_send_unencrypted(failure): + if failure.check(KeyNotFound): + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._from_address) + d = self._sign(origmsg, from_address) + d.addCallback(signal_sign) + return d + else: + return failure + + log.msg("Will encrypt the message with %s and sign with %s." + % (to_address, from_address)) + signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + d = self._encrypt_and_sign(origmsg, to_address, from_address) + d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) + return d - def _encrypt_and_sign(self, origmsg, pubkey, signkey): + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): """ Create an RFC 3156 compliang PGP encrypted and signed message using - C{pubkey} to encrypt and C{signkey} to sign. + C{encrypt_address} to encrypt and C{sign_address} to sign. :param origmsg: The original message :type origmsg: email.message.Message - :param pubkey: The public key used to encrypt the message. - :type pubkey: OpenPGPKey - :param signkey: The private key used to sign the message. - :type signkey: OpenPGPKey - :return: The encrypted and signed message - :rtype: MultipartEncrypted + :param encrypt_address: The address used to encrypt the message. + :type encrypt_address: str + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartEncrypted message + :rtype: Deferred """ # create new multipart/encrypted message with 'pgp-encrypted' protocol - newmsg = MultipartEncrypted('application/pgp-encrypted') - # move (almost) all headers from original message to the new message - self._fix_headers(origmsg, newmsg, signkey) - # create 'application/octet-stream' encrypted message - encmsg = MIMEApplication( - self._keymanager.encrypt(origmsg.as_string(unixfrom=False), pubkey, - sign=signkey), - _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - return newmsg - - - def _sign(self, origmsg, signkey): + + def encrypt(res): + newmsg, origmsg = res + d = self._keymanager.encrypt( + origmsg.as_string(unixfrom=False), + encrypt_address, OpenPGPKey, sign=sign_address) + d.addCallback(lambda encstr: (newmsg, encstr)) + return d + + def create_encrypted_message(res): + newmsg, encstr = res + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + d = self._fix_headers( + origmsg, + MultipartEncrypted('application/pgp-encrypted'), + sign_address) + d.addCallback(encrypt) + d.addCallback(create_encrypted_message) + return d + + def _sign(self, origmsg, sign_address): """ - Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. + Create an RFC 3156 compliant PGP signed MIME message using + C{sign_address}. :param origmsg: The original message :type origmsg: email.message.Message - :param signkey: The private key used to sign the message. - :type signkey: leap.common.keymanager.openpgp.OpenPGPKey - :return: The signed message. - :rtype: MultipartSigned + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartSigned message. + :rtype: Deferred """ - # create new multipart/signed message - newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') - # move (almost) all headers from original message to the new message - self._fix_headers(origmsg, newmsg, signkey) # apply base64 content-transfer-encoding encode_base64_rec(origmsg) # get message text with headers and replace \n for \r\n @@ -316,17 +332,27 @@ class OutgoingMail: if origmsg.is_multipart(): if not msgtext.endswith("\r\n"): msgtext += "\r\n" - # calculate signature - signature = self._keymanager.sign(msgtext, signkey, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - newmsg.attach(origmsg) - newmsg.attach(sigmsg) - return newmsg + def create_signed_message(res): + (msg, _), signature = res + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + msg.attach(origmsg) + msg.attach(sigmsg) + return msg + + dh = self._fix_headers( + origmsg, + MultipartSigned('application/pgp-signature', 'pgp-sha512'), + sign_address) + ds = self._keymanager.sign( + msgtext, sign_address, OpenPGPKey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + d = defer.gatherResults([dh, ds]) + d.addCallback(create_signed_message) + return d - def _fix_headers(self, origmsg, newmsg, signkey): + def _fix_headers(self, origmsg, newmsg, sign_address): """ Move some headers from C{origmsg} to C{newmsg}, delete unwanted headers from C{origmsg} and add new headers to C{newms}. @@ -360,8 +386,13 @@ class OutgoingMail: :type origmsg: email.message.Message :param newmsg: The new message being created. :type newmsg: email.message.Message - :param signkey: The key used to sign C{newmsg} - :type signkey: OpenPGPKey + :param sign_address: The address used to sign C{newmsg} + :type sign_address: str + + :return: A Deferred with a touple: + (new Message with the unencrypted headers, + original Message with headers removed) + :rtype: Deferred """ # move headers from origmsg to newmsg headers = origmsg.items() @@ -375,11 +406,17 @@ class OutgoingMail: del (origmsg[hkey]) # add a new message-id to newmsg newmsg.add_header('Message-Id', smtp.messageid()) - # add openpgp header to newmsg - username, domain = signkey.address.split('@') - newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, - url='https://%s/key/%s' % (domain, username), - preference='signencrypt') # delete user-agent from origmsg del (origmsg['user-agent']) + + def add_openpgp_header(signkey): + username, domain = sign_address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + return newmsg, origmsg + + d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True) + d.addCallback(add_openpgp_header) + return d diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index b022091..d58c581 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -48,7 +48,6 @@ from leap.mail.smtp.rfc3156 import ( RFC3156CompliantGenerator, ) -from leap.mail.service import OutgoingMail # replace email generator with a RFC 3156 compliant one. from email import generator @@ -197,22 +196,31 @@ class SMTPDelivery(object): accepted. """ # try to find recipient's public key - try: - address = validate_address(user.dest.addrstr) - # verify if recipient key is available in keyring - self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound + address = validate_address(user.dest.addrstr) + + # verify if recipient key is available in keyring + def found(_): log.msg("Accepting mail for %s..." % user.dest.addrstr) signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) - except KeyNotFound: - # if key was not found, check config to see if will send anyway. - if self._encrypted_only: - signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) - raise smtp.SMTPBadRcpt(user.dest.addrstr) - log.msg("Warning: will send an unencrypted message (because " - "encrypted_only' is set to False).") - signal( - proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) - return lambda: EncryptedMessage(user, self._outgoing_mail) + + def not_found(failure): + if failure.check(KeyNotFound): + # if key was not found, check config to see if will send anyway + if self._encrypted_only: + signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + raise smtp.SMTPBadRcpt(user.dest.addrstr) + log.msg("Warning: will send an unencrypted message (because " + "encrypted_only' is set to False).") + signal( + proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, + user.dest.addrstr) + else: + return failure + + d = self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound + d.addCallbacks(found, not_found) + d.addCallbac(lambda _: EncryptedMessage(user, self._outgoing_mail)) + return d def validateFrom(self, helo, origin): """ diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index aeace4a..8cbff8f 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -23,6 +23,7 @@ SMTP gateway tests. import re from datetime import datetime +from twisted.internet.defer import inlineCallbacks, fail from twisted.test import proto_helpers from mock import Mock @@ -34,7 +35,7 @@ from leap.mail.tests import ( ADDRESS, ADDRESS_2, ) -from leap.keymanager import openpgp +from leap.keymanager import openpgp, errors # some regexps @@ -87,7 +88,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): proto = SMTPFactory( u'anotheruser@leap.se', self._km, - self._config['encrypted_only'], outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + self._config['encrypted_only'], + outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) # snip... transport = proto_helpers.StringTransport() proto.makeConnection(transport) @@ -98,23 +100,26 @@ class TestSmtpGateway(TestCaseWithKeyManager): 'Did not get expected answer from gateway.') proto.setTimeout(None) + @inlineCallbacks def test_missing_key_rejects_address(self): """ Test if server rejects to send unencrypted when 'encrypted_only' is True. """ # remove key from key manager - pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) + pubkey = yield self._km.get_key(ADDRESS, openpgp.OpenPGPKey) pgp = openpgp.OpenPGPScheme( self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.delete_key(pubkey) + yield pgp.delete_key(pubkey) # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) # prepare the SMTP factory proto = SMTPFactory( u'anotheruser@leap.se', self._km, - self._config['encrypted_only'], outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + self._config['encrypted_only'], + outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') @@ -127,18 +132,20 @@ class TestSmtpGateway(TestCaseWithKeyManager): lines[-1], 'Address should have been rejecetd with appropriate message.') + @inlineCallbacks def test_missing_key_accepts_address(self): """ Test if server accepts to send unencrypted when 'encrypted_only' is False. """ # remove key from key manager - pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) + pubkey = yield self._km.get_key(ADDRESS, openpgp.OpenPGPKey) pgp = openpgp.OpenPGPScheme( self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.delete_key(pubkey) + yield pgp.delete_key(pubkey) # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) # prepare the SMTP factory with encrypted only equal to false proto = SMTPFactory( u'anotheruser@leap.se', diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index 10bc5fe..b35107d 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -19,16 +19,14 @@ Base classes and keys for leap.mail tests. """ import os import distutils.spawn -import shutil -import tempfile from mock import Mock +from twisted.internet.defer import gatherResults +from twisted.trial import unittest from leap.soledad.client import Soledad -from leap.keymanager import ( - KeyManager, - openpgp, -) +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey from leap.common.testing.basetest import BaseLeapTest @@ -39,22 +37,12 @@ def _find_gpg(): return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" -class TestCaseWithKeyManager(BaseLeapTest): +class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest): GPG_BINARY_PATH = _find_gpg() def setUp(self): - # mimic BaseLeapTest.setUpClass behaviour, because this is deprecated - # in Twisted: http://twistedmatrix.com/trac/ticket/1870 - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir + self.setUpEnv() # setup our own stuff address = 'leap@leap.se' # user's address in the form user@provider @@ -65,36 +53,7 @@ class TestCaseWithKeyManager(BaseLeapTest): server_url = 'http://provider/' cert_file = '' - self._soledad = self._soledad_instance( - uuid, passphrase, secrets_path, local_db_path, server_url, - cert_file) - self._km = self._keymanager_instance(address) - - def _soledad_instance(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file): - """ - Return a Soledad instance for tests. - """ - # mock key fetching and storing so Soledad doesn't fail when trying to - # reach the server. - Soledad._fetch_keys_from_shared_db = Mock(return_value=None) - Soledad._assert_keys_in_shared_db = Mock(return_value=None) - - # instantiate soledad - def _put_doc_side_effect(doc): - self._doc_put = doc - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=_put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - soledad = Soledad( + self._soledad = Soledad( uuid, passphrase, secrets_path=secrets_path, @@ -103,13 +62,11 @@ class TestCaseWithKeyManager(BaseLeapTest): cert_file=cert_file, syncable=False ) + return self._setup_keymanager(address) - soledad._shared_db = MockSharedDB() - return soledad - - def _keymanager_instance(self, address): + def _setup_keymanager(self, address): """ - Return a Key Manager instance for tests. + Set up Key Manager and return a Deferred that will be fired when done. """ self._config = { 'host': 'https://provider/', @@ -132,26 +89,17 @@ class TestCaseWithKeyManager(BaseLeapTest): pass nickserver_url = '' # the url of the nickserver - km = KeyManager(address, nickserver_url, self._soledad, - ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH) - km._fetcher.put = Mock() - km._fetcher.get = Mock(return_value=Response()) - - # insert test keys in key manager. - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.put_ascii_key(PRIVATE_KEY) - pgp.put_ascii_key(PRIVATE_KEY_2) + self._km = KeyManager(address, nickserver_url, self._soledad, + ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH) + self._km._fetcher.put = Mock() + self._km._fetcher.get = Mock(return_value=Response()) - return km + d1 = self._km.put_raw_key(PRIVATE_KEY, OpenPGPKey, ADDRESS) + d2 = self._km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2) + return gatherResults([d1, d2]) def tearDown(self): - # mimic LeapBaseTest.tearDownClass behaviour - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) + self.tearDownEnv() # Key material for testing diff --git a/src/leap/mail/tests/test_service.py b/src/leap/mail/tests/test_service.py index f0a807d..43f354d 100644 --- a/src/leap/mail/tests/test_service.py +++ b/src/leap/mail/tests/test_service.py @@ -22,7 +22,8 @@ SMTP gateway tests. import re from datetime import datetime -from twisted.mail.smtp import User, Address +from twisted.internet.defer import fail +from twisted.mail.smtp import User from mock import Mock @@ -33,7 +34,7 @@ from leap.mail.tests import ( ADDRESS, ADDRESS_2, ) -from leap.keymanager import openpgp +from leap.keymanager import openpgp, errors class TestOutgoingMail(TestCaseWithKeyManager): @@ -54,71 +55,126 @@ class TestOutgoingMail(TestCaseWithKeyManager): 'QUIT'] def setUp(self): - TestCaseWithKeyManager.setUp(self) self.lines = [line for line in self.EMAIL_DATA[4:12]] self.lines.append('') # add a trailing newline self.raw = '\r\n'.join(self.lines) + self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + + "\r\n\r\n--\r\nI prefer encrypted email - " + "https://leap.se/key/anotheruser\r\n") self.fromAddr = ADDRESS_2 - self.outgoing_mail = OutgoingMail(self.fromAddr, self._km, self._config['cert'], self._config['key'], - self._config['host'], self._config['port']) - self.proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) - self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) - - def test_openpgp_encrypt_decrypt(self): - "Test if openpgp can encrypt and decrypt." - text = "simple raw text" - pubkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=False) - encrypted = self._km.encrypt(text, pubkey) - self.assertNotEqual( - text, encrypted, "Ciphertext is equal to plaintext.") - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt(encrypted, privkey) - self.assertEqual(text, decrypted, - "Decrypted text differs from plaintext.") + + def init_outgoing_and_proto(_): + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], + self._config['key'], self._config['host'], + self._config['port']) + self.proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, + self._config['encrypted_only'], + self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) + + d = TestCaseWithKeyManager.setUp(self) + d.addCallback(init_outgoing_and_proto) + return d def test_message_encrypt(self): """ Test if message gets encrypted to destination email. """ - - message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - - # assert structure of encrypted message - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/encrypted', message.get_content_type()) - self.assertEqual('application/pgp-encrypted', - message.get_param('protocol')) - self.assertEqual(2, len(message.get_payload())) - self.assertEqual('application/pgp-encrypted', - message.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - message.get_payload(1).get_content_type()) - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt( - message.get_payload(1).get_payload(), privkey) - - expected = '\n' + '\r\n'.join( - self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n' - self.assertEqual( - expected, - decrypted, - 'Decrypted text differs from plaintext.') + def check_decryption(res): + decrypted, _ = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) + d.addCallback(check_decryption) + return d def test_message_encrypt_sign(self): """ Test if message gets encrypted to destination email and signed with sender key. - """ - message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + '""" + def check_decryption_and_verify(res): + decrypted, signkey = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + self.assertTrue(ADDRESS_2 in signkey.address, + "Verification failed") + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, + verify=ADDRESS_2)) + d.addCallback(check_decryption_and_verify) + return d - # assert structure of encrypted message + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], self._config['key'], + self._config['host'], self._config['port']) + + def check_signed(res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + self.assertEqual(self.expected_body, + message.get_payload(0).get_payload(decode=True)) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + return message + + def verify(message): + # replace EOL before verifying (according to rfc3156) + signed_text = re.sub('\r?\n', '\r\n', + message.get_payload(0).as_string()) + + def assert_verify(key): + self.assertTrue(ADDRESS_2 in key.address, + 'Signature could not be verified.') + + d = self._km.verify( + signed_text, ADDRESS_2, openpgp.OpenPGPKey, + detached_sig=message.get_payload(1).get_payload()) + d.addCallback(assert_verify) + return d + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + d.addCallback(check_signed) + d.addCallback(verify) + return d + + def _assert_encrypted(self, res): + message, _ = res self.assertTrue('Content-Type' in message) self.assertEqual('multipart/encrypted', message.get_content_type()) self.assertEqual('application/pgp-encrypted', @@ -128,58 +184,4 @@ class TestOutgoingMail(TestCaseWithKeyManager): message.get_payload(0).get_content_type()) self.assertEqual('application/octet-stream', message.get_payload(1).get_content_type()) - # decrypt and verify - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - decrypted = self._km.decrypt( - message.get_payload(1).get_payload(), privkey, verify=pubkey) - self.assertEqual( - '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - decrypted, - 'Decrypted text differs from plaintext.') - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) - recipient = User('ihavenopubkey@nonleap.se', - 'gateway.leap.se', self.proto, ADDRESS) - self.outgoing_mail = OutgoingMail(self.fromAddr, self._km, self._config['cert'], self._config['key'], - self._config['host'], self._config['port']) - - message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) - - # assert structure of signed message - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/signed', message.get_content_type()) - self.assertEqual('application/pgp-signature', - message.get_param('protocol')) - self.assertEqual('pgp-sha512', message.get_param('micalg')) - # assert content of message - self.assertEqual( - '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - message.get_payload(0).get_payload(decode=True)) - # assert content of signature - self.assertTrue( - message.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - message.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - # assert signature is valid - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - # replace EOL before verifying (according to rfc3156) - signed_text = re.sub('\r?\n', '\r\n', - message.get_payload(0).as_string()) - self.assertTrue( - self._km.verify(signed_text, - pubkey, - detached_sig=message.get_payload(1).get_payload()), - 'Signature could not be verified.') + return message -- cgit v1.2.3 From ea4373132458f906b6270744bcfd3e76b64dbd0a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Nov 2014 15:04:26 +0100 Subject: Serializable Models + Soledad Adaptor --- src/leap/mail/adaptors/__init__.py | 0 src/leap/mail/adaptors/models.py | 125 ++++ src/leap/mail/adaptors/soledad.py | 723 +++++++++++++++++++++ src/leap/mail/adaptors/soledad_indexes.py | 112 ++++ src/leap/mail/adaptors/tests/__init__.py | 0 src/leap/mail/adaptors/tests/rfc822.message | 86 +++ src/leap/mail/adaptors/tests/test_models.py | 103 +++ .../mail/adaptors/tests/test_soledad_adaptor.py | 583 +++++++++++++++++ src/leap/mail/constants.py | 21 + src/leap/mail/imap/account.py | 223 +++---- src/leap/mail/imap/fields.py | 132 +--- src/leap/mail/imap/index.py | 90 --- src/leap/mail/imap/interfaces.py | 2 + src/leap/mail/imap/mailbox.py | 76 ++- src/leap/mail/imap/messages.py | 484 +++++--------- src/leap/mail/imap/parser.py | 45 -- src/leap/mail/imap/tests/test_imap.py | 2 + src/leap/mail/imap/tests/utils.py | 15 +- src/leap/mail/interfaces.py | 113 ++++ src/leap/mail/mail.py | 248 +++++++ 20 files changed, 2415 insertions(+), 768 deletions(-) create mode 100644 src/leap/mail/adaptors/__init__.py create mode 100644 src/leap/mail/adaptors/models.py create mode 100644 src/leap/mail/adaptors/soledad.py create mode 100644 src/leap/mail/adaptors/soledad_indexes.py create mode 100644 src/leap/mail/adaptors/tests/__init__.py create mode 100644 src/leap/mail/adaptors/tests/rfc822.message create mode 100644 src/leap/mail/adaptors/tests/test_models.py create mode 100644 src/leap/mail/adaptors/tests/test_soledad_adaptor.py create mode 100644 src/leap/mail/constants.py delete mode 100644 src/leap/mail/imap/index.py delete mode 100644 src/leap/mail/imap/parser.py create mode 100644 src/leap/mail/interfaces.py create mode 100644 src/leap/mail/mail.py diff --git a/src/leap/mail/adaptors/__init__.py b/src/leap/mail/adaptors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py new file mode 100644 index 0000000..1648059 --- /dev/null +++ b/src/leap/mail/adaptors/models.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Models to be used by the Document Adaptors. +""" +import copy + + +class SerializableModel(object): + """ + A Generic document model, that can be serialized into a dictionary. + + Subclasses of this `SerializableModel` are meant to be added as class + attributes of classes inheriting from DocumentWrapper. + + A subclass __meta__ of this SerializableModel might exist, and contain info + relative to particularities of this model. + + For instance, the use of `__meta__.index` marks the existence of a primary + index in the model, which will be used to do unique queries (in which case + all the other indexed fields in the underlying document will be filled with + the default info contained in the model definition). + """ + + @classmethod + def serialize(klass): + """ + Get a dictionary representation of the public attributes in the model + class. To avoid collisions with builtin functions, any occurrence of an + attribute ended in '_' (like 'type_') will be normalized by removing + the trailing underscore. + + This classmethod is used from within the serialized method of a + DocumentWrapper instance: it provides defaults for the + empty document. + """ + assert isinstance(klass, type) + return _normalize_dict(klass.__dict__) + + +class DocumentWrapper(object): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the store understands. + It is related to a SerializableModel, which must be specified as the + ``model`` class attribute. The instance of this DocumentWrapper will not + allow any other *public* attributes than those defined in the corresponding + model. + """ + # TODO we could do some very basic type checking here + # TODO set a dirty flag (on __setattr__, whenever the value is != from + # before) + # TODO we could enforce the existence of a correct "model" attribute + # in some other way (other than in the initializer) + + def __init__(self, **kwargs): + if not getattr(self, 'model', None): + raise RuntimeError( + 'DocumentWrapper class needs a model attribute') + + defaults = self.model.serialize() + + if kwargs: + values = copy.deepcopy(defaults) + values.update(kwargs) + else: + values = defaults + + for k, v in values.items(): + k = k.replace('-', '_') + setattr(self, k, v) + + def __setattr__(self, attr, value): + normalized = _normalize_dict(self.model.__dict__) + if not attr.startswith('_') and attr not in normalized: + raise RuntimeError( + "Cannot set attribute because it's not defined " + "in the model: %s" % attr) + object.__setattr__(self, attr, value) + + def serialize(self): + return _normalize_dict(self.__dict__) + + def create(self): + raise NotImplementedError() + + def update(self): + raise NotImplementedError() + + def delete(self): + raise NotImplementedError() + + @classmethod + def get_or_create(self): + raise NotImplementedError() + + @classmethod + def get_all(self): + raise NotImplementedError() + + +def _normalize_dict(_dict): + items = _dict.items() + not_callable = lambda (k, v): not callable(v) + not_private = lambda(k, v): not k.startswith('_') + for cond in not_callable, not_private: + items = filter(cond, items) + items = [(k, v) if not k.endswith('_') else (k[:-1], v) + for (k, v) in items] + items = [(k.replace('-', '_'), v) for (k, v) in items] + return dict(items) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py new file mode 100644 index 0000000..2e25f04 --- /dev/null +++ b/src/leap/mail/adaptors/soledad.py @@ -0,0 +1,723 @@ +# -*- coding: utf-8 -*- +# soledad.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledadad MailAdaptor module. +""" +import re +from collections import defaultdict +from email import message_from_string + +from pycryptopp.hash import sha256 +from twisted.internet import defer +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail import walk +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors import models +from leap.mail.imap.mailbox import normalize_mailbox +from leap.mail.utils import lowerdict, first +from leap.mail.utils import stringify_parts_map +from leap.mail.interfaces import IMailAdaptor, IMessageWrapper + + +# TODO +# [ ] Convenience function to create mail specifying subject, date, etc? + + +_MSGID_PATTERN = r"""<([\w@.]+)>""" +_MSGID_RE = re.compile(_MSGID_PATTERN) + + +class DuplicatedDocumentError(Exception): + """ + Raised when a duplicated document is detected. + """ + pass + + +class SoledadDocumentWrapper(models.DocumentWrapper): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the Soledad Store understands. + + It ensures atomicity of the document operations on creation, update and + deletion. + """ + + # TODO we could also use a _dirty flag (in models) + + # We keep a dictionary with DeferredLocks, that will be + # unique to every subclass of SoledadDocumentWrapper. + _k_locks = defaultdict(defer.DeferredLock) + + @classmethod + def _get_klass_lock(cls): + """ + Get a DeferredLock that is unique for this subclass name. + Used to lock the access to indexes in the `get_or_create` call + for a particular DocumentWrapper. + """ + return cls._k_locks[cls.__name__] + + def __init__(self, **kwargs): + doc_id = kwargs.pop('doc_id', None) + self._doc_id = doc_id + self._lock = defer.DeferredLock() + super(SoledadDocumentWrapper, self).__init__(**kwargs) + + @property + def doc_id(self): + return self._doc_id + + def create(self, store): + """ + Create the documents for this wrapper. + Since this method will not check for duplication, the + responsibility of avoiding duplicates is left to the caller. + + You might be interested in using `get_or_create` classmethod + instead (that's the preferred way of creating documents from + the wrapper object). + + :return: a deferred that will fire when the underlying + Soledad document has been created. + :rtype: Deferred + """ + leap_assert(self._doc_id is None, + "This document already has a doc_id!") + + def update_doc_id(doc): + self._doc_id = doc.doc_id + return doc + d = store.create_doc(self.serialize()) + d.addCallback(update_doc_id) + return d + + def update(self, store): + """ + Update the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been updated. + :rtype: Deferred + """ + # the deferred lock guards against revision conflicts + return self._lock.run(self._update, store) + + def _update(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before updating") + + def update_and_put_doc(doc): + doc.content.update(self.serialize()) + return store.put_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(update_and_put_doc) + return d + + def delete(self, store): + """ + Delete the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been deleted. + :rtype: Deferred + """ + # the deferred lock guards against conflicts while updating + return self._lock.run(self._delete, store) + + def _delete(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before deleting") + # XXX might want to flag this DocumentWrapper to avoid + # updating it by mistake. This could go in models.DocumentWrapper + + def delete_doc(doc): + return store.delete_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(delete_doc) + return d + + @classmethod + def get_or_create(cls, store, index, value): + """ + Get a unique DocumentWrapper by index, or create a new one if the + matching query does not exist. + + :param index: the primary index for the model. + :type index: str + :param value: the value to query the primary index. + :type value: str + + :return: a deferred that will be fired with the SoledadDocumentWrapper + matching the index query, either existing or just created. + :rtype: Deferred + """ + return cls._get_klass_lock().run( + cls._get_or_create, store, index, value) + + @classmethod + def _get_or_create(cls, store, index, value): + assert store is not None + assert index is not None + assert value is not None + + def get_main_index(): + try: + return cls.model.__meta__.index + except AttributeError: + raise RuntimeError("The model is badly defined") + + def try_to_get_doc_from_index(indexes): + values = [] + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + values = [value] + else: + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + values.append(value) + else: + values.append(fields[field]) + d = store.get_from_index(index, *values) + return d + + def get_first_doc_if_any(docs): + if not docs: + return None + if len(docs) > 1: + raise DuplicatedDocumentError + return docs[0] + + def wrap_existing_or_create_new(doc): + if doc: + return cls(doc_id=doc.doc_id, **doc.content) + else: + return create_and_wrap_new_doc() + + def create_and_wrap_new_doc(): + # XXX use closure to store indexes instead of + # querying for them again. + d = store.list_indexes() + d.addCallback(get_wrapper_instance_from_index) + d.addCallback(return_wrapper_when_created) + return d + + def get_wrapper_instance_from_index(indexes): + init_values = {} + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + init_value = {idx_def[0]: value} + return cls(**init_value) + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + init_values[field] = value + else: + init_values[field] = fields[field] + return cls(**init_values) + + def return_wrapper_when_created(wrapper): + d = wrapper.create(store) + d.addCallback(lambda doc: wrapper) + return d + + d = store.list_indexes() + d.addCallback(try_to_get_doc_from_index) + d.addCallback(get_first_doc_if_any) + d.addCallback(wrap_existing_or_create_new) + return d + + @classmethod + def get_all(cls, store): + """ + Get a collection of wrappers around all the documents belonging + to this kind. + + For this to work, the model.__meta__ needs to include a tuple with + the index to be used for listing purposes, and which is the field to be + used to query the index. + + Note that this method only supports indexes of a single field at the + moment. It also might be too expensive to return all the documents + matching the query, so handle with care. + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + :return: a deferred that will be fired with an iterable containing + as many SoledadDocumentWrapper are matching the index defined + in the model as the `list_index`. + :rtype: Deferred + """ + # TODO + # [ ] extend support to indexes with n-ples + # [ ] benchmark the cost of querying and returning indexes in a big + # database. This might badly need pagination before being put to + # serious use. + return cls._get_klass_lock().run(cls._get_all, store) + + @classmethod + def _get_all(cls, store): + try: + list_index, list_attr = cls.model.__meta__.list_index + except AttributeError: + raise RuntimeError("The model is badly defined: no list_index") + try: + index_value = getattr(cls.model, list_attr) + except AttributeError: + raise RuntimeError("The model is badly defined: " + "no attribute matching list_index") + + def wrap_docs(docs): + return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs) + + d = store.get_from_index(list_index, index_value) + d.addCallback(wrap_docs) + return d + + # TODO + # [ ] get_count() ??? + + def __repr__(self): + try: + idx = getattr(self, self.model.__meta__.index) + except AttributeError: + idx = "" + return "<%s: %s (%s)>" % (self.__class__.__name__, + idx, self._doc_id) + + +# +# Message documents +# + +class FlagsDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "flags" + chash = "" + + mbox = "inbox" + seen = False + deleted = False + recent = False + multi = False + flags = [] + tags = [] + size = 0 + + class __meta__(object): + index = "mbox" + + +class HeaderDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "head" + chash = "" + + date = "" + subject = "" + headers = {} + part_map = {} + body = "" # link to phash of body + msgid = "" + multi = False + + class __meta__(object): + index = "chash" + + +class ContentDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "cnt" + phash = "" + + ctype = "" # XXX index by ctype too? + lkf = [] # XXX not implemented yet! + raw = "" + + content_disposition = "" + content_transfer_encoding = "" + content_type = "" + + class __meta__(object): + index = "phash" + + +class MessageWrapper(object): + + # TODO generalize wrapper composition? + # This could benefit of a DeferredLock to create/update all the + # documents at the same time maybe, and defend against concurrent updates? + + implements(IMessageWrapper) + + def __init__(self, fdoc, hdoc, cdocs=None): + """ + Need at least a flag-document and a header-document to instantiate a + MessageWrapper. Content-documents can be retrieved lazily. + + cdocs, if any, should be a dictionary in which the keys are ascending + integers, beginning at one, and the values are dictionaries with the + content of the content-docs. + """ + self.fdoc = FlagsDocWrapper(**fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + if cdocs is None: + cdocs = {} + cdocs_keys = cdocs.keys() + assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) + self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in + cdocs.items()]) + + def create(self, store): + """ + Create all the parts for this message in the store. + """ + leap_assert(self.cdocs, + "Need non empty cdocs to create the " + "MessageWrapper documents") + leap_assert(self.fdoc.doc_id is None, + "Cannot create: fdoc has a doc_id") + + # TODO I think we need to tolerate the no hdoc.doc_id case, for when we + # are doing a copy to another mailbox. + leap_assert(self.hdoc.doc_id is None, + "Cannot create: hdoc has a doc_id") + d = [] + d.append(self.fdoc.create(store)) + d.append(self.hdoc.create(store)) + for cdoc in self.cdocs.values(): + if cdoc.doc_id is not None: + # we could be just linking to an existing + # content-doc. + continue + d.append(cdoc.create(store)) + return defer.gatherResults(d) + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + return self.fdoc.update(store) + + def delete(self, store): + # Eventually this would have to do the duplicate search or send for the + # garbage collector. At least the fdoc can be unlinked. + raise NotImplementedError() + +# +# Mailboxes +# + + +class MailboxWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "mbox" + mbox = INBOX_NAME + flags = [] + closed = False + subscribed = False + rw = True + + class __meta__(object): + index = "mbox" + list_index = (indexes.TYPE_IDX, 'type_') + + +# +# Soledad Adaptor +# + +# TODO make this an interface? +class SoledadIndexMixin(object): + """ + this will need a class attribute `indexes`, that is a dictionary containing + the index definitions for the underlying u1db store underlying soledad. + + It needs to be in the following format: + {'index-name': ['field1', 'field2']} + """ + # TODO could have a wrapper class for indexes, supporting introspection + # and __getattr__ + indexes = {} + + store_ready = False + _index_creation_deferreds = [] + + # TODO we might want to move this logic to soledad itself + # so that each application can pass a set of indexes for their data model. + # TODO check also the decorator used in keymanager for waiting for indexes + # to be ready. + + def initialize_store(self, store): + """ + Initialize the indexes in the database. + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + # TODO I think we *should* get another deferredLock in here, but + # global to the soledad namespace, to protect from several points + # initializing soledad indexes at the same time. + + leap_assert(store, "Need a store") + leap_assert_type(self.indexes, dict) + self._index_creation_deferreds = [] + + def _on_indexes_created(ignored): + self.store_ready = True + + def _create_index(name, expression): + d = store.create_index(name, *expression) + self._index_creation_deferreds.append(d) + + def _create_indexes(db_indexes): + db_indexes = dict(db_indexes) + + for name, expression in self.indexes.items(): + if name not in db_indexes: + # The index does not yet exist. + _create_index(name, expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so + # we delete it and add the proper index expression. + d1 = store.delete_index(name) + d1.addCallback(lambda _: _create_index(name, expression)) + + all_created = defer.gatherResults(self._index_creation_deferreds) + all_created.addCallback(_on_indexes_created) + return all_created + + # Ask the database for currently existing indexes, and create them + # if not found. + d = store.list_indexes() + d.addCallback(_create_indexes) + return d + + +class SoledadMailAdaptor(SoledadIndexMixin): + + implements(IMailAdaptor) + store = None + + indexes = indexes.MAIL_INDEXES + + # Message handling + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param raw_msg: a string containing the raw email message. + :type raw_msg: str + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + return self.get_msg_from_docs( + MessageClass, fdoc, hdoc, cdocs) + + def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param fdoc: a dictionary containing values from which a + FlagsDocWrapper can be initialized + :type fdoc: dict + :param hdoc: a dictionary containing values from which a + HeaderDocWrapper can be initialized + :type hdoc: dict + :param cdocs: None, or a dictionary mapping integers (1-indexed) to + dicts from where a ContentDocWrapper can be initialized. + :type cdocs: dict, or None + + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :type store: + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.create(store) + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :type store: + :param msg: a Message object. + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.update(store) + + # Mailbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or creatre one if it does not + exist. + + :param name: the name of the mailbox + :type name: str + """ + index = indexes.TYPE_MBOX_IDX + mbox = normalize_mailbox(name) + return MailboxWrapper.get_or_create(store, index, mbox) + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + return mbox_wrapper.update(store) + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ + return MailboxWrapper.get_all(store) + + +def _split_into_parts(raw): + # TODO signal that we can delete the original message!----- + # when all the processing is done. + # TODO add the linked-from info ! + # TODO add reference to the original message? + # TODO populate Default FLAGS/TAGS (unseen?) + # TODO seed propely the content_docs with defaults?? + + msg, parts, chash, size, multi = _parse_msg(raw) + body_phash_fun = [walk.get_body_phash_simple, + walk.get_body_phash_multi][int(multi)] + body_phash = body_phash_fun(walk.get_payloads(msg)) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + + fdoc = _build_flags_doc(chash, size, multi) + hdoc = _build_headers_doc(msg, chash, parts_map) + + # The MessageWrapper expects a dict, one-indexed + cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + + # XXX convert each to_dicts... + return fdoc, hdoc, cdocs + + +def _parse_msg(raw): + msg = message_from_string(raw) + parts = walk.get_parts(msg) + size = len(raw) + chash = sha256.SHA256(raw).hexdigest() + multi = msg.is_multipart() + return msg, parts, chash, size, multi + + +def _build_flags_doc(chash, size, multi): + _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) + return _fdoc.serialize() + + +def _build_headers_doc(msg, chash, parts_map): + """ + Assemble a headers document from the original parsed message, the + content-hash, and the parts map. + + It takes into account possibly repeated headers. + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + + # "fix" for repeated headers. + for k, v in headers.items(): + newline = "\n%s: " % (k,) + headers[k] = newline.join(v) + + lower_headers = lowerdict(headers) + msgid = first(_MSGID_RE.findall( + lower_headers.get('message-id', ''))) + + _hdoc = HeaderDocWrapper( + chash=chash, headers=lower_headers, msgid=msgid) + + def copy_attr(headers, key, doc): + if key in headers: + setattr(doc, key, headers[key]) + + copy_attr(lower_headers, "subject", _hdoc) + copy_attr(lower_headers, "date", _hdoc) + + hdoc = _hdoc.serialize() + # add parts map to header doc + # (body, multi, part_map) + for key in parts_map: + hdoc[key] = parts_map[key] + return stringify_parts_map(hdoc) diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py new file mode 100644 index 0000000..f3e990d --- /dev/null +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# soledad_indexes.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Indexes for Mail Documents. +""" + +# TODO +# [ ] hide most of the constants here + +# Document Type, for indexing + +TYPE = "type" +MBOX = "mbox" +FLAGS = "flags" +HEADERS = "head" +CONTENT = "cnt" +RECENT = "rct" +HDOCS_SET = "hdocset" + +INCOMING_KEY = "incoming" +ERROR_DECRYPTING_KEY = "errdecr" + +# indexing keys +CONTENT_HASH = "chash" +PAYLOAD_HASH = "phash" +MSGID = "msgid" +UID = "uid" + + +# Index types +# -------------- + +TYPE_IDX = 'by-type' +TYPE_MBOX_IDX = 'by-type-and-mbox' +#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' +TYPE_SUBS_IDX = 'by-type-and-subscribed' +TYPE_MSGID_IDX = 'by-type-and-message-id' +TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' +TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' +TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' +TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' +TYPE_C_HASH_IDX = 'by-type-and-contenthash' +TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' +TYPE_P_HASH_IDX = 'by-type-and-payloadhash' + +# Soledad index for incoming mail, without decrypting errors. +# and the backward-compatible index, will be deprecated at 0.7 +JUST_MAIL_IDX = "just-mail" +JUST_MAIL_COMPAT_IDX = "just-mail-compat" + +# Tomas created the `recent and seen index`, but the semantic is not too +# correct since the recent flag is volatile --- XXX review and delete. +#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + +# TODO +# it would be nice to measure the cost of indexing +# by many fields. + +# TODO +# make the indexes dict more readable! + +MAIL_INDEXES = { + # generic + TYPE_IDX: [TYPE], + TYPE_MBOX_IDX: [TYPE, MBOX], + + # XXX deprecate 0.4.0 + # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], + + # mailboxes + TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'], + + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH], + + # headers doc - search by msgid. + TYPE_MSGID_IDX: [TYPE, MSGID], + + # content, headers doc + TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH], + + # attachment payload dedup + TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], + + # messages + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'], + #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX, + #'bool(recent)', 'bool(seen)'], + + # incoming queue + JUST_MAIL_IDX: [INCOMING_KEY, + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], + + # the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], +} diff --git a/src/leap/mail/adaptors/tests/__init__.py b/src/leap/mail/adaptors/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py new file mode 100644 index 0000000..efe0bf2 --- /dev/null +++ b/src/leap/mail/adaptors/tests/test_models.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# test_models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the leap.mail.adaptors.models module. +""" +from twisted.trial import unittest + +from leap.mail.adaptors import models + + +class SerializableModelsTestCase(unittest.TestCase): + + def test_good_serialized_model(self): + + class M(models.SerializableModel): + foo = 42 + bar = 33 + baaz_ = None + _nope = 0 + __nope = 0 + + def not_today(self): + pass + + class IgnoreMe(object): + pass + + killmeplease = lambda x: x + + serialized = M.serialize() + expected = {'foo': 42, 'bar': 33, 'baaz': None} + self.assertEqual(serialized, expected) + + +class DocumentWrapperTestCase(unittest.TestCase): + + def test_wrapper_defaults(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar = 11 + + wrapper = Wrapper() + wrapper._ignored = True + serialized = wrapper.serialize() + expected = {'foo': 42, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar_ = 11 + + wrapper = Wrapper(foo=0, bar=-1) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper.foo = 23 + serialized = wrapper.serialize() + expected = {'foo': 23, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper = Wrapper(foo=0) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_invalid_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + getwrapper = lambda: Wrapper(bar=1) + self.assertRaises(RuntimeError, getwrapper) + + def test_no_model_wrapper(self): + + class Wrapper(models.DocumentWrapper): + pass + + def getwrapper(): + w = Wrapper() + w.foo = None + + self.assertRaises(RuntimeError, getwrapper) diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py new file mode 100644 index 0000000..657a602 --- /dev/null +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -0,0 +1,583 @@ +# -*- coding: utf-8 -*- +# test_soledad_adaptor.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad +""" +import os +import shutil +import tempfile + +from functools import partial + +from twisted.internet import defer +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest +from leap.mail.adaptors import models +from leap.mail.adaptors.soledad import SoledadDocumentWrapper +from leap.mail.adaptors.soledad import SoledadIndexMixin +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.soledad.client import Soledad + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + +# DEBUG +# import logging +# logging.basicConfig(level=logging.DEBUG) + + +def initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +# TODO move to common module +# XXX remove duplication +class SoledadTestMixin(BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception as exc: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + finally: + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) + + +class CounterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + counter = 0 + flag = None + + +class CharacterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + name = "" + age = 20 + + +class ActorWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + type_ = "actor" + name = None + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + +class TestAdaptor(SoledadIndexMixin): + indexes = {'by-name': ['name'], + 'by-type-and-name': ['type', 'name'], + 'by-type': ['type']} + + +class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_create_single(self): + + store = self._soledad + wrapper = CounterWrapper() + + def assert_one_doc(docs): + self.assertEqual(docs[0], 1) + + d = wrapper.create(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(assert_one_doc) + return d + + def test_create_many(self): + + store = self._soledad + w1 = CounterWrapper() + w2 = CounterWrapper(counter=1) + w3 = CounterWrapper(counter=2) + w4 = CounterWrapper(counter=3) + w5 = CounterWrapper(counter=4) + + d1 = [w1.create(store), + w2.create(store), + w3.create(store), + w4.create(store), + w5.create(store)] + + d = defer.gatherResults(d1) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 5)) + return d + + def test_multiple_updates(self): + + store = self._soledad + wrapper = CounterWrapper(counter=1) + MAX = 100 + + def assert_doc_id(doc): + self.assertTrue(wrapper._doc_id is not None) + return doc + + def assert_counter_initial_ok(doc): + self.assertEqual(wrapper.counter, 1) + + def increment_counter(ignored): + d1 = [] + + def record_revision(revision): + rev = int(revision.split(':')[1]) + self.results.append(rev) + + for i in list(range(MAX)): + wrapper.counter += 1 + wrapper.flag = i % 2 == 0 + d = wrapper.update(store) + d.addCallback(record_revision) + d1.append(d) + + return defer.gatherResults(d1) + + def assert_counter_final_ok(doc): + self.assertEqual(doc.content['counter'], MAX + 1) + self.assertEqual(doc.content['flag'], False) + + def assert_results_ordered_list(ignored): + self.assertEqual(self.results, sorted(range(2, MAX + 2))) + + d = wrapper.create(store) + d.addCallback(assert_doc_id) + d.addCallback(assert_counter_initial_ok) + d.addCallback(increment_counter) + d.addCallback(lambda _: store.get_doc(wrapper._doc_id)) + d.addCallback(assert_counter_final_ok) + d.addCallback(assert_results_ordered_list) + return d + + def test_delete(self): + adaptor = TestAdaptor() + store = self._soledad + + wrapper_list = [] + + def get_or_create_bob(ignored): + def add_to_list(wrapper): + wrapper_list.append(wrapper) + return wrapper + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + wrapper.addCallback(add_to_list) + return wrapper + + def delete_bob(ignored): + wrapper = wrapper_list[0] + return wrapper.delete(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + d.addCallback(delete_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + return d + + def test_get_or_create(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_bob(ignored): + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + return wrapper + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_get_or_create_multi_index(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_actor_harry(ignored): + wrapper = ActorWrapper.get_or_create( + store, 'by-type-and-name', 'harrison') + return wrapper + + def create_director_harry(ignored): + wrapper = ActorWrapper(name="harrison", type="director") + return wrapper.create(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # create director harry, should create new doc + d.addCallback(create_director_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + + # this should get us harrison document, still 2 docs + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + return d + + def test_get_all(self): + adaptor = TestAdaptor() + store = self._soledad + actor_names = ["harry", "carrie", "mark", "david"] + + def create_some_actors(ignored): + deferreds = [] + for name in actor_names: + dw = ActorWrapper.get_or_create( + store, 'by-type-and-name', name) + deferreds.append(dw) + return defer.gatherResults(deferreds) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + d.addCallback(create_some_actors) + + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + + def assert_actor_list_is_expected(res): + got = set([actor.name for actor in res]) + expected = set(actor_names) + self.assertEqual(got, expected) + + d.addCallback(lambda _: ActorWrapper.get_all(store)) + d.addCallback(assert_actor_list_is_expected) + return d + +here = os.path.split(os.path.abspath(__file__))[0] + + +class TestMessageClass(object): + def __init__(self, wrapper): + self.wrapper = wrapper + + def get_wrapper(self): + return self.wrapper + + +class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadMailAdaptor. + """ + + def get_adaptor(self): + adaptor = SoledadMailAdaptor() + adaptor.store = self._soledad + return adaptor + + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_mail_adaptor_init(self): + adaptor = self.get_adaptor() + self.assertTrue(isinstance(adaptor.indexes, dict)) + self.assertTrue(len(adaptor.indexes) != 0) + + # Messages + + def test_get_msg_from_string(self): + adaptor = self.get_adaptor() + + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + + chash = ("D27B2771C0DCCDCB468EE65A4540438" + "09DBD11588E87E951545BE0CBC321C308") + phash = ("64934534C1C80E0D4FA04BE1CCBA104" + "F07BCA5F469C86E2C0ABE1D41310B7299") + subject = ("[Twisted-commits] rebuild now works on " + "python versions from 2.2.0 and up.") + self.assertTrue(msg.wrapper.fdoc is not None) + self.assertTrue(msg.wrapper.hdoc is not None) + self.assertTrue(msg.wrapper.cdocs is not None) + self.assertEquals(len(msg.wrapper.cdocs), 1) + self.assertEquals(msg.wrapper.fdoc.chash, chash) + self.assertEquals(msg.wrapper.fdoc.size, 3834) + self.assertEquals(msg.wrapper.hdoc.chash, chash) + self.assertEqual(msg.wrapper.hdoc.headers['subject'], + subject) + self.assertEqual(msg.wrapper.hdoc.subject, subject) + self.assertEqual(msg.wrapper.cdocs[1].phash, phash) + + def test_get_msg_from_docs(self): + adaptor = self.get_adaptor() + fdoc = dict( + mbox="Foobox", + flags=('\Seen', '\Nice'), + tags=('Personal', 'TODO'), + seen=False, deleted=False, + recent=False, multi=False) + hdoc = dict( + subject="Test Msg") + cdocs = { + 1: dict( + raw='This is a test message')} + + msg = adaptor.get_msg_from_docs( + TestMessageClass, fdoc, hdoc, cdocs=cdocs) + self.assertEqual(msg.wrapper.fdoc.flags, + ('\Seen', '\Nice')) + self.assertEqual(msg.wrapper.fdoc.tags, + ('Personal', 'TODO')) + self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox") + self.assertEqual(msg.wrapper.hdoc.multi, False) + self.assertEqual(msg.wrapper.hdoc.subject, + "Test Msg") + self.assertEqual(msg.wrapper.cdocs[1].raw, + "This is a test message") + + def test_create_msg(self): + adaptor = self.get_adaptor() + + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + + def check_create_result(created): + self.assertEqual(len(created), 3) + for doc in created: + self.assertTrue( + doc.__class__.__name__, + "SoledadDocument") + + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(check_create_result) + return d + + def test_update_msg(self): + adaptor = self.get_adaptor() + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + + def assert_msg_has_doc_id(ignored, msg): + wrapper = msg.get_wrapper() + self.assertTrue(wrapper.fdoc.doc_id is not None) + + def assert_msg_has_no_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, []) + + def update_msg_flags(ignored, msg): + wrapper = msg.get_wrapper() + wrapper.fdoc.flags = ["This", "That"] + return wrapper.update(adaptor.store) + + def assert_msg_has_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, ["This", "That"]) + + def get_fdoc_and_check_flags(ignored): + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['flags'], + ['This', 'That']) + wrapper = msg.get_wrapper() + d = adaptor.store.get_doc(wrapper.fdoc.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(assert_msg_has_doc_id, msg) + d.addCallback(assert_msg_has_no_flags, msg) + + # update it! + d.addCallback(update_msg_flags, msg) + d.addCallback(assert_msg_has_flags, msg) + d.addCallback(get_fdoc_and_check_flags) + return d + + # Mailboxes + + def test_get_or_create_mbox(self): + adaptor = self.get_adaptor() + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def assert_good_doc(mbox_wrapper): + self.assertTrue(mbox_wrapper.doc_id is not None) + self.assertEqual(mbox_wrapper.mbox, "Trash") + self.assertEqual(mbox_wrapper.type, "mbox") + self.assertEqual(mbox_wrapper.closed, False) + self.assertEqual(mbox_wrapper.subscribed, False) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(assert_good_doc) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_update_mbox(self): + adaptor = self.get_adaptor() + + wrapper_ref = [] + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def update_wrapper(wrapper, wrapper_ref): + wrapper_ref.append(wrapper) + wrapper.subscribed = True + wrapper.closed = True + d = adaptor.update_mbox(adaptor.store, wrapper) + return d + + def get_mbox_doc_and_check_flags(res, wrapper_ref): + wrapper = wrapper_ref[0] + + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['subscribed'], True) + self.assertEqual(doc.content['closed'], True) + d = adaptor.store.get_doc(wrapper.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(update_wrapper, wrapper_ref) + d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref) + return d + + def test_get_all_mboxes(self): + adaptor = self.get_adaptor() + mboxes = ("Sent", "Trash", "Personal", "ListFoo") + + def get_or_create_mboxes(ignored): + d = [] + for mbox in mboxes: + d.append(adaptor.get_or_create_mbox( + adaptor.store, mbox)) + return defer.gatherResults(d) + + def get_all_mboxes(ignored): + return adaptor.get_all_mboxes(adaptor.store) + + def assert_mboxes_match_expected(wrappers): + names = [m.mbox for m in wrappers] + self.assertEqual(set(names), set(mboxes)) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mboxes) + d.addCallback(get_all_mboxes) + d.addCallback(assert_mboxes_match_expected) + return d diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py new file mode 100644 index 0000000..55bf1da --- /dev/null +++ b/src/leap/mail/constants.py @@ -0,0 +1,21 @@ +# *- coding: utf-8 -*- +# constants.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Constants for leap.mail. +""" + +INBOX_NAME = "INBOX" diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index fe466cb..7dfbbd1 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -28,10 +28,10 @@ from twisted.python import log from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.mail.imap.index import IndexedDB + +from leap.mail.mail import Account from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.parser import MBoxParser -from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -39,7 +39,6 @@ logger = logging.getLogger(__name__) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") @@ -47,96 +46,43 @@ if PROFILE_CMD: ####################################### -# Soledad Account +# Soledad IMAP Account ####################################### +# TODO remove MsgFields too -# TODO change name to LeapIMAPAccount, since we're using -# the memstore. -# IndexedDB should also not be here anymore. - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): +class IMAPAccount(WithMsgFields): """ - An implementation of IAccount and INamespacePresenteer + An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. """ implements(imap4.IAccount, imap4.INamespacePresenter) - _soledad = None selected = None closed = False - _initialized = False - def __init__(self, account_name, soledad, memstore=None): + def __init__(self, user_id, store): """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. + Keeps track of the mailboxes and subscriptions handled by this account. - :param acct_name: The name of the account (user id). - :type acct_name: str + :param account: The name of the account (user id). + :type account: str - :param soledad: a Soledad instance. - :type soledad: Soledad - :param memstore: a MemoryStore instance. - :type memstore: MemoryStore + :param store: a Soledad instance. + :type store: Soledad """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) + # XXX assert a generic store interface instead, so that we + # can plug the memory store wrapper seamlessly. + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) # XXX SHOULD assert too that the name matches the user/uuid with which # soledad has been initialized. + self.user_id = user_id + self.account = Account(store) - # XXX ??? why is this parsing mailbox name??? it's account... - # userid? homogenize. - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - self._memstore = memstore - - self.__mailboxes = set([]) - - self._deferred_initialization = defer.Deferred() - self._initialize_storage() - - def _initialize_storage(self): - - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def finish_initialization(result): - self._initialized = True - self._deferred_initialization.callback(None) - - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - - d = self.initialize_db() - - d.addCallback(load_mbox_cache) - d.addCallback(add_mailbox_if_none) - d.addCallback(finish_initialization) - - def callWhenReady(self, cb): - if self._initialized: - cb(self) - return defer.succeed(None) - else: - self._deferred_initialization.addCallback(cb) - return self._deferred_initialization - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - + # XXX should hide this in the adaptor... def _get_mailbox_by_name(self, name): """ Return an mbox document by name. @@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ - # XXX use soledadstore instead ...; def get_first_if_any(docs): return docs[0] if docs else None - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) + normalize_mailbox(name)) d.addCallback(get_first_if_any) return d - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) - - def _load_mailboxes(self): - def update_mailboxes(db_indexes): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] for doc in db_indexes]) - d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) - d.addCallback(update_mailboxes) - return d - + # XXX move to Account? + # XXX needed? def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) - if name not in self.mailboxes: + if name not in self.account.mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) - return SoledadMailbox(name, self._soledad, - memstore=self._memstore) + # XXX Does mailbox really need reference to soledad? + return SoledadMailbox(name, self._store) # # IAccount # + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + # XXX move to mailbox module + return copy.deepcopy(mailbox.EMPTY_MBOX) + + # TODO use mail.Account.add_mailbox def addMailbox(self, name, creation_ts=None): """ Add a mailbox to the account. @@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a Deferred that will contain the document if successful. :rtype: bool """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) leap_assert(name, "Need a mailbox name to create a mailbox") @@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d.addCallback(lambda _: result) return d - d = self._soledad.create_doc(mbox) + d = self._store.create_doc(mbox) d.addCallback(load_mbox_cache) return d + # TODO use mail.Account.create_mailbox? + # Watch out, imap specific exceptions raised here. def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. @@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) + paths = filter(None, normalize_mailbox(pathspec).split('/')) subs = [] sep = '/' @@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d1.addCallback(load_mbox_cache) return d1 + # TODO use mail.Account.get_collection_by_mailbox def select(self, name, readwrite=1): """ Selects a mailbox. @@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadMailbox """ - if PROFILE_CMD: - start = time.time() - - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: logger.warning("No such mailbox!") return None self.selected = name - sm = SoledadMailbox( - name, self._soledad, self._memstore, readwrite) - if PROFILE_CMD: - _debugProfiling(None, "SELECT", start) + sm = SoledadMailbox(name, self._store, readwrite) return sm + # TODO use mail.Account.delete_mailbox def delete(self, name, force=False): """ Deletes a mailbox. @@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type force: bool :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # ??! -- can this be rite? # self._index.removeMailbox(name) + # TODO use mail.Account.rename_mailbox def rename(self, oldname, newname): """ Renames a mailbox. @@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param newname: new name of the mailbox :type newname: str """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) if oldname not in self.mailboxes: raise imap4.NoSuchMailbox(repr(oldname)) @@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO use mail.Account.list_mailboxes + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames(normalize_mailbox(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + # TODO ------------------ can we preserve the attr? # maybe add to memory store. def isSubscribed(self, name): @@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: Deferred (will fire with bool) """ + # TODO use Flags class subscribed = self.SUBSCRIBED_KEY def is_subscribed(mbox): @@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def get_docs_content(docs): return [doc.content[self.MBOX_KEY] for doc in docs] - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') d.addCallback(get_docs_content) return d @@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def update_subscribed_value(mbox): mbox.content[subscribed] = value - return self._soledad.put_doc(mbox) + return self._store.put_doc(mbox) # maybe we should store subscriptions in another # document... @@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_subscribe(subscriptions): if name not in subscriptions: @@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_unsubscribe(subscriptions): if name not in subscriptions: @@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def getSubscriptions(self): return self._get_subscriptions() - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - # # INamespacePresenter # @@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): """ Representation string for this object. """ - return "" % self._account_name + return "" % self.user_id diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 4576939..a751c6d 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -17,7 +17,9 @@ """ Fields for Mailbox and Message. """ -from leap.mail.imap.parser import MBoxParser + +# TODO deprecate !!! (move all to constants maybe?) +# Flags -> foo class WithMsgFields(object): @@ -25,55 +27,12 @@ class WithMsgFields(object): Container class for class-attributes to be shared by several message-related classes. """ - # indexing - CONTENT_HASH_KEY = "chash" - PAYLOAD_HASH_KEY = "phash" - - # Internal representation of Message - - # flags doc - UID_KEY = "uid" - MBOX_KEY = "mbox" - SEEN_KEY = "seen" - DEL_KEY = "deleted" - RECENT_KEY = "recent" - FLAGS_KEY = "flags" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # headers - HEADERS_KEY = "headers" - DATE_KEY = "date" - SUBJECT_KEY = "subject" - PARTS_MAP_KEY = "part_map" - BODY_KEY = "body" # link to phash of body - MSGID_KEY = "msgid" - - # content - LINKED_FROM_KEY = "lkf" # XXX not implemented yet! - RAW_KEY = "raw" - CTYPE_KEY = "ctype" - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" + CREATED_KEY = "created" # used??? + RECENTFLAGS_KEY = "rct" HDOCS_SET_KEY = "hdocset" - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - TYPE_HEADERS_VAL = "head" - TYPE_CONTENT_VAL = "cnt" - TYPE_RECENT_VAL = "rct" - TYPE_HDOCS_SET_VAL = "hdocset" - - INBOX_VAL = "inbox" - # Flags in Mailbox and Message SEEN_FLAG = "\\Seen" RECENT_FLAG = "\\Recent" @@ -88,86 +47,5 @@ class WithMsgFields(object): SUBJECT_FIELD = "Subject" DATE_FIELD = "Date" - # Index types - # -------------- - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MSGID_IDX = 'by-type-and-message-id' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' - TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' - TYPE_C_HASH_IDX = 'by-type-and-contenthash' - TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' - TYPE_P_HASH_IDX = 'by-type-and-payloadhash' - - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - # Soledad index for incoming mail, without decrypting errors. - JUST_MAIL_IDX = "just-mail" - # XXX the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX = "just-mail-compat" - - INCOMING_KEY = "incoming" - ERROR_DECRYPTING_KEY = "errdecr" - - KTYPE = TYPE_KEY - MBOX_VAL = TYPE_MBOX_VAL - CHASH_VAL = CONTENT_HASH_KEY - PHASH_VAL = PAYLOAD_HASH_KEY - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # fdocs uniqueness - TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], - - # headers doc - search by msgid. - TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], - - # content, headers doc - TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - - # attachment payload dedup - TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - - # incoming queue - JUST_MAIL_IDX: [INCOMING_KEY, - "bool(%s)" % (ERROR_DECRYPTING_KEY,)], - - # the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - TYPE_KEY: MBOX_KEY, - TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - SUBJECT_KEY: "", - FLAGS_KEY: [], - CLOSED_KEY: False, - SUBSCRIBED_KEY: False, - RW_KEY: 1, - LAST_UID_KEY: 0 - } fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py deleted file mode 100644 index ea35fff..0000000 --- a/src/leap/mail/imap/index.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# index.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Index for SoledadBackedAccount, Mailbox and Messages. -""" -import logging - -from twisted.internet import defer - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail.imap.fields import fields - - -logger = logging.getLogger(__name__) - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - _index_creation_deferreds = [] - index_ready = False - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - self._index_creation_deferreds = [] - - def _on_indexes_created(ignored): - self.index_ready = True - - def _create_index(name, expression): - d = self._soledad.create_index(name, *expression) - self._index_creation_deferreds.append(d) - - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - _create_index(name, expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = self._soledad.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - - all_created = defer.gatherResults(self._index_creation_deferreds) - all_created.addCallback(_on_indexes_created) - return all_created - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - if self._soledad is not None: - d = self._soledad.list_indexes() - d.addCallback(_create_indexes) - return d diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py index c906278..f8f25fa 100644 --- a/src/leap/mail/imap/interfaces.py +++ b/src/leap/mail/imap/interfaces.py @@ -20,6 +20,7 @@ Interfaces for the IMAP module. from zope.interface import Interface, Attribute +# TODO remove ---------------- class IMessageContainer(Interface): """ I am a container around the different documents that a message @@ -38,6 +39,7 @@ class IMessageContainer(Interface): """ +# TODO remove -------------------- class IMessageStore(Interface): """ I represent a generic storage for LEAP Messages. diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3c1769a..ea54d33 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ Soledad Mailbox. """ import copy +import re import threading import logging import StringIO @@ -27,6 +28,7 @@ import os from collections import defaultdict from twisted.internet import defer +from twisted.internet import reactor from twisted.internet.task import deferLater from twisted.python import log @@ -36,15 +38,18 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import INBOX_NAME from leap.mail.decorators import deferred_to_thread from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) +# TODO +# [ ] Restore profile_cmd instrumentation + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. @@ -71,7 +76,9 @@ if PROFILE_CMD: d.addErrback(lambda f: log.msg(f.getTraceback())) -class SoledadMailbox(WithMsgFields, MBoxParser): +# TODO Rename to Mailbox +# TODO Remove WithMsgFields +class SoledadMailbox(WithMsgFields): """ A Soledad-backed IMAP mailbox. @@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): _last_uid_primed = {} _known_uids_primed = {} - def __init__(self, mbox, soledad, memstore, rw=1): + # TODO pass the collection to the constructor + # TODO pass the mbox_doc too + def __init__(self, mbox, store, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. @@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param mbox: the mailbox name :type mbox: str - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + :param store: + :type store: Soledad :param rw: read-and-write flag for this mailbox :type rw: int """ leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert(store, "Need a store instance to initialize") - from twisted.internet import reactor - self.reactor = reactor - - self.mbox = self._parse_mailbox_name(mbox) + self.mbox = normalize_mailbox(mbox) self.rw = rw - self._soledad = soledad - self._memstore = memstore - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad, memstore=self._memstore) + self.store = store + self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None # XXX careful with this get/set (it would be @@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._memstore.get_mbox_doc(self.mbox) - # XXX the memstore->soledadstore method in memstore is not complete def getFlags(self): """ Returns the flags defined for this mailbox. @@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = self.INIT_FLAGS return map(str, flags) - # XXX the memstore->soledadstore method in memstore is not complete def setFlags(self, flags): """ Sets flags for this mailbox. @@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date, notify_on_disk=notify_on_disk) - if PROFILE_CMD: - do_profile_cmd(d, "APPEND") + #if PROFILE_CMD: + #do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # to work fine for now. def notifyCallback(x): - self.reactor.callLater(0, self.notify_new) + reactor.callLater(0, self.notify_new) return x d.addCallback(notifyCallback) @@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: deferred """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH") + + # XXX do not need no thread... + reactor.callInThread(self._do_fetch, messages_asked, uid, d) d.addCallback(self.cb_signal_unread_to_ui) return d @@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self.__cb_signal_unread_to_ui) return result - @deferred_to_thread def _get_unseen_deferred(self): return self.getUnseenCount() @@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: C{list} or C{Deferred} """ # TODO see if we can raise w/o interrupting flow - #:raise IllegalQueryError: Raised when query is not valid. + # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', # '52D44F11.9060107@dev.bitmask.net'] @@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(createCopy) d.addErrback(lambda f: log.msg(f.getTraceback())) - @deferred_to_thread + #@deferred_to_thread def _get_msg_copy(self, message): """ Get a copy of the fdoc for this message, and check whether @@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return u"" % ( self.mbox, self.messages.count()) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index c761091..d47c8eb 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # messages.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,30 +19,25 @@ LeapMessage and MessageCollection. """ import copy import logging -import re import threading import StringIO from collections import defaultdict -from email import message_from_string from functools import partial -from pycryptopp.hash import sha256 from twisted.mail import imap4 -from twisted.internet import defer, reactor +from twisted.internet import reactor from zope.interface import implements from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset -from leap.mail import walk -from leap.mail.utils import first, find_charset, lowerdict, empty -from leap.mail.utils import stringify_parts_map -from leap.mail.decorators import deferred_to_thread +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.utils import find_charset, empty from leap.mail.imap.index import IndexedDB from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.memorystore import MessageWrapper from leap.mail.imap.messageparts import MessagePart, MessagePartDoc from leap.mail.imap.parser import MBoxParser @@ -59,9 +54,6 @@ logger = logging.getLogger(__name__) # [ ] Delete incoming mail only after successful write! # [ ] Remove UID from syncable db. Store only those indexes locally. -MSGID_PATTERN = r"""<([\w@.]+)>""" -MSGID_RE = re.compile(MSGID_PATTERN) - def try_unique_query(curried): """ @@ -90,28 +82,18 @@ def try_unique_query(curried): logger.exception("Unhandled error %r" % exc) -""" -A dictionary that keeps one lock per mbox and uid. -""" -# XXX too much overhead? -fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# FIXME remove-me +#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) -class LeapMessage(fields, MBoxParser): +class IMAPMessage(fields, MBoxParser): """ The main representation of a message. - - It indexes the messages in one mailbox by a combination - of uid+mailbox name. """ - # TODO this has to change. - # Should index primarily by chash, and keep a local-only - # UID table. - implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox, collection=None, container=None): + def __init__(self, soledad, uid, mbox): """ Initializes a LeapMessage. @@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser): self._soledad = soledad self._uid = int(uid) if uid is not None else None self._mbox = self._parse_mailbox_name(mbox) - self._collection = collection - self._container = container self.__chash = None self.__bdoc = None - # XXX make these properties public - - # XXX FIXME ------ the documents can be - # deferreds too.... niice. - - @property - def fdoc(self): - """ - An accessor to the flags document. - """ - if all(map(bool, (self._uid, self._mbox))): - fdoc = None - if self._container is not None: - fdoc = self._container.fdoc - if not fdoc: - fdoc = self._get_flags_doc() - if fdoc: - fdoc_content = fdoc.content - self.__chash = fdoc_content.get( - fields.CONTENT_HASH_KEY, None) - return fdoc - - @property - def hdoc(self): - """ - An accessor to the headers document. - """ - container = self._container - if container is not None: - hdoc = self._container.hdoc - if hdoc and not empty(hdoc.content): - return hdoc - hdoc = self._get_headers_doc() - - if container and not empty(hdoc.content): + # TODO collection and container are deprecated. + + # TODO move to adaptor + + #@property + #def fdoc(self): + #""" + #An accessor to the flags document. + #""" + #if all(map(bool, (self._uid, self._mbox))): + #fdoc = None + #if self._container is not None: + #fdoc = self._container.fdoc + #if not fdoc: + #fdoc = self._get_flags_doc() + #if fdoc: + #fdoc_content = fdoc.content + #self.__chash = fdoc_content.get( + #fields.CONTENT_HASH_KEY, None) + #return fdoc +# + #@property + #def hdoc(self): + #""" + #An accessor to the headers document. + #""" + #container = self._container + #if container is not None: + #hdoc = self._container.hdoc + #if hdoc and not empty(hdoc.content): + #return hdoc + #hdoc = self._get_headers_doc() +# + #if container and not empty(hdoc.content): # mem-cache it - hdoc_content = hdoc.content - chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - hdocs = {chash: hdoc_content} - container.memstore.load_header_docs(hdocs) - return hdoc - - @property - def chash(self): - """ - An accessor to the content hash for this message. - """ - if not self.fdoc: - return None - if not self.__chash and self.fdoc: - self.__chash = self.fdoc.content.get( - fields.CONTENT_HASH_KEY, None) - return self.__chash - - @property - def bdoc(self): - """ - An accessor to the body document. - """ - if not self.hdoc: - return None - if not self.__bdoc: - self.__bdoc = self._get_body_doc() - return self.__bdoc + #hdoc_content = hdoc.content + #chash = hdoc_content.get(fields.CONTENT_HASH_KEY) + #hdocs = {chash: hdoc_content} + #container.memstore.load_header_docs(hdocs) + #return hdoc +# + #@property + #def chash(self): + #""" + #An accessor to the content hash for this message. + #""" + #if not self.fdoc: + #return None + #if not self.__chash and self.fdoc: + #self.__chash = self.fdoc.content.get( + #fields.CONTENT_HASH_KEY, None) + #return self.__chash + + #@property + #def bdoc(self): + #""" + #An accessor to the body document. + #""" + #if not self.hdoc: + #return None + #if not self.__bdoc: + #self.__bdoc = self._get_body_doc() + #return self.__bdoc # IMessage implementation @@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ + # TODO ----> return lookup in local sqlcipher table. return self._uid + # -------------------------------------------------------------- + # TODO -- from here on, all the methods should be proxied to the + # instance of leap.mail.mail.Message + def getFlags(self): """ Retrieve the flags associated with this Message. @@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser): REMOVE = -1 SET = 0 - with fdoc_locks[mbox][uid]: - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) + doc = self.fdoc + if not doc: + logger.warning( + "Could not find FDOC for %r:%s while setting flags!" % + (mbox, uid)) + return + current = doc.content[self.FLAGS_KEY] + if mode == APPEND: + newflags = tuple(set(tuple(current) + flags)) + elif mode == REMOVE: + newflags = tuple(set(current).difference(set(flags))) + elif mode == SET: + newflags = flags + new_fdoc = { + self.FLAGS_KEY: newflags, + self.SEEN_KEY: self.SEEN_FLAG in newflags, + self.DEL_KEY: self.DELETED_FLAG in newflags} + self._collection.memstore.update_flags(mbox, uid, new_fdoc) return map(str, newflags) @@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser): else: logger.warning("No FLAGS doc for %s:%s" % (self._mbox, self._uid)) - if not size: + #if not size: # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len + #size = self.getBodyFile().len return size def getHeaders(self, negate, *names): @@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser): # XXX refactor together with MessagePart method headers = self._get_headers() + + # XXX keep this in the imap imessage implementation, + # because the server impl. expects content-type to be present. if not headers: logger.warning("No headers found") return {str('content-type'): str('')} @@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): (the u1db index) for all the headers documents for a given mailbox. We use it to prefetch massively all the headers for a mailbox. This is the second massive query, after fetching all the FLAGS, that - a MUA will do in a case where we do not have local disk cache. + a typical IMAP MUA will do in a case where we do not have local disk cache. """ HDOCS_SET_DOC = "HDOCS_SET" templates = { - # Message Level - - FLAGS_DOC: { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, # XXX moe to a local table - fields.MBOX_KEY: fields.INBOX_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.SEEN_KEY: False, - fields.DEL_KEY: False, - fields.FLAGS_KEY: [], - fields.MULTIPART_KEY: False, - fields.SIZE_KEY: 0 - }, - - HEADERS_DOC: { - fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.DATE_KEY: "", - fields.SUBJECT_KEY: "", - - fields.HEADERS_KEY: {}, - fields.PARTS_MAP_KEY: {}, - }, - - CONTENT_DOC: { - fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, - fields.PAYLOAD_HASH_KEY: "", - fields.LINKED_FROM_KEY: [], - fields.CTYPE_KEY: "", # should index by this too - - # should only get inmutable headers parts - # (for indexing) - fields.HEADERS_KEY: {}, - fields.RAW_KEY: "", - fields.PARTS_MAP_KEY: {}, - fields.HEADERS_KEY: {}, - fields.MULTIPART_KEY: False, - }, - # Mailbox Level RECENT_DOC: { - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.RECENT, + "mbox": INBOX_NAME, fields.RECENTFLAGS_KEY: [], }, HDOCS_SET_DOC: { - fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.HDOCS_SET, + "mbox": INBOX_NAME, fields.HDOCS_SET_KEY: [], } @@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # Different locks for wrapping both the u1db document getting/setting # and the property getting/settting in an atomic operation. - # TODO we would abstract this to a SoledadProperty class - + # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks _rdoc_lock = defaultdict(lambda: threading.Lock()) _rdoc_write_lock = defaultdict(lambda: threading.Lock()) _rdoc_read_lock = defaultdict(lambda: threading.Lock()) @@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): rdoc[fields.MBOX_KEY] = self.mbox self._soledad.create_doc(rdoc) - @deferred_to_thread - def _do_parse(self, raw): - """ - Parse raw message and return it along with - relevant information about its outer level. - - This is done in a separate thread, and the callback is passed - to `_do_add_msg` method. + # -------------------------------------------------------------------- - :param raw: the raw message - :type raw: StringIO or basestring - :return: msg, parts, chash, size, multi - :rtype: tuple - """ - msg = message_from_string(raw) - parts = walk.get_parts(msg) - size = len(raw) - chash = sha256.SHA256(raw).hexdigest() - multi = msg.is_multipart() - return msg, parts, chash, size, multi - - def _populate_flags(self, flags, uid, chash, size, multi): - """ - Return a flags doc. - - XXX Missing DOC ----------- - """ - fd = self._get_empty_doc(self.FLAGS_DOC) - - fd[self.MBOX_KEY] = self.mbox - fd[self.UID_KEY] = uid - fd[self.CONTENT_HASH_KEY] = chash - fd[self.SIZE_KEY] = size - fd[self.MULTIPART_KEY] = multi - if flags: - fd[self.FLAGS_KEY] = flags - fd[self.SEEN_KEY] = self.SEEN_FLAG in flags - fd[self.DEL_KEY] = self.DELETED_FLAG in flags - fd[self.RECENT_KEY] = True # set always by default - return fd - - def _populate_headr(self, msg, chash, subject, date): - """ - Return a headers doc. - - XXX Missing DOC ----------- - """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) - - lower_headers = lowerdict(headers) - msgid = first(MSGID_RE.findall( - lower_headers.get('message-id', ''))) - - hd = self._get_empty_doc(self.HEADERS_DOC) - hd[self.CONTENT_HASH_KEY] = chash - hd[self.HEADERS_KEY] = headers - hd[self.MSGID_KEY] = msgid - - if not subject and self.SUBJECT_FIELD in headers: - hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - else: - hd[self.SUBJECT_KEY] = subject - - if not date and self.DATE_FIELD in headers: - hd[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - hd[self.DATE_KEY] = date - return hd + # ----------------------------------------------------------------------- def _fdoc_already_exists(self, chash): """ @@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) - # TODO return soledad deferred instead - observer = defer.Deferred() - d = self._do_parse(raw) - d.addCallback(lambda result: reactor.callInThread( - self._do_add_msg, result, flags, subject, date, - notify_on_disk, observer)) - return observer + # TODO ---- proxy to MessageCollection addMessage + + #observer = defer.Deferred() + #d = self._do_parse(raw) + #d.addCallback(lambda result: reactor.callInThread( + #self._do_add_msg, result, flags, subject, date, + #notify_on_disk, observer)) + #return observer + + # TODO --------------------------------------------------- + # move this to leap.mail.adaptors.soledad - # Called in thread def _do_add_msg(self, parse_result, flags, subject, date, notify_on_disk, observer): """ - Helper that creates a new message document. - Here lives the magic of the leap mail. Well, in soledad, really. - - See `add_msg` docstring for parameter info. - - :param parse_result: a tuple with the results of `self._do_parse` - :type parse_result: tuple - :param observer: a deferred that will be fired with the message - uid when the adding succeed. - :type observer: deferred """ - # TODO signal that we can delete the original message!----- - # when all the processing is done. - - # TODO add the linked-from info ! - # TODO add reference to the original message - msg, parts, chash, size, multi = parse_result + # XXX move to SoledadAdaptor write operation ... ??? # check for uniqueness -------------------------------- # Watch out! We're reserving a UID right after this! existing_uid = self._fdoc_already_exists(chash) if existing_uid: msg = self.get_msg_by_uid(existing_uid) - - # We can say the observer that we're done - # TODO return soledad deferred instead reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return + # TODO move UID autoincrement to MessageCollection.addMessage(mailbox) # TODO S2 -- get FUCKING UID from autoincremental table - uid = self.memstore.increment_last_soledad_uid(self.mbox) - - # We can say the observer that we're done at this point, but - # before that we should make sure it has no serious consequences - # if we're issued, for instance, a fetch command right after... - # reactor.callFromThread(observer.callback, uid) - # if we did the notify, we need to invalidate the deferred - # so not to try to fire it twice. - # observer = None - - fd = self._populate_flags(flags, uid, chash, size, multi) - hd = self._populate_headr(msg, chash, subject, date) - - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) - parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) - - # add parts map to header doc - # (body, multi, part_map) - for key in parts_map: - hd[key] = parts_map[key] - del parts_map + #uid = self.memstore.increment_last_soledad_uid(self.mbox) + #self.set_recent_flag(uid) - hd = stringify_parts_map(hd) - # The MessageContainer expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) - - self.set_recent_flag(uid) - msg_container = MessageWrapper(fd, hd, cdocs) - - # TODO S1 -- just pass this to memstore and return that deferred. - self.memstore.create_message( - self.mbox, uid, msg_container, - observer=observer, notify_on_disk=notify_on_disk) + # ------------------------------------------------------------ # # getters: specific queries @@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # USED from: + # [ ] duplicated fdoc detection + # [ ] _get_uid_from_msgidCb + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, @@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): if msg_container is not None: if mem_only: - msg = LeapMessage(None, uid, self.mbox, collection=self, + msg = IMAPMessage(None, uid, self.mbox, collection=self, container=msg_container) else: # We pass a reference to soledad just to be able to retrieve # missing parts that cannot be found in the container, like # the content docs after a copy. - msg = LeapMessage(self._soledad, uid, self.mbox, + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self, container=msg_container) else: - msg = LeapMessage(self._soledad, uid, self.mbox, collection=self) + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self) if not msg.does_exist(): return None return msg - def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - """ - Get all documents for the selected mailbox of the - passed type. By default, it returns the flag docs. - - If you want acess to the content, use __iter__ instead - - :return: a Deferred, that will fire with a list of u1db documents - :rtype: Deferred (promise of list of SoledadDocument) - """ - if _type not in fields.__dict__.values(): - raise TypeError("Wrong type passed to get_all_docs") - + # FIXME --- used where ? --------------------------------------------- + #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + #""" + #Get all documents for the selected mailbox of the + #passed type. By default, it returns the flag docs. +# + #If you want acess to the content, use __iter__ instead +# + #:return: a Deferred, that will fire with a list of u1db documents + #:rtype: Deferred (promise of list of SoledadDocument) + #""" + #if _type not in fields.__dict__.values(): + #raise TypeError("Wrong type passed to get_all_docs") +# # FIXME ----- either raise or return a deferred wrapper. - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - def get_sorted_docs(docs): - all_docs = [doc for doc in docs] + #if sameProxiedObjects(self._soledad, None): + #logger.warning('Tried to get messages but soledad is None!') + #return [] +# + #def get_sorted_docs(docs): + #all_docs = [doc for doc in docs] # inneficient, but first let's grok it and then # let's worry about efficiency. # XXX FIXINDEX -- should implement order by in soledad # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, _type, self.mbox) - d.addCallback(get_sorted_docs) - return d + #return sorted(all_docs, key=lambda item: item.content['uid']) +# + #d = self._soledad.get_from_index( + #fields.TYPE_MBOX_IDX, _type, self.mbox) + #d.addCallback(get_sorted_docs) + #return d def all_soledad_uid_iter(self): """ @@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(self._soledad, docid, self.mbox, collection=self) + return [IMAPMessage(self._soledad, docid, self.mbox, collection=self) for docid in self.unseen_iter()] # recent messages @@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: iterator of dicts with content for all messages. :rtype: iterable """ - return (LeapMessage(self._soledad, docuid, self.mbox, collection=self) + return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self) for docuid in self.all_uid_iter()) def __repr__(self): diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py deleted file mode 100644 index 4a801b0..0000000 --- a/src/leap/mail/imap/parser.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# parser.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Mail parser mixin. -""" -import re - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - Return a normalized representation of the mailbox C{name}. - - This method ensures that an eventual initial 'inbox' part of a - mailbox name is made uppercase. - - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dd4294c..5af499f 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): MessageCollection interface in this particular TestCase """ super(MessageCollectionTestCase, self).setUp() + + # TODO deprecate memstore memstore = MemoryStore() self.messages = MessageCollection("testmbox%s" % (self.count,), self._soledad, memstore=memstore) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 9a3868c..920eeb0 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() +# XXX move to common helper def initialize_soledad(email, gnupg_home, tempdir): """ Initializes soledad by hand @@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest): """ Setup method for each test. - Initializes and run a LEAP IMAP4 Server, - but passing the same Soledad instance (it's costly to initialize), - so we have to be sure to restore state across tests. + Initializes and run a LEAP IMAP4 Server. """ self.old_path = os.environ['PATH'] self.old_home = os.environ['HOME'] @@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest): def tearDown(self): """ tearDown method called after each test. - - Deletes all documents in the Index, and deletes - instances of server and client. """ try: self._soledad.close() + except Exception: + print "ERROR WHILE CLOSING SOLEDAD" + finally: os.environ["PATH"] = self.old_path os.environ["HOME"] = self.old_home # safety check assert 'leap_tests-' in self.tempdir shutil.rmtree(self.tempdir) - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" def populateMessages(self): """ @@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest): def loopback(self): return loopback.loopbackAsync(self.server, self.client) - - diff --git a/src/leap/mail/interfaces.py b/src/leap/mail/interfaces.py new file mode 100644 index 0000000..5838ce9 --- /dev/null +++ b/src/leap/mail/interfaces.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Interfaces for the leap.mail module. +""" +from zope.interface import Interface, Attribute + + +class IMessageWrapper(Interface): + """ + I know how to access the different parts into which a given message is + splitted into. + """ + + fdoc = Attribute('A dictionaly-like containing the flags document ' + '(mutable)') + hdoc = Attribute('A dictionary-like containing the headers docuemnt ' + '(immutable)') + cdocs = Attribute('A dictionary with the content-docs, one-indexed') + + +class IMailAdaptor(Interface): + """ + I know how to store the standard representation for messages and mailboxes, + and how to update the relevant mutable parts when needed. + """ + + def initialize_store(self, store): + """ + Performs whatever initialization is needed before the store can be + used (creating indexes, sanity checks, etc). + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + + # TODO is staticmethod valid with an interface? + # @staticmethod + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Return IMessageWrapper implementor from a raw mail string + + :param MessageClass: an implementor of IMessage + :type raw_msg: str + :rtype: implementor of leap.mail.IMessage + """ + + # TODO is staticmethod valid with an interface? + # @staticmethod + def get_msg_from_docs(self, MessageClass, msg_wrapper): + """ + Return an IMessage implementor from its parts. + + :param MessageClass: an implementor of IMessage + :param msg_wrapper: an implementor of IMessageWrapper + :rtype: implementor of leap.mail.IMessage + """ + + # ------------------------------------------------------------------- + # XXX unsure about the following part yet ........................... + + # the idea behind these three methods is that the adaptor also offers a + # fixed interface to create the documents the first time (using + # soledad.create_docs or whatever method maps to it in a similar store, and + # also allows to update flags and tags, hiding the actual implementation of + # where the flags/tags live in behind the concrete MailWrapper in use + # by this particular adaptor. In our impl it will be put_doc(fdoc) after + # locking the getting + updating of that fdoc for atomicity. + + # 'store' must be an instance of something that offers a minimal subset of + # the document API that Soledad currently implements (create_doc, put_doc) + # I *think* store should belong to Account/Collection and be passed as + # param here instead of relying on it being an attribute of the instance. + + def create_msg_docs(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ + + def update_msg_flags(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ + + def update_msg_tags(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py new file mode 100644 index 0000000..ea9c95e --- /dev/null +++ b/src/leap/mail/mail.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Access to Mail objects: Public LEAP Mail API. +""" +from twisted.internet import defer + +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors.soledad import SoledadMailAdaptor + + +# TODO +# [ ] Probably change the name of this module to "api" or "account", mail is +# too generic (there's also IncomingMail, and OutgoingMail + + +class Message(object): + + def __init__(self, wrapper): + """ + :param wrapper: an instance of an implementor of IMessageWrapper + """ + self._wrapper = wrapper + + def get_wrapper(self): + return self._wrapper + + # imap.IMessage methods + + def get_flags(): + """ + """ + + def get_internal_date(): + """ + """ + + # imap.IMessageParts + + def get_headers(): + """ + """ + + def get_body_file(): + """ + """ + + def get_size(): + """ + """ + + def is_multipart(): + """ + """ + + def get_subpart(part): + """ + """ + + # Custom methods. + + def get_tags(): + """ + """ + + +class MessageCollection(object): + """ + A generic collection of messages. It can be messages sharing the same + mailbox, tag, the result of a given query, or just a bunch of ids for + master documents. + + Since LEAP Mail is primarily oriented to store mail in Soledad, the default + (and, so far, only) implementation of the store is contained in this + Soledad Mail Adaptor. If you need to use a different adaptor, change the + adaptor class attribute in your Account object. + + Store is a reference to a particular instance of the message store (soledad + instance or proxy, for instance). + """ + + # TODO look at IMessageSet methods + + # Account should provide an adaptor instance when creating this collection. + adaptor = None + store = None + + def get_message_by_doc_id(self, doc_id): + # ... get from soledad etc + # ... but that should be part of adaptor/store too... :/ + fdoc, hdoc = None + return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + + # TODO review if this is the best place for: + + def create_docs(): + pass + + def udpate_flags(): + # 1. update the flags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + def update_tags(): + # 1. update the tags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + # TODO add delete methods here? + + +class Account(object): + """ + Account is the top level abstraction to access collections of messages + associated with a LEAP Mail Account. + + It primarily handles creation and access of Mailboxes, which will be the + basic collection handled by traditional MUAs, but it can also handle other + types of Collections (tag based, for instance). + + leap.mail.imap.SoledadBackedAccount partially proxies methods in this + class. + """ + + # Adaptor is passed to the returned MessageCollections, so if you want to + # use a different adaptor this is the place to change it, by subclassing + # the Account class. + + adaptor_class = SoledadMailAdaptor + store = None + mailboxes = None + + def __init__(self, store): + self.store = store + self.adaptor = self.adaptor_class() + + self.__mailboxes = set([]) + self._initialized = False + self._deferred_initialization = defer.Deferred() + + self._initialize_storage() + + def _initialize_storage(self): + + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.add_mailbox(INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.adaptor.initialize_store(self.store) + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + # XXX this could use adaptor.store_ready instead...?? + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + :rtype: set + """ + return sorted(self.__mailboxes) + + def _load_mailboxes(self): + + def update_mailboxes(mbox_names): + self.__mailboxes.update(mbox_names) + + d = self.adaptor.get_all_mboxes(self.store) + d.addCallback(update_mailboxes) + return d + + # + # Public API Starts + # + + # XXX params for IMAP only??? + def list_mailboxes(self, ref, wildcard): + self.adaptor.get_all_mboxes(self.store) + + def add_mailbox(self, name, mbox=None): + pass + + def create_mailbox(self, pathspec): + pass + + def delete_mailbox(self, name): + pass + + def rename_mailbox(self, oldname, newname): + pass + + # FIXME yet to be decided if it belongs here... + + def get_collection_by_mailbox(self, name): + """ + :rtype: MessageCollection + """ + # imap select will use this, passing the collection to SoledadMailbox + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_docs(self, docs): + """ + :rtype: MessageCollection + """ + # get a collection of docs by a list of doc_id + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_tag(self, tag): + """ + :rtype: MessageCollection + """ + # is this a good idea? + pass -- cgit v1.2.3 From c8dbbb6bad0463ba84ca9186693e21e9c99161a0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 26 Dec 2014 18:25:36 -0400 Subject: MessageCollections + MailboxIndexer --- src/leap/mail/adaptors/soledad.py | 173 ++++++++++-- src/leap/mail/adaptors/tests/rfc822.message | 87 +----- .../mail/adaptors/tests/test_soledad_adaptor.py | 110 ++------ src/leap/mail/constants.py | 17 ++ src/leap/mail/mail.py | 296 ++++++++++++++++----- src/leap/mail/mailbox_indexer.py | 254 ++++++++++++++++++ src/leap/mail/tests/common.py | 106 ++++++++ src/leap/mail/tests/rfc822.message | 86 ++++++ src/leap/mail/tests/test_mail.py | 95 +++++++ src/leap/mail/tests/test_mailbox_indexer.py | 241 +++++++++++++++++ 10 files changed, 1204 insertions(+), 261 deletions(-) mode change 100644 => 120000 src/leap/mail/adaptors/tests/rfc822.message create mode 100644 src/leap/mail/mailbox_indexer.py create mode 100644 src/leap/mail/tests/common.py create mode 100644 src/leap/mail/tests/rfc822.message create mode 100644 src/leap/mail/tests/test_mail.py create mode 100644 src/leap/mail/tests/test_mailbox_indexer.py diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 2e25f04..0b97869 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # soledad.py # Copyright (C) 2014 LEAP # @@ -20,6 +19,7 @@ Soledadad MailAdaptor module. import re from collections import defaultdict from email import message_from_string +from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer @@ -27,6 +27,7 @@ from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type +from leap.mail import constants from leap.mail import walk from leap.mail.adaptors import soledad_indexes as indexes from leap.mail.constants import INBOX_NAME @@ -60,7 +61,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper): It ensures atomicity of the document operations on creation, update and deletion. """ - # TODO we could also use a _dirty flag (in models) # We keep a dictionary with DeferredLocks, that will be @@ -79,6 +79,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def __init__(self, **kwargs): doc_id = kwargs.pop('doc_id', None) self._doc_id = doc_id + self._future_doc_id = kwargs.pop('future_doc_id', None) self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs) @@ -86,6 +87,13 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def doc_id(self): return self._doc_id + @property + def future_doc_id(self): + return self._future_doc_id + + def set_future_doc_id(self, doc_id): + self._future_doc_id = doc_id + def create(self, store): """ Create the documents for this wrapper. @@ -105,8 +113,14 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_doc_id(doc): self._doc_id = doc.doc_id + self._future_doc_id = None return doc - d = store.create_doc(self.serialize()) + + if self.future_doc_id is None: + d = store.create_doc(self.serialize()) + else: + d = store.create_doc(self.serialize(), + doc_id=self.future_doc_id) d.addCallback(update_doc_id) return d @@ -333,6 +347,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): class __meta__(object): index = "mbox" + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash) + self._future_doc_id = new_id + self.mbox = mbox + class HeaderDocWrapper(SoledadDocumentWrapper): @@ -370,6 +390,23 @@ class ContentDocWrapper(SoledadDocumentWrapper): index = "phash" +class MetaMsgDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "meta" + fdoc = "" + hdoc = "" + cdocs = [] + + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] + new_id = constants.METAMSGID.format(mbox=mbox, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash) + self._future_doc_id = new_id + self.fdoc = new_fdoc_id + + class MessageWrapper(object): # TODO generalize wrapper composition? @@ -378,23 +415,32 @@ class MessageWrapper(object): implements(IMessageWrapper) - def __init__(self, fdoc, hdoc, cdocs=None): + def __init__(self, mdoc, fdoc, hdoc, cdocs=None): """ - Need at least a flag-document and a header-document to instantiate a - MessageWrapper. Content-documents can be retrieved lazily. + Need at least a metamsg-document, a flag-document and a header-document + to instantiate a MessageWrapper. Content-documents can be retrieved + lazily. cdocs, if any, should be a dictionary in which the keys are ascending integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ + self.mdoc = MetaMsgDocWrapper(**mdoc) + self.fdoc = FlagsDocWrapper(**fdoc) + self.fdoc.set_future_doc_id(self.mdoc.fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + self.hdoc.set_future_doc_id(self.mdoc.hdoc) + if cdocs is None: cdocs = {} cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in cdocs.items()]) + for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + cdoc.set_future_doc_id(doc_id) def create(self, store): """ @@ -403,16 +449,21 @@ class MessageWrapper(object): leap_assert(self.cdocs, "Need non empty cdocs to create the " "MessageWrapper documents") + leap_assert(self.mdoc.doc_id is None, + "Cannot create: mdoc has a doc_id") leap_assert(self.fdoc.doc_id is None, "Cannot create: fdoc has a doc_id") + # TODO check that the doc_ids in the mdoc are coherent # TODO I think we need to tolerate the no hdoc.doc_id case, for when we # are doing a copy to another mailbox. - leap_assert(self.hdoc.doc_id is None, - "Cannot create: hdoc has a doc_id") + # leap_assert(self.hdoc.doc_id is None, + # "Cannot create: hdoc has a doc_id") d = [] + d.append(self.mdoc.create(store)) d.append(self.fdoc.create(store)) - d.append(self.hdoc.create(store)) + if self.hdoc.doc_id is None: + d.append(self.hdoc.create(store)) for cdoc in self.cdocs.values(): if cdoc.doc_id is not None: # we could be just linking to an existing @@ -432,6 +483,25 @@ class MessageWrapper(object): # garbage collector. At least the fdoc can be unlinked. raise NotImplementedError() + def copy(self, store, newmailbox): + """ + Return a copy of this MessageWrapper in a new mailbox. + """ + # 1. copy the fdoc, mdoc + # 2. remove the doc_id of that fdoc + # 3. create it (with new doc_id) + # 4. return new wrapper (new meta too!) + raise NotImplementedError() + + def set_mbox(self, mbox): + """ + Set the mailbox for this wrapper. + This method should only be used before the Documents for the + MessageWrapper have been created, will raise otherwise. + """ + self.mdoc.set_mbox(mbox) + self.fdoc.set_mbox(mbox) + # # Mailboxes # @@ -535,6 +605,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + mboxwrapper_klass = MailboxWrapper # Message handling @@ -552,11 +623,11 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) return self.get_msg_from_docs( - MessageClass, fdoc, hdoc, cdocs) + MessageClass, mdoc, fdoc, hdoc, cdocs) - def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. @@ -582,7 +653,62 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + + def _get_msg_from_variable_doc_list(self, doc_list, msg_class): + if len(doc_list) == 2: + fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 2: + fdoc, hdoc = doc_list[:2] + cdocs = dict(enumerate(doc_list[2:], 1)) + return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + + def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, + get_cdocs=False): + metamsg_id = doc_id + + def wrap_meta_doc(doc): + cls = MetaMsgDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_part_docs_from_mdoc_wrapper(wrapper): + d_docs = [] + d_docs.append(store.get_doc(wrapper.fdoc)) + d_docs.append(store.get_doc(wrapper.hdoc)) + for cdoc in wrapper.cdocs: + d_docs.append(store.get_doc(cdoc)) + d = defer.gatherResults(d_docs) + return d + + def get_parts_doc_from_mdoc_id(): + mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + def _get_hdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + d_docs = [] + fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id) + hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id) + d_docs.append(store.get_doc(fdoc_id)) + d_docs.append(store.get_doc(hdoc_id)) + d = defer.gatherResults(d_docs) + return d + + if get_cdocs: + d = store.get_doc(metamsg_id) + d.addCallback(wrap_meta_doc) + d.addCallback(get_part_docs_from_mdoc_wrapper) + else: + d = get_parts_doc_from_mdoc_id() + + d.addCallback(partial(self._get_msg_from_variable_doc_list, + msg_class=MessageClass)) + return d def create_msg(self, store, msg): """ @@ -615,7 +741,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): def get_or_create_mbox(self, store, name): """ - Get the mailbox with the given name, or creatre one if it does not + Get the mailbox with the given name, or create one if it does not exist. :param name: the name of the mailbox @@ -636,6 +762,9 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ return mbox_wrapper.update(store) + def delete_mbox(self, store, mbox_wrapper): + return mbox_wrapper.delete(store) + def get_all_mboxes(self, store): """ Retrieve a list with wrappers for all the mailboxes. @@ -660,15 +789,17 @@ def _split_into_parts(raw): walk.get_body_phash_multi][int(multi)] body_phash = body_phash_fun(walk.get_payloads(msg)) parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + cdocs_list = list(walk.get_raw_docs(msg, parts)) + cdocs_phashes = [c['phash'] for c in cdocs_list] + mdoc = _build_meta_doc(chash, cdocs_phashes) fdoc = _build_flags_doc(chash, size, multi) hdoc = _build_headers_doc(msg, chash, parts_map) # The MessageWrapper expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + cdocs = dict(enumerate(cdocs_list, 1)) - # XXX convert each to_dicts... - return fdoc, hdoc, cdocs + return mdoc, fdoc, hdoc, cdocs def _parse_msg(raw): @@ -680,6 +811,14 @@ def _parse_msg(raw): return msg, parts, chash, size, multi +def _build_meta_doc(chash, cdocs_phashes): + _mdoc = MetaMsgDocWrapper() + _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash) + _mdoc.hdoc = constants.HDOCID.format(chash=chash) + _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] + return _mdoc.serialize() + + def _build_flags_doc(chash, size, multi): _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) return _fdoc.serialize() diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message deleted file mode 100644 index ee97ab9..0000000 --- a/src/leap/mail/adaptors/tests/rfc822.message +++ /dev/null @@ -1,86 +0,0 @@ -Return-Path: -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for ; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: -List-Post: -List-Subscribe: , - -List-Id: -List-Unsubscribe: , - -List-Archive: -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message new file mode 120000 index 0000000..b19cc28 --- /dev/null +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message \ No newline at end of file diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 657a602..0cca5ef 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -18,106 +18,22 @@ Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad """ import os -import shutil -import tempfile - from functools import partial from twisted.internet import defer from twisted.trial import unittest -from leap.common.testing.basetest import BaseLeapTest from leap.mail.adaptors import models from leap.mail.adaptors.soledad import SoledadDocumentWrapper from leap.mail.adaptors.soledad import SoledadIndexMixin from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.soledad.client import Soledad - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" +from leap.mail.tests.common import SoledadTestMixin # DEBUG # import logging # logging.basicConfig(level=logging.DEBUG) -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return soledad - - -# TODO move to common module -# XXX remove duplication -class SoledadTestMixin(BaseLeapTest): - """ - It is **VERY** important that this base is added *AFTER* unittest.TestCase - """ - - def setUp(self): - self.results = [] - - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - - def tearDown(self): - """ - tearDown method called after each test. - """ - self.results = [] - try: - self._soledad.close() - except Exception as exc: - print "ERROR WHILE CLOSING SOLEDAD" - # logging.exception(exc) - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - class CounterWrapper(SoledadDocumentWrapper): class model(models.SerializableModel): counter = 0 @@ -357,7 +273,7 @@ class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): d.addCallback(assert_actor_list_is_expected) return d -here = os.path.split(os.path.abspath(__file__))[0] +HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): @@ -391,7 +307,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_string(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) @@ -416,6 +332,10 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_docs(self): adaptor = self.get_adaptor() + mdoc = dict( + fdoc="F-Foobox-deadbeef", + hdoc="H-deadbeef", + cdocs=["C-deadabad"]) fdoc = dict( mbox="Foobox", flags=('\Seen', '\Nice'), @@ -423,13 +343,14 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): seen=False, deleted=False, recent=False, multi=False) hdoc = dict( + chash="deadbeef", subject="Test Msg") cdocs = { 1: dict( raw='This is a test message')} msg = adaptor.get_msg_from_docs( - TestMessageClass, fdoc, hdoc, cdocs=cdocs) + TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) self.assertEqual(msg.wrapper.fdoc.flags, ('\Seen', '\Nice')) self.assertEqual(msg.wrapper.fdoc.tags, @@ -441,15 +362,20 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): self.assertEqual(msg.wrapper.cdocs[1].raw, "This is a test message") + def test_get_msg_from_metamsg_doc_id(self): + # XXX complete-me! + self.fail() + def test_create_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) def check_create_result(created): - self.assertEqual(len(created), 3) + # that's one mdoc, one hdoc, one fdoc, one cdoc + self.assertEqual(len(created), 4) for doc in created: self.assertTrue( doc.__class__.__name__, @@ -461,7 +387,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_update_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() def assert_msg_has_doc_id(ignored, msg): @@ -493,7 +419,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): msg = adaptor.get_msg_from_string(TestMessageClass, raw) d = adaptor.create_msg(adaptor.store, msg) d.addCallback(lambda _: adaptor.store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(partial(self.assert_num_docs, 4)) d.addCallback(assert_msg_has_doc_id, msg) d.addCallback(assert_msg_has_no_flags, msg) diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index 55bf1da..bf1db7f 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -19,3 +19,20 @@ Constants for leap.mail. """ INBOX_NAME = "INBOX" + +# Regular expressions for the identifiers to be used in the Message Data Layer. + +METAMSGID = "M-{mbox}-{chash}" +METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+" +METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" +METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" + +FDOCID = "F-{mbox}-{chash}" +FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+" +FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" + +HDOCID = "H-{chash}" +HDOCID_RE = "H\-[0-9a-fA-F]+" + +CDOCID = "C-{phash}" +CDOCID_RE = "C\-[0-9a-fA-F]+" diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ea9c95e..ca07f67 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -27,8 +28,17 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +def _get_mdoc_id(mbox, chash): + """ + Get the doc_id for the metamsg document. + """ + return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) + class Message(object): + """ + Represents a single message, and gives access to all its attributes. + """ def __init__(self, wrapper): """ @@ -37,45 +47,56 @@ class Message(object): self._wrapper = wrapper def get_wrapper(self): + """ + Get the wrapper for this message. + """ return self._wrapper # imap.IMessage methods - def get_flags(): + def get_flags(self): """ """ + return tuple(self._wrapper.fdoc.flags) - def get_internal_date(): + def get_internal_date(self): """ """ + return self._wrapper.fdoc.date # imap.IMessageParts - def get_headers(): + def get_headers(self): """ """ + # XXX process here? from imap.messages + return self._wrapper.hdoc.headers - def get_body_file(): + def get_body_file(self): """ """ - def get_size(): + def get_size(self): """ """ + return self._wrapper.fdoc.size - def is_multipart(): + def is_multipart(self): """ """ + return self._wrapper.fdoc.multi - def get_subpart(part): + def get_subpart(self, part): """ """ + # XXX ??? return MessagePart? # Custom methods. - def get_tags(): + def get_tags(self): """ """ + return tuple(self._wrapper.fdoc.tags) class MessageCollection(object): @@ -85,43 +106,174 @@ class MessageCollection(object): master documents. Since LEAP Mail is primarily oriented to store mail in Soledad, the default - (and, so far, only) implementation of the store is contained in this - Soledad Mail Adaptor. If you need to use a different adaptor, change the + (and, so far, only) implementation of the store is contained in the + Soledad Mail Adaptor, which is passed to every collection on creation by + the root Account object. If you need to use a different adaptor, change the adaptor class attribute in your Account object. Store is a reference to a particular instance of the message store (soledad instance or proxy, for instance). """ - # TODO look at IMessageSet methods + # TODO + # [ ] look at IMessageSet methods + # [ ] make constructor with a per-instance deferredLock to use on + # creation/deletion? + # [ ] instead of a mailbox, we could pass an arbitrary container with + # pointers to different doc_ids (type: foo) + # [ ] To guarantee synchronicity of the documents sent together during a + # sync, we could get hold of a deferredLock that inhibits + # synchronization while we are updating (think more about this!) # Account should provide an adaptor instance when creating this collection. adaptor = None store = None + messageklass = Message - def get_message_by_doc_id(self, doc_id): - # ... get from soledad etc - # ... but that should be part of adaptor/store too... :/ - fdoc, hdoc = None - return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + """ + """ + self.adaptor = adaptor + self.store = store - # TODO review if this is the best place for: + # TODO I have to think about what to do when there is no mbox passed to + # the initialization. We could still get the MetaMsg by index, instead + # of by doc_id. See get_message_by_content_hash + self.mbox_indexer = mbox_indexer + self.mbox_wrapper = mbox_wrapper - def create_docs(): - pass + def is_mailbox_collection(self): + """ + Return True if this collection represents a Mailbox. + :rtype: bool + """ + return bool(self.mbox_wrapper) + + # Get messages + + def get_message_by_content_hash(self, chash, get_cdocs=False): + """ + Retrieve a message by its content hash. + :rtype: Deferred + """ + + if not self.is_mailbox_collection(): + # instead of getting the metamsg by chash, query by (meta) index + # or use the internal collection of pointers-to-docs. + raise NotImplementedError() + + metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + metamsg_id, get_cdocs=get_cdocs) + + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): + """ + Retrieve a message by its Unique Identifier. + + If this is a Mailbox collection, that is the message UID, unique for a + given mailbox, or a relative sequence number depending on the absolute + flag. For now, only absolute identifiers are supported. + :rtype: Deferred + """ + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_msg_from_mdoc_id(doc_id): + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + doc_id, get_cdocs=get_cdocs) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d.addCallback(get_msg_from_mdoc_id) + return d + + def count(self): + """ + Count the messages in this collection. + :rtype: int + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.mbox_indexer.count(self.mbox_wrapper.mbox) + + # Manipulate messages + + def add_msg(self, raw_msg): + """ + Add a message to this collection. + """ + msg = self.adaptor.get_msg_from_string(Message, raw_msg) + wrapper = msg.get_wrapper() + + if self.is_mailbox_collection(): + mbox = self.mbox_wrapper.mbox + wrapper.set_mbox(mbox) + + def insert_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.insert_doc( + self.mbox_wrapper.mbox, doc_id) + + d = wrapper.create(self.store) + d.addCallback(insert_mdoc_id) + return d + + def copy_msg(self, msg, newmailbox): + """ + Copy the message to another collection. (it only makes sense for + mailbox collections) + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def insert_copied_mdoc_id(wrapper): + return self.mbox_indexer.insert_doc( + newmailbox, wrapper.mdoc.doc_id) - def udpate_flags(): + wrapper = msg.get_wrapper() + d = wrapper.copy(self.store, newmailbox) + d.addCallback(insert_copied_mdoc_id) + return d + + def delete_msg(self, msg): + """ + Delete this message. + """ + wrapper = msg.get_wrapper() + + def delete_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.delete_doc_by_hash( + self.mbox_wrapper.mbox, doc_id) + d = wrapper.delete(self.store) + d.addCallback(delete_mdoc_id) + return d + + # TODO should add a delete-by-uid to collection? + + def udpate_flags(self, msg, flags, mode): + """ + Update flags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the flags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) + # 2. update the special flags in the wrapper (seen, etc) + # 3. call adaptor.update_msg(store) pass - def update_tags(): + def update_tags(self, msg, tags, mode): + """ + Update tags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the tags in the message wrapper --- stored where??? # 2. call adaptor.update_msg(store) pass - # TODO add delete methods here? - class Account(object): """ @@ -147,8 +299,8 @@ class Account(object): def __init__(self, store): self.store = store self.adaptor = self.adaptor_class() + self.mbox_indexer = MailboxIndexer(self.store) - self.__mailboxes = set([]) self._initialized = False self._deferred_initialization = defer.Deferred() @@ -156,23 +308,16 @@ class Account(object): def _initialize_storage(self): - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: + def add_mailbox_if_none(mboxes): + if not mboxes: self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True self._deferred_initialization.callback(None) - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - d = self.adaptor.initialize_store(self.store) - d.addCallback(load_mbox_cache) + d.addCallback(self.list_all_mailbox_names) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) @@ -185,64 +330,83 @@ class Account(object): self._deferred_initialization.addCallback(cb) return self._deferred_initialization - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) + # + # Public API Starts + # - def _load_mailboxes(self): + def list_all_mailbox_names(self): + def filter_names(mboxes): + return [m.name for m in mboxes] - def update_mailboxes(mbox_names): - self.__mailboxes.update(mbox_names) + d = self.get_all_mailboxes() + d.addCallback(filter_names) + return d + def get_all_mailboxes(self): d = self.adaptor.get_all_mboxes(self.store) - d.addCallback(update_mailboxes) return d - # - # Public API Starts - # + def add_mailbox(self, name): - # XXX params for IMAP only??? - def list_mailboxes(self, ref, wildcard): - self.adaptor.get_all_mboxes(self.store) - - def add_mailbox(self, name, mbox=None): - pass + def create_uid_table_cb(res): + d = self.mbox_uid.create_table(name) + d.addCallback(lambda _: res) + return d - def create_mailbox(self, pathspec): - pass + d = self.adaptor.__class__.get_or_create(name) + d.addCallback(create_uid_table_cb) + return d def delete_mailbox(self, name): - pass + def delete_uid_table_cb(res): + d = self.mbox_uid.delete_table(name) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.delete_mbox(self.store) + d.addCallback(delete_uid_table_cb) + return d def rename_mailbox(self, oldname, newname): - pass + def _rename_mbox(wrapper): + wrapper.mbox = newname + return wrapper.update() - # FIXME yet to be decided if it belongs here... + def rename_uid_table_cb(res): + d = self.mbox_uid.rename_table(oldname, newname) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.__class__.get_or_create(oldname) + d.addCallback(_rename_mbox) + d.addCallback(rename_uid_table_cb) + return d def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection """ # imap select will use this, passing the collection to SoledadMailbox - # XXX pass adaptor to MessageCollection - pass + def get_collection_for_mailbox(mbox_wrapper): + return MessageCollection( + self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + + mboxwrapper_klass = self.adaptor.mboxwrapper_klass + d = mboxwrapper_klass.get_or_create(name) + d.addCallback(get_collection_for_mailbox) + return d def get_collection_by_docs(self, docs): """ :rtype: MessageCollection """ # get a collection of docs by a list of doc_id - # XXX pass adaptor to MessageCollection - pass + # get.docs(...) --> it should be a generator. does it behave in the + # threadpool? + raise NotImplementedError() def get_collection_by_tag(self, tag): """ :rtype: MessageCollection """ - # is this a good idea? - pass + raise NotImplementedError() diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py new file mode 100644 index 0000000..bc298ea --- /dev/null +++ b/src/leap/mail/mailbox_indexer.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Local tables to store the message Unique Identifiers for a given mailbox. +""" +import re + +from leap.mail.constants import METAMSGID_RE + + +class WrongMetaDocIDError(Exception): + pass + + +class MailboxIndexer(object): + """ + This class contains the commands needed to create, modify and alter the + local-only UID tables for a given mailbox. + + Its purpouse is to keep a local-only index with the messages in each + mailbox, mainly to satisfy the demands of the IMAP specification, but + useful too for any effective listing of the messages in a mailbox. + + Since the incoming mail can be processed at any time in any replica, it's + preferred not to attempt to maintain a global chronological global index. + + These indexes are Message Attributes needed for the IMAP specification (rfc + 3501), although they can be useful for other non-imap store + implementations. + """ + # The uids are expected to be 32-bits values, but the ROWIDs in sqlite + # are 64-bit values. I *don't* think it really matters for any + # practical use, but it's good to remmeber we've got that difference going + # on. + + store = None + table_preffix = "leapmail_uid_" + + def __init__(self, store): + self.store = store + + def _query(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_query(*args, **kw) + + def create_table(self, mailbox): + """ + Create the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("CREATE TABLE if not exists {preffix}{name}( " + "uid INTEGER PRIMARY KEY AUTOINCREMENT, " + "hash TEXT UNIQUE NOT NULL)".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def delete_table(self, mailbox): + """ + Delete the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("DROP TABLE if exists {preffix}{name}".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=oldmailbox, new=newmailbox)) + return self._query(sql) + + def insert_doc(self, mailbox, doc_id): + """ + Insert the doc_id for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M++ + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + + if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") + + def get_rowid(result): + return result[0][0] + + sql = ("INSERT INTO {preffix}{name} VALUES (" + "NULL, ?)".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + + sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + d = self._query(sql, values) + d.addCallback(lambda _: self._query(sql_last)) + d.addCallback(get_rowid) + return d + + def delete_doc_by_uid(self, mailbox, uid): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the UID of the message. + :type uid: int + :rtype: Deferred + """ + assert mailbox + assert uid + sql = ("DELETE FROM {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + return self._query(sql, values) + + def delete_doc_by_hash(self, mailbox, doc_id): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M++ + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + sql = ("DELETE FROM {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + return self._query(sql, values) + + def get_doc_id_from_uid(self, mailbox, uid): + """ + Get the doc_id for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the uid for the MetaMsg for this mailbox + :type uid: int + :rtype: Deferred + """ + def get_hash(result): + return result[0][0] + + sql = ("SELECT hash from {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + d = self._query(sql, values) + d.addCallback(get_hash) + return d + + def get_doc_ids_from_uids(self, mailbox, uids): + # For IMAP relative numbering /sequences. + # XXX dereference the range (n,*) + raise NotImplementedError() + + def count(self, mailbox): + """ + Get the number of entries in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the count. + :rtype: Deferred + """ + def get_count(result): + return result[0][0] + + sql = ("SELECT Count(*) FROM {preffix}{name};".format( + preffix=self.table_preffix, name=mailbox)) + d = self._query(sql) + d.addCallback(get_count) + return d + + def get_next_uid(self, mailbox): + """ + Get the next integer beyond the highest UID count for a given mailbox. + + This is expected by the IMAP implementation. There are no guarantees + that a document to be inserted in the future gets the returned UID: the + only thing that can be assured is that it will be equal or greater than + the value returned. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the next + uid. + :rtype: Deferred + """ + assert mailbox + + def increment(result): + return result[0][0] + 1 + + sql = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + + d = self._query(sql) + d.addCallback(increment) + return d diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py new file mode 100644 index 0000000..fefa7ee --- /dev/null +++ b/src/leap/mail/tests/common.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# common.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Common utilities for testing Soledad. +""" +import os +import shutil +import tempfile + +from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.client import Soledad + +# TODO move to common module, or Soledad itself +# XXX remove duplication + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def _initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +class SoledadTestMixin(BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = _initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception as exc: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + finally: + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) diff --git a/src/leap/mail/tests/rfc822.message b/src/leap/mail/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/mail/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py new file mode 100644 index 0000000..ce2366c --- /dev/null +++ b/src/leap/mail/tests/test_mail.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# test_mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the mail module. +""" +import os +from functools import partial + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.tests.common import SoledadTestMixin + +from twisted.internet import defer +from twisted.trial import unittest + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + + def get_collection(self, mbox_collection=True): + """ + Get a collection for tests. + """ + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + if mbox_collection: + mbox_indexer = MailboxIndexer(store) + mbox_name = "TestMbox" + else: + mbox_indexer = mbox_name = None + + def get_collection_from_mbox_wrapper(wrapper): + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + if mbox_collection: + d.addCallback(lambda _: mbox_indexer.create_table(mbox_name)) + d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + def test_is_mailbox_collection(self): + + def assert_is_mbox_collection(collection): + self.assertTrue(collection.is_mailbox_collection()) + + d = self.get_collection() + d.addCallback(assert_is_mbox_collection) + return d + + def assert_collection_count(self, _, expected, collection): + + def _assert_count(count): + self.assertEqual(count, expected) + d = collection.count() + d.addCallback(_assert_count) + return d + + def test_add_msg(self): + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + def add_msg_to_collection_and_assert_count(collection): + d = collection.add_msg(raw) + d.addCallback(partial( + self.assert_collection_count, + expected=1, collection=collection)) + return d + + d = self.get_collection() + d.addCallback(add_msg_to_collection_and_assert_count) + return d diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py new file mode 100644 index 0000000..47a3bdc --- /dev/null +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +# test_mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the mailbox_indexer module. +""" +from functools import partial + +from twisted.trial import unittest + +from leap.mail import mailbox_indexer as mi +from leap.mail.tests.common import SoledadTestMixin + +hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e' +hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014' +hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752' +hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' +hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' + + +def fmt_hash(mailbox, hash): + return "M-" + mailbox + "-" + hash + + +class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the MailboxUID class. + """ + def get_mbox_uid(self): + m_uid = mi.MailboxIndexer(self._soledad) + return m_uid + + def list_mail_tables_cb(self, ignored): + def filter_mailuid_tables(tables): + filtered = [ + table[0] for table in tables if + table[0].startswith(mi.MailboxIndexer.table_preffix)] + return filtered + + sql = "SELECT name FROM sqlite_master WHERE type='table';" + d = self._soledad.raw_sqlcipher_query(sql) + d.addCallback(filter_mailuid_tables) + return d + + def select_uid_rows(self, mailbox): + sql = "SELECT * FROM %s%s;" % ( + mi.MailboxIndexer.table_preffix, mailbox) + d = self._soledad.raw_sqlcipher_query(sql) + return d + + def test_create_table(self): + def assert_table_created(tables): + self.assertEqual( + tables, ["leapmail_uid_inbox"]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_created) + return d + + def test_create_and_delete_table(self): + def assert_table_deleted(tables): + self.assertEqual(tables, []) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(lambda _: m_uid.delete_table('inbox')) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_deleted) + return d + + def test_rename_table(self): + def assert_table_renamed(tables): + self.assertEqual( + tables, ["leapmail_uid_foomailbox"]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_renamed) + return d + + def test_insert_doc(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + def assert_uid_rows(rows): + expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(assert_uid_rows) + return d + + def test_insert_doc_return(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + def assert_rowid(rowid, expected=None): + self.assertEqual(rowid, expected) + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(partial(assert_rowid, expected=1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(partial(assert_rowid, expected=2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(partial(assert_rowid, expected=3)) + return d + + def test_delete_doc(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + def assert_uid_rows(rows): + expected = [(4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3)) + + d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(assert_uid_rows) + return d + + def test_get_doc_id_from_uid(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + + def assert_doc_hash(res): + self.assertEqual(res, h1) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1)) + d.addCallback(assert_doc_hash) + return d + + def test_count(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + def assert_count_after_inserts(count): + self.assertEquals(count, 5) + + d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(assert_count_after_inserts) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + + def assert_count_after_deletions(count): + self.assertEquals(count, 3) + + d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(assert_count_after_deletions) + return d + + def test_get_next_uid(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + def assert_next_uid(result, expected=1): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.get_next_uid(mbox)) + d.addCallback(partial(assert_next_uid, expected=6)) + return d -- cgit v1.2.3 From 4067ccbb9a9e56df0d19063dcd329d33dcb9e17b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 2 Jan 2015 10:58:43 -0400 Subject: make outgoing a new submodule --- src/leap/mail/outgoing/__init__.py | 0 src/leap/mail/outgoing/service.py | 429 ++++++++++++++++++++++++++ src/leap/mail/outgoing/tests/__init__.py | 0 src/leap/mail/outgoing/tests/test_outgoing.py | 187 +++++++++++ src/leap/mail/service.py | 422 ------------------------- src/leap/mail/smtp/__init__.py | 2 +- src/leap/mail/smtp/gateway.py | 6 +- src/leap/mail/tests/test_service.py | 187 ----------- src/leap/mail/walk.py | 64 ++-- 9 files changed, 650 insertions(+), 647 deletions(-) create mode 100644 src/leap/mail/outgoing/__init__.py create mode 100644 src/leap/mail/outgoing/service.py create mode 100644 src/leap/mail/outgoing/tests/__init__.py create mode 100644 src/leap/mail/outgoing/tests/test_outgoing.py delete mode 100644 src/leap/mail/service.py delete mode 100644 src/leap/mail/tests/test_service.py diff --git a/src/leap/mail/outgoing/__init__.py b/src/leap/mail/outgoing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py new file mode 100644 index 0000000..b70b3b1 --- /dev/null +++ b/src/leap/mail/outgoing/service.py @@ -0,0 +1,429 @@ +# -*- coding: utf-8 -*- +# outgoing/service.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import re +from StringIO import StringIO +from email.parser import Parser +from email.mime.application import MIMEApplication + +from OpenSSL import SSL + +from twisted.mail import smtp +from twisted.internet import reactor +from twisted.internet import defer +from twisted.protocols.amp import ssl +from twisted.python import log + +from leap.common.check import leap_assert_type, leap_assert +from leap.common.events import proto, signal +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey +from leap.keymanager.errors import KeyNotFound +from leap.mail import __version__ +from leap.mail.utils import validate_address +from leap.mail.smtp.rfc3156 import MultipartEncrypted +from leap.mail.smtp.rfc3156 import MultipartSigned +from leap.mail.smtp.rfc3156 import encode_base64_rec +from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator +from leap.mail.smtp.rfc3156 import PGPSignature +from leap.mail.smtp.rfc3156 import PGPEncrypted + +# TODO +# [ ] rename this module to something else, service should be the implementor +# of IService + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def getContext(self): + # FIXME -- we should use sslv23 to allow for tlsv1.2 + # and, if possible, explicitely disable sslv3 clientside. + # Servers should avoid sslv3 + self.method = SSL.TLSv1_METHOD # SSLv23_METHOD + ctx = ssl.ClientContextFactory.getContext(self) + ctx.use_certificate_file(self.cert) + ctx.use_privatekey_file(self.key) + return ctx + + +class OutgoingMail: + """ + A service for handling encrypted outgoing mail. + """ + + FOOTER_STRING = "I prefer encrypted email" + + def __init__(self, from_address, keymanager, cert, key, host, port): + """ + Initialize the mail service. + + :param from_address: The sender address. + :type from_address: str + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param cert: The client certificate for SSL authentication. + :type cert: str + :param key: The client private key for SSL authentication. + :type key: str + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + """ + + # assert params + leap_assert_type(from_address, str) + leap_assert('@' in from_address) + leap_assert_type(keymanager, KeyManager) + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + + self._port = port + self._host = host + self._key = key + self._cert = cert + self._from_address = from_address + self._keymanager = keymanager + + def send_message(self, raw, recipient): + """ + Sends a message to a recipient. Maybe encrypts and signs. + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type recipient: smtp.User + :return: a deferred which delivers the message when fired + """ + d = self._maybe_encrypt_and_sign(raw, recipient) + d.addCallback(self._route_msg) + d.addErrback(self.sendError) + return d + + def sendSuccess(self, smtp_sender_result): + """ + Callback for a successful send. + + :param smtp_sender_result: The result from the ESMTPSender from + _route_msg + :type smtp_sender_result: tuple(int, list(tuple)) + """ + dest_addrstr = smtp_sender_result[1][0][0] + log.msg('Message sent to %s' % dest_addrstr) + signal(proto.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) + + def sendError(self, failure): + """ + Callback for an unsuccessfull send. + + :param e: The result from the last errback. + :type e: anything + """ + # XXX: need to get the address from the exception to send signal + # signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + err = failure.value + log.err(err) + raise err + + def _route_msg(self, encrypt_and_sign_result): + """ + Sends the msg using the ESMTPSenderFactory. + + :param encrypt_and_sign_result: A tuple containing the 'maybe' + encrypted message and the recipient + :type encrypt_and_sign_result: tuple + """ + message, recipient = encrypt_and_sign_result + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + msg = message.as_string(False) + + # we construct a defer to pass to the ESMTPSenderFactory + d = defer.Deferred() + d.addCallbacks(self.sendSuccess, self.sendError) + # we don't pass an ssl context factory to the ESMTPSenderFactory + # because ssl will be handled by reactor.connectSSL() below. + factory = smtp.ESMTPSenderFactory( + "", # username is blank because client auth is done on SSL protocol level + "", # password is blank because client auth is done on SSL protocol level + self._from_address, + recipient.dest.addrstr, + StringIO(msg), + d, + heloFallback=True, + requireAuthentication=False, + requireTransportSecurity=True) + factory.domain = __version__ + signal(proto.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) + reactor.connectSSL( + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) + + def _maybe_encrypt_and_sign(self, raw, recipient): + """ + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type: recipient: smtp.User + + :return: A Deferred that will be fired with a MIMEMultipart message + and the original recipient Message + :rtype: Deferred + """ + # pass if the original message's content-type is "multipart/encrypted" + lines = raw.split('\r\n') + origmsg = Parser().parsestr(raw) + + if origmsg.get_content_type() == 'multipart/encrypted': + return defer.success((origmsg, recipient)) + + from_address = validate_address(self._from_address) + username, domain = from_address.split('@') + to_address = validate_address(recipient.dest.addrstr) + + # add a nice footer to the outgoing message + # XXX: footer will eventually optional or be removed + if origmsg.get_content_type() == 'text/plain': + lines.append('--') + lines.append('%s - https://%s/key/%s' % + (self.FOOTER_STRING, domain, username)) + lines.append('') + + origmsg = Parser().parsestr('\r\n'.join(lines)) + + def signal_encrypt_sign(newmsg): + signal(proto.SMTP_END_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + return newmsg, recipient + + def signal_sign(newmsg): + signal(proto.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + def if_key_not_found_send_unencrypted(failure): + if failure.check(KeyNotFound): + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._from_address) + d = self._sign(origmsg, from_address) + d.addCallback(signal_sign) + return d + else: + return failure + + log.msg("Will encrypt the message with %s and sign with %s." + % (to_address, from_address)) + signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + d = self._encrypt_and_sign(origmsg, to_address, from_address) + d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) + return d + + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): + """ + Create an RFC 3156 compliang PGP encrypted and signed message using + C{encrypt_address} to encrypt and C{sign_address} to sign. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param encrypt_address: The address used to encrypt the message. + :type encrypt_address: str + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartEncrypted message + :rtype: Deferred + """ + # create new multipart/encrypted message with 'pgp-encrypted' protocol + + def encrypt(res): + newmsg, origmsg = res + d = self._keymanager.encrypt( + origmsg.as_string(unixfrom=False), + encrypt_address, OpenPGPKey, sign=sign_address) + d.addCallback(lambda encstr: (newmsg, encstr)) + return d + + def create_encrypted_message(res): + newmsg, encstr = res + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + d = self._fix_headers( + origmsg, + MultipartEncrypted('application/pgp-encrypted'), + sign_address) + d.addCallback(encrypt) + d.addCallback(create_encrypted_message) + return d + + def _sign(self, origmsg, sign_address): + """ + Create an RFC 3156 compliant PGP signed MIME message using + C{sign_address}. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartSigned message. + :rtype: Deferred + """ + # apply base64 content-transfer-encoding + encode_base64_rec(origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + + def create_signed_message(res): + (msg, _), signature = res + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + msg.attach(origmsg) + msg.attach(sigmsg) + return msg + + dh = self._fix_headers( + origmsg, + MultipartSigned('application/pgp-signature', 'pgp-sha512'), + sign_address) + ds = self._keymanager.sign( + msgtext, sign_address, OpenPGPKey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + d = defer.gatherResults([dh, ds]) + d.addCallback(create_signed_message) + return d + + def _fix_headers(self, origmsg, newmsg, sign_address): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param origmsg: The original message. + :type origmsg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param sign_address: The address used to sign C{newmsg} + :type sign_address: str + + :return: A Deferred with a touple: + (new Message with the unencrypted headers, + original Message with headers removed) + :rtype: Deferred + """ + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del (origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # delete user-agent from origmsg + del (origmsg['user-agent']) + + def add_openpgp_header(signkey): + username, domain = sign_address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + return newmsg, origmsg + + d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True) + d.addCallback(add_openpgp_header) + return d diff --git a/src/leap/mail/outgoing/tests/__init__.py b/src/leap/mail/outgoing/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py new file mode 100644 index 0000000..fa50c30 --- /dev/null +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +SMTP gateway tests. +""" + +import re +from datetime import datetime +from twisted.internet.defer import fail +from twisted.mail.smtp import User + +from mock import Mock + +from leap.mail.smtp.gateway import SMTPFactory +from leap.mail.outgoing.service import OutgoingMail +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, + ADDRESS_2, +) +from leap.keymanager import openpgp, errors + + +class TestOutgoingMail(TestCaseWithKeyManager): + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + self.lines = [line for line in self.EMAIL_DATA[4:12]] + self.lines.append('') # add a trailing newline + self.raw = '\r\n'.join(self.lines) + self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + + "\r\n\r\n--\r\nI prefer encrypted email - " + "https://leap.se/key/anotheruser\r\n") + self.fromAddr = ADDRESS_2 + + def init_outgoing_and_proto(_): + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], + self._config['key'], self._config['host'], + self._config['port']) + self.proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, + self._config['encrypted_only'], + self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) + + d = TestCaseWithKeyManager.setUp(self) + d.addCallback(init_outgoing_and_proto) + return d + + def test_message_encrypt(self): + """ + Test if message gets encrypted to destination email. + """ + def check_decryption(res): + decrypted, _ = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) + d.addCallback(check_decryption) + return d + + def test_message_encrypt_sign(self): + """ + Test if message gets encrypted to destination email and signed with + sender key. + '""" + def check_decryption_and_verify(res): + decrypted, signkey = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + self.assertTrue(ADDRESS_2 in signkey.address, + "Verification failed") + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, + verify=ADDRESS_2)) + d.addCallback(check_decryption_and_verify) + return d + + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], self._config['key'], + self._config['host'], self._config['port']) + + def check_signed(res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + self.assertEqual(self.expected_body, + message.get_payload(0).get_payload(decode=True)) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + return message + + def verify(message): + # replace EOL before verifying (according to rfc3156) + signed_text = re.sub('\r?\n', '\r\n', + message.get_payload(0).as_string()) + + def assert_verify(key): + self.assertTrue(ADDRESS_2 in key.address, + 'Signature could not be verified.') + + d = self._km.verify( + signed_text, ADDRESS_2, openpgp.OpenPGPKey, + detached_sig=message.get_payload(1).get_payload()) + d.addCallback(assert_verify) + return d + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + d.addCallback(check_signed) + d.addCallback(verify) + return d + + def _assert_encrypted(self, res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/encrypted', message.get_content_type()) + self.assertEqual('application/pgp-encrypted', + message.get_param('protocol')) + self.assertEqual(2, len(message.get_payload())) + self.assertEqual('application/pgp-encrypted', + message.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + message.get_payload(1).get_content_type()) + return message diff --git a/src/leap/mail/service.py b/src/leap/mail/service.py deleted file mode 100644 index a99f13a..0000000 --- a/src/leap/mail/service.py +++ /dev/null @@ -1,422 +0,0 @@ -# -*- coding: utf-8 -*- -# service.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import re -from StringIO import StringIO -from email.parser import Parser -from email.mime.application import MIMEApplication - -from OpenSSL import SSL - -from twisted.mail import smtp -from twisted.internet import reactor -from twisted.internet import defer -from twisted.protocols.amp import ssl -from twisted.python import log - -from leap.common.check import leap_assert_type, leap_assert -from leap.common.events import proto, signal -from leap.keymanager import KeyManager -from leap.keymanager.openpgp import OpenPGPKey -from leap.keymanager.errors import KeyNotFound -from leap.mail import __version__ -from leap.mail.utils import validate_address -from leap.mail.smtp.rfc3156 import MultipartEncrypted -from leap.mail.smtp.rfc3156 import MultipartSigned -from leap.mail.smtp.rfc3156 import encode_base64_rec -from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator -from leap.mail.smtp.rfc3156 import PGPSignature -from leap.mail.smtp.rfc3156 import PGPEncrypted - - -class SSLContextFactory(ssl.ClientContextFactory): - def __init__(self, cert, key): - self.cert = cert - self.key = key - - def getContext(self): - self.method = SSL.TLSv1_METHOD # SSLv23_METHOD - ctx = ssl.ClientContextFactory.getContext(self) - ctx.use_certificate_file(self.cert) - ctx.use_privatekey_file(self.key) - return ctx - - -class OutgoingMail: - """ - A service for handling encrypted mail. - """ - - FOOTER_STRING = "I prefer encrypted email" - - def __init__(self, from_address, keymanager, cert, key, host, port): - """ - Initialize the mail service. - - :param from_address: The sender address. - :type from_address: str - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param cert: The client certificate for SSL authentication. - :type cert: str - :param key: The client private key for SSL authentication. - :type key: str - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - """ - - # assert params - leap_assert_type(from_address, str) - leap_assert('@' in from_address) - leap_assert_type(keymanager, KeyManager) - leap_assert_type(host, str) - leap_assert(host != '') - leap_assert_type(port, int) - leap_assert(port is not 0) - leap_assert_type(cert, unicode) - leap_assert(cert != '') - leap_assert_type(key, unicode) - leap_assert(key != '') - - self._port = port - self._host = host - self._key = key - self._cert = cert - self._from_address = from_address - self._keymanager = keymanager - - def send_message(self, raw, recipient): - """ - Sends a message to a recipient. Maybe encrypts and signs. - - :param raw: The raw message - :type raw: str - :param recipient: The recipient for the message - :type recipient: smtp.User - :return: a deferred which delivers the message when fired - """ - d = self._maybe_encrypt_and_sign(raw, recipient) - d.addCallback(self._route_msg) - d.addErrback(self.sendError) - return d - - def sendSuccess(self, smtp_sender_result): - """ - Callback for a successful send. - - :param smtp_sender_result: The result from the ESMTPSender from - _route_msg - :type smtp_sender_result: tuple(int, list(tuple)) - """ - dest_addrstr = smtp_sender_result[1][0][0] - log.msg('Message sent to %s' % dest_addrstr) - signal(proto.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) - - def sendError(self, failure): - """ - Callback for an unsuccessfull send. - - :param e: The result from the last errback. - :type e: anything - """ - # XXX: need to get the address from the exception to send signal - # signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) - err = failure.value - log.err(err) - raise err - - def _route_msg(self, encrypt_and_sign_result): - """ - Sends the msg using the ESMTPSenderFactory. - - :param encrypt_and_sign_result: A tuple containing the 'maybe' - encrypted message and the recipient - :type encrypt_and_sign_result: tuple - """ - message, recipient = encrypt_and_sign_result - log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) - msg = message.as_string(False) - - # we construct a defer to pass to the ESMTPSenderFactory - d = defer.Deferred() - d.addCallbacks(self.sendSuccess, self.sendError) - # we don't pass an ssl context factory to the ESMTPSenderFactory - # because ssl will be handled by reactor.connectSSL() below. - factory = smtp.ESMTPSenderFactory( - "", # username is blank because client auth is done on SSL protocol level - "", # password is blank because client auth is done on SSL protocol level - self._from_address, - recipient.dest.addrstr, - StringIO(msg), - d, - heloFallback=True, - requireAuthentication=False, - requireTransportSecurity=True) - factory.domain = __version__ - signal(proto.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) - reactor.connectSSL( - self._host, self._port, factory, - contextFactory=SSLContextFactory(self._cert, self._key)) - - def _maybe_encrypt_and_sign(self, raw, recipient): - """ - Attempt to encrypt and sign the outgoing message. - - The behaviour of this method depends on: - - 1. the original message's content-type, and - 2. the availability of the recipient's public key. - - If the original message's content-type is "multipart/encrypted", then - the original message is not altered. For any other content-type, the - method attempts to fetch the recipient's public key. If the - recipient's public key is available, the message is encrypted and - signed; otherwise it is only signed. - - Note that, if the C{encrypted_only} configuration is set to True and - the recipient's public key is not available, then the recipient - address would have been rejected in SMTPDelivery.validateTo(). - - The following table summarizes the overall behaviour of the gateway: - - +---------------------------------------------------+----------------+ - | content-type | rcpt pubkey | enforce encr. | action | - +---------------------+-------------+---------------+----------------+ - | multipart/encrypted | any | any | pass | - | other | available | any | encrypt + sign | - | other | unavailable | yes | reject | - | other | unavailable | no | sign | - +---------------------+-------------+---------------+----------------+ - - :param raw: The raw message - :type raw: str - :param recipient: The recipient for the message - :type: recipient: smtp.User - - :return: A Deferred that will be fired with a MIMEMultipart message - and the original recipient Message - :rtype: Deferred - """ - # pass if the original message's content-type is "multipart/encrypted" - lines = raw.split('\r\n') - origmsg = Parser().parsestr(raw) - - if origmsg.get_content_type() == 'multipart/encrypted': - return defer.success((origmsg, recipient)) - - from_address = validate_address(self._from_address) - username, domain = from_address.split('@') - to_address = validate_address(recipient.dest.addrstr) - - # add a nice footer to the outgoing message - # XXX: footer will eventually optional or be removed - if origmsg.get_content_type() == 'text/plain': - lines.append('--') - lines.append('%s - https://%s/key/%s' % - (self.FOOTER_STRING, domain, username)) - lines.append('') - - origmsg = Parser().parsestr('\r\n'.join(lines)) - - def signal_encrypt_sign(newmsg): - signal(proto.SMTP_END_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) - return newmsg, recipient - - def signal_sign(newmsg): - signal(proto.SMTP_END_SIGN, self._from_address) - return newmsg, recipient - - def if_key_not_found_send_unencrypted(failure): - if failure.check(KeyNotFound): - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._from_address) - d = self._sign(origmsg, from_address) - d.addCallback(signal_sign) - return d - else: - return failure - - log.msg("Will encrypt the message with %s and sign with %s." - % (to_address, from_address)) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) - d = self._encrypt_and_sign(origmsg, to_address, from_address) - d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) - return d - - def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): - """ - Create an RFC 3156 compliang PGP encrypted and signed message using - C{encrypt_address} to encrypt and C{sign_address} to sign. - - :param origmsg: The original message - :type origmsg: email.message.Message - :param encrypt_address: The address used to encrypt the message. - :type encrypt_address: str - :param sign_address: The address used to sign the message. - :type sign_address: str - - :return: A Deferred with the MultipartEncrypted message - :rtype: Deferred - """ - # create new multipart/encrypted message with 'pgp-encrypted' protocol - - def encrypt(res): - newmsg, origmsg = res - d = self._keymanager.encrypt( - origmsg.as_string(unixfrom=False), - encrypt_address, OpenPGPKey, sign=sign_address) - d.addCallback(lambda encstr: (newmsg, encstr)) - return d - - def create_encrypted_message(res): - newmsg, encstr = res - encmsg = MIMEApplication( - encstr, _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - return newmsg - - d = self._fix_headers( - origmsg, - MultipartEncrypted('application/pgp-encrypted'), - sign_address) - d.addCallback(encrypt) - d.addCallback(create_encrypted_message) - return d - - def _sign(self, origmsg, sign_address): - """ - Create an RFC 3156 compliant PGP signed MIME message using - C{sign_address}. - - :param origmsg: The original message - :type origmsg: email.message.Message - :param sign_address: The address used to sign the message. - :type sign_address: str - - :return: A Deferred with the MultipartSigned message. - :rtype: Deferred - """ - # apply base64 content-transfer-encoding - encode_base64_rec(origmsg) - # get message text with headers and replace \n for \r\n - fp = StringIO() - g = RFC3156CompliantGenerator( - fp, mangle_from_=False, maxheaderlen=76) - g.flatten(origmsg) - msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) - # make sure signed message ends with \r\n as per OpenPGP stantard. - if origmsg.is_multipart(): - if not msgtext.endswith("\r\n"): - msgtext += "\r\n" - - def create_signed_message(res): - (msg, _), signature = res - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - msg.attach(origmsg) - msg.attach(sigmsg) - return msg - - dh = self._fix_headers( - origmsg, - MultipartSigned('application/pgp-signature', 'pgp-sha512'), - sign_address) - ds = self._keymanager.sign( - msgtext, sign_address, OpenPGPKey, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - d = defer.gatherResults([dh, ds]) - d.addCallback(create_signed_message) - return d - - def _fix_headers(self, origmsg, newmsg, sign_address): - """ - Move some headers from C{origmsg} to C{newmsg}, delete unwanted - headers from C{origmsg} and add new headers to C{newms}. - - Outgoing messages are either encrypted and signed or just signed - before being sent. Because of that, they are packed inside new - messages and some manipulation has to be made on their headers. - - Allowed headers for passing through: - - - From - - Date - - To - - Subject - - Reply-To - - References - - In-Reply-To - - Cc - - Headers to be added: - - - Message-ID (i.e. should not use origmsg's Message-Id) - - Received (this is added automatically by twisted smtp API) - - OpenPGP (see #4447) - - Headers to be deleted: - - - User-Agent - - :param origmsg: The original message. - :type origmsg: email.message.Message - :param newmsg: The new message being created. - :type newmsg: email.message.Message - :param sign_address: The address used to sign C{newmsg} - :type sign_address: str - - :return: A Deferred with a touple: - (new Message with the unencrypted headers, - original Message with headers removed) - :rtype: Deferred - """ - # move headers from origmsg to newmsg - headers = origmsg.items() - passthrough = [ - 'from', 'date', 'to', 'subject', 'reply-to', 'references', - 'in-reply-to', 'cc' - ] - headers = filter(lambda x: x[0].lower() in passthrough, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del (origmsg[hkey]) - # add a new message-id to newmsg - newmsg.add_header('Message-Id', smtp.messageid()) - # delete user-agent from origmsg - del (origmsg['user-agent']) - - def add_openpgp_header(signkey): - username, domain = sign_address.split('@') - newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, - url='https://%s/key/%s' % (domain, username), - preference='signencrypt') - return newmsg, origmsg - - d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True) - d.addCallback(add_openpgp_header) - return d diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 72b26ed..24402b4 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -22,7 +22,7 @@ import logging from twisted.internet import reactor from twisted.internet.error import CannotListenError -from leap.mail.service import OutgoingMail +from leap.mail.outgoing.service import OutgoingMail logger = logging.getLogger(__name__) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index d58c581..222ef3f 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -93,7 +93,7 @@ class SMTPFactory(ServerFactory): mail or not. :type encrypted_only: bool :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.service.OutgoingMail + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail """ leap_assert_type(encrypted_only, bool) @@ -141,7 +141,7 @@ class SMTPDelivery(object): mail or not. :type encrypted_only: bool :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.service.OutgoingMail + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail """ self._userid = userid self._outgoing_mail = outgoing_mail @@ -266,7 +266,7 @@ class EncryptedMessage(object): :param user: The recipient of this message. :type user: twisted.mail.smtp.User :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.service.OutgoingMail + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail """ # assert params leap_assert_type(user, smtp.User) diff --git a/src/leap/mail/tests/test_service.py b/src/leap/mail/tests/test_service.py deleted file mode 100644 index 43f354d..0000000 --- a/src/leap/mail/tests/test_service.py +++ /dev/null @@ -1,187 +0,0 @@ -# -*- coding: utf-8 -*- -# test_gateway.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -SMTP gateway tests. -""" - -import re -from datetime import datetime -from twisted.internet.defer import fail -from twisted.mail.smtp import User - -from mock import Mock - -from leap.mail.smtp.gateway import SMTPFactory -from leap.mail.service import OutgoingMail -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, -) -from leap.keymanager import openpgp, errors - - -class TestOutgoingMail(TestCaseWithKeyManager): - EMAIL_DATA = ['HELO gateway.leap.se', - 'MAIL FROM: <%s>' % ADDRESS_2, - 'RCPT TO: <%s>' % ADDRESS, - 'DATA', - 'From: User <%s>' % ADDRESS_2, - 'To: Leap <%s>' % ADDRESS, - 'Date: ' + datetime.now().strftime('%c'), - 'Subject: test message', - '', - 'This is a secret message.', - 'Yours,', - 'A.', - '', - '.', - 'QUIT'] - - def setUp(self): - self.lines = [line for line in self.EMAIL_DATA[4:12]] - self.lines.append('') # add a trailing newline - self.raw = '\r\n'.join(self.lines) - self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + - "\r\n\r\n--\r\nI prefer encrypted email - " - "https://leap.se/key/anotheruser\r\n") - self.fromAddr = ADDRESS_2 - - def init_outgoing_and_proto(_): - self.outgoing_mail = OutgoingMail( - self.fromAddr, self._km, self._config['cert'], - self._config['key'], self._config['host'], - self._config['port']) - self.proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) - self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) - - d = TestCaseWithKeyManager.setUp(self) - d.addCallback(init_outgoing_and_proto) - return d - - def test_message_encrypt(self): - """ - Test if message gets encrypted to destination email. - """ - def check_decryption(res): - decrypted, _ = res - self.assertEqual( - '\n' + self.expected_body, - decrypted, - 'Decrypted text differs from plaintext.') - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - d.addCallback(self._assert_encrypted) - d.addCallback(lambda message: self._km.decrypt( - message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) - d.addCallback(check_decryption) - return d - - def test_message_encrypt_sign(self): - """ - Test if message gets encrypted to destination email and signed with - sender key. - '""" - def check_decryption_and_verify(res): - decrypted, signkey = res - self.assertEqual( - '\n' + self.expected_body, - decrypted, - 'Decrypted text differs from plaintext.') - self.assertTrue(ADDRESS_2 in signkey.address, - "Verification failed") - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - d.addCallback(self._assert_encrypted) - d.addCallback(lambda message: self._km.decrypt( - message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, - verify=ADDRESS_2)) - d.addCallback(check_decryption_and_verify) - return d - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self._km._fetch_keys_from_server = Mock( - return_value=fail(errors.KeyNotFound())) - recipient = User('ihavenopubkey@nonleap.se', - 'gateway.leap.se', self.proto, ADDRESS) - self.outgoing_mail = OutgoingMail( - self.fromAddr, self._km, self._config['cert'], self._config['key'], - self._config['host'], self._config['port']) - - def check_signed(res): - message, _ = res - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/signed', message.get_content_type()) - self.assertEqual('application/pgp-signature', - message.get_param('protocol')) - self.assertEqual('pgp-sha512', message.get_param('micalg')) - # assert content of message - self.assertEqual(self.expected_body, - message.get_payload(0).get_payload(decode=True)) - # assert content of signature - self.assertTrue( - message.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - message.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - return message - - def verify(message): - # replace EOL before verifying (according to rfc3156) - signed_text = re.sub('\r?\n', '\r\n', - message.get_payload(0).as_string()) - - def assert_verify(key): - self.assertTrue(ADDRESS_2 in key.address, - 'Signature could not be verified.') - - d = self._km.verify( - signed_text, ADDRESS_2, openpgp.OpenPGPKey, - detached_sig=message.get_payload(1).get_payload()) - d.addCallback(assert_verify) - return d - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) - d.addCallback(check_signed) - d.addCallback(verify) - return d - - def _assert_encrypted(self, res): - message, _ = res - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/encrypted', message.get_content_type()) - self.assertEqual('application/pgp-encrypted', - message.get_param('protocol')) - self.assertEqual(2, len(message.get_payload())) - self.assertEqual('application/pgp-encrypted', - message.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - message.get_payload(1).get_content_type()) - return message diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index f747377..5172837 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -85,6 +85,35 @@ get_raw_docs = lambda msg, parts: ( for payload, headers in get_payloads(msg) if not isinstance(payload, list)) +""" +Groucho Marx: Now pay particular attention to this first clause, because it's + most important. There's the party of the first part shall be + known in this contract as the party of the first part. How do you + like that, that's pretty neat eh? + +Chico Marx: No, that's no good. +Groucho Marx: What's the matter with it? + +Chico Marx: I don't know, let's hear it again. +Groucho Marx: So the party of the first part shall be known in this contract as + the party of the first part. + +Chico Marx: Well it sounds a little better this time. +Groucho Marx: Well, it grows on you. Would you like to hear it once more? + +Chico Marx: Just the first part. +Groucho Marx: All right. It says the first part of the party of the first part + shall be known in this contract as the first part of the party of + the first part, shall be known in this contract - look, why + should we quarrel about a thing like this, we'll take it right + out, eh? + +Chico Marx: Yes, it's too long anyhow. Now what have we got left? +Groucho Marx: Well I've got about a foot and a half. Now what's the matter? + +Chico Marx: I don't like the second party either. +""" + def walk_msg_tree(parts, body_phash=None): """ @@ -162,7 +191,7 @@ def walk_msg_tree(parts, body_phash=None): outer = parts[0] outer.pop(HEADERS) - if not PART_MAP in outer: + if PART_MAP not in outer: # we have a multipart with 1 part only, so kind of fix it # although it would be prettier if I take this special case at # the beginning of the walk. @@ -177,36 +206,3 @@ def walk_msg_tree(parts, body_phash=None): pdoc = outer pdoc[BODY] = body_phash return pdoc - -""" -Groucho Marx: Now pay particular attention to this first clause, because it's - most important. There's the party of the first part shall be - known in this contract as the party of the first part. How do you - like that, that's pretty neat eh? - -Chico Marx: No, that's no good. -Groucho Marx: What's the matter with it? - -Chico Marx: I don't know, let's hear it again. -Groucho Marx: So the party of the first part shall be known in this contract as - the party of the first part. - -Chico Marx: Well it sounds a little better this time. -Groucho Marx: Well, it grows on you. Would you like to hear it once more? - -Chico Marx: Just the first part. -Groucho Marx: All right. It says the first part of the party of the first part - shall be known in this contract as the first part of the party of - the first part, shall be known in this contract - look, why - should we quarrel about a thing like this, we'll take it right - out, eh? - -Chico Marx: Yes, it's too long anyhow. Now what have we got left? -Groucho Marx: Well I've got about a foot and a half. Now what's the matter? - -Chico Marx: I don't like the second party either. -""" - -""" -I feel you deserved it after reading the above and try to debug your problem ;) -""" -- cgit v1.2.3 From 30387b83756ba078d04f4af701e3f595e30d9c50 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 1 Jan 2015 18:21:44 -0400 Subject: cleanup imap implementation --- src/leap/mail/adaptors/soledad.py | 13 +- src/leap/mail/constants.py | 14 + src/leap/mail/imap/account.py | 306 +++----- src/leap/mail/imap/fields.py | 51 -- src/leap/mail/imap/interfaces.py | 96 --- src/leap/mail/imap/mailbox.py | 472 ++++--------- src/leap/mail/imap/memorystore.py | 1340 ------------------------------------ src/leap/mail/imap/messageparts.py | 586 ---------------- src/leap/mail/imap/messages.py | 1007 +++++---------------------- src/leap/mail/imap/soledadstore.py | 617 ----------------- src/leap/mail/mail.py | 93 ++- src/leap/mail/messageflow.py | 200 ------ 12 files changed, 522 insertions(+), 4273 deletions(-) delete mode 100644 src/leap/mail/imap/fields.py delete mode 100644 src/leap/mail/imap/interfaces.py delete mode 100644 src/leap/mail/imap/memorystore.py delete mode 100644 src/leap/mail/imap/messageparts.py delete mode 100644 src/leap/mail/imap/soledadstore.py delete mode 100644 src/leap/mail/messageflow.py diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 0b97869..bf8f7e9 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -513,9 +513,13 @@ class MailboxWrapper(SoledadDocumentWrapper): type_ = "mbox" mbox = INBOX_NAME flags = [] + recent = [] + created = 1 closed = False subscribed = False - rw = True + + # I think we don't need to store this one. + # rw = True class __meta__(object): index = "mbox" @@ -655,6 +659,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): assert(MessageClass is not None) return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + # XXX pass UID too? def _get_msg_from_variable_doc_list(self, doc_list, msg_class): if len(doc_list) == 2: fdoc, hdoc = doc_list @@ -664,12 +669,14 @@ class SoledadMailAdaptor(SoledadIndexMixin): cdocs = dict(enumerate(doc_list[2:], 1)) return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + # XXX pass UID too ? def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, get_cdocs=False): metamsg_id = doc_id def wrap_meta_doc(doc): cls = MetaMsgDocWrapper + # XXX pass UID? return cls(doc_id=doc.doc_id, **doc.content) def get_part_docs_from_mdoc_wrapper(wrapper): @@ -692,8 +699,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return constants.FDOCID.format(mbox=mbox, chash=chash) d_docs = [] - fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id) - hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id) + fdoc_id = _get_fdoc_id_from_mdoc_id() + hdoc_id = _get_hdoc_id_from_mdoc_id() d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) d = defer.gatherResults(d_docs) diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index bf1db7f..d76e652 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -36,3 +36,17 @@ HDOCID_RE = "H\-[0-9a-fA-F]+" CDOCID = "C-{phash}" CDOCID_RE = "C\-[0-9a-fA-F]+" + + +class MessageFlags(object): + """ + Flags used in Message and Mailbox. + """ + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 7dfbbd1..0baf078 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # account.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,12 +15,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Soledad Backed Account. +Soledad Backed IMAP Account. """ -import copy import logging import os import time +from functools import partial from twisted.internet import defer from twisted.mail import imap4 @@ -29,9 +29,9 @@ from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import MessageFlags from leap.mail.mail import Account -from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox +from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -49,9 +49,10 @@ if PROFILE_CMD: # Soledad IMAP Account ####################################### -# TODO remove MsgFields too +# XXX watchout, account needs to be ready... so we should maybe return +# a deferred to the IMAP service when it's initialized -class IMAPAccount(WithMsgFields): +class IMAPAccount(object): """ An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. @@ -72,37 +73,20 @@ class IMAPAccount(WithMsgFields): :param store: a Soledad instance. :type store: Soledad """ - # XXX assert a generic store interface instead, so that we - # can plug the memory store wrapper seamlessly. leap_assert(store, "Need a store instance to initialize") leap_assert_type(store, Soledad) - # XXX SHOULD assert too that the name matches the user/uuid with which + # TODO assert too that the name matches the user/uuid with which # soledad has been initialized. self.user_id = user_id self.account = Account(store) - # XXX should hide this in the adaptor... - def _get_mailbox_by_name(self, name): - """ - Return an mbox document by name. - - :param name: the name of the mailbox - :type name: str - - :rtype: SoledadDocument - """ - def get_first_if_any(docs): - return docs[0] if docs else None - - d = self._store.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, - normalize_mailbox(name)) - d.addCallback(get_first_if_any) - return d + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + return IMAPMailbox(collection, rw=readwrite) - # XXX move to Account? - # XXX needed? + # XXX Where's this used from? -- self.delete... def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -110,31 +94,25 @@ class IMAPAccount(WithMsgFields): :param name: name of the mailbox :type name: str - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox + :returns: an IMAPMailbox instance + :rtype: IMAPMailbox """ name = normalize_mailbox(name) - if name not in self.account.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) + def check_it_exists(mailboxes): + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) - # XXX Does mailbox really need reference to soledad? - return SoledadMailbox(name, self._store) + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallbacK(self._return_mailbox_from_collection) + return d # # IAccount # - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - # XXX move to mailbox module - return copy.deepcopy(mailbox.EMPTY_MBOX) - - # TODO use mail.Account.add_mailbox def addMailbox(self, name, creation_ts=None): """ Add a mailbox to the account. @@ -154,8 +132,9 @@ class IMAPAccount(WithMsgFields): leap_assert(name, "Need a mailbox name to create a mailbox") - if name in self.mailboxes: - raise imap4.MailboxCollision(repr(name)) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision(repr(name)) if creation_ts is None: # by default, we pass an int value @@ -164,21 +143,18 @@ class IMAPAccount(WithMsgFields): # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) + def set_mbox_creation_ts(collection): + d = collection.set_mbox_attr("created") + d.addCallback(lambda _: collection) return d - d = self._store.create_doc(mbox) - d.addCallback(load_mbox_cache) + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_does_not_exist) + d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallback(set_mbox_creation_ts) + d.addCallback(self._return_mailbox_from_collection) return d - # TODO use mail.Account.create_mailbox? - # Watch out, imap specific exceptions raised here. def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. @@ -204,9 +180,10 @@ class IMAPAccount(WithMsgFields): for accum in range(1, len(paths)): try: - partial = sep.join(paths[:accum]) - d = self.addMailbox(partial) + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) subs.append(d) + # XXX should this be handled by the deferred? except imap4.MailboxCollision: pass try: @@ -222,21 +199,13 @@ class IMAPAccount(WithMsgFields): def all_good(result): return all(result) - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - if subs: d1 = defer.gatherResults(subs, consumeErrors=True) - d1.addCallback(load_mbox_cache) d1.addCallback(all_good) else: d1 = defer.succeed(False) - d1.addCallback(load_mbox_cache) return d1 - # TODO use mail.Account.get_collection_by_mailbox def select(self, name, readwrite=1): """ Selects a mailbox. @@ -250,15 +219,28 @@ class IMAPAccount(WithMsgFields): :rtype: SoledadMailbox """ name = normalize_mailbox(name) - if name not in self.mailboxes: - logger.warning("No such mailbox!") - return None - self.selected = name - sm = SoledadMailbox(name, self._store, readwrite) - return sm + def check_it_exists(mailboxes): + if name not in mailboxes: + logger.warning("SELECT: No such mailbox!") + return None + return name + + def set_selected(_): + self.selected = name + + def get_collection(name): + if name is None: + return None + return self.account.get_collection_by_mailbox(name) + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_collection) + d.addCallback(partial( + self._return_mailbox_from_collection, readwrite=readwrite)) + return d - # TODO use mail.Account.delete_mailbox def delete(self, name, force=False): """ Deletes a mailbox. @@ -276,37 +258,52 @@ class IMAPAccount(WithMsgFields): :rtype: Deferred """ name = normalize_mailbox(name) + _mboxes = [] - if name not in self.mailboxes: - err = imap4.MailboxException("No such mailbox: %r" % name) - return defer.fail(err) - mbox = self.getMailbox(name) + def check_it_exists(mailboxes): + # FIXME works? -- pass variable ref to outer scope + _mboxes = mailboxes + if name not in mailboxes: + err = imap4.MailboxException("No such mailbox: %r" % name) + return defer.fail(err) - if not force: + def get_mailbox(_): + return self.getMailbox(name) + + def destroy_mailbox(mbox): + return mbox.destroy() + + def check_can_be_deleted(mbox): # See if this box is flagged \Noselect - # XXX use mbox.flags instead? mbox_flags = mbox.getFlags() - if self.NOSELECT_FLAG in mbox_flags: + if MessageFlags.NOSELECT_FLAG in mbox_flags: # Check for hierarchically inferior mailboxes with this one # as part of their root. - for others in self.mailboxes: + for others in _mboxes: if others != name and others.startswith(name): err = imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") return defer.fail(err) - self.__mailboxes.discard(name) - return mbox.destroy() + return mbox - # XXX FIXME --- not honoring the inferior names... + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_mailbox) + if not force: + d.addCallback(check_can_be_deleted) + d.addCallback(destroy_mailbox) + return d + # FIXME --- not honoring the inferior names... # if there are no hierarchically inferior names, we will # delete it from our ken. + # XXX is this right? # if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - # self._index.removeMailbox(name) + # self._index.removeMailbox(name) # TODO use mail.Account.rename_mailbox + # TODO finish conversion to deferreds def rename(self, oldname, newname): """ Renames a mailbox. @@ -320,6 +317,9 @@ class IMAPAccount(WithMsgFields): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) + # FIXME check that scope works (test) + _mboxes = [] + if oldname not in self.mailboxes: raise imap4.NoSuchMailbox(repr(oldname)) @@ -327,34 +327,19 @@ class IMAPAccount(WithMsgFields): inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] for (old, new) in inferiors: - if new in self.mailboxes: + if new in _mboxes: raise imap4.MailboxCollision(repr(new)) rename_deferreds = [] - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - - def update_mbox_doc_name(mbox, oldname, newname, update_deferred): - mbox.content[self.MBOX_KEY] = newname - d = self._soledad.put_doc(mbox) - d.addCallback(lambda r: update_deferred.callback(True)) - for (old, new) in inferiors: - self.__mailboxes.discard(old) - self._memstore.rename_fdocs_mailbox(old, new) - - d0 = defer.Deferred() - d = self._get_mailbox_by_name(old) - d.addCallback(update_mbox_doc_name, old, new, d0) - rename_deferreds.append(d0) + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - d1.addCallback(load_mbox_cache) return d1 + # FIXME use deferreds (list_all_mailbox_names, etc) def _inferiorNames(self, name): """ Return hierarchically inferior mailboxes. @@ -387,16 +372,15 @@ class IMAPAccount(WithMsgFields): :type wildcard: str """ # XXX use wildcard in index query - ref = self._inferiorNames(normalize_mailbox(ref)) + # TODO get deferreds wildcard = imap4.wildcardToRegexp(wildcard, '/') + ref = self._inferiorNames(normalize_mailbox(ref)) return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] # # The rest of the methods are specific for leap.mail.imap.account.Account # - # TODO ------------------ can we preserve the attr? - # maybe add to memory store. def isSubscribed(self, name): """ Returns True if user is subscribed to this mailbox. @@ -406,63 +390,13 @@ class IMAPAccount(WithMsgFields): :rtype: Deferred (will fire with bool) """ - # TODO use Flags class - subscribed = self.SUBSCRIBED_KEY - - def is_subscribed(mbox): - subs_bool = bool(mbox.content.get(subscribed, False)) - return subs_bool - - d = self._get_mailbox_by_name(name) - d.addCallback(is_subscribed) - return d - - # TODO ------------------ can we preserve the property? - # maybe add to memory store. - - def _get_subscriptions(self): - """ - Return a list of the current subscriptions for this account. - - :returns: A deferred that will fire with the subscriptions. - :rtype: Deferred - """ - def get_docs_content(docs): - return [doc.content[self.MBOX_KEY] for doc in docs] - - d = self._store.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') - d.addCallback(get_docs_content) - return d - - def _set_subscription(self, name, value): - """ - Sets the subscription value for a given mailbox - - :param name: the mailbox - :type name: str - - :param value: the boolean value - :type value: bool - """ - # XXX Note that this kind of operation has - # no guarantees of atomicity. We should not be accessing mbox - # documents concurrently. - - subscribed = self.SUBSCRIBED_KEY + name = normalize_mailbox(name) - def update_subscribed_value(mbox): - mbox.content[subscribed] = value - return self._store.put_doc(mbox) + def get_subscribed(mbox): + return mbox.get_mbox_attr("subscribed") - # maybe we should store subscriptions in another - # document... - if name not in self.mailboxes: - d = self.addMailbox(name) - d.addCallback(lambda v: self._get_mailbox_by_name(name)) - else: - d = self._get_mailbox_by_name(name) - d.addCallback(update_subscribed_value) + d = self.getMailbox(name) + d.addCallback(get_subscribed) return d def subscribe(self, name): @@ -475,11 +409,11 @@ class IMAPAccount(WithMsgFields): """ name = normalize_mailbox(name) - def check_and_subscribe(subscriptions): - if name not in subscriptions: - return self._set_subscription(name, True) - d = self._get_subscriptions() - d.addCallback(check_and_subscribe) + def set_subscribed(mbox): + return mbox.set_mbox_attr("subscribed", True) + + d = self.getMailbox(name) + d.addCallback(set_subscribed) return d def unsubscribe(self, name): @@ -492,17 +426,17 @@ class IMAPAccount(WithMsgFields): """ name = normalize_mailbox(name) - def check_and_unsubscribe(subscriptions): - if name not in subscriptions: - raise imap4.MailboxException( - "Not currently subscribed to %r" % name) - return self._set_subscription(name, False) - d = self._get_subscriptions() - d.addCallback(check_and_unsubscribe) + def set_unsubscribed(mbox): + return mbox.set_mbox_attr("subscribed", False) + + d = self.getMailbox(name) + d.addCallback(set_unsubscribed) return d + # TODO -- get__all_mboxes, return tuple + # with ... name? and subscribed bool... def getSubscriptions(self): - return self._get_subscriptions() + raise NotImplementedError() # # INamespacePresenter @@ -517,20 +451,6 @@ class IMAPAccount(WithMsgFields): def getOtherNamespaces(self): return None - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - def __repr__(self): """ Representation string for this object. diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py deleted file mode 100644 index a751c6d..0000000 --- a/src/leap/mail/imap/fields.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# fields.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Fields for Mailbox and Message. -""" - -# TODO deprecate !!! (move all to constants maybe?) -# Flags -> foo - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # Mailbox specific keys - CREATED_KEY = "created" # used??? - - RECENTFLAGS_KEY = "rct" - HDOCS_SET_KEY = "hdocset" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - - -fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py deleted file mode 100644 index f8f25fa..0000000 --- a/src/leap/mail/imap/interfaces.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -# interfaces.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Interfaces for the IMAP module. -""" -from zope.interface import Interface, Attribute - - -# TODO remove ---------------- -class IMessageContainer(Interface): - """ - I am a container around the different documents that a message - is split into. - """ - fdoc = Attribute('The flags document for this message, if any.') - hdoc = Attribute('The headers document for this message, if any.') - cdocs = Attribute('The dict of content documents for this message, ' - 'if any.') - - def walk(self): - """ - Return an iterator to the docs for all the parts. - - :rtype: iterator - """ - - -# TODO remove -------------------- -class IMessageStore(Interface): - """ - I represent a generic storage for LEAP Messages. - """ - - def create_message(self, mbox, uid, message): - """ - Put the passed message into this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :param message: a IMessageContainer implementor. - """ - - def put_message(self, mbox, uid, message): - """ - Put the passed message into this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :param message: a IMessageContainer implementor. - """ - - def remove_message(self, mbox, uid): - """ - Remove the given message from this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - """ - - def get_message(self, mbox, uid): - """ - Get a IMessageContainer for the given mbox and uid combination. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :return: IMessageContainer - """ - - -class IMessageStoreWriter(Interface): - """ - I represent a storage that is able to write its contents to another - different IMessageStore. - """ - - def write_messages(self, store): - """ - Write the documents in this IMessageStore to a different - storage. Usually this will be done from a MemoryStorage to a DbStorage. - - :param store: another IMessageStore implementor. - """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index ea54d33..faeba9d 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013, 2014 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,11 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Soledad Mailbox. +IMAP Mailbox. """ -import copy import re -import threading import logging import StringIO import cStringIO @@ -29,7 +27,6 @@ from collections import defaultdict from twisted.internet import defer from twisted.internet import reactor -from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -38,17 +35,15 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type -from leap.mail.constants import INBOX_NAME -from leap.mail.decorators import deferred_to_thread -from leap.mail.utils import empty -from leap.mail.imap.fields import WithMsgFields, fields -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.messageparts import MessageWrapper +from leap.mail.constants import INBOX_NAME, MessageFlags logger = logging.getLogger(__name__) -# TODO +# TODO LIST # [ ] Restore profile_cmd instrumentation +# [ ] finish the implementation of IMailboxListener +# [ ] implement the rest of ISearchableMailbox + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid @@ -75,16 +70,20 @@ if PROFILE_CMD: d.addCallback(_debugProfiling, name, time.time()) d.addErrback(lambda f: log.msg(f.getTraceback())) +INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, + MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, + MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, + MessageFlags.LIST_FLAG) + -# TODO Rename to Mailbox -# TODO Remove WithMsgFields -class SoledadMailbox(WithMsgFields): +class IMAPMailbox(object): """ A Soledad-backed IMAP mailbox. Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. + The low-level database methods are contained in IMAPMessageCollection + class, which we instantiate and make accessible in the `messages` + attribute. """ implements( imap4.IMailbox, @@ -93,17 +92,7 @@ class SoledadMailbox(WithMsgFields): imap4.ISearchableMailbox, imap4.IMessageCopier) - # XXX should finish the implementation of IMailboxListener - # XXX should completely implement ISearchableMailbox too - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None + init_flags = INIT_FLAGS CMD_MSG = "MESSAGES" CMD_RECENT = "RECENT" @@ -111,58 +100,31 @@ class SoledadMailbox(WithMsgFields): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" - # FIXME we should turn this into a datastructure with limited capacity + # TODO we should turn this into a datastructure with limited capacity _listeners = defaultdict(set) - next_uid_lock = threading.Lock() - last_uid_lock = threading.Lock() - - # TODO unify all the `primed` dicts - _fdoc_primed = {} - _last_uid_primed = {} - _known_uids_primed = {} - - # TODO pass the collection to the constructor - # TODO pass the mbox_doc too - def __init__(self, mbox, store, rw=1): + def __init__(self, collection, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. - :param mbox: the mailbox name - :type mbox: str - - :param store: - :type store: Soledad + :param collection: instance of IMAPMessageCollection + :type collection: IMAPMessageCollection :param rw: read-and-write flag for this mailbox :type rw: int """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(store, "Need a store instance to initialize") - - self.mbox = normalize_mailbox(mbox) self.rw = rw - self.store = store - - self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None + self.collection = collection - # XXX careful with this get/set (it would be - # hitting db unconditionally, move to memstore too) - # Now it's returning a fixed amount of flags from mem - # as a workaround. if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - if self._memstore: - self.prime_known_uids_to_memstore() - self.prime_last_uid_to_memstore() - self.prime_flag_docs_to_memstore() + self.setFlags(self.init_flags) - # purge memstore from empty fdocs. - self._memstore.purge_fdoc_store(mbox) + @property + def mbox_name(self): + return self.collection.mbox_name @property def listeners(self): @@ -175,11 +137,12 @@ class SoledadMailbox(WithMsgFields): :rtype: set """ - return self._listeners[self.mbox] + return self._listeners[self.mbox_name] - # TODO this grows too crazily when many instances are fired, like + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. + def addListener(self, listener): """ Add a listener to the listeners queue. @@ -204,16 +167,6 @@ class SoledadMailbox(WithMsgFields): """ self.listeners.remove(listener) - def _get_mbox_doc(self): - """ - Return mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - return self._memstore.get_mbox_doc(self.mbox) - def getFlags(self): """ Returns the flags defined for this mailbox. @@ -221,10 +174,11 @@ class SoledadMailbox(WithMsgFields): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - flags = self._memstore.get_mbox_flags(self.mbox) + flags = self.collection.mbox_wrapper.flags if not flags: - flags = self.INIT_FLAGS - return map(str, flags) + flags = self.init_flags + flags_str = map(str, flags) + return flags_str def setFlags(self, flags): """ @@ -234,98 +188,31 @@ class SoledadMailbox(WithMsgFields): :type flags: tuple of str """ # XXX this is setting (overriding) old flags. + # Better pass a mode flag leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") - self._memstore.set_mbox_flags(self.mbox, flags) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + return self.collection.set_mbox_attr("flags", flags) - def _get_closed(self): + @property + def is_closed(self): """ Return the closed attribute for this mailbox. :return: True if the mailbox is closed :rtype: bool """ - return self._memstore.get_mbox_closed(self.mbox) + return self.collection.get_mbox_attr("closed") - def _set_closed(self, closed): + def set_closed(self, closed): """ Set the closed attribute for this mailbox. :param closed: the state to be set :type closed: bool - """ - self._memstore.set_mbox_closed(self.mbox, closed) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - If we have a memory store, the last UID will be the highest - recorded UID in the message store, or a counter cached from - the mailbox document in soledad if this is higher. - - :return: the last uid for messages in this mailbox - :rtype: int - """ - last = self._memstore.get_last_uid(self.mbox) - logger.debug("last uid for %s: %s (from memstore)" % ( - repr(self.mbox), last)) - return last - - last_uid = property( - _get_last_uid, doc="Last_UID attribute.") - def prime_last_uid_to_memstore(self): - """ - Prime memstore with last_uid value - """ - primed = self._last_uid_primed.get(self.mbox, False) - if not primed: - mbox = self._get_mbox_doc() - if mbox is None: - # memory-only store - return - last = mbox.content.get('lastuid', 0) - logger.info("Priming Soledad last_uid to %s" % (last,)) - self._memstore.set_last_soledad_uid(self.mbox, last) - self._last_uid_primed[self.mbox] = True - - def prime_known_uids_to_memstore(self): - """ - Prime memstore with the set of all known uids. - - We do this to be able to filter the requests efficiently. - """ - primed = self._known_uids_primed.get(self.mbox, False) - # XXX handle the maybeDeferred - - def set_primed(known_uids): - self._memstore.set_known_uids(self.mbox, known_uids) - self._known_uids_primed[self.mbox] = True - - if not primed: - d = self.messages.all_soledad_uid_iter() - d.addCallback(set_primed) - return d - - def prime_flag_docs_to_memstore(self): - """ - Prime memstore with all the flags documents. + :rtype: Deferred """ - primed = self._fdoc_primed.get(self.mbox, False) - - def set_flag_docs(flag_docs): - self._memstore.load_flag_docs(self.mbox, flag_docs) - self._fdoc_primed[self.mbox] = True - - if not primed: - d = self.messages.get_all_soledad_flag_docs() - d.addCallback(set_flag_docs) - return d + return self.collection.set_mbox_attr("closed", closed) def getUIDValidity(self): """ @@ -334,12 +221,7 @@ class SoledadMailbox(WithMsgFields): :return: unique validity identifier :rtype: int """ - if self._uidvalidity is None: - mbox = self._get_mbox_doc() - if mbox is None: - return 0 - self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) - return self._uidvalidity + return self.collection.get_mbox_attr("created") def getUID(self, message): """ @@ -354,9 +236,9 @@ class SoledadMailbox(WithMsgFields): :rtype: int """ - msg = self.messages.get_msg_by_uid(message) - if msg is not None: - return msg.getUID() + d = self.collection.get_msg_by_uid(message) + d.addCallback(lambda m: m.getUID()) + return d def getUIDNext(self): """ @@ -364,23 +246,20 @@ class SoledadMailbox(WithMsgFields): mailbox. Currently it returns the higher UID incremented by one. - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int + :return: deferred with int + :rtype: Deferred """ - with self.next_uid_lock: - return self.last_uid + 1 + d = self.collection.get_uid_next() + return d def getMessageCount(self): """ Returns the total count of messages in this mailbox. - :rtype: int + :return: deferred with int + :rtype: Deferred """ - return self.messages.count() + return self.collection.count() def getUnseenCount(self): """ @@ -389,7 +268,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `unseen` :rtype: int """ - return self.messages.count_unseen() + return self.collection.count_unseen() def getRecentCount(self): """ @@ -398,7 +277,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `recent` :rtype: int """ - return self.messages.count_recent() + return self.collection.count_recent() def isWriteable(self): """ @@ -407,6 +286,8 @@ class SoledadMailbox(WithMsgFields): :return: 1 if mailbox is read-writeable, 0 otherwise. :rtype: int """ + # XXX We don't need to store it in the mbox doc, do we? + # return int(self.collection.get_mbox_attr('rw')) return self.rw def getHierarchicalDelimiter(self): @@ -431,14 +312,14 @@ class SoledadMailbox(WithMsgFields): if self.CMD_RECENT in names: r[self.CMD_RECENT] = self.getRecentCount() if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 + r[self.CMD_UIDNEXT] = self.getUIDNext() if self.CMD_UIDVALIDITY in names: r[self.CMD_UIDVALIDITY] = self.getUIDValidity() if self.CMD_UNSEEN in names: r[self.CMD_UNSEEN] = self.getUnseenCount() return defer.succeed(r) - def addMessage(self, message, flags, date=None, notify_on_disk=False): + def addMessage(self, message, flags, date=None): """ Adds a message to this mailbox. @@ -464,10 +345,8 @@ class SoledadMailbox(WithMsgFields): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - #if PROFILE_CMD: - #do_profile_cmd(d, "APPEND") + # if PROFILE_CMD: + # do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -478,19 +357,11 @@ class SoledadMailbox(WithMsgFields): reactor.callLater(0, self.notify_new) return x + d = self.collection.add_message(flags=flags, date=date) d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d - def _do_add_message(self, message, flags, date, notify_on_disk=False): - """ - Calls to the messageCollection add_msg method. - Invoked from addMessage. - """ - d = self.messages.add_msg(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - return d - def notify_new(self, *args): """ Notify of new messages to all the listeners. @@ -502,26 +373,34 @@ class SoledadMailbox(WithMsgFields): def cbNotifyNew(result): exists, recent = result - for l in self.listeners: - l.newMessages(exists, recent) + for listener in self.listeners: + listener.newMessages(exists, recent) + d = self._get_notify_count() d.addCallback(cbNotifyNew) d.addCallback(self.cb_signal_unread_to_ui) - @deferred_to_thread def _get_notify_count(self): """ Get message count and recent count for this mailbox Executed in a separate thread. Called from notify_new. - :return: number of messages and number of recent messages. - :rtype: tuple + :return: a deferred that will fire with a tuple, with number of + messages and number of recent messages. + :rtype: Deferred """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( - self.mbox, exists, recent)) - return exists, recent + d_exists = self.getMessageCount() + d_recent = self.getRecentCount() + d_list = [d_exists, d_recent] + + def log_num_msg(result): + exists, recent = result + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox_name, exists, recent)) + + d = defer.gatherResults(d_list) + d.addCallback(log_num_msg) + return d # commands, do not rename methods @@ -533,27 +412,18 @@ class SoledadMailbox(WithMsgFields): on the mailbox. """ - # XXX this will overwrite all the existing flags! + # XXX this will overwrite all the existing flags # should better simply addFlag - self.setFlags((self.NOSELECT_FLAG,)) - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - - def remove_mbox_doc(ignored): - # XXX move to memory store?? + self.setFlags((MessageFlags.NOSELECT_FLAG,)) - def _remove_mbox_doc(doc): - if doc is None: - # memory-only store! - return defer.succeed(True) - return self._soledad.delete_doc(doc) - - doc = self._get_mbox_doc() - return _remove_mbox_doc(doc) + def remove_mbox(_): + # FIXME collection does not have a delete_mbox method, + # it's in account. + # XXX should take care of deleting the uid table too. + return self.collection.delete_mbox(self.mbox_name) d = self.deleteAllDocs() - d.addCallback(remove_mbox_doc) + d.addCallback(remove_mbox) return d def _close_cb(self, result): @@ -574,9 +444,11 @@ class SoledadMailbox(WithMsgFields): if not self.isWriteable(): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self._memstore.expunge(self.mbox, d) + # FIXME actually broken. + # Iterate through index, and do a expunge. return d + # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): """ Put an upper bound to a messages sequence if this is open. @@ -596,6 +468,7 @@ class SoledadMailbox(WithMsgFields): pass return messages_asked + # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -627,29 +500,6 @@ class SoledadMailbox(WithMsgFields): :rtype: deferred """ - d = defer.Deferred() - - # XXX do not need no thread... - reactor.callInThread(self._do_fetch, messages_asked, uid, d) - d.addCallback(self.cb_signal_unread_to_ui) - return d - - # called in thread - def _do_fetch(self, messages_asked, uid, d): - """ - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - :param d: deferred whose callback will be called with result. - :type d: Deferred - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we @@ -660,18 +510,23 @@ class SoledadMailbox(WithMsgFields): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - getmsg = lambda uid: self.messages.get_msg_by_uid(uid) + getmsg = self.collection.get_msg_by_uid # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") + # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) result = ((msgid, msg) for msgid, msg in got_msg if msg is not None) - self.reactor.callLater(0, self.unset_recent_flags, seq_messg) - self.reactor.callFromThread(d.callback, result) + reactor.callLater(0, self.unset_recent_flags, seq_messg) + + # TODO -- call signal_to_ui + # d.addCallback(self.cb_signal_unread_to_ui) + + return result def fetch_flags(self, messages_asked, uid): """ @@ -698,12 +553,11 @@ class SoledadMailbox(WithMsgFields): :rtype: tuple """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d) + reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: do_profile_cmd(d, "FETCH-ALL-FLAGS") return d - # called in thread def _do_fetch_flags(self, messages_asked, uid, d): """ :param messages_asked: IDs of the messages to retrieve information @@ -733,10 +587,11 @@ class SoledadMailbox(WithMsgFields): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - all_flags = self._memstore.all_flags(self.mbox) + # FIXME use deferreds here + all_flags = self.collection.get_all_flags(self.mbox_name) result = ((msgid, flagsPart( msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - self.reactor.callFromThread(d.callback, result) + d.callback(result) def fetch_headers(self, messages_asked, uid): """ @@ -843,8 +698,8 @@ class SoledadMailbox(WithMsgFields): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self.reactor.callLater(0, self._do_store, messages_asked, flags, - mode, uid, d) + reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") d.addCallback(self.cb_signal_unread_to_ui) @@ -853,7 +708,7 @@ class SoledadMailbox(WithMsgFields): def _do_store(self, messages_asked, flags, mode, uid, observer): """ - Helper method, invoke set_flags method in the MessageCollection. + Helper method, invoke set_flags method in the IMAPMessageCollection. See the documentation for the `store` method for the parameters. @@ -869,7 +724,8 @@ class SoledadMailbox(WithMsgFields): flags = tuple(flags) messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer) + self.collection.set_flags( + self.mbox_name, seq_messg, flags, mode, observer) # ISearchableMailbox @@ -908,6 +764,7 @@ class SoledadMailbox(WithMsgFields): msgid = str(query[3]).strip() logger.debug("Searching for %s" % (msgid,)) d = self.messages._get_uid_from_msgid(str(msgid)) + # XXX remove gatherResults d1 = defer.gatherResults([d]) # we want a list, so return it all the same return d1 @@ -928,94 +785,18 @@ class SoledadMailbox(WithMsgFields): uid when the copy succeed. :rtype: Deferred """ - d = defer.Deferred() if PROFILE_CMD: do_profile_cmd(d, "COPY") # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. - d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - deferLater(self.reactor, 0, self._do_copy, message, d) - return d + #d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + #deferLater(self.reactor, 0, self._do_copy, message, d) + #return d - def _do_copy(self, message, observer): - """ - Call invoked from the deferLater in `copy`. This will - copy the flags and header documents, and pass them to the - `create_message` method in the MemoryStore, together with - the observer deferred that we've been passed along. - - :param message: an IMessage implementor - :type message: LeapMessage - :param observer: the deferred that will fire with the - UID of the message - :type observer: Deferred - """ - memstore = self._memstore - - def createCopy(result): - exist, new_fdoc = result - if exist: - # Should we signal error on the callback? - logger.warning("Destination message already exists!") - - # XXX I'm not sure if we should raise the - # errback. This actually rases an ugly warning - # in some muas like thunderbird. - # UID 0 seems a good convention for no uid. - observer.callback(0) - else: - mbox = self.mbox - uid_next = memstore.increment_last_soledad_uid(mbox) - - new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = mbox - - flags = list(new_fdoc[self.FLAGS_KEY]) - flags.append(fields.RECENT_FLAG) - new_fdoc[self.FLAGS_KEY] = tuple(set(flags)) - - # FIXME set recent! - - self._memstore.create_message( - self.mbox, uid_next, - MessageWrapper(new_fdoc), - observer=observer, - notify_on_disk=False) - - d = self._get_msg_copy(message) - d.addCallback(createCopy) - d.addErrback(lambda f: log.msg(f.getTraceback())) - - #@deferred_to_thread - def _get_msg_copy(self, message): - """ - Get a copy of the fdoc for this message, and check whether - it already exists. - - :param message: an IMessage implementor - :type message: LeapMessage - :return: exist, new_fdoc - :rtype: tuple - """ - # XXX for clarity, this could be delegated to a - # MessageCollection mixin that implements copy too, and - # moved out of here. - msg = message - memstore = self._memstore - - if empty(msg.fdoc): - logger.warning("Tried to copy a MSG with no fdoc") - return - new_fdoc = copy.deepcopy(msg.fdoc.content) - fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] - - dest_fdoc = memstore.get_fdoc_from_chash( - fdoc_chash, self.mbox) - - exist = not empty(dest_fdoc) - return exist, new_fdoc + # FIXME not implemented !!! --- + return self.collection.copy_msg(message, self.mbox_name) # convenience fun @@ -1023,29 +804,25 @@ class SoledadMailbox(WithMsgFields): """ Delete all docs in this mailbox """ - def del_all_docs(docs): - todelete = [] - for doc in docs: - d = self.messages._soledad.delete_doc(doc) - todelete.append(d) - return defer.gatherResults(todelete) - - d = self.messages.get_all_docs() - d.addCallback(del_all_docs) - return d + # FIXME not implemented + return self.collection.delete_all_docs() def unset_recent_flags(self, uid_seq): """ Unset Recent flag for a sequence of UIDs. """ - self.messages.unset_recent_flags(uid_seq) + # FIXME not implemented + return self.collection.unset_recent_flags(uid_seq) def __repr__(self): """ Representation string for this mailbox. """ - return u"" % ( - self.mbox, self.messages.count()) + return u"" % ( + self.mbox_name, self.messages.count()) + + +_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) def normalize_mailbox(name): @@ -1060,7 +837,8 @@ def normalize_mailbox(name): :rtype: unicode """ - _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + # XXX maybe it would make sense to normalize common folders too: + # trash, sent, drafts, etc... if _INBOX_RE.match(name): # ensure inital INBOX is uppercase return INBOX_NAME + name[len(INBOX_NAME):] diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py deleted file mode 100644 index eda5b96..0000000 --- a/src/leap/mail/imap/memorystore.py +++ /dev/null @@ -1,1340 +0,0 @@ - -# memorystore.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -In-memory transient store for a LEAPIMAPServer. -""" -import contextlib -import logging -import threading -import weakref - -from collections import defaultdict -from copy import copy - -from enum import Enum -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet.task import LoopingCall -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert_type -from leap.mail import size -from leap.mail.utils import empty, phash_iter -from leap.mail.messageflow import MessageProducer -from leap.mail.imap import interfaces -from leap.mail.imap.fields import fields -from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc -from leap.mail.imap.messageparts import RecentFlagsDoc -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.messageparts import ReferenciableDict - -from leap.mail.decorators import deferred_to_thread - -logger = logging.getLogger(__name__) - - -# The default period to do writebacks to the permanent -# soledad storage, in seconds. -SOLEDAD_WRITE_PERIOD = 15 - -FDOC = MessagePartType.fdoc.name -HDOC = MessagePartType.hdoc.name -CDOCS = MessagePartType.cdocs.name -DOCS_ID = MessagePartType.docs_id.name - - -@contextlib.contextmanager -def set_bool_flag(obj, att): - """ - Set a boolean flag to True while we're doing our thing. - Just to let the world know. - """ - setattr(obj, att, True) - try: - yield True - except RuntimeError as exc: - logger.exception(exc) - finally: - setattr(obj, att, False) - - -DirtyState = Enum("DirtyState", "none dirty new") - - -class MemoryStore(object): - """ - An in-memory store to where we can write the different parts that - we split the messages into and buffer them until we write them to the - permanent storage. - - It uses MessageWrapper instances to represent the message-parts, which are - indexed by mailbox name and UID. - - It also can be passed a permanent storage as a paremeter (any implementor - of IMessageStore, in this case a SoledadStore). In this case, a periodic - dump of the messages stored in memory will be done. The period of the - writes to the permanent storage is controled by the write_period parameter - in the constructor. - """ - implements(interfaces.IMessageStore, - interfaces.IMessageStoreWriter) - - # TODO We will want to index by chash when we transition to local-only - # UIDs. - - WRITING_FLAG = "_writing" - _last_uid_lock = threading.Lock() - _fdoc_docid_lock = threading.Lock() - - def __init__(self, permanent_store=None, - write_period=SOLEDAD_WRITE_PERIOD): - """ - Initialize a MemoryStore. - - :param permanent_store: a IMessageStore implementor to dump - messages to. - :type permanent_store: IMessageStore - :param write_period: the interval to dump messages to disk, in seconds. - :type write_period: int - """ - self._permanent_store = permanent_store - self._write_period = write_period - - if permanent_store is None: - self._mbox_closed = defaultdict(lambda: False) - - # Internal Storage: messages - """ - flags document store. - _fdoc_store[mbox][uid] = { 'content': 'aaa' } - """ - self._fdoc_store = defaultdict(lambda: defaultdict( - lambda: ReferenciableDict({}))) - - # Sizes - """ - {'mbox, uid': } - """ - self._sizes = {} - - # Internal Storage: payload-hash - """ - fdocs:doc-id store, stores document IDs for putting - the dirty flags-docs. - """ - self._fdoc_id_store = defaultdict(lambda: defaultdict( - lambda: '')) - - # Internal Storage: content-hash:hdoc - """ - hdoc-store keeps references to - the header-documents indexed by content-hash. - - {'chash': { dict-stuff } - } - """ - self._hdoc_store = defaultdict(lambda: ReferenciableDict({})) - - # Internal Storage: payload-hash:cdoc - """ - content-docs stored by payload-hash - {'phash': { dict-stuff } } - """ - self._cdoc_store = defaultdict(lambda: ReferenciableDict({})) - - # Internal Storage: content-hash:fdoc - """ - chash-fdoc-store keeps references to - the flag-documents indexed by content-hash. - - {'chash': {'mbox-a': weakref.proxy(dict), - 'mbox-b': weakref.proxy(dict)} - } - """ - self._chash_fdoc_store = defaultdict(lambda: defaultdict(lambda: None)) - - # Internal Storage: recent-flags store - """ - recent-flags store keeps one dict per mailbox, - with the document-id of the u1db document - and the set of the UIDs that have the recent flag. - - {'mbox-a': {'doc_id': 'deadbeef', - 'set': {1,2,3,4} - } - } - """ - # TODO this will have to transition to content-hash - # indexes after we move to local-only UIDs. - - self._rflags_store = defaultdict( - lambda: {'doc_id': None, 'set': set([])}) - - """ - last-uid store keeps the count of the highest UID - per mailbox. - - {'mbox-a': 42, - 'mbox-b': 23} - """ - self._last_uid = defaultdict(lambda: 0) - - """ - known-uids keeps a count of the uids that soledad knows for a given - mailbox - - {'mbox-a': set([1,2,3])} - """ - self._known_uids = defaultdict(set) - - """ - mbox-flags is a dict containing flags for each mailbox. this is - modified from mailbox.getFlags / mailbox.setFlags - """ - self._mbox_flags = defaultdict(set) - - # New and dirty flags, to set MessageWrapper State. - self._new = set([]) - self._new_queue = set([]) - self._new_deferreds = {} - - self._dirty = set([]) - self._dirty_queue = set([]) - self._dirty_deferreds = {} - - self._rflags_dirty = set([]) - - # Flag for signaling we're busy writing to the disk storage. - setattr(self, self.WRITING_FLAG, False) - - if self._permanent_store is not None: - # this producer spits its messages to the permanent store - # consumer using a queue. We will use that to put - # our messages to be written. - self.producer = MessageProducer(permanent_store, - period=0.1) - # looping call for dumping to SoledadStore - self._write_loop = LoopingCall(self.write_messages, - permanent_store) - - # We can start the write loop right now, why wait? - self._start_write_loop() - else: - # We have a memory-only store. - self.producer = None - self._write_loop = None - - # TODO -- remove - def _start_write_loop(self): - """ - Start loop for writing to disk database. - """ - if self._write_loop is None: - return - if not self._write_loop.running: - self._write_loop.start(self._write_period, now=True) - - # TODO -- remove - def _stop_write_loop(self): - """ - Stop loop for writing to disk database. - """ - if self._write_loop is None: - return - if self._write_loop.running: - self._write_loop.stop() - - # IMessageStore - - # XXX this would work well for whole message operations. - # We would have to add a put_flags operation to modify only - # the flags doc (and set the dirty flag accordingly) - - def create_message(self, mbox, uid, message, observer, - notify_on_disk=True): - """ - Create the passed message into this MemoryStore. - - By default we consider that any message is a new message. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :param message: a message to be added - :type message: MessageWrapper - :param observer: - the deferred that will fire with the UID of the message. If - notify_on_disk is True, this will happen when the message is - written to Soledad. Otherwise it will fire as soon as we've added - the message to the memory store. - :type observer: Deferred - :param notify_on_disk: - whether the `observer` deferred should wait until the message is - written to disk to be fired. - :type notify_on_disk: bool - """ - # TODO -- return a deferred - log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) - key = mbox, uid - - self._add_message(mbox, uid, message, notify_on_disk) - self._new.add(key) - - if observer is not None: - if notify_on_disk: - # We store this deferred so we can keep track of the pending - # operations internally. - # TODO this should fire with the UID !!! -- change that in - # the soledad store code. - self._new_deferreds[key] = observer - - else: - # Caller does not care, just fired and forgot, so we pass - # a defer that will inmediately have its callback triggered. - reactor.callFromThread(observer.callback, uid) - - def put_message(self, mbox, uid, message, notify_on_disk=True): - """ - Put an existing message. - - This will also set the dirty flag on the MemoryStore. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :param message: a message to be added - :type message: MessageWrapper - :param notify_on_disk: whether the deferred that is returned should - wait until the message is written to disk to - be fired. - :type notify_on_disk: bool - - :return: a Deferred. if notify_on_disk is True, will be fired - when written to the db on disk. - Otherwise will fire inmediately - :rtype: Deferred - """ - key = mbox, uid - d = defer.Deferred() - d.addCallback(lambda result: log.msg("message PUT save: %s" % result)) - - self._dirty.add(key) - self._dirty_deferreds[key] = d - self._add_message(mbox, uid, message, notify_on_disk) - return d - - def _add_message(self, mbox, uid, message, notify_on_disk=True): - """ - Helper method, called by both create_message and put_message. - See those for parameter documentation. - """ - msg_dict = message.as_dict() - - fdoc = msg_dict.get(FDOC, None) - if fdoc is not None: - fdoc_store = self._fdoc_store[mbox][uid] - fdoc_store.update(fdoc) - chash_fdoc_store = self._chash_fdoc_store - - # content-hash indexing - chash = fdoc.get(fields.CONTENT_HASH_KEY) - chash_fdoc_store[chash][mbox] = weakref.proxy( - self._fdoc_store[mbox][uid]) - - hdoc = msg_dict.get(HDOC, None) - if hdoc is not None: - chash = hdoc.get(fields.CONTENT_HASH_KEY) - hdoc_store = self._hdoc_store[chash] - hdoc_store.update(hdoc) - - cdocs = message.cdocs - for cdoc in cdocs.values(): - phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None) - if not phash: - continue - cdoc_store = self._cdoc_store[phash] - cdoc_store.update(cdoc) - - # Update memory store size - # XXX this should use [mbox][uid] - # TODO --- this has to be deferred to thread, - # TODO add hdoc and cdocs sizes too - # it's slowing things down here. - # key = mbox, uid - # self._sizes[key] = size.get_size(self._fdoc_store[key]) - - def purge_fdoc_store(self, mbox): - """ - Purge the empty documents from a fdoc store. - Called during initialization of the SoledadMailbox - - :param mbox: the mailbox - :type mbox: str or unicode - """ - # XXX This is really a workaround until I find the conditions - # that are making the empty items remain there. - # This happens, for instance, after running several times - # the regression test, that issues a store deleted + expunge + select - # The items are being correclty deleted, but in succesive appends - # the empty items with previously deleted uids reappear as empty - # documents. I suspect it's a timing condition with a previously - # evaluated sequence being used after the items has been removed. - - for uid, value in self._fdoc_store[mbox].items(): - if empty(value): - del self._fdoc_store[mbox][uid] - - def get_docid_for_fdoc(self, mbox, uid): - """ - Return Soledad document id for the flags-doc for a given mbox and uid, - or None of no flags document could be found. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - :rtype: unicode or None - """ - with self._fdoc_docid_lock: - doc_id = self._fdoc_id_store[mbox][uid] - - if empty(doc_id): - fdoc = self._permanent_store.get_flags_doc(mbox, uid) - if empty(fdoc) or empty(fdoc.content): - return None - doc_id = fdoc.doc_id - self._fdoc_id_store[mbox][uid] = doc_id - - return doc_id - - def get_message(self, mbox, uid, dirtystate=DirtyState.none, - flags_only=False): - """ - Get a MessageWrapper for the given mbox and uid combination. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - :param dirtystate: DirtyState enum: one of `dirty`, `new` - or `none` (default) - :type dirtystate: enum - :param flags_only: whether the message should carry only a reference - to the flags document. - :type flags_only: bool - : - - :return: MessageWrapper or None - """ - # TODO -- return deferred - if dirtystate == DirtyState.dirty: - flags_only = True - - key = mbox, uid - - fdoc = self._fdoc_store[mbox][uid] - if empty(fdoc): - return None - - new, dirty = False, False - if dirtystate == DirtyState.none: - new, dirty = self._get_new_dirty_state(key) - if dirtystate == DirtyState.dirty: - new, dirty = False, True - if dirtystate == DirtyState.new: - new, dirty = True, False - - if flags_only: - return MessageWrapper(fdoc=fdoc, - new=new, dirty=dirty, - memstore=weakref.proxy(self)) - else: - chash = fdoc.get(fields.CONTENT_HASH_KEY) - hdoc = self._hdoc_store[chash] - if empty(hdoc): - # XXX this will be a deferred - hdoc = self._permanent_store.get_headers_doc(chash) - if empty(hdoc): - return None - if not empty(hdoc.content): - self._hdoc_store[chash] = hdoc.content - hdoc = hdoc.content - cdocs = None - - pmap = hdoc.get(fields.PARTS_MAP_KEY, None) - if new and pmap is not None: - # take the different cdocs for write... - cdoc_store = self._cdoc_store - cdocs_list = phash_iter(hdoc) - cdocs = dict(enumerate( - [cdoc_store[phash] for phash in cdocs_list], 1)) - - return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs, - new=new, dirty=dirty, - memstore=weakref.proxy(self)) - - def remove_message(self, mbox, uid): - """ - Remove a Message from this MemoryStore. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - # XXX For the moment we are only removing the flags and headers - # docs. The rest we leave there polluting your hard disk, - # until we think about a good way of deorphaning. - - # XXX implement elijah's idea of using a PUT document as a - # token to ensure consistency in the removal. - - try: - del self._fdoc_store[mbox][uid] - except KeyError: - pass - - try: - key = mbox, uid - self._new.discard(key) - self._dirty.discard(key) - if key in self._sizes: - del self._sizes[key] - self._known_uids[mbox].discard(uid) - except KeyError: - pass - except Exception as exc: - logger.error("error while removing message!") - logger.exception(exc) - try: - with self._fdoc_docid_lock: - del self._fdoc_id_store[mbox][uid] - except KeyError: - pass - except Exception as exc: - logger.error("error while removing message!") - logger.exception(exc) - - # IMessageStoreWriter - - # TODO -- I think we don't need this anymore. - # instead, we can have - def write_messages(self, store): - """ - Write the message documents in this MemoryStore to a different store. - - :param store: the IMessageStore to write to - :rtype: False if queue is not empty, None otherwise. - """ - # For now, we pass if the queue is not empty, to avoid duplicate - # queuing. - # We would better use a flag to know when we've already enqueued an - # item. - - # XXX this could return the deferred for all the enqueued operations - - if not self.producer.is_queue_empty(): - return False - - if any(map(lambda i: not empty(i), (self._new, self._dirty))): - logger.info("Writing messages to Soledad...") - - # TODO change for lock, and make the property access - # is accquired - with set_bool_flag(self, self.WRITING_FLAG): - for rflags_doc_wrapper in self.all_rdocs_iter(): - self.producer.push(rflags_doc_wrapper, - state=self.producer.STATE_DIRTY) - for msg_wrapper in self.all_new_msg_iter(): - self.producer.push(msg_wrapper, - state=self.producer.STATE_NEW) - for msg_wrapper in self.all_dirty_msg_iter(): - self.producer.push(msg_wrapper, - state=self.producer.STATE_DIRTY) - - # MemoryStore specific methods. - - def get_uids(self, mbox): - """ - Get all uids for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: list - """ - return self._fdoc_store[mbox].keys() - - def get_soledad_known_uids(self, mbox): - """ - Get all uids that soledad knows about, from the memory cache. - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: list - """ - return self._known_uids.get(mbox, []) - - # last_uid - - def get_last_uid(self, mbox): - """ - Return the highest UID for a given mbox. - It will be the highest between the highest uid in the message store for - the mailbox, and the soledad integer cache. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: int - """ - uids = self.get_uids(mbox) - last_mem_uid = uids and max(uids) or 0 - last_soledad_uid = self.get_last_soledad_uid(mbox) - return max(last_mem_uid, last_soledad_uid) - - def get_last_soledad_uid(self, mbox): - """ - Get last uid for a given mbox from the soledad integer cache. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - return self._last_uid.get(mbox, 0) - - def set_last_soledad_uid(self, mbox, value): - """ - Set last uid for a given mbox in the soledad integer cache. - SoledadMailbox should prime this value during initialization. - Other methods (during message adding) SHOULD call - `increment_last_soledad_uid` instead. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - # can be long??? - # leap_assert_type(value, int) - logger.info("setting last soledad uid for %s to %s" % - (mbox, value)) - # if we already have a value here, don't do anything - with self._last_uid_lock: - if not self._last_uid.get(mbox, None): - self._last_uid[mbox] = value - - def set_known_uids(self, mbox, value): - """ - Set the value fo the known-uids set for this mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: a sequence of integers to be added to the set. - :type value: tuple - """ - current = self._known_uids[mbox] - self._known_uids[mbox] = current.union(set(value)) - - def increment_last_soledad_uid(self, mbox): - """ - Increment by one the soledad integer cache for the last_uid for - this mbox, and fire a defer-to-thread to update the soledad value. - The caller should lock the call tho this method. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - with self._last_uid_lock: - self._last_uid[mbox] += 1 - value = self._last_uid[mbox] - reactor.callInThread(self.write_last_uid, mbox, value) - return value - - def write_last_uid(self, mbox, value): - """ - Increment the soledad integer cache for the highest uid value. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - leap_assert_type(value, int) - if self._permanent_store: - self._permanent_store.write_last_uid(mbox, value) - - def load_flag_docs(self, mbox, flag_docs): - """ - Load the flag documents for the given mbox. - Used during initial flag docs prefetch. - - :param mbox: the mailbox - :type mbox: str or unicode - :param flag_docs: a dict with the content for the flag docs, indexed - by uid. - :type flag_docs: dict - """ - # We can do direct assignments cause we know this will only - # be called during initialization of the mailbox. - # TODO could hook here a sanity-check - # for duplicates - - fdoc_store = self._fdoc_store[mbox] - chash_fdoc_store = self._chash_fdoc_store - for uid in flag_docs: - rdict = ReferenciableDict(flag_docs[uid]) - fdoc_store[uid] = rdict - # populate chash dict too, to avoid fdoc duplication - chash = flag_docs[uid]["chash"] - chash_fdoc_store[chash][mbox] = weakref.proxy( - self._fdoc_store[mbox][uid]) - - def update_flags(self, mbox, uid, fdoc): - """ - Update the flag document for a given mbox and uid combination, - and set the dirty flag. - We could use put_message, but this is faster. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the uid of the message - :type uid: int - - :param fdoc: a dict with the content for the flag docs - :type fdoc: dict - """ - key = mbox, uid - self._fdoc_store[mbox][uid].update(fdoc) - self._dirty.add(key) - - def load_header_docs(self, header_docs): - """ - Load the flag documents for the given mbox. - Used during header docs prefetch, and during cache after - a read from soledad if the hdoc property in message did not - find its value in here. - - :param flag_docs: a dict with the content for the flag docs. - :type flag_docs: dict - """ - hdoc_store = self._hdoc_store - for chash in header_docs: - hdoc_store[chash] = ReferenciableDict(header_docs[chash]) - - def all_flags(self, mbox): - """ - Return a dictionary with all the flags for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - fdict = {} - uids = self.get_uids(mbox) - fstore = self._fdoc_store[mbox] - - for uid in uids: - try: - fdict[uid] = fstore[uid][fields.FLAGS_KEY] - except KeyError: - continue - return fdict - - def all_headers(self, mbox): - """ - Return a dictionary with all the header docs for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - headers_dict = {} - uids = self.get_uids(mbox) - fdoc_store = self._fdoc_store[mbox] - hdoc_store = self._hdoc_store - - for uid in uids: - try: - chash = fdoc_store[uid][fields.CONTENT_HASH_KEY] - hdoc = hdoc_store[chash] - if not empty(hdoc): - headers_dict[uid] = hdoc - except KeyError: - continue - return headers_dict - - # Counting sheeps... - - def count_new_mbox(self, mbox): - """ - Count the new messages by mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: number of new messages - :rtype: int - """ - return len([(m, uid) for m, uid in self._new if mbox == mbox]) - - # XXX used at all? - def count_new(self): - """ - Count all the new messages in the MemoryStore. - - :rtype: int - """ - return len(self._new) - - def count(self, mbox): - """ - Return the count of messages for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: number of messages - :rtype: int - """ - return len(self._fdoc_store[mbox]) - - def unseen_iter(self, mbox): - """ - Get an iterator for the message UIDs with no `seen` flag - for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - fdocs = self._fdoc_store[mbox] - - return [uid for uid, value - in fdocs.items() - if fields.SEEN_FLAG not in value.get(fields.FLAGS_KEY, [])] - - def get_cdoc_from_phash(self, phash): - """ - Return a content-document by its payload-hash. - - :param phash: the payload hash to check against - :type phash: str or unicode - :rtype: MessagePartDoc - """ - doc = self._cdoc_store.get(phash, None) - - # XXX return None for consistency? - - # XXX have to keep a mapping between phash and its linkage - # info, to know if this payload is been already saved or not. - # We will be able to get this from the linkage-docs, - # not yet implemented. - new = True - dirty = False - return MessagePartDoc( - new=new, dirty=dirty, store="mem", - part=MessagePartType.cdoc, - content=doc, - doc_id=None) - - def get_fdoc_from_chash(self, chash, mbox): - """ - Return a flags-document by its content-hash and a given mailbox. - Used during content-duplication detection while copying or adding a - message. - - :param chash: the content hash to check against - :type chash: str or unicode - :param mbox: the mailbox - :type mbox: str or unicode - - :return: MessagePartDoc. It will return None if the flags document - has empty content or it is flagged as \\Deleted. - """ - fdoc = self._chash_fdoc_store[chash][mbox] - - # a couple of special cases. - # 1. We might have a doc with empty content... - if empty(fdoc): - return None - - # 2. ...Or the message could exist, but being flagged for deletion. - # We want to create a new one in this case. - # Hmmm what if the deletion is un-done?? We would end with a - # duplicate... - if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []): - return None - - uid = fdoc[fields.UID_KEY] - key = mbox, uid - new = key in self._new - dirty = key in self._dirty - - return MessagePartDoc( - new=new, dirty=dirty, store="mem", - part=MessagePartType.fdoc, - content=fdoc, - doc_id=None) - - def iter_fdoc_keys(self): - """ - Return a generator through all the mbox, uid keys in the flags-doc - store. - """ - fdoc_store = self._fdoc_store - for mbox in fdoc_store: - for uid in fdoc_store[mbox]: - yield mbox, uid - - def all_new_msg_iter(self): - """ - Return generator that iterates through all new messages. - - :return: generator of MessageWrappers - :rtype: generator - """ - gm = self.get_message - # need to freeze, set can change during iteration - new = [gm(*key, dirtystate=DirtyState.new) for key in tuple(self._new)] - # move content from new set to the queue - self._new_queue.update(self._new) - self._new.difference_update(self._new) - return new - - def all_dirty_msg_iter(self): - """ - Return generator that iterates through all dirty messages. - - :return: generator of MessageWrappers - :rtype: generator - """ - gm = self.get_message - # need to freeze, set can change during iteration - dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty) - for key in tuple(self._dirty)] - # move content from new and dirty sets to the queue - - self._dirty_queue.update(self._dirty) - self._dirty.difference_update(self._dirty) - return dirty - - def all_deleted_uid_iter(self, mbox): - """ - Return a list with the UIDs for all messags - with deleted flag in a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: list of integers - :rtype: list - """ - # This *needs* to return a fixed sequence. Otherwise the dictionary len - # will change during iteration, when we modify it - fdocs = self._fdoc_store[mbox] - return [uid for uid, value - in fdocs.items() - if fields.DELETED_FLAG in value.get(fields.FLAGS_KEY, [])] - - # new, dirty flags - - def _get_new_dirty_state(self, key): - """ - Return `new` and `dirty` flags for a given message. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - :return: tuple of bools - :rtype: tuple - """ - # TODO change indexing of sets to [mbox][key] too. - # XXX should return *first* the news, and *then* the dirty... - - # TODO should query in queues too , true? - # - return map(lambda _set: key in _set, (self._new, self._dirty)) - - def set_new_queued(self, key): - """ - Add the key value to the `new-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._new_queue.add(key) - - def unset_new_queued(self, key): - """ - Remove the key value from the `new-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._new_queue.discard(key) - deferreds = self._new_deferreds - d = deferreds.get(key, None) - if d: - # XXX use a namedtuple for passing the result - # when we check it in the other side. - d.callback('%s, ok' % str(key)) - deferreds.pop(key) - - def set_dirty_queued(self, key): - """ - Add the key value to the `dirty-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._dirty_queue.add(key) - - def unset_dirty_queued(self, key): - """ - Remove the key value from the `dirty-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._dirty_queue.discard(key) - deferreds = self._dirty_deferreds - d = deferreds.get(key, None) - if d: - # XXX use a namedtuple for passing the result - # when we check it in the other side. - d.callback('%s, ok' % str(key)) - deferreds.pop(key) - - # Recent Flags - - def set_recent_flag(self, mbox, uid): - """ - Set the `Recent` flag for a given mailbox and UID. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - self._rflags_dirty.add(mbox) - self._rflags_store[mbox]['set'].add(uid) - - # TODO --- nice but unused - def unset_recent_flag(self, mbox, uid): - """ - Unset the `Recent` flag for a given mailbox and UID. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - self._rflags_store[mbox]['set'].discard(uid) - - def set_recent_flags(self, mbox, value): - """ - Set the value for the set of the recent flags. - Used from the property in the MessageCollection. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: a sequence of flags to set - :type value: sequence - """ - self._rflags_dirty.add(mbox) - self._rflags_store[mbox]['set'] = set(value) - - def load_recent_flags(self, mbox, flags_doc): - """ - Load the passed flags document in the recent flags store, for a given - mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param flags_doc: A dictionary containing the `doc_id` of the Soledad - flags-document for this mailbox, and the `set` - of uids marked with that flag. - """ - self._rflags_store[mbox] = flags_doc - - def get_recent_flags(self, mbox): - """ - Return the set of UIDs with the `Recent` flag for this mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: set, or None - """ - rflag_for_mbox = self._rflags_store.get(mbox, None) - if not rflag_for_mbox: - return None - return self._rflags_store[mbox]['set'] - - # XXX -- remove - def all_rdocs_iter(self): - """ - Return an iterator through all in-memory recent flag dicts, wrapped - under a RecentFlagsDoc namedtuple. - Used for saving to disk. - - :return: a generator of RecentFlagDoc - :rtype: generator - """ - # XXX use enums - DOC_ID = "doc_id" - SET = "set" - - rflags_store = self._rflags_store - - def get_rdoc(mbox, rdict): - mbox_rflag_set = rdict[SET] - recent_set = copy(mbox_rflag_set) - # zero it! - mbox_rflag_set.difference_update(mbox_rflag_set) - return RecentFlagsDoc( - doc_id=rflags_store[mbox][DOC_ID], - content={ - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: mbox, - fields.RECENTFLAGS_KEY: list(recent_set) - }) - - return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items() - if not empty(rdict[SET])) - - # Methods that mirror the IMailbox interface - - def remove_all_deleted(self, mbox): - """ - Remove all messages flagged \\Deleted from this Memory Store only. - Called from `expunge` - - :param mbox: the mailbox - :type mbox: str or unicode - :return: a list of UIDs - :rtype: list - """ - mem_deleted = self.all_deleted_uid_iter(mbox) - for uid in mem_deleted: - self.remove_message(mbox, uid) - return mem_deleted - - # TODO -- remove - def stop_and_flush(self): - """ - Stop the write loop and trigger a write to the producer. - """ - self._stop_write_loop() - if self._permanent_store is not None: - # XXX we should check if we did get a True value on this - # operation. If we got False we should retry! (queue was not empty) - self.write_messages(self._permanent_store) - self.producer.flush() - - def expunge(self, mbox, observer): - """ - Remove all messages flagged \\Deleted, from the Memory Store - and from the permanent store also. - - It first queues up a last write, and wait for the deferreds to be done - before continuing. - - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - soledad_store = self._permanent_store - if soledad_store is None: - # just-in memory store, easy then. - self._delete_from_memory(mbox, observer) - return - - # We have a soledad storage. - try: - # Stop and trigger last write - self.stop_and_flush() - # Wait on the writebacks to finish - - # XXX what if pending deferreds is empty? - pending_deferreds = (self._new_deferreds.get(mbox, []) + - self._dirty_deferreds.get(mbox, [])) - d1 = defer.gatherResults(pending_deferreds, consumeErrors=True) - d1.addCallback( - self._delete_from_soledad_and_memory, mbox, observer) - except Exception as exc: - logger.exception(exc) - - def _delete_from_memory(self, mbox, observer): - """ - Remove all messages marked as deleted from soledad and memory. - - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - mem_deleted = self.remove_all_deleted(mbox) - # TODO return a DeferredList - observer.callback(mem_deleted) - - def _delete_from_soledad_and_memory(self, result, mbox, observer): - """ - Remove all messages marked as deleted from soledad and memory. - - :param result: ignored. the result of the deferredList that triggers - this as a callback from `expunge`. - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - all_deleted = [] - soledad_store = self._permanent_store - - try: - # 1. Delete all messages marked as deleted in soledad. - logger.debug("DELETING FROM SOLEDAD ALL FOR %r" % (mbox,)) - sol_deleted = soledad_store.remove_all_deleted(mbox) - - try: - self._known_uids[mbox].difference_update(set(sol_deleted)) - except Exception as exc: - logger.exception(exc) - - # 2. Delete all messages marked as deleted in memory. - logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,)) - mem_deleted = self.remove_all_deleted(mbox) - - all_deleted = set(mem_deleted).union(set(sol_deleted)) - logger.debug("deleted %r" % all_deleted) - except Exception as exc: - logger.exception(exc) - finally: - self._start_write_loop() - - observer.callback(all_deleted) - - # Mailbox documents and attributes - - # This could be also be cached in memstore, but proxying directly - # to soledad since it's not too performance-critical. - - def get_mbox_doc(self, mbox): - """ - Return the soledad document for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: SoledadDocument or None. - """ - if self.permanent_store is not None: - return self.permanent_store.get_mbox_document(mbox) - else: - return None - - def get_mbox_closed(self, mbox): - """ - Return the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: bool - """ - if self.permanent_store is not None: - return self.permanent_store.get_mbox_closed(mbox) - else: - return self._mbox_closed[mbox] - - def set_mbox_closed(self, mbox, closed): - """ - Set the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - if self.permanent_store is not None: - self.permanent_store.set_mbox_closed(mbox, closed) - else: - self._mbox_closed[mbox] = closed - - def get_mbox_flags(self, mbox): - """ - Get the flags for a given mbox. - :rtype: list - """ - return sorted(self._mbox_flags[mbox]) - - def set_mbox_flags(self, mbox, flags): - """ - Set the mbox flags - """ - self._mbox_flags[mbox] = set(flags) - # TODO - # This should write to the permanent store!!! - - # Rename flag-documents - - def rename_fdocs_mailbox(self, old_mbox, new_mbox): - """ - Change the mailbox name for all flag documents in a given mailbox. - Used from account.rename - - :param old_mbox: name for the old mbox - :type old_mbox: str or unicode - :param new_mbox: name for the new mbox - :type new_mbox: str or unicode - """ - fs = self._fdoc_store - keys = fs[old_mbox].keys() - for k in keys: - fdoc = fs[old_mbox][k] - fdoc['mbox'] = new_mbox - fs[new_mbox][k] = fdoc - fs[old_mbox].pop(k) - self._dirty.add((new_mbox, k)) - - # Dump-to-disk controls. - - @property - def is_writing(self): - """ - Property that returns whether the store is currently writing its - internal state to a permanent storage. - - Used to evaluate whether the CHECK command can inform that the field - is clear to proceed, or waiting for the write operations to complete - is needed instead. - - :rtype: bool - """ - # FIXME this should return a deferred !!! - # TODO this should be moved to soledadStore instead - # (all pending deferreds) - return getattr(self, self.WRITING_FLAG) - - @property - def permanent_store(self): - return self._permanent_store - - # Memory management. - - def get_size(self): - """ - Return the size of the internal storage. - Use for calculating the limit beyond which we should flush the store. - - :rtype: int - """ - return reduce(lambda x, y: x + y, self._sizes, 0) diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py deleted file mode 100644 index fb1d75a..0000000 --- a/src/leap/mail/imap/messageparts.py +++ /dev/null @@ -1,586 +0,0 @@ -# messageparts.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -MessagePart implementation. Used from LeapMessage. -""" -import logging -import StringIO -import weakref - -from collections import namedtuple - -from enum import Enum -from zope.interface import implements -from twisted.mail import imap4 - -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset -from leap.mail.imap import interfaces -from leap.mail.imap.fields import fields -from leap.mail.utils import empty, first, find_charset - -MessagePartType = Enum("MessagePartType", "hdoc fdoc cdoc cdocs docs_id") - - -logger = logging.getLogger(__name__) - - -""" -A MessagePartDoc is a light wrapper around the dictionary-like -data that we pass along for message parts. It can be used almost everywhere -that you would expect a SoledadDocument, since it has a dict under the -`content` attribute. - -We also keep some metadata on it, relative in part to the message as a whole, -and sometimes to a part in particular only. - -* `new` indicates that the document has just been created. SoledadStore - should just create a new doc for all the related message parts. -* `store` indicates the type of store a given MessagePartDoc lives in. - We currently use this to indicate that the document comes from memeory, - but we should probably get rid of it as soon as we extend the use of the - SoledadStore interface along LeapMessage, MessageCollection and Mailbox. -* `part` is one of the MessagePartType enums. - -* `dirty` indicates that, while we already have the document in Soledad, - we have modified its state in memory, so we need to put_doc instead while - dumping the MemoryStore contents. - `dirty` attribute would only apply to flags-docs and linkage-docs. -* `doc_id` is the identifier for the document in the u1db database, if any. - -""" - -MessagePartDoc = namedtuple( - 'MessagePartDoc', - ['new', 'dirty', 'part', 'store', 'content', 'doc_id']) - -""" -A RecentFlagsDoc is used to send the recent-flags document payload to the -SoledadWriter during dumps. -""" -RecentFlagsDoc = namedtuple( - 'RecentFlagsDoc', - ['content', 'doc_id']) - - -class ReferenciableDict(dict): - """ - A dict that can be weak-referenced. - - Some builtin objects are not weak-referenciable unless - subclassed. So we do. - - Used to return pointers to the items in the MemoryStore. - """ - - -class MessageWrapper(object): - """ - A simple nested dictionary container around the different message subparts. - """ - implements(interfaces.IMessageContainer) - - FDOC = "fdoc" - HDOC = "hdoc" - CDOCS = "cdocs" - DOCS_ID = "docs_id" - - # Using slots to limit some the memory use, - # Add your attribute here. - - __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"] - - def __init__(self, fdoc=None, hdoc=None, cdocs=None, - from_dict=None, memstore=None, - new=True, dirty=False, docs_id={}): - """ - Initialize a MessageWrapper. - """ - # TODO add optional reference to original message in the incoming - self._dict = {} - self.memstore = memstore - - self._new = new - self._dirty = dirty - - self._storetype = "mem" - - if from_dict is not None: - self.from_dict(from_dict) - else: - if fdoc is not None: - self._dict[self.FDOC] = ReferenciableDict(fdoc) - if hdoc is not None: - self._dict[self.HDOC] = ReferenciableDict(hdoc) - if cdocs is not None: - self._dict[self.CDOCS] = ReferenciableDict(cdocs) - - # This will keep references to the doc_ids to be able to put - # messages to soledad. It will be populated during the walk() to avoid - # the overhead of reading from the db. - - # XXX it really *only* make sense for the FDOC, the other parts - # should not be "dirty", just new...!!! - self._dict[self.DOCS_ID] = docs_id - - # properties - - # TODO Could refactor new and dirty properties together. - - def _get_new(self): - """ - Get the value for the `new` flag. - - :rtype: bool - """ - return self._new - - def _set_new(self, value=False): - """ - Set the value for the `new` flag, and propagate it - to the memory store if any. - - :param value: the value to set - :type value: bool - """ - self._new = value - if self.memstore: - mbox = self.fdoc.content.get('mbox', None) - uid = self.fdoc.content.get('uid', None) - if not mbox or not uid: - logger.warning("Malformed fdoc") - return - key = mbox, uid - fun = [self.memstore.unset_new_queued, - self.memstore.set_new_queued][int(value)] - fun(key) - else: - logger.warning("Could not find a memstore referenced from this " - "MessageWrapper. The value for new will not be " - "propagated") - - new = property(_get_new, _set_new, - doc="The `new` flag for this MessageWrapper") - - def _get_dirty(self): - """ - Get the value for the `dirty` flag. - - :rtype: bool - """ - return self._dirty - - def _set_dirty(self, value=True): - """ - Set the value for the `dirty` flag, and propagate it - to the memory store if any. - - :param value: the value to set - :type value: bool - """ - self._dirty = value - if self.memstore: - mbox = self.fdoc.content.get('mbox', None) - uid = self.fdoc.content.get('uid', None) - if not mbox or not uid: - logger.warning("Malformed fdoc") - return - key = mbox, uid - fun = [self.memstore.unset_dirty_queued, - self.memstore.set_dirty_queued][int(value)] - fun(key) - else: - logger.warning("Could not find a memstore referenced from this " - "MessageWrapper. The value for new will not be " - "propagated") - - dirty = property(_get_dirty, _set_dirty) - - # IMessageContainer - - @property - def fdoc(self): - """ - Return a MessagePartDoc wrapping around a weak reference to - the flags-document in this MemoryStore, if any. - - :rtype: MessagePartDoc - """ - _fdoc = self._dict.get(self.FDOC, None) - if _fdoc: - content_ref = weakref.proxy(_fdoc) - else: - logger.warning("NO FDOC!!!") - content_ref = {} - - return MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.fdoc, - content=content_ref, - doc_id=self._dict[self.DOCS_ID].get( - self.FDOC, None)) - - @property - def hdoc(self): - """ - Return a MessagePartDoc wrapping around a weak reference to - the headers-document in this MemoryStore, if any. - - :rtype: MessagePartDoc - """ - _hdoc = self._dict.get(self.HDOC, None) - if _hdoc: - content_ref = weakref.proxy(_hdoc) - else: - content_ref = {} - return MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.hdoc, - content=content_ref, - doc_id=self._dict[self.DOCS_ID].get( - self.HDOC, None)) - - @property - def cdocs(self): - """ - Return a weak reference to a zero-indexed dict containing - the content-documents, or an empty dict if none found. - If you want access to the MessagePartDoc for the individual - parts, use the generator returned by `walk` instead. - - :rtype: dict - """ - _cdocs = self._dict.get(self.CDOCS, None) - if _cdocs: - return weakref.proxy(_cdocs) - else: - return {} - - def walk(self): - """ - Generator that iterates through all the parts, returning - MessagePartDoc. Used for writing to SoledadStore. - - :rtype: generator - """ - if self._dirty: - try: - mbox = self.fdoc.content[fields.MBOX_KEY] - uid = self.fdoc.content[fields.UID_KEY] - docid_dict = self._dict[self.DOCS_ID] - docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc( - mbox, uid) - except Exception as exc: - logger.debug("Error while walking message...") - logger.exception(exc) - - if not empty(self.fdoc.content) and 'uid' in self.fdoc.content: - yield self.fdoc - if not empty(self.hdoc.content): - yield self.hdoc - for cdoc in self.cdocs.values(): - if not empty(cdoc): - content_ref = weakref.proxy(cdoc) - yield MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.cdoc, - content=content_ref, - doc_id=None) - - # i/o - - def as_dict(self): - """ - Return a dict representation of the parts contained. - - :rtype: dict - """ - return self._dict - - def from_dict(self, msg_dict): - """ - Populate MessageWrapper parts from a dictionary. - It expects the same format that we use in a - MessageWrapper. - - - :param msg_dict: a dictionary containing the parts to populate - the MessageWrapper from - :type msg_dict: dict - """ - fdoc, hdoc, cdocs = map( - lambda part: msg_dict.get(part, None), - [self.FDOC, self.HDOC, self.CDOCS]) - - for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc), - (self.CDOCS, cdocs)): - self._dict[t] = ReferenciableDict(doc) if doc else None - - -class MessagePart(object): - """ - IMessagePart implementor, to be passed to several methods - of the IMAP4Server. - It takes a subpart message and is able to find - the inner parts. - - See the interface documentation. - """ - - implements(imap4.IMessagePart) - - def __init__(self, soledad, part_map): - """ - Initializes the MessagePart. - - :param soledad: Soledad instance. - :type soledad: Soledad - :param part_map: a dictionary containing the parts map for this - message - :type part_map: dict - """ - # TODO - # It would be good to pass the uid/mailbox also - # for references while debugging. - - # We have a problem on bulk moves, and is - # that when the fetch on the new mailbox is done - # the parts maybe are not complete. - # So we should be able to fail with empty - # docs until we solve that. The ideal would be - # to gather the results of the deferred operations - # to signal the operation is complete. - #leap_assert(part_map, "part map dict cannot be null") - - self._soledad = soledad - self._pmap = part_map - - def getSize(self): - """ - Return the total size, in octets, of this message part. - - :return: size of the message, in octets - :rtype: int - """ - if empty(self._pmap): - return 0 - size = self._pmap.get('size', None) - if size is None: - logger.error("Message part cannot find size in the partmap") - size = 0 - return size - - def getBodyFile(self): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO - """ - fd = StringIO.StringIO() - if not empty(self._pmap): - multi = self._pmap.get('multi') - if not multi: - phash = self._pmap.get("phash", None) - else: - pmap = self._pmap.get('part_map') - first_part = pmap.get('1', None) - if not empty(first_part): - phash = first_part['phash'] - else: - phash = None - - if phash is None: - logger.warning("Could not find phash for this subpart!") - payload = "" - else: - payload = self._get_payload_from_document_memoized(phash) - if empty(payload): - payload = self._get_payload_from_document(phash) - - else: - logger.warning("Message with no part_map!") - payload = "" - - if payload: - content_type = self._get_ctype_from_document(phash) - charset = find_charset(content_type) - if charset is None: - charset = self._get_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - - fd.write(payload) - fd.seek(0) - return fd - - # TODO should memory-bound this memoize!!! - @memoized_method - def _get_payload_from_document_memoized(self, phash): - """ - Memoized method call around the regular method, to be able - to call the non-memoized method in case we got a None. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode or None - """ - return self._get_payload_from_document(phash) - - def _get_payload_from_document(self, phash): - """ - Return the message payload from the content document. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode or None - """ - cdocs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - - cdoc = first(cdocs) - if cdoc is None: - logger.warning( - "Could not find the content doc " - "for phash %s" % (phash,)) - payload = "" - else: - payload = cdoc.content.get(fields.RAW_KEY, "") - return payload - - # TODO should memory-bound this memoize!!! - @memoized_method - def _get_ctype_from_document(self, phash): - """ - Reeturn the content-type from the content document. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode - """ - cdocs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - - cdoc = first(cdocs) - if not cdoc: - logger.warning( - "Could not find the content doc " - "for phash %s" % (phash,)) - ctype = cdoc.content.get('ctype', "") - return ctype - - @memoized_method - def _get_charset(self, stuff): - # TODO put in a common class with LeapMessage - """ - Gets (guesses?) the charset of a payload. - - :param stuff: the stuff to guess about. - :type stuff: str or unicode - :return: charset - :rtype: unicode - """ - # XXX existential doubt 2. shouldn't we make the scope - # of the decorator somewhat more persistent? - # ah! yes! and put memory bounds. - return get_email_charset(stuff) - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - # XXX refactor together with MessagePart method - if not self._pmap: - logger.warning("No pmap in Subpart!") - return {} - headers = dict(self._pmap.get("headers", [])) - - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # default to most likely standard - charset = find_charset(headers, "utf-8") - headers2 = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() - - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - # filter original dict by negate-condition - if cond(key): - headers2[key] = value - return headers2 - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - if empty(self._pmap): - logger.warning("Could not get part map!") - return False - multi = self._pmap.get("multi", False) - return multi - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - if not self.isMultipart(): - raise TypeError - - sub_pmap = self._pmap.get("part_map", {}) - try: - part_map = sub_pmap[str(part + 1)] - except KeyError: - logger.debug("getSubpart for %s: KeyError" % (part,)) - raise IndexError - - # XXX check for validity - return MessagePart(self._soledad, part_map) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index d47c8eb..7e0f973 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# messages.py -# Copyright (C) 2013, 2014 LEAP +# imap/messages.py +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,85 +15,41 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -LeapMessage and MessageCollection. +IMAPMessage and IMAPMessageCollection. """ -import copy import logging -import threading -import StringIO - -from collections import defaultdict -from functools import partial - +# import StringIO from twisted.mail import imap4 -from twisted.internet import reactor from zope.interface import implements -from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset -from leap.mail.adaptors import soledad_indexes as indexes -from leap.mail.constants import INBOX_NAME -from leap.mail.utils import find_charset, empty -from leap.mail.imap.index import IndexedDB -from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.messageparts import MessagePart, MessagePartDoc -from leap.mail.imap.parser import MBoxParser - -logger = logging.getLogger(__name__) - -# TODO ------------------------------------------------------------ -# [ ] Add ref to incoming message during add_msg -# [ ] Add linked-from info. -# * Need a new type of documents: linkage info. -# * HDOCS are linked from FDOCs (ref to chash) -# * CDOCS are linked from HDOCS (ref to chash) +from leap.mail.utils import find_charset -# [ ] Delete incoming mail only after successful write! -# [ ] Remove UID from syncable db. Store only those indexes locally. +from leap.mail.imap.messageparts import MessagePart +# from leap.mail.imap.messagepargs import MessagePartDoc +logger = logging.getLogger(__name__) -def try_unique_query(curried): - """ - Try to execute a query that is expected to have a - single outcome, and log a warning if more than one document found. - - :param curried: a curried function - :type curried: callable - """ - # XXX FIXME ---------- convert to deferreds - leap_assert(callable(curried), "A callable is expected") - try: - query = curried() - if query: - if len(query) > 1: - # TODO we could take action, like trigger a background - # process to kill dupes. - name = getattr(curried, 'expected', 'doc') - logger.warning( - "More than one %s found for this mbox, " - "we got a duplicate!!" % (name,)) - return query.pop() - else: - return None - except Exception as exc: - logger.exception("Unhandled error %r" % exc) - +# TODO ------------------------------------------------------------ -# FIXME remove-me -#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# [ ] Add ref to incoming message during add_msg. +# [ ] Delete incoming mail only after successful write. -class IMAPMessage(fields, MBoxParser): +class IMAPMessage(object): """ The main representation of a message. """ implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox): + # TODO ---- see what should we pass here instead + # where's UID added to the message? + # def __init__(self, soledad, uid, mbox): + def __init__(self, message, collection): """ Initializes a LeapMessage. @@ -103,81 +59,14 @@ class IMAPMessage(fields, MBoxParser): :type uid: int or basestring :param mbox: the mbox this message belongs to :type mbox: str or unicode - :param collection: a reference to the parent collection object - :type collection: MessageCollection - :param container: a IMessageContainer implementor instance - :type container: IMessageContainer """ - self._soledad = soledad - self._uid = int(uid) if uid is not None else None - self._mbox = self._parse_mailbox_name(mbox) - - self.__chash = None - self.__bdoc = None - - # TODO collection and container are deprecated. - - # TODO move to adaptor - - #@property - #def fdoc(self): - #""" - #An accessor to the flags document. - #""" - #if all(map(bool, (self._uid, self._mbox))): - #fdoc = None - #if self._container is not None: - #fdoc = self._container.fdoc - #if not fdoc: - #fdoc = self._get_flags_doc() - #if fdoc: - #fdoc_content = fdoc.content - #self.__chash = fdoc_content.get( - #fields.CONTENT_HASH_KEY, None) - #return fdoc -# - #@property - #def hdoc(self): - #""" - #An accessor to the headers document. - #""" - #container = self._container - #if container is not None: - #hdoc = self._container.hdoc - #if hdoc and not empty(hdoc.content): - #return hdoc - #hdoc = self._get_headers_doc() -# - #if container and not empty(hdoc.content): - # mem-cache it - #hdoc_content = hdoc.content - #chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - #hdocs = {chash: hdoc_content} - #container.memstore.load_header_docs(hdocs) - #return hdoc -# - #@property - #def chash(self): - #""" - #An accessor to the content hash for this message. - #""" - #if not self.fdoc: - #return None - #if not self.__chash and self.fdoc: - #self.__chash = self.fdoc.content.get( - #fields.CONTENT_HASH_KEY, None) - #return self.__chash - - #@property - #def bdoc(self): - #""" - #An accessor to the body document. - #""" - #if not self.hdoc: - #return None - #if not self.__bdoc: - #self.__bdoc = self._get_body_doc() - #return self.__bdoc + #self._uid = int(uid) if uid is not None else None + #self._mbox = normalize_mailbox(mbox) + + self.message = message + + # TODO maybe not needed, see setFlags below + self.collection = collection # IMessage implementation @@ -188,12 +77,7 @@ class IMAPMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ - # TODO ----> return lookup in local sqlcipher table. - return self._uid - - # -------------------------------------------------------------- - # TODO -- from here on, all the methods should be proxied to the - # instance of leap.mail.mail.Message + return self.message.get_uid() def getFlags(self): """ @@ -202,24 +86,14 @@ class IMAPMessage(fields, MBoxParser): :return: The flags, represented as strings :rtype: tuple """ - uid = self._uid - - flags = set([]) - fdoc = self.fdoc - if fdoc: - flags = set(fdoc.content.get(self.FLAGS_KEY, None)) + return self.message.get_flags() - msgcol = self._collection + # setFlags not in the interface spec but we use it with store command. - # We treat the recent flag specially: gotten from - # a mailbox-level document. - if msgcol and uid in msgcol.recent_flags: - flags.add(fields.RECENT_FLAG) - if flags: - flags = map(str, flags) - return tuple(flags) + # XXX if we can move it to a collection method, we don't need to pass + # collection to the IMAPMessage - # setFlags not in the interface spec but we use it with store command. + # lookup method? IMAPMailbox? def setFlags(self, flags, mode): """ @@ -231,32 +105,11 @@ class IMAPMessage(fields, MBoxParser): :type mode: int """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - mbox, uid = self._mbox, self._uid - - APPEND = 1 - REMOVE = -1 - SET = 0 - - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) - - return map(str, newflags) + # XXX + # return new flags + # map to str + #self.message.set_flags(flags, mode) + self.collection.update_flags(self.message, flags, mode) def getInternalDate(self): """ @@ -273,8 +126,7 @@ class IMAPMessage(fields, MBoxParser): :return: An RFC822-formatted date string. :rtype: str """ - date = self.hdoc.content.get(fields.DATE_KEY, '') - return date + return self.message.get_internal_date() # # IMessagePart @@ -290,42 +142,40 @@ class IMAPMessage(fields, MBoxParser): :return: file-like object opened for reading :rtype: StringIO """ - def write_fd(body): - fd.write(body) - fd.seek(0) - return fd - + #def write_fd(body): + #fd.write(body) + #fd.seek(0) + #return fd +# # TODO refactor with getBodyFile in MessagePart - - fd = StringIO.StringIO() - - if self.bdoc is not None: - bdoc_content = self.bdoc.content - if empty(bdoc_content): - logger.warning("No BDOC content found for message!!!") - return write_fd("") - - body = bdoc_content.get(self.RAW_KEY, "") - content_type = bdoc_content.get('content-type', "") - charset = find_charset(content_type) - if charset is None: - charset = self._get_charset(body) - try: - if isinstance(body, unicode): - body = body.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - logger.debug("Attempted to encode with: %s" % charset) - body = body.encode(charset, 'replace') - finally: - return write_fd(body) - - # We are still returning funky characters from here. - else: - logger.warning("No BDOC found for message.") - return write_fd("") - +# + #fd = StringIO.StringIO() +# + #if self.bdoc is not None: + #bdoc_content = self.bdoc.content + #if empty(bdoc_content): + #logger.warning("No BDOC content found for message!!!") + #return write_fd("") +# + #body = bdoc_content.get(self.RAW_KEY, "") + #content_type = bdoc_content.get('content-type', "") + #charset = find_charset(content_type) + #if charset is None: + #charset = self._get_charset(body) + #try: + #if isinstance(body, unicode): + #body = body.encode(charset) + #except UnicodeError as exc: + #logger.error( + #"Unicode error, using 'replace'. {0!r}".format(exc)) + #logger.debug("Attempted to encode with: %s" % charset) + #body = body.encode(charset, 'replace') + #finally: + #return write_fd(body) + + return self.message.get_body_file() + + # TODO move to mail.mail @memoized_method def _get_charset(self, stuff): """ @@ -337,7 +187,7 @@ class IMAPMessage(fields, MBoxParser): """ # XXX shouldn't we make the scope # of the decorator somewhat more persistent? - # ah! yes! and put memory bounds. + # and put memory bounds. return get_email_charset(stuff) def getSize(self): @@ -347,17 +197,11 @@ class IMAPMessage(fields, MBoxParser): :return: size of the message, in octets :rtype: int """ - size = None - if self.fdoc is not None: - fdoc_content = self.fdoc.content - size = fdoc_content.get(self.SIZE_KEY, False) - else: - logger.warning("No FLAGS doc for %s:%s" % (self._mbox, - self._uid)) - #if not size: - # XXX fallback, should remove when all migrated. - #size = self.getBodyFile().len - return size + #size = None + #fdoc_content = self.fdoc.content + #size = fdoc_content.get(self.SIZE_KEY, False) + #return size + return self.message.get_size() def getHeaders(self, negate, *names): """ @@ -374,10 +218,10 @@ class IMAPMessage(fields, MBoxParser): :return: A mapping of header field names to header field values :rtype: dict """ - # TODO split in smaller methods + # TODO split in smaller methods -- format_headers()? # XXX refactor together with MessagePart method - headers = self._get_headers() + headers = self.message.get_headers() # XXX keep this in the imap imessage implementation, # because the server impl. expects content-type to be present. @@ -417,34 +261,15 @@ class IMAPMessage(fields, MBoxParser): headers2[key] = value return headers2 - def _get_headers(self): - """ - Return the headers dict for this message. - """ - if self.hdoc is not None: - hdoc_content = self.hdoc.content - headers = hdoc_content.get(self.HEADERS_KEY, {}) - return headers - - else: - logger.warning( - "No HEADERS doc for msg %s:%s" % ( - self._mbox, - self._uid)) - def isMultipart(self): """ Return True if this message is multipart. """ - if self.fdoc: - fdoc_content = self.fdoc.content - is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) - return is_multipart - else: - logger.warning( - "No FLAGS doc for msg %s:%s" % ( - self._mbox, - self._uid)) + #fdoc_content = self.fdoc.content + #is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) + #return is_multipart + + return self.message.fdoc.is_multi def getSubPart(self, part): """ @@ -463,12 +288,16 @@ class IMAPMessage(fields, MBoxParser): pmap_dict = self._get_part_from_parts_map(part + 1) except KeyError: raise IndexError + + # TODO move access to adaptor ---- return MessagePart(self._soledad, pmap_dict) # # accessors # + # FIXME + # -- move to wrapper/adaptor def _get_part_from_parts_map(self, part): """ Get a part map from the headers doc @@ -476,100 +305,44 @@ class IMAPMessage(fields, MBoxParser): :raises: KeyError if key does not exist :rtype: dict """ - if not self.hdoc: - logger.warning("Tried to get part but no HDOC found!") - return None - - hdoc_content = self.hdoc.content - pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) + raise NotImplementedError() + #hdoc_content = self.hdoc.content + #pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) +# # remember, lads, soledad is using strings in its keys, # not integers! - return pmap[str(part)] + #return pmap[str(part)] - # XXX moved to memory store - # move the rest too. ------------------------------------------ - def _get_flags_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - def get_first_if_any(docs): - result = first(docs) - return result if result else {} - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - d.addCallback(get_first_if_any) - return d - - # TODO move to soledadstore instead of accessing soledad directly - def _get_headers_doc(self): - """ - Return the document that keeps the headers for this - message. - """ - d = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(self.chash)) - d.addCallback(lambda docs: first(docs)) - return d - - # TODO move to soledadstore instead of accessing soledad directly + # TODO move to wrapper/adaptor def _get_body_doc(self): """ Return the document that keeps the body for this message. """ - # XXX FIXME --- this might need a maybedeferred - # on the receiving side... - hdoc_content = self.hdoc.content - body_phash = hdoc_content.get( - fields.BODY_KEY, None) - if not body_phash: - logger.warning("No body phash for this document!") - return None - - # XXX get from memstore too... - # if memstore: memstore.get_phrash - # memstore should keep a dict with weakrefs to the - # phash doc... - - if self._container is not None: - bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) - if not empty(bdoc) and not empty(bdoc.content): - return bdoc - + # FIXME + # -- just get the body and retrieve the cdoc P- + #hdoc_content = self.hdoc.content + #body_phash = hdoc_content.get( + #fields.BODY_KEY, None) + #if not body_phash: + #logger.warning("No body phash for this document!") + #return None +# + #if self._container is not None: + #bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) + #if not empty(bdoc) and not empty(bdoc.content): + #return bdoc +# # no memstore, or no body doc found there - d = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(body_phash)) - d.addCallback(lambda docs: first(docs)) - return d - - def __getitem__(self, key): - """ - Return an item from the content of the flags document, - for convenience. - - :param key: The key - :type key: str - - :return: The content value indexed by C{key} or None - :rtype: str - """ - return self.fdoc.content.get(key, None) - - def does_exist(self): - """ - Return True if there is actually a flags document for this - UID and mbox. - """ - return not empty(self.fdoc) + #d = self._soledad.get_from_index( + #fields.TYPE_P_HASH_IDX, + #fields.TYPE_CONTENT_VAL, str(body_phash)) + #d.addCallback(lambda docs: first(docs)) + #return d -class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): +class IMAPMessageCollection(object): """ A collection of messages, surprisingly. @@ -578,9 +351,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): database. """ - # XXX this should be able to produce a MessageSet methinks - # could validate these kinds of objects turning them - # into a template for the class. + messageklass = IMAPMessage + + # TODO + # [ ] Add RECENT flags docs to mailbox-doc attributes (list-of-uids) + # [ ] move Query for all the headers documents to Collection + + # TODO this should be able to produce a MessageSet methinks + # TODO --- reimplement, review and prune documentation below. + FLAGS_DOC = "FLAGS" HEADERS_DOC = "HEADERS" CONTENT_DOC = "CONTENT" @@ -604,145 +383,40 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): """ HDOCS_SET_DOC = "HDOCS_SET" - templates = { - - # Mailbox Level - - RECENT_DOC: { - "type": indexes.RECENT, - "mbox": INBOX_NAME, - fields.RECENTFLAGS_KEY: [], - }, - - HDOCS_SET_DOC: { - "type": indexes.HDOCS_SET, - "mbox": INBOX_NAME, - fields.HDOCS_SET_KEY: [], - } - - - } - - # Different locks for wrapping both the u1db document getting/setting - # and the property getting/settting in an atomic operation. - - # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks - _rdoc_lock = defaultdict(lambda: threading.Lock()) - _rdoc_write_lock = defaultdict(lambda: threading.Lock()) - _rdoc_read_lock = defaultdict(lambda: threading.Lock()) - _rdoc_property_lock = defaultdict(lambda: threading.Lock()) - - _initialized = {} - - def __init__(self, mbox=None, soledad=None, memstore=None): - """ - Constructor for MessageCollection. - - On initialization, we ensure that we have a document for - storing the recent flags. The nature of this flag make us wanting - to store the set of the UIDs with this flag at the level of the - MessageCollection for each mailbox, instead of treating them - as a property of each message. - - We are passed an instance of MemoryStore, the same for the - SoledadBackedAccount, that we use as a read cache and a buffer - for writes. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database. - :type mbox: str - :param soledad: Soledad database - :type soledad: Soledad instance - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + def __init__(self, collection): """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") + Constructor for IMAPMessageCollection. - # okay, all in order, keep going... - - self.mbox = self._parse_mailbox_name(mbox) - - # XXX get a SoledadStore passed instead - self._soledad = soledad - self.memstore = memstore - - self.__rflags = None - - if not self._initialized.get(mbox, False): - try: - self.initialize_db() - # ensure that we have a recent-flags doc - self._get_or_create_rdoc() - except Exception: - logger.debug("Error initializing %r" % (mbox,)) - else: - self._initialized[mbox] = True - - def _get_empty_doc(self, _type=FLAGS_DOC): - """ - Returns an empty doc for storing different message parts. - Defaults to returning a template for a flags document. - :return: a dict with the template - :rtype: dict - """ - if _type not in self.templates.keys(): - raise TypeError("Improper type passed to _get_empty_doc") - return copy.deepcopy(self.templates[_type]) - - def _get_or_create_rdoc(self): - """ - Try to retrieve the recent-flags doc for this MessageCollection, - and create one if not found. + :param collection: an instance of a MessageCollection + :type collection: MessageCollection """ - # XXX should move this to memstore too - with self._rdoc_write_lock[self.mbox]: - rdoc = self._get_recent_doc_from_soledad() - if rdoc is None: - rdoc = self._get_empty_doc(self.RECENT_DOC) - if self.mbox != fields.INBOX_VAL: - rdoc[fields.MBOX_KEY] = self.mbox - self._soledad.create_doc(rdoc) - - # -------------------------------------------------------------------- + leap_assert( + collection.is_mailbox_collection(), + "Need a mailbox name to initialize") + mbox_name = collection.mbox_name + leap_assert(mbox_name.strip() != "", "mbox cannot be blank space") + leap_assert(isinstance(mbox_name, (str, unicode)), + "mbox needs to be a string") + self.collection = collection - # ----------------------------------------------------------------------- + # XXX this has to be done in IMAPAccount + # (Where the collection must be instantiated and passed to us) + # self.mbox = normalize_mailbox(mbox) - def _fdoc_already_exists(self, chash): + @property + def mbox_name(self): """ - Check whether we can find a flags doc for this mailbox with the - given content-hash. It enforces that we can only have the same maessage - listed once for a a given mailbox. - - :param chash: the content-hash to check about. - :type chash: basestring - :return: False, if it does not exist, or UID. + Return the string that identifies this mailbox. """ - exist = False - exist = self.memstore.get_fdoc_from_chash(chash, self.mbox) + return self.collection.mbox_name - if not exist: - exist = self._get_fdoc_from_chash(chash) - if exist and exist.content is not None: - return exist.content.get(fields.UID_KEY, "unknown-uid") - else: - return False - - def add_msg(self, raw, subject=None, flags=None, date=None, - notify_on_disk=False): + def add_msg(self, raw, flags=None, date=None): """ Creates a new message document. :param raw: the raw message :type raw: str - :param subject: subject of the message. - :type subject: str - :param flags: flags :type flags: list @@ -756,212 +430,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): if flags is None: flags = tuple() leap_assert_type(flags, tuple) + return self.collection.add_msg(raw, flags, date) - # TODO ---- proxy to MessageCollection addMessage - - #observer = defer.Deferred() - #d = self._do_parse(raw) - #d.addCallback(lambda result: reactor.callInThread( - #self._do_add_msg, result, flags, subject, date, - #notify_on_disk, observer)) - #return observer - - # TODO --------------------------------------------------- - # move this to leap.mail.adaptors.soledad - - def _do_add_msg(self, parse_result, flags, subject, - date, notify_on_disk, observer): - """ - """ - msg, parts, chash, size, multi = parse_result - - # XXX move to SoledadAdaptor write operation ... ??? - # check for uniqueness -------------------------------- - # Watch out! We're reserving a UID right after this! - existing_uid = self._fdoc_already_exists(chash) - if existing_uid: - msg = self.get_msg_by_uid(existing_uid) - reactor.callFromThread(observer.callback, existing_uid) - msg.setFlags((fields.DELETED_FLAG,), -1) - return - - # TODO move UID autoincrement to MessageCollection.addMessage(mailbox) - # TODO S2 -- get FUCKING UID from autoincremental table - #uid = self.memstore.increment_last_soledad_uid(self.mbox) - #self.set_recent_flag(uid) - - - # ------------------------------------------------------------ - - # - # getters: specific queries - # - - # recent flags - - def _get_recent_flags(self): - """ - An accessor for the recent-flags set for this mailbox. + def get_msg_by_uid(self, uid, absolute=True): """ - # XXX check if we should remove this - if self.__rflags is not None: - return self.__rflags - - if self.memstore is not None: - with self._rdoc_lock[self.mbox]: - rflags = self.memstore.get_recent_flags(self.mbox) - if not rflags: - # not loaded in the memory store yet. - # let's fetch them from soledad... - rdoc = self._get_recent_doc_from_soledad() - if rdoc is None: - return set([]) - rflags = set(rdoc.content.get( - fields.RECENTFLAGS_KEY, [])) - # ...and cache them now. - self.memstore.load_recent_flags( - self.mbox, - {'doc_id': rdoc.doc_id, 'set': rflags}) - return rflags - - def _set_recent_flags(self, value): - """ - Setter for the recent-flags set for this mailbox. - """ - if self.memstore is not None: - self.memstore.set_recent_flags(self.mbox, value) - - recent_flags = property( - _get_recent_flags, _set_recent_flags, - doc="Set of UIDs with the recent flag for this mailbox.") - - def _get_recent_doc_from_soledad(self): - """ - Get recent-flags document from Soledad for this mailbox. - :rtype: SoledadDocument or None - """ - # FIXME ----- use deferreds. - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MBOX_IDX, - fields.TYPE_RECENT_VAL, self.mbox) - curried.expected = "rdoc" - with self._rdoc_read_lock[self.mbox]: - return try_unique_query(curried) - - # Property-set modification (protected by a different - # lock to give atomicity to the read/write operation) - - def unset_recent_flags(self, uids): - """ - Unset Recent flag for a sequence of uids. - - :param uids: the uids to unset - :type uid: sequence - """ - # FIXME ----- use deferreds. - with self._rdoc_property_lock[self.mbox]: - self.recent_flags.difference_update( - set(uids)) - - # Individual flags operations - - def unset_recent_flag(self, uid): - """ - Unset Recent flag for a given uid. - - :param uid: the uid to unset - :type uid: int - """ - # FIXME ----- use deferreds. - with self._rdoc_property_lock[self.mbox]: - self.recent_flags.difference_update( - set([uid])) - - def set_recent_flag(self, uid): - """ - Set Recent flag for a given uid. + Retrieves a IMAPMessage by UID. + This is used primarity in the Mailbox fetch and store methods. - :param uid: the uid to set + :param uid: the message uid to query by :type uid: int - """ - # FIXME ----- use deferreds. - with self._rdoc_property_lock[self.mbox]: - self.recent_flags = self.recent_flags.union( - set([uid])) - - # individual doc getters, message layer. - def _get_fdoc_from_chash(self, chash): + :rtype: IMAPMessage """ - Return a flags document for this mailbox with a given chash. + def make_imap_msg(msg): + kls = self.messageklass + # TODO --- remove ref to collection + return kls(msg, self.collection) - :return: A SoledadDocument containing the Flags Document, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - # USED from: - # [ ] duplicated fdoc detection - # [ ] _get_uid_from_msgidCb - - # FIXME ----- use deferreds. - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MBOX_C_HASH_IDX, - fields.TYPE_FLAGS_VAL, self.mbox, chash) - curried.expected = "fdoc" - fdoc = try_unique_query(curried) - if fdoc is not None: - return fdoc - else: - # probably this should be the other way round, - # ie, try fist on memstore... - cf = self.memstore._chash_fdoc_store - fdoc = cf[chash][self.mbox] - # hey, I just needed to wrap fdoc thing into - # a "content" attribute, look a better way... - if not empty(fdoc): - return MessagePartDoc( - new=None, dirty=None, part=None, - store=None, doc_id=None, - content=fdoc) - - def _get_uid_from_msgidCb(self, msgid): - hdoc = None - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MSGID_IDX, - fields.TYPE_HEADERS_VAL, msgid) - curried.expected = "hdoc" - hdoc = try_unique_query(curried) - - # XXX this is only a quick hack to avoid regression - # on the "multiple copies of the draft" issue, but - # this is currently broken since it's not efficient to - # look for this. Should lookup better. - # FIXME! - - if hdoc is not None: - hdoc_dict = hdoc.content + d = self.collection.get_msg_by_uid(uid, absolute=absolute) + d.addCalback(make_imap_msg) + return d - else: - hdocstore = self.memstore._hdoc_store - match = [x for _, x in hdocstore.items() if x['msgid'] == msgid] - hdoc_dict = first(match) - - if hdoc_dict is None: - logger.warning("Could not find hdoc for msgid %s" - % (msgid,)) - return None - msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY) - - fdoc = self._get_fdoc_from_chash(msg_chash) - if not fdoc: - logger.warning("Could not find fdoc for msgid %s" - % (msgid,)) - return None - return fdoc.content.get(fields.UID_KEY, None) + # TODO -- move this to collection too + # Used for the Search (Drafts) queries? def _get_uid_from_msgid(self, msgid): """ Return a UID for a given message-id. @@ -972,15 +464,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :return: A UID, or None """ - # We need to wait a little bit, cause in some of the cases - # the query is received right after we've saved the document, - # and we cannot find it otherwise. This seems to be enough. - - # XXX do a deferLater instead ?? - # XXX is this working? return self._get_uid_from_msgidCb(msgid) - def set_flags(self, mbox, messages, flags, mode, observer): + # TODO handle deferreds + def set_flags(self, messages, flags, mode): """ Set flags for a sequence of messages. @@ -1000,142 +487,27 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): getmsg = self.get_msg_by_uid def set_flags(uid, flags, mode): - msg = getmsg(uid, mem_only=True, flags_only=True) + msg = getmsg(uid) if msg is not None: + # XXX IMAPMessage needs access to the collection + # to be able to set flags. Better if we make use + # of collection... here. return uid, msg.setFlags(flags, mode) setted_flags = [set_flags(uid, flags, mode) for uid in messages] result = dict(filter(None, setted_flags)) + # XXX return gatherResults or something + return result - # TODO -- remove - reactor.callFromThread(observer.callback, result) - - # getters: generic for a mailbox - - def get_msg_by_uid(self, uid, mem_only=False, flags_only=False): - """ - Retrieves a LeapMessage by UID. - This is used primarity in the Mailbox fetch and store methods. - - :param uid: the message uid to query by - :type uid: int - :param mem_only: a flag that indicates whether this Message should - pass a reference to soledad to retrieve missing pieces - or not. - :type mem_only: bool - :param flags_only: whether the message should carry only a reference - to the flags document. - :type flags_only: bool - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - msg_container = self.memstore.get_message( - self.mbox, uid, flags_only=flags_only) - - if msg_container is not None: - if mem_only: - msg = IMAPMessage(None, uid, self.mbox, collection=self, - container=msg_container) - else: - # We pass a reference to soledad just to be able to retrieve - # missing parts that cannot be found in the container, like - # the content docs after a copy. - msg = IMAPMessage(self._soledad, uid, self.mbox, - collection=self, container=msg_container) - else: - msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self) - - if not msg.does_exist(): - return None - return msg - - # FIXME --- used where ? --------------------------------------------- - #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - #""" - #Get all documents for the selected mailbox of the - #passed type. By default, it returns the flag docs. -# - #If you want acess to the content, use __iter__ instead -# - #:return: a Deferred, that will fire with a list of u1db documents - #:rtype: Deferred (promise of list of SoledadDocument) - #""" - #if _type not in fields.__dict__.values(): - #raise TypeError("Wrong type passed to get_all_docs") -# - # FIXME ----- either raise or return a deferred wrapper. - #if sameProxiedObjects(self._soledad, None): - #logger.warning('Tried to get messages but soledad is None!') - #return [] -# - #def get_sorted_docs(docs): - #all_docs = [doc for doc in docs] - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - # FIXME ---------------------------------------------- - #return sorted(all_docs, key=lambda item: item.content['uid']) -# - #d = self._soledad.get_from_index( - #fields.TYPE_MBOX_IDX, _type, self.mbox) - #d.addCallback(get_sorted_docs) - #return d - - def all_soledad_uid_iter(self): - """ - Return an iterator through the UIDs of all messages, sorted in - ascending order. - """ - # XXX FIXME ------ sorted??? - - def get_uids(docs): - return set([ - doc.content[self.UID_KEY] for doc in docs if not empty(doc)]) - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) - d.addCallback(get_uids) - return d - - def all_uid_iter(self): - """ - Return an iterator through the UIDs of all messages, from memory. + def count(self): """ - mem_uids = self.memstore.get_uids(self.mbox) - soledad_known_uids = self.memstore.get_soledad_known_uids( - self.mbox) - combined = tuple(set(mem_uids).union(soledad_known_uids)) - return combined + Return the count of messages for this mailbox. - def get_all_soledad_flag_docs(self): + :rtype: int """ - Return a dict with the content of all the flag documents - in soledad store for the given mbox. + return self.collection.count() - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - # XXX we really could return a reduced version with - # just {'uid': (flags-tuple,) since the prefetch is - # only oriented to get the flag tuples. - - def get_content(docs): - all_docs = [( - doc.content[self.UID_KEY], - dict(doc.content)) - for doc in docs - if not empty(doc.content)] - all_flags = dict(all_docs) - return all_flags - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - d.addCallback(get_content) - return d + # headers query def all_headers(self): """ @@ -1144,15 +516,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :rtype: dict """ - return self.memstore.all_headers(self.mbox) - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - return self.memstore.count(self.mbox) + # Use self.collection.mbox_indexer + # and derive all the doc_ids for the hdocs + raise NotImplementedError() # unseen messages @@ -1164,7 +530,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :return: iterator through unseen message doc UIDs :rtype: iterable """ - return self.memstore.unseen_iter(self.mbox) + raise NotImplementedError() def count_unseen(self): """ @@ -1182,13 +548,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: a list of LeapMessages :rtype: list """ - return [IMAPMessage(self._soledad, docid, self.mbox, collection=self) - for docid in self.unseen_iter()] + raise NotImplementedError() + #return [self.messageklass(self._soledad, doc_id, self.mbox) + #for doc_id in self.unseen_iter()] # recent messages - # XXX take it from memstore - # XXX Used somewhere? def count_recent(self): """ Count all messages with the `Recent` flag. @@ -1199,32 +564,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: count :rtype: int """ - return len(self.recent_flags) + raise NotImplementedError() + + # magic def __len__(self): """ Returns the number of messages on this mailbox. - :rtype: int """ return self.count() - def __iter__(self): - """ - Returns an iterator over all messages. - - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self) - for docuid in self.all_uid_iter()) - def __repr__(self): """ Representation string for this object. """ - return u"" % ( - self.mbox, self.count()) + return u"" % ( + self.mbox_name, self.count()) - # XXX should implement __eq__ also !!! - # use chash... + # TODO implement __iter__ ? diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py deleted file mode 100644 index fc8ea55..0000000 --- a/src/leap/mail/imap/soledadstore.py +++ /dev/null @@ -1,617 +0,0 @@ -# -*- coding: utf-8 -*- -# soledadstore.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -A MessageStore that writes to Soledad. -""" -import logging -import threading - -from collections import defaultdict -from itertools import chain - -from u1db import errors as u1db_errors -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert_type, leap_assert -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.messageparts import MessagePartType -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.messageparts import RecentFlagsDoc -from leap.mail.imap.fields import fields -from leap.mail.imap.interfaces import IMessageStore -from leap.mail.messageflow import IMessageConsumer -from leap.mail.utils import first, empty, accumulator_queue - -logger = logging.getLogger(__name__) - - -class ContentDedup(object): - """ - Message deduplication. - - We do a query for the content hashes before writing to our beloved - sqlcipher backend of Soledad. This means, by now, that: - - 1. We will not store the same body/attachment twice, only the hash of it. - 2. We will not store the same message header twice, only the hash of it. - - The first case is useful if you are always receiving the same old memes - from unwary friends that still have not discovered that 4chan is the - generator of the internet. The second will save your day if you have - initiated session with the same account in two different machines. I also - wonder why would you do that, but let's respect each other choices, like - with the religious celebrations, and assume that one day we'll be able - to run Bitmask in completely free phones. Yes, I mean that, the whole GSM - Stack. - """ - # TODO refactor using unique_query - - def _header_does_exist(self, doc): - """ - Check whether we already have a header document for this - content hash in our database. - - :param doc: tentative header for document - :type doc: dict - :returns: True if it exists, False otherwise. - """ - if not doc: - return False - chash = doc[fields.CONTENT_HASH_KEY] - header_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(chash)) - if not header_docs: - return False - - # FIXME enable only to debug this problem. - #if len(header_docs) != 1: - #logger.warning("Found more than one copy of chash %s!" - #% (chash,)) - - #logger.debug("Found header doc with that hash! Skipping save!") - return True - - def _content_does_exist(self, doc): - """ - Check whether we already have a content document for a payload - with this hash in our database. - - :param doc: tentative content for document - :type doc: dict - :returns: True if it exists, False otherwise. - """ - if not doc: - return False - phash = doc[fields.PAYLOAD_HASH_KEY] - attach_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - if not attach_docs: - return False - - # FIXME enable only to debug this problem - #if len(attach_docs) != 1: - #logger.warning("Found more than one copy of phash %s!" - #% (phash,)) - #logger.debug("Found attachment doc with that hash! Skipping save!") - return True - - -class MsgWriteError(Exception): - """ - Raised if any exception is found while saving message parts. - """ - pass - - -""" -A lock per document. -""" -# TODO should bound the space of this!!! -# http://stackoverflow.com/a/2437645/1157664 -# Setting this to twice the number of threads in the threadpool -# should be safe. - -put_locks = defaultdict(lambda: threading.Lock()) -mbox_doc_locks = defaultdict(lambda: threading.Lock()) - - -class SoledadStore(ContentDedup): - """ - This will create docs in the local Soledad database. - """ - _remove_lock = threading.Lock() - - implements(IMessageConsumer, IMessageStore) - - def __init__(self, soledad): - """ - Initialize the permanent store that writes to Soledad database. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - from twisted.internet import reactor - self.reactor = reactor - - self._soledad = soledad - - self._CREATE_DOC_FUN = self._soledad.create_doc - self._PUT_DOC_FUN = self._soledad.put_doc - self._GET_DOC_FUN = self._soledad.get_doc - - # we instantiate an accumulator to batch the notifications - self.docs_notify_queue = accumulator_queue( - lambda item: reactor.callFromThread(self._unset_new_dirty, item), - 20) - - # IMessageStore - - # ------------------------------------------------------------------- - # We are not yet using this interface, but it would make sense - # to implement it. - - def create_message(self, mbox, uid, message): - """ - Create the passed message into this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - :param message: a IMessageContainer implementor. - """ - raise NotImplementedError() - - def put_message(self, mbox, uid, message): - """ - Put the passed existing message into this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - :param message: a IMessageContainer implementor. - """ - raise NotImplementedError() - - def remove_message(self, mbox, uid): - """ - Remove the given message from this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - """ - raise NotImplementedError() - - def get_message(self, mbox, uid): - """ - Get a IMessageContainer for the given mbox and uid combination. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - """ - raise NotImplementedError() - - # IMessageConsumer - - # TODO should handle the delete case - # TODO should handle errors better - # TODO could generalize this method into a generic consumer - # and only implement `process` here - - def consume(self, queue): - """ - Creates a new document in soledad db. - - :param queue: a tuple of queues to get item from, with content of the - document to be inserted. - :type queue: tuple of Queues - """ - new, dirty = queue - while not new.empty(): - doc_wrapper = new.get() - self.reactor.callInThread(self._consume_doc, doc_wrapper, - self.docs_notify_queue) - while not dirty.empty(): - doc_wrapper = dirty.get() - self.reactor.callInThread(self._consume_doc, doc_wrapper, - self.docs_notify_queue) - - # Queue empty, flush the notifications queue. - self.docs_notify_queue(None, flush=True) - - def _unset_new_dirty(self, doc_wrapper): - """ - Unset the `new` and `dirty` flags for this document wrapper in the - memory store. - - :param doc_wrapper: a MessageWrapper instance - :type doc_wrapper: MessageWrapper - """ - if isinstance(doc_wrapper, MessageWrapper): - # XXX still needed for debug quite often - #logger.info("unsetting new flag!") - doc_wrapper.new = False - doc_wrapper.dirty = False - - @deferred_to_thread - def _consume_doc(self, doc_wrapper, notify_queue): - """ - Consume each document wrapper in a separate thread. - We pass an instance of an accumulator that handles the notifications - to the memorystore when the write has been done. - - :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance - :type doc_wrapper: MessageWrapper or RecentFlagsDoc - :param notify_queue: a callable that handles the writeback - notifications to the memstore. - :type notify_queue: callable - """ - def queueNotifyBack(failed, doc_wrapper): - if failed: - log.msg("There was an error writing the mesage...") - else: - notify_queue(doc_wrapper) - - def doSoledadCalls(items): - # we prime the generator, that should return the - # message or flags wrapper item in the first place. - try: - doc_wrapper = items.next() - except StopIteration: - pass - else: - failed = self._soledad_write_document_parts(items) - queueNotifyBack(failed, doc_wrapper) - - doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper)) - - # - # SoledadStore specific methods. - # - - def _soledad_write_document_parts(self, items): - """ - Write the document parts to soledad in a separate thread. - - :param items: the iterator through the different document wrappers - payloads. - :type items: iterator - :return: whether the write was successful or not - :rtype: bool - """ - failed = False - for item, call in items: - if empty(item): - continue - try: - self._try_call(call, item) - except Exception as exc: - logger.debug("ITEM WAS: %s" % repr(item)) - if hasattr(item, 'content'): - logger.debug("ITEM CONTENT WAS: %s" % - repr(item.content)) - logger.exception(exc) - failed = True - continue - return failed - - def _iter_wrapper_subparts(self, doc_wrapper): - """ - Return an iterator that will yield the doc_wrapper in the first place, - followed by the subparts item and the proper call type for every - item in the queue, if any. - - :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance - :type doc_wrapper: MessageWrapper or RecentFlagsDoc - """ - if isinstance(doc_wrapper, MessageWrapper): - return chain((doc_wrapper,), - self._get_calls_for_msg_parts(doc_wrapper)) - elif isinstance(doc_wrapper, RecentFlagsDoc): - return chain((doc_wrapper,), - self._get_calls_for_rflags_doc(doc_wrapper)) - else: - logger.warning("CANNOT PROCESS ITEM!") - return (i for i in []) - - def _try_call(self, call, item): - """ - Try to invoke a given call with item as a parameter. - - :param call: the function to call - :type call: callable - :param item: the payload to pass to the call as argument - :type item: object - """ - if call is None: - return - - if call == self._PUT_DOC_FUN: - doc_id = item.doc_id - if doc_id is None: - logger.warning("BUG! Dirty doc but has no doc_id!") - return - with put_locks[doc_id]: - doc = self._GET_DOC_FUN(doc_id) - - if doc is None: - logger.warning("BUG! Dirty doc but could not " - "find document %s" % (doc_id,)) - return - - doc.content = dict(item.content) - - item = doc - try: - call(item) - except u1db_errors.RevisionConflict as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - except Exception as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - - else: - try: - call(item) - except u1db_errors.RevisionConflict as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - except Exception as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - - def _get_calls_for_msg_parts(self, msg_wrapper): - """ - Generator that return the proper call type for a given item. - - :param msg_wrapper: A MessageWrapper - :type msg_wrapper: IMessageContainer - :return: a generator of tuples with recent-flags doc payload - and callable - :rtype: generator - """ - call = None - - if msg_wrapper.new: - call = self._CREATE_DOC_FUN - - # item is expected to be a MessagePartDoc - for item in msg_wrapper.walk(): - if item.part == MessagePartType.fdoc: - yield dict(item.content), call - - elif item.part == MessagePartType.hdoc: - if not self._header_does_exist(item.content): - yield dict(item.content), call - - elif item.part == MessagePartType.cdoc: - if not self._content_does_exist(item.content): - yield dict(item.content), call - - # For now, the only thing that will be dirty is - # the flags doc. - - elif msg_wrapper.dirty: - call = self._PUT_DOC_FUN - # item is expected to be a MessagePartDoc - for item in msg_wrapper.walk(): - # XXX FIXME Give error if dirty and not doc_id !!! - doc_id = item.doc_id # defend! - if not doc_id: - logger.warning("Dirty item but no doc_id!") - continue - - if item.part == MessagePartType.fdoc: - yield item, call - - # XXX also for linkage-doc !!! - else: - logger.error("Cannot delete documents yet from the queue...!") - - def _get_calls_for_rflags_doc(self, rflags_wrapper): - """ - We always put these documents. - - :param rflags_wrapper: A wrapper around recent flags doc. - :type rflags_wrapper: RecentFlagsWrapper - :return: a tuple with recent-flags doc payload and callable - :rtype: tuple - """ - call = self._PUT_DOC_FUN - - payload = rflags_wrapper.content - if payload: - logger.debug("Saving RFLAGS to Soledad...") - yield rflags_wrapper, call - - # Mbox documents and attributes - - def get_mbox_document(self, mbox): - """ - Return mailbox document. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - with mbox_doc_locks[mbox]: - return self._get_mbox_document(mbox) - - def _get_mbox_document(self, mbox): - """ - Helper for returning the mailbox document. - """ - try: - query = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_MBOX_VAL, mbox) - if query: - return query.pop() - else: - logger.error("Could not find mbox document for %r" % - (mbox,)) - except Exception as exc: - logger.exception("Unhandled error %r" % exc) - - def get_mbox_closed(self, mbox): - """ - Return the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: bool - """ - mbox_doc = self.get_mbox_document() - return mbox_doc.content.get(fields.CLOSED_KEY, False) - - def set_mbox_closed(self, mbox, closed): - """ - Set the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param closed: the value to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - with mbox_doc_locks[mbox]: - mbox_doc = self._get_mbox_document(mbox) - if mbox_doc is None: - logger.error( - "Could not find mbox document for %r" % (mbox,)) - return - mbox_doc.content[fields.CLOSED_KEY] = closed - self._soledad.put_doc(mbox_doc) - - def write_last_uid(self, mbox, value): - """ - Write the `last_uid` integer to the proper mailbox document - in Soledad. - This is called from the deferred triggered by - memorystore.increment_last_soledad_uid, which is expected to - run in a separate thread. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - leap_assert_type(value, int) - key = fields.LAST_UID_KEY - - # XXX use accumulator to reduce number of hits - with mbox_doc_locks[mbox]: - mbox_doc = self._get_mbox_document(mbox) - old_val = mbox_doc.content[key] - if value > old_val: - mbox_doc.content[key] = value - try: - self._soledad.put_doc(mbox_doc) - except Exception as exc: - logger.error("Error while setting last_uid for %r" - % (mbox,)) - logger.exception(exc) - - def get_flags_doc(self, mbox, uid): - """ - Return the SoledadDocument for the given mbox and uid. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :rtype: SoledadDocument or None - """ - # TODO -- inlineCallbacks - result = None - try: - # TODO -- yield - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, mbox, str(uid)) - if len(flag_docs) != 1: - logger.warning("More than one flag doc for %r:%s" % - (mbox, uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % uid) - logger.exception(exc) - finally: - return result - - def get_headers_doc(self, chash): - """ - Return the document that keeps the headers for a message - indexed by its content-hash. - - :param chash: the content-hash to retrieve the document from. - :type chash: str or unicode - :rtype: SoledadDocument or None - """ - head_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(chash)) - return first(head_docs) - - # deleted messages - - def deleted_iter(self, mbox): - """ - Get an iterator for the the doc_id for SoledadDocuments for messages - with \\Deleted flag for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: iterator through deleted message docs - :rtype: iterable - """ - return [doc.doc_id for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_DEL_IDX, - fields.TYPE_FLAGS_VAL, mbox, '1')] - - def remove_all_deleted(self, mbox): - """ - Remove from Soledad all messages flagged as deleted for a given - mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - deleted = [] - for doc_id in self.deleted_iter(mbox): - with self._remove_lock: - doc = self._soledad.get_doc(doc_id) - if doc is not None: - self._soledad.delete_doc(doc) - try: - deleted.append(doc.content[fields.UID_KEY]) - except TypeError: - # empty content - pass - return deleted diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ca07f67..482b64d 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -61,6 +62,18 @@ class Message(object): def get_internal_date(self): """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str """ return self._wrapper.fdoc.date @@ -99,6 +112,15 @@ class Message(object): return tuple(self._wrapper.fdoc.tags) +class Flagsmode(object): + """ + Modes for setting the flags/tags. + """ + APPEND = 1 + REMOVE = -1 + SET = 0 + + class MessageCollection(object): """ A generic collection of messages. It can be messages sharing the same @@ -132,6 +154,7 @@ class MessageCollection(object): def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ + Constructore for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -149,6 +172,20 @@ class MessageCollection(object): """ return bool(self.mbox_wrapper) + @property + def mbox_name(self): + wrapper = getattr(self, "mbox_wrapper", None) + if not wrapper: + return None + return wrapper.mbox + + def get_mbox_attr(self, attr): + return getattr(self.mbox_wrapper, attr) + + def set_mbox_attr(self, attr, value): + setattr(self.mbox_wrapper, attr, value) + return self.mbox_wrapper.update(self.store) + # Get messages def get_message_by_content_hash(self, chash, get_cdocs=False): @@ -162,7 +199,7 @@ class MessageCollection(object): # or use the internal collection of pointers-to-docs. raise NotImplementedError() - metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + metamsg_id = _get_mdoc_id(self.mbox_name, chash) return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, @@ -181,25 +218,37 @@ class MessageCollection(object): raise NotImplementedError("Does not support relative ids yet") def get_msg_from_mdoc_id(doc_id): + # XXX pass UID? return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, doc_id, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) d.addCallback(get_msg_from_mdoc_id) return d def count(self): """ Count the messages in this collection. - :rtype: int + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_wrapper.mbox) + return self.mbox_indexer.count(self.mbox_name) + + def get_uid_next(self): + """ + Get the next integer beyond the highest UID count for this mailbox. + + :return: a Deferred that will fire with the integer for the next uid. + :rtype: Deferred + """ + return self.mbox_indexer.get_uid_next(self.mbox_name) # Manipulate messages + # TODO pass flags, date too... def add_msg(self, raw_msg): """ Add a message to this collection. @@ -208,14 +257,14 @@ class MessageCollection(object): wrapper = msg.get_wrapper() if self.is_mailbox_collection(): - mbox = self.mbox_wrapper.mbox + mbox = self.mbox_name wrapper.set_mbox(mbox) def insert_mdoc_id(_): # XXX does this work? doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( - self.mbox_wrapper.mbox, doc_id) + self.mbox_name, doc_id) d = wrapper.create(self.store) d.addCallback(insert_mdoc_id) @@ -248,31 +297,45 @@ class MessageCollection(object): # XXX does this work? doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( - self.mbox_wrapper.mbox, doc_id) + self.mbox_name, doc_id) d = wrapper.delete(self.store) d.addCallback(delete_mdoc_id) return d # TODO should add a delete-by-uid to collection? + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + def udpate_flags(self, msg, flags, mode): """ Update flags for a given message. """ wrapper = msg.get_wrapper() - # 1. update the flags in the message wrapper --- stored where??? - # 2. update the special flags in the wrapper (seen, etc) - # 3. call adaptor.update_msg(store) - pass + current = wrapper.fdoc.flags + newflags = self._update_flags_or_tags(current, flags, mode) + wrapper.fdoc.flags = newflags + + wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags + wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags + + return self.adaptor.update_msg(self.store, msg) def update_tags(self, msg, tags, mode): """ Update tags for a given message. """ wrapper = msg.get_wrapper() - # 1. update the tags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) - pass + current = wrapper.fdoc.tags + newtags = self._update_flags_or_tags(current, tags, mode) + wrapper.fdoc.tags = newtags + return self.adaptor.update_msg(self.store, msg) class Account(object): @@ -382,6 +445,8 @@ class Account(object): d.addCallback(rename_uid_table_cb) return d + # Get Collections + def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py deleted file mode 100644 index c8f224c..0000000 --- a/src/leap/mail/messageflow.py +++ /dev/null @@ -1,200 +0,0 @@ -# -*- coding: utf-8 -*- -# messageflow.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Message Producers and Consumers for flow control. -""" -import Queue - -from twisted.internet.task import LoopingCall - -from zope.interface import Interface, implements - - -class IMessageConsumer(Interface): - """ - I consume messages from a queue. - """ - - def consume(self, queue): - """ - Consumes the passed item. - - :param item: a queue where we put the object to be consumed. - :type item: object - """ - # TODO we could add an optional type to be passed - # for doing type check. - - # TODO in case of errors, we could return the object to - # the queue, maybe wrapped in an object with a retries attribute. - - -class IMessageProducer(Interface): - """ - I produce messages and put them in a store to be consumed by other - entities. - """ - - def push(self, item, state=None): - """ - Push a new item in the queue. - """ - - def start(self): - """ - Start producing items. - """ - - def stop(self): - """ - Stop producing items. - """ - - def flush(self): - """ - Flush queued messages to consumer. - """ - - -class DummyMsgConsumer(object): - - implements(IMessageConsumer) - - def consume(self, queue): - """ - Just prints the passed item. - """ - if not queue.empty(): - print "got item %s" % queue.get() - - -class MessageProducer(object): - """ - A Producer class that we can use to temporarily buffer the production - of messages so that different objects can consume them. - - This is useful for serializing the consumption of the messages stream - in the case of an slow resource (db), or for returning early from a - deferred chain and leave further processing detached from the calling loop, - as in the case of smtp. - """ - implements(IMessageProducer) - - # TODO this can be seen as a first step towards properly implementing - # components that implement IPushProducer / IConsumer interfaces. - # However, I need to think more about how to pause the streaming. - # In any case, the differential rate between message production - # and consumption is not likely (?) to consume huge amounts of memory in - # our current settings, so the need to pause the stream is not urgent now. - - # TODO use enum - STATE_NEW = 1 - STATE_DIRTY = 2 - - def __init__(self, consumer, queue=Queue.Queue, period=1): - """ - Initializes the MessageProducer - - :param consumer: an instance of a IMessageConsumer that will consume - the new messages. - :param queue: any queue implementation to be used as the temporary - buffer for new items. Default is a FIFO Queue. - :param period: the period to check for new items, in seconds. - """ - # XXX should assert it implements IConsumer / IMailConsumer - # it should implement a `consume` method - self._consumer = consumer - - self._queue_new = queue() - self._queue_dirty = queue() - self._period = period - - self._loop = LoopingCall(self._check_for_new) - - # private methods - - def _check_for_new(self): - """ - Check for new items in the internal queue, and calls the consume - method in the consumer. - - If the queue is found empty, the loop is stopped. It will be started - again after the addition of new items. - """ - self._consumer.consume((self._queue_new, self._queue_dirty)) - if self.is_queue_empty(): - self.stop() - - def is_queue_empty(self): - """ - Return True if queue is empty, False otherwise. - """ - new = self._queue_new - dirty = self._queue_dirty - return new.empty() and dirty.empty() - - # public methods: IMessageProducer - - def push(self, item, state=None): - """ - Push a new item in the queue. - - If the queue was empty, we will start the loop again. - """ - # XXX this might raise if the queue does not accept any new - # items. what to do then? - queue = self._queue_new - - if state == self.STATE_NEW: - queue = self._queue_new - if state == self.STATE_DIRTY: - queue = self._queue_dirty - - queue.put(item) - self.start() - - def start(self): - """ - Start polling for new items. - """ - if not self._loop.running: - self._loop.start(self._period, now=True) - - def stop(self): - """ - Stop polling for new items. - """ - if self._loop.running: - self._loop.stop() - - def flush(self): - """ - Flush queued messages to consumer. - """ - self._check_for_new() - - -if __name__ == "__main__": - from twisted.internet import reactor - producer = MessageProducer(DummyMsgConsumer()) - producer.start() - - for delay, item in ((2, 1), (3, 2), (4, 3), - (6, 4), (7, 5), (8, 6), (8.2, 7), - (15, 'a'), (16, 'b'), (17, 'c')): - reactor.callLater(delay, producer.put, item) - reactor.run() -- cgit v1.2.3 From 60eecafcc8a516985f900ef81bc7941a5234930a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 4 Jan 2015 03:37:18 -0400 Subject: tests for mail.mail module: Message --- src/leap/mail/adaptors/models.py | 2 +- src/leap/mail/adaptors/soledad.py | 131 ++++++++--- src/leap/mail/imap/messages.py | 172 +++----------- src/leap/mail/imap/tests/rfc822.message | 87 +------ .../mail/imap/tests/rfc822.multi-minimal.message | 17 +- .../mail/imap/tests/rfc822.multi-signed.message | 239 +------------------ src/leap/mail/imap/tests/rfc822.multi.message | 97 +------- src/leap/mail/imap/tests/rfc822.plain.message | 67 +----- src/leap/mail/mail.py | 165 +++++++++++-- src/leap/mail/mailbox_indexer.py | 22 +- src/leap/mail/tests/rfc822.multi-minimal.message | 16 ++ src/leap/mail/tests/rfc822.multi-signed.message | 238 +++++++++++++++++++ src/leap/mail/tests/rfc822.multi.message | 96 ++++++++ src/leap/mail/tests/rfc822.plain.message | 66 ++++++ src/leap/mail/tests/test_mail.py | 255 +++++++++++++++++++-- src/leap/mail/walk.py | 18 +- 16 files changed, 955 insertions(+), 733 deletions(-) mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.multi-minimal.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.multi-signed.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.multi.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.plain.message create mode 100644 src/leap/mail/tests/rfc822.multi-minimal.message create mode 100644 src/leap/mail/tests/rfc822.multi-signed.message create mode 100644 src/leap/mail/tests/rfc822.multi.message create mode 100644 src/leap/mail/tests/rfc822.plain.message diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py index 1648059..88e0e4e 100644 --- a/src/leap/mail/adaptors/models.py +++ b/src/leap/mail/adaptors/models.py @@ -89,7 +89,7 @@ class DocumentWrapper(object): if not attr.startswith('_') and attr not in normalized: raise RuntimeError( "Cannot set attribute because it's not defined " - "in the model: %s" % attr) + "in the model %s: %s" % (self.__class__, attr)) object.__setattr__(self, attr, value) def serialize(self): diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index bf8f7e9..f0808af 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -23,6 +23,7 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer +from twisted.python import util from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -37,6 +38,8 @@ from leap.mail.utils import lowerdict, first from leap.mail.utils import stringify_parts_map from leap.mail.interfaces import IMailAdaptor, IMessageWrapper +from leap.soledad.common.document import SoledadDocument + # TODO # [ ] Convenience function to create mail specifying subject, date, etc? @@ -339,10 +342,10 @@ class FlagsDocWrapper(SoledadDocumentWrapper): seen = False deleted = False recent = False - multi = False flags = [] tags = [] size = 0 + multi = False class __meta__(object): index = "mbox" @@ -409,7 +412,6 @@ class MetaMsgDocWrapper(SoledadDocumentWrapper): class MessageWrapper(object): - # TODO generalize wrapper composition? # This could benefit of a DeferredLock to create/update all the # documents at the same time maybe, and defend against concurrent updates? @@ -425,11 +427,19 @@ class MessageWrapper(object): integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ + if isinstance(mdoc, SoledadDocument): + mdoc = mdoc.content + if not mdoc: + mdoc = {} self.mdoc = MetaMsgDocWrapper(**mdoc) + if isinstance(fdoc, SoledadDocument): + fdoc = fdoc.content self.fdoc = FlagsDocWrapper(**fdoc) self.fdoc.set_future_doc_id(self.mdoc.fdoc) + if isinstance(hdoc, SoledadDocument): + hdoc = hdoc.content self.hdoc = HeaderDocWrapper(**hdoc) self.hdoc.set_future_doc_id(self.mdoc.hdoc) @@ -502,6 +512,43 @@ class MessageWrapper(object): self.mdoc.set_mbox(mbox) self.fdoc.set_mbox(mbox) + def set_flags(self, flags): + # TODO serialize the get + update + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + self.fdoc.flags = list(flags) + + def set_tags(self, tags): + # TODO serialize the get + update + if tags is None: + tags = tuple() + leap_assert_type(tags, tuple) + self.fdoc.tags = list(tags) + + def set_date(self, date): + # XXX assert valid date format + self.hdoc.date = date + + def get_subpart_dict(self, index): + """ + :param index: index, 1-indexed + :type index: int + """ + return self.hdoc.part_map[str(index)] + + def get_body(self, store): + """ + :rtype: deferred + """ + body_phash = self.hdoc.body + if not body_phash: + return None + d = store.get_doc('C-' + body_phash) + d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) + return d + + # # Mailboxes # @@ -631,7 +678,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return self.get_msg_from_docs( MessageClass, mdoc, fdoc, hdoc, cdocs) - def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None): + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, + uid=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. @@ -657,26 +705,24 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) - # XXX pass UID too? - def _get_msg_from_variable_doc_list(self, doc_list, msg_class): - if len(doc_list) == 2: - fdoc, hdoc = doc_list + def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): + if len(doc_list) == 3: + mdoc, fdoc, hdoc = doc_list cdocs = None - elif len(doc_list) > 2: - fdoc, hdoc = doc_list[:2] - cdocs = dict(enumerate(doc_list[2:], 1)) - return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + elif len(doc_list) > 3: + fdoc, hdoc = doc_list[:3] + cdocs = dict(enumerate(doc_list[3:], 1)) + return self.get_msg_from_docs( + msg_class, mdoc, fdoc, hdoc, cdocs, uid=None) - # XXX pass UID too ? def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, - get_cdocs=False): + uid=None, get_cdocs=False): metamsg_id = doc_id def wrap_meta_doc(doc): cls = MetaMsgDocWrapper - # XXX pass UID? return cls(doc_id=doc.doc_id, **doc.content) def get_part_docs_from_mdoc_wrapper(wrapper): @@ -685,7 +731,12 @@ class SoledadMailAdaptor(SoledadIndexMixin): d_docs.append(store.get_doc(wrapper.hdoc)) for cdoc in wrapper.cdocs: d_docs.append(store.get_doc(cdoc)) + + def add_mdoc(doc_list): + return [wrapper.serialize()] + doc_list + d = defer.gatherResults(d_docs) + d.addCallback(add_mdoc) return d def get_parts_doc_from_mdoc_id(): @@ -696,25 +747,31 @@ class SoledadMailAdaptor(SoledadIndexMixin): return constants.FDOCID.format(mbox=mbox, chash=chash) def _get_hdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox=mbox, chash=chash) + return constants.HDOCID.format(mbox=mbox, chash=chash) d_docs = [] fdoc_id = _get_fdoc_id_from_mdoc_id() hdoc_id = _get_hdoc_id_from_mdoc_id() + d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) + d = defer.gatherResults(d_docs) return d + def add_mdoc_id_placeholder(docs_list): + return [None] + docs_list + if get_cdocs: d = store.get_doc(metamsg_id) d.addCallback(wrap_meta_doc) d.addCallback(get_part_docs_from_mdoc_wrapper) else: d = get_parts_doc_from_mdoc_id() + d.addCallback(add_mdoc_id_placeholder) d.addCallback(partial(self._get_msg_from_variable_doc_list, - msg_class=MessageClass)) + msg_class=MessageClass, uid=uid)) return d def create_msg(self, store, msg): @@ -791,17 +848,17 @@ def _split_into_parts(raw): # TODO populate Default FLAGS/TAGS (unseen?) # TODO seed propely the content_docs with defaults?? - msg, parts, chash, size, multi = _parse_msg(raw) - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) + msg, parts, chash, multi = _parse_msg(raw) + size = len(msg.as_string()) + body_phash = walk.get_body_phash(msg) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) cdocs_list = list(walk.get_raw_docs(msg, parts)) cdocs_phashes = [c['phash'] for c in cdocs_list] mdoc = _build_meta_doc(chash, cdocs_phashes) fdoc = _build_flags_doc(chash, size, multi) - hdoc = _build_headers_doc(msg, chash, parts_map) + hdoc = _build_headers_doc(msg, chash, body_phash, parts_map) # The MessageWrapper expects a dict, one-indexed cdocs = dict(enumerate(cdocs_list, 1)) @@ -812,10 +869,9 @@ def _split_into_parts(raw): def _parse_msg(raw): msg = message_from_string(raw) parts = walk.get_parts(msg) - size = len(raw) chash = sha256.SHA256(raw).hexdigest() multi = msg.is_multipart() - return msg, parts, chash, size, multi + return msg, parts, chash, multi def _build_meta_doc(chash, cdocs_phashes): @@ -831,28 +887,31 @@ def _build_flags_doc(chash, size, multi): return _fdoc.serialize() -def _build_headers_doc(msg, chash, parts_map): +def _build_headers_doc(msg, chash, body_phash, parts_map): """ Assemble a headers document from the original parsed message, the content-hash, and the parts map. It takes into account possibly repeated headers. """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) - - lower_headers = lowerdict(headers) + headers = msg.items() + + # TODO move this manipulation to IMAP + #headers = defaultdict(list) + #for k, v in msg.items(): + #headers[k].append(v) + ## "fix" for repeated headers. + #for k, v in headers.items(): + #newline = "\n%s: " % (k,) + #headers[k] = newline.join(v) + + lower_headers = lowerdict(dict(headers)) msgid = first(_MSGID_RE.findall( lower_headers.get('message-id', ''))) _hdoc = HeaderDocWrapper( - chash=chash, headers=lower_headers, msgid=msgid) + chash=chash, headers=headers, body=body_phash, + msgid=msgid) def copy_attr(headers, key, doc): if key in headers: diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 7e0f973..883da35 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -46,28 +46,12 @@ class IMAPMessage(object): implements(imap4.IMessage) - # TODO ---- see what should we pass here instead - # where's UID added to the message? - # def __init__(self, soledad, uid, mbox): - def __init__(self, message, collection): + def __init__(self, message): """ Initializes a LeapMessage. - - :param soledad: a Soledad instance - :type soledad: Soledad - :param uid: the UID for the message. - :type uid: int or basestring - :param mbox: the mbox this message belongs to - :type mbox: str or unicode """ - #self._uid = int(uid) if uid is not None else None - #self._mbox = normalize_mailbox(mbox) - self.message = message - # TODO maybe not needed, see setFlags below - self.collection = collection - # IMessage implementation def getUID(self): @@ -95,21 +79,21 @@ class IMAPMessage(object): # lookup method? IMAPMailbox? - def setFlags(self, flags, mode): - """ - Sets the flags for this message - - :param flags: the flags to update in the message. - :type flags: tuple of str - :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - :type mode: int - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + #def setFlags(self, flags, mode): + #""" + #Sets the flags for this message +# + #:param flags: the flags to update in the message. + #:type flags: tuple of str + #:param mode: the mode for setting. 1 is append, -1 is remove, 0 set. + #:type mode: int + #""" + #leap_assert(isinstance(flags, tuple), "flags need to be a tuple") # XXX # return new flags # map to str #self.message.set_flags(flags, mode) - self.collection.update_flags(self.message, flags, mode) + #self.collection.update_flags(self.message, flags, mode) def getInternalDate(self): """ @@ -132,9 +116,6 @@ class IMAPMessage(object): # IMessagePart # - # XXX we should implement this interface too for the subparts - # so we allow nested parts... - def getBodyFile(self): """ Retrieve a file object containing only the body of this message. @@ -142,53 +123,25 @@ class IMAPMessage(object): :return: file-like object opened for reading :rtype: StringIO """ - #def write_fd(body): - #fd.write(body) - #fd.seek(0) - #return fd -# # TODO refactor with getBodyFile in MessagePart -# - #fd = StringIO.StringIO() -# - #if self.bdoc is not None: - #bdoc_content = self.bdoc.content - #if empty(bdoc_content): - #logger.warning("No BDOC content found for message!!!") - #return write_fd("") -# - #body = bdoc_content.get(self.RAW_KEY, "") - #content_type = bdoc_content.get('content-type', "") - #charset = find_charset(content_type) - #if charset is None: - #charset = self._get_charset(body) - #try: - #if isinstance(body, unicode): - #body = body.encode(charset) - #except UnicodeError as exc: - #logger.error( - #"Unicode error, using 'replace'. {0!r}".format(exc)) - #logger.debug("Attempted to encode with: %s" % charset) - #body = body.encode(charset, 'replace') - #finally: - #return write_fd(body) - return self.message.get_body_file() + #body = bdoc_content.get(self.RAW_KEY, "") + #content_type = bdoc_content.get('content-type', "") + #charset = find_charset(content_type) + #if charset is None: + #charset = self._get_charset(body) + #try: + #if isinstance(body, unicode): + #body = body.encode(charset) + #except UnicodeError as exc: + #logger.error( + #"Unicode error, using 'replace'. {0!r}".format(exc)) + #logger.debug("Attempted to encode with: %s" % charset) + #body = body.encode(charset, 'replace') + #finally: + #return write_fd(body) - # TODO move to mail.mail - @memoized_method - def _get_charset(self, stuff): - """ - Gets (guesses?) the charset of a payload. - - :param stuff: the stuff to guess about. - :type stuff: basestring - :returns: charset - """ - # XXX shouldn't we make the scope - # of the decorator somewhat more persistent? - # and put memory bounds. - return get_email_charset(stuff) + return self.message.get_body_file() def getSize(self): """ @@ -197,10 +150,6 @@ class IMAPMessage(object): :return: size of the message, in octets :rtype: int """ - #size = None - #fdoc_content = self.fdoc.content - #size = fdoc_content.get(self.SIZE_KEY, False) - #return size return self.message.get_size() def getHeaders(self, negate, *names): @@ -265,11 +214,7 @@ class IMAPMessage(object): """ Return True if this message is multipart. """ - #fdoc_content = self.fdoc.content - #is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) - #return is_multipart - - return self.message.fdoc.is_multi + return self.message.is_multipart() def getSubPart(self, part): """ @@ -282,64 +227,7 @@ class IMAPMessage(object): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - if not self.isMultipart(): - raise TypeError - try: - pmap_dict = self._get_part_from_parts_map(part + 1) - except KeyError: - raise IndexError - - # TODO move access to adaptor ---- - return MessagePart(self._soledad, pmap_dict) - - # - # accessors - # - - # FIXME - # -- move to wrapper/adaptor - def _get_part_from_parts_map(self, part): - """ - Get a part map from the headers doc - - :raises: KeyError if key does not exist - :rtype: dict - """ - raise NotImplementedError() - - #hdoc_content = self.hdoc.content - #pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) -# - # remember, lads, soledad is using strings in its keys, - # not integers! - #return pmap[str(part)] - - # TODO move to wrapper/adaptor - def _get_body_doc(self): - """ - Return the document that keeps the body for this - message. - """ - # FIXME - # -- just get the body and retrieve the cdoc P- - #hdoc_content = self.hdoc.content - #body_phash = hdoc_content.get( - #fields.BODY_KEY, None) - #if not body_phash: - #logger.warning("No body phash for this document!") - #return None -# - #if self._container is not None: - #bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) - #if not empty(bdoc) and not empty(bdoc.content): - #return bdoc -# - # no memstore, or no body doc found there - #d = self._soledad.get_from_index( - #fields.TYPE_P_HASH_IDX, - #fields.TYPE_CONTENT_VAL, str(body_phash)) - #d.addCallback(lambda docs: first(docs)) - #return d + return self.message.get_subpart(part) class IMAPMessageCollection(object): diff --git a/src/leap/mail/imap/tests/rfc822.message b/src/leap/mail/imap/tests/rfc822.message deleted file mode 100644 index ee97ab9..0000000 --- a/src/leap/mail/imap/tests/rfc822.message +++ /dev/null @@ -1,86 +0,0 @@ -Return-Path: -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for ; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: -List-Post: -List-Subscribe: , - -List-Id: -List-Unsubscribe: , - -List-Archive: -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/imap/tests/rfc822.message b/src/leap/mail/imap/tests/rfc822.message new file mode 120000 index 0000000..b19cc28 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message deleted file mode 100644 index 582297c..0000000 --- a/src/leap/mail/imap/tests/rfc822.multi-minimal.message +++ /dev/null @@ -1,16 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============6203542367371144092==" -MIME-Version: 1.0 -Subject: [TEST] 010 - Inceptos cum lorem risus congue -From: testmailbitmaskspam@gmail.com -To: test_c5@dev.bitmask.net - ---===============6203542367371144092== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Howdy from python! -The subject: [TEST] 010 - Inceptos cum lorem risus congue -Current date & time: Wed Jan 8 16:36:21 2014 -Trying to attach: [] ---===============6203542367371144092==-- diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message new file mode 120000 index 0000000..e0aa678 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-minimal.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message deleted file mode 100644 index 9907c2d..0000000 --- a/src/leap/mail/imap/tests/rfc822.multi-signed.message +++ /dev/null @@ -1,238 +0,0 @@ -Date: Mon, 6 Jan 2014 04:40:47 -0400 -From: Kali Kaneko -To: penguin@example.com -Subject: signed message -Message-ID: <20140106084047.GA21317@samsara.lan> -MIME-Version: 1.0 -Content-Type: multipart/signed; micalg=pgp-sha1; - protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" -Content-Disposition: inline -User-Agent: Mutt/1.5.21 (2012-12-30) - - ---z9ECzHErBrwFF8sy -Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" -Content-Disposition: inline - - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=utf-8 -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -This is an example of a signed message, -with attachments. - - ---=20 -Nihil sine chao! =E2=88=B4 - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=us-ascii -Content-Disposition: attachment; filename="attach.txt" - -this is attachment in plain text. - ---z0eOaCaDLjvTGF2l -Content-Type: application/octet-stream -Content-Disposition: attachment; filename="hack.ico" -Content-Transfer-Encoding: base64 - -AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA -KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG -RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA -PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl -5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA -/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ -yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A -Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK -ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK -LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP -QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy -AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs -AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA -AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA -gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d -HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA -x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 -+wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA -AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 -+QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA -OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK -igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA -JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra -2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA -xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj -owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB -AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA -AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d -XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d -XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA -AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB -AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm -X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC -AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B -bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ -S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu -J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y -AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N -KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB -XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A -AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA -AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d -XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA -AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr -RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA -AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A -Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI -yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA -CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys -rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA -vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d -HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA -urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx -cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA -CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo -6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA -2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 -OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA -UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp -qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA -lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa -WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB -AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB -AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA -ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA -AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB -AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB -AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA -tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA -AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB -AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB -AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA -AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd -AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB -AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB -AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 -ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 -NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF -RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB -lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA -AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa -WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA -AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX -AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB -AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB -AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA -AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA -AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA -AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA -AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB -AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB -AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA -ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA -AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 -LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA -AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA -ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 -RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi -JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 -NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK -T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN -UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA -AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA -W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA -AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB -l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB -AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ -WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA -AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv -RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA -AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj -AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB -AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA -AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA -AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA -dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A -AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB -AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW -pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= - ---z0eOaCaDLjvTGF2l-- - ---z9ECzHErBrwFF8sy -Content-Type: application/pgp-signature - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.15 (GNU/Linux) - -iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv -kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl -vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK -PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC -w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw -sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr -BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN -QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt -mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ -jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 -gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X -sSdfcAhT7Tno7PB/Acoh -=+okv ------END PGP SIGNATURE----- - ---z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message new file mode 120000 index 0000000..4172244 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-signed.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message deleted file mode 100644 index 30f74e5..0000000 --- a/src/leap/mail/imap/tests/rfc822.multi.message +++ /dev/null @@ -1,96 +0,0 @@ -Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) -From: Doug Sauder -To: Joe Blow -Subject: Test message from PINE -Message-ID: -MIME-Version: 1.0 -Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" - - This message is in MIME format. The first part should be readable text, - while the remaining parts are likely unreadable without MIME-aware tools. - Send mail to mime@docserver.cac.washington.edu for more info. - ----1463757054-952513540-958744548=:8452 -Content-Type: TEXT/PLAIN; charset=US-ASCII - -This is a test message from PINE MUA. - - ----1463757054-952513540-958744548=:8452 -Content-Type: APPLICATION/octet-stream; name="redball.png" -Content-Transfer-Encoding: BASE64 -Content-ID: -Content-Description: A PNG graphic file -Content-Disposition: attachment; filename="redball.png" - -iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A -AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj -AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv -AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 -GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf -hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl -rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL -ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS -AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP -AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP -AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI -AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR -AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 -AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO -AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 -AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 -LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu -MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF -BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp -6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow -8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G -ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn -OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 -a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T -VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 -Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f -lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ -joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms -1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA -JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 -vAAAAABJRU5ErkJggg== ----1463757054-952513540-958744548=:8452 -Content-Type: APPLICATION/octet-stream; name="blueball.png" -Content-Transfer-Encoding: BASE64 -Content-ID: -Content-Description: A PNG graphic file -Content-Disposition: attachment; filename="blueball.png" - -iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A -AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI -IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y -Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S -hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM -vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB -Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu -MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b -fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo -Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp -LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX -P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M -jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 -+VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE -1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 -YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY -mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp -Z3VldDZzO7wAAAAASUVORK5CYII= ----1463757054-952513540-958744548=:8452-- diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message new file mode 120000 index 0000000..62057d2 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi.message @@ -0,0 +1 @@ +../../tests/rfc822.multi.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message deleted file mode 100644 index fc627c3..0000000 --- a/src/leap/mail/imap/tests/rfc822.plain.message +++ /dev/null @@ -1,66 +0,0 @@ -From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 -Return-Path: -X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net -X-Spam-Level: ** -X-Spam-Pyzor: Reported 0 times. -X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, - CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, - NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled - version=3.3.2 -Delivered-To: kali@leap.se -Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) - (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) - (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) - by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F - for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) -Received: from pyar.usla.org.ar (unknown [190.228.30.157]) - by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 - for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) -Received: from [127.0.0.1] (localhost [127.0.0.1]) - by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F - for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) -MIME-Version: 1.0 -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: quoted-printable -From: pyar-request@python.org.ar -To: kali@leap.se -Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 -Reply-To: pyar-request@python.org.ar -Auto-Submitted: auto-replied -Message-ID: -Date: Wed, 08 Jan 2014 15:45:59 -0300 -Precedence: bulk -X-BeenThere: pyar@python.org.ar -X-Mailman-Version: 2.1.15 -List-Id: Python Argentina -X-List-Administrivia: yes -Errors-To: pyar-bounces@python.org.ar -Sender: "pyar" -X-Virus-Scanned: clamav-milter 0.97.8 at mx1 -X-Virus-Status: Clean - -Mailing list subscription confirmation notice for mailing list pyar - -We have received a request de kaliyuga@riseup.net for subscription of -your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar -mailing list. To confirm that you want to be added to this mailing -list, simply reply to this message, keeping the Subject: header -intact. Or visit this web page: - - http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= -3377148ac2 - - -Or include the following line -- and only the following line -- in a -message to pyar-request@python.org.ar: - - confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 - -Note that simply sending a `reply' to this message should work from -most mail readers, since that usually leaves the Subject: line in the -right form (additional "Re:" text in the Subject: is okay). - -If you do not wish to be subscribed to this list, please simply -disregard this message. If you think you are being maliciously -subscribed to the list, or have any other questions, send them to -pyar-owner@python.org.ar. diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message new file mode 120000 index 0000000..5bab0e8 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.plain.message @@ -0,0 +1 @@ +../../tests/rfc822.plain.message \ No newline at end of file diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 482b64d..55e50f7 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,15 +17,24 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import logging +import StringIO + from twisted.internet import defer +from leap.common.check import leap_assert_type +from leap.common.mail import get_email_charset + +from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.utils import empty, find_charset + +logger = logging.getLogger(name=__name__) -# TODO +# TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail @@ -36,16 +45,92 @@ def _get_mdoc_id(mbox, chash): return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) +def _write_and_rewind(payload): + fd = StringIO.StringIO() + fd.write(payload) + fd.seek(0) + return fd + + +class MessagePart(object): + + # TODO pass cdocs in init + + def __init__(self, part_map, cdocs={}): + self._pmap = part_map + self._cdocs = cdocs + + def get_size(self): + return self._pmap['size'] + + def get_body_file(self): + pmap = self._pmap + multi = pmap.get('multi') + if not multi: + phash = pmap.get("phash") + else: + pmap_ = pmap.get('part_map') + first_part = pmap_.get('1', None) + if not empty(first_part): + phash = first_part['phash'] + else: + phash = "" + + payload = self._get_payload(phash) + + if payload: + # FIXME + # content_type = self._get_ctype_from_document(phash) + # charset = find_charset(content_type) + charset = None + if charset is None: + charset = get_email_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + + return _write_and_rewind(payload) + + def get_headers(self): + return self._pmap.get("headers", []) + + def is_multipart(self): + multi = self._pmap.get("multi", False) + return multi + + def get_subpart(self, part): + if not self.is_multipart(): + raise TypeError + + sub_pmap = self._pmap.get("part_map", {}) + try: + part_map = sub_pmap[str(part + 1)] + except KeyError: + logger.debug("getSubpart for %s: KeyError" % (part,)) + raise IndexError + return MessagePart(self._soledad, part_map) + + def _get_payload(self, phash): + return self._cdocs.get(phash, "") + + class Message(object): """ Represents a single message, and gives access to all its attributes. """ - def __init__(self, wrapper): + def __init__(self, wrapper, uid=None): """ :param wrapper: an instance of an implementor of IMessageWrapper + :param uid: + :type uid: int """ self._wrapper = wrapper + self._uid = uid def get_wrapper(self): """ @@ -53,10 +138,18 @@ class Message(object): """ return self._wrapper + def get_uid(self): + """ + Get the (optional) UID. + """ + return self._uid + # imap.IMessage methods def get_flags(self): """ + Get flags for this message. + :rtype: tuple """ return tuple(self._wrapper.fdoc.flags) @@ -81,28 +174,50 @@ class Message(object): def get_headers(self): """ + Get the raw headers document. """ - # XXX process here? from imap.messages - return self._wrapper.hdoc.headers + return [tuple(item) for item in self._wrapper.hdoc.headers] - def get_body_file(self): + def get_body_file(self, store): """ """ + def write_and_rewind_if_found(cdoc): + if not cdoc: + return None + return _write_and_rewind(cdoc.raw) + + d = defer.maybeDeferred(self._wrapper.get_body, store) + d.addCallback(write_and_rewind_if_found) + return d def get_size(self): """ + Size, in octets. """ return self._wrapper.fdoc.size def is_multipart(self): """ + Return True if this message is multipart. """ return self._wrapper.fdoc.multi def get_subpart(self, part): """ - """ - # XXX ??? return MessagePart? + :param part: The number of the part to retrieve, indexed from 0. + :type part: int + :rtype: MessagePart + """ + if not self.is_multipart(): + raise TypeError + part_index = part + 1 + try: + subpart_dict = self._wrapper.get_subpart_dict( + part_index) + except KeyError: + raise TypeError + # XXX pass cdocs + return MessagePart(subpart_dict) # Custom methods. @@ -137,7 +252,7 @@ class MessageCollection(object): instance or proxy, for instance). """ - # TODO + # TODO LIST # [ ] look at IMessageSet methods # [ ] make constructor with a per-instance deferredLock to use on # creation/deletion? @@ -159,7 +274,7 @@ class MessageCollection(object): self.adaptor = adaptor self.store = store - # TODO I have to think about what to do when there is no mbox passed to + # XXX I have to think about what to do when there is no mbox passed to # the initialization. We could still get the MetaMsg by index, instead # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer @@ -218,10 +333,11 @@ class MessageCollection(object): raise NotImplementedError("Does not support relative ids yet") def get_msg_from_mdoc_id(doc_id): - # XXX pass UID? + if doc_id is None: + return None return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, - doc_id, get_cdocs=get_cdocs) + doc_id, uid=uid, get_cdocs=get_cdocs) d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) d.addCallback(get_msg_from_mdoc_id) @@ -248,26 +364,37 @@ class MessageCollection(object): # Manipulate messages - # TODO pass flags, date too... - def add_msg(self, raw_msg): + def add_msg(self, raw_msg, flags=None, tags=None, date=None): """ Add a message to this collection. """ + if not flags: + flags = tuple() + if not tags: + tags = tuple() + leap_assert_type(flags, tuple) + leap_assert_type(date, str) + msg = self.adaptor.get_msg_from_string(Message, raw_msg) wrapper = msg.get_wrapper() - if self.is_mailbox_collection(): + if not self.is_mailbox_collection(): + raise NotImplementedError() + + else: mbox = self.mbox_name + wrapper.set_flags(flags) + wrapper.set_tags(tags) + wrapper.set_date(date) wrapper.set_mbox(mbox) - def insert_mdoc_id(_): - # XXX does this work? + def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( self.mbox_name, doc_id) d = wrapper.create(self.store) - d.addCallback(insert_mdoc_id) + d.addCallback(insert_mdoc_id, wrapper) return d def copy_msg(self, msg, newmailbox): @@ -338,6 +465,8 @@ class MessageCollection(object): return self.adaptor.update_msg(self.store, msg) +# TODO -------------------- split into account object? + class Account(object): """ Account is the top level abstraction to access collections of messages diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index bc298ea..1ceaec0 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -22,6 +22,17 @@ import re from leap.mail.constants import METAMSGID_RE +def _maybe_first_query_item(thing): + """ + Return the first item the returned query result, or None + if empty. + """ + try: + return thing[0][0] + except IndexError: + return None + + class WrongMetaDocIDError(Exception): pass @@ -124,7 +135,7 @@ class MailboxIndexer(object): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( @@ -192,7 +203,7 @@ class MailboxIndexer(object): :rtype: Deferred """ def get_hash(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( @@ -217,7 +228,7 @@ class MailboxIndexer(object): :rtype: Deferred """ def get_count(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( preffix=self.table_preffix, name=mailbox)) @@ -243,7 +254,10 @@ class MailboxIndexer(object): assert mailbox def increment(result): - return result[0][0] + 1 + uid = _maybe_first_query_item(result) + if uid is None: + return None + return uid + 1 sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( diff --git a/src/leap/mail/tests/rfc822.multi-minimal.message b/src/leap/mail/tests/rfc822.multi-minimal.message new file mode 100644 index 0000000..582297c --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/src/leap/mail/tests/rfc822.multi-signed.message b/src/leap/mail/tests/rfc822.multi-signed.message new file mode 100644 index 0000000..9907c2d --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/tests/rfc822.multi.message b/src/leap/mail/tests/rfc822.multi.message new file mode 100644 index 0000000..30f74e5 --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) +From: Doug Sauder +To: Joe Blow +Subject: Test message from PINE +Message-ID: +MIME-Version: 1.0 +Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" + + This message is in MIME format. The first part should be readable text, + while the remaining parts are likely unreadable without MIME-aware tools. + Send mail to mime@docserver.cac.washington.edu for more info. + +---1463757054-952513540-958744548=:8452 +Content-Type: TEXT/PLAIN; charset=US-ASCII + +This is a test message from PINE MUA. + + +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="redball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="redball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj +AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv +AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 +GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf +hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl +rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL +ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS +AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP +AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP +AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI +AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR +AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 +AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO +AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 +AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 +LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF +BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp +6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow +8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G +ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn +OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 +a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T +VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 +Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f +lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ +joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms +1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA +JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 +vAAAAABJRU5ErkJggg== +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="blueball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="blueball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI +IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y +Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S +hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM +vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB +Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b +fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo +Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp +LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX +P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M +jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 ++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE +1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 +YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY +mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp +Z3VldDZzO7wAAAAASUVORK5CYII= +---1463757054-952513540-958744548=:8452-- diff --git a/src/leap/mail/tests/rfc822.plain.message b/src/leap/mail/tests/rfc822.plain.message new file mode 100644 index 0000000..fc627c3 --- /dev/null +++ b/src/leap/mail/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index ce2366c..cb97be5 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -17,24 +17,48 @@ """ Tests for the mail module. """ +import time import os from functools import partial +from email.parser import Parser +from email.Utils import formatdate + +from twisted.python import util from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.mail import MessageCollection from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin -from twisted.internet import defer +# from twisted.internet import defer from twisted.trial import unittest HERE = os.path.split(os.path.abspath(__file__))[0] -class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): - """ - Tests for the SoledadDocumentWrapper. - """ +def _get_raw_msg(multi=False): + if multi: + sample = "rfc822.multi.message" + else: + sample = "rfc822.message" + with open(os.path.join(HERE, sample)) as f: + raw = f.read() + return raw + + +def _get_parsed_msg(multi=False): + mail_parser = Parser() + raw = _get_raw_msg(multi=multi) + return mail_parser.parsestr(raw) + + +def _get_msg_time(): + timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1)) + return formatdate(timestamp) + + + +class CollectionMixin(object): def get_collection(self, mbox_collection=True): """ @@ -61,35 +85,224 @@ class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): d.addCallback(get_collection_from_mbox_wrapper) return d - def test_is_mailbox_collection(self): - def assert_is_mbox_collection(collection): - self.assertTrue(collection.is_mailbox_collection()) +class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): + """ + Tests for the Message class. + """ + msg_flags = ('\Recent', '\Unseen', '\TestFlag') + msg_tags = ('important', 'todo', 'wonderful') + internal_date = "19-Mar-2015 19:22:21 -0500" + maxDiff = None + + def _do_insert_msg(self, multi=False): + """ + Inserts and return a regular message, for tests. + """ + raw = _get_raw_msg(multi=multi) d = self.get_collection() - d.addCallback(assert_is_mbox_collection) + d.addCallback(lambda col: col.add_msg( + raw, flags=self.msg_flags, tags=self.msg_tags, + date=self.internal_date)) + return d + + def get_inserted_msg(self, multi=False): + d = self._do_insert_msg(multi=multi) + d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda col: col.get_message_by_uid(1)) + return d + + def test_get_flags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_flags_cb) + return d + + def _test_get_flags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(msg.get_flags(), self.msg_flags) + + def test_get_internal_date(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_internal_date_cb) + + def _test_get_internal_date_cb(self, msg): + self.assertTrue(msg is not None) + self.assertDictEqual(msg.get_internal_date(), + self.internal_date) + + def test_get_headers(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_headers_cb) + return d + + def _test_get_headers_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg().items() + self.assertEqual(msg.get_headers(), expected) + + def test_get_body_file(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_body_file_cb) + return d + + def _test_get_body_file_cb(self, msg): + self.assertTrue(msg is not None) + orig = _get_parsed_msg(multi=True) + expected = orig.get_payload()[0].get_payload() + d = msg.get_body_file(self._soledad) + + def assert_body(fd): + self.assertTrue(fd is not None) + self.assertEqual(fd.read(), expected) + d.addCallback(assert_body) + return d + + def test_get_size(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_size_cb) + return d + + def _test_get_size_cb(self, msg): + self.assertTrue(msg is not None) + expected = len(_get_parsed_msg().as_string()) + self.assertEqual(msg.get_size(), expected) + + def test_is_multipart_no(self): + d = self.get_inserted_msg() + d.addCallback(self._test_is_multipart_no_cb) + return d + + def _test_is_multipart_no_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg().is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_is_multipart_yes(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_is_multipart_yes_cb) + return d + + def _test_is_multipart_yes_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg(multi=True).is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_get_subpart(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_subpart_cb) + return d + + def _test_get_subpart_cb(self, msg): + self.assertTrue(msg is not None) + + def test_get_tags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_tags_cb) return d - def assert_collection_count(self, _, expected, collection): + def _test_get_tags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(msg.get_tags(), self.msg_tags) + +class MessageCollectionTestCase(unittest.TestCase, + SoledadTestMixin, CollectionMixin): + """ + Tests for the MessageCollection class. + """ + def assert_collection_count(self, _, expected): def _assert_count(count): self.assertEqual(count, expected) - d = collection.count() + + d = self.get_collection() + d.addCallback(lambda col: col.count()) d.addCallback(_assert_count) return d - def test_add_msg(self): + def add_msg_to_collection(self): + raw = _get_raw_msg() - with open(os.path.join(HERE, "rfc822.message")) as f: - raw = f.read() - - def add_msg_to_collection_and_assert_count(collection): - d = collection.add_msg(raw) - d.addCallback(partial( - self.assert_collection_count, - expected=1, collection=collection)) + def add_msg_to_collection(collection): + d = collection.add_msg(raw, date=_get_msg_time()) return d + d = self.get_collection() + d.addCallback(add_msg_to_collection) + return d + def test_is_mailbox_collection(self): d = self.get_collection() - d.addCallback(add_msg_to_collection_and_assert_count) + d.addCallback(self._test_is_mailbox_collection_cb) + return d + + def _test_is_mailbox_collection_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def test_get_uid_next(self): + d = self.add_msg_to_collection() + d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda col: col.get_uid_next()) + d.addCallback(self._test_get_uid_next_cb) + + def _test_get_uid_next_cb(self, next_uid): + self.assertEqual(next_uid, 2) + + def test_add_and_count_msg(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_add_and_count_msg_cb) + return d + + def _test_add_and_count_msg_cb(self, _): + return partial(self.assert_collection_count, expected=1) + + def test_coppy_msg(self): + self.fail() + + def test_delete_msg(self): + self.fail() + + def test_update_flags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_flags_cb) + return d + + def _test_update_flags_cb(self, msg): + pass + + def test_update_tags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_tags_cb) return d + + def _test_update_tags_cb(self, msg): + pass + + +class AccountTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the Account class. + """ + + def test_add_mailbox(self): + self.fail() + + def test_delete_mailbox(self): + self.fail() + + def test_rename_mailbox(self): + self.fail() + + def test_list_all_mailbox_names(self): + self.fail() + + def test_get_all_mailboxes(self): + self.fail() + + def test_get_collection_by_docs(self): + self.fail() + + def test_get_collection_by_mailbox(self): + self.fail() + + def test_get_collection_by_tag(self): + self.fail() diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 5172837..8653a5f 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # walk.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,15 +56,15 @@ get_payloads = lambda msg: ((x.get_payload(), dict(((str.lower(k), v) for k, v in (x.items())))) for x in msg.walk()) -get_body_phash_simple = lambda payloads: first( - [get_hash(payload) for payload, headers in payloads - if payloads]) -get_body_phash_multi = lambda payloads: (first( - [get_hash(payload) for payload, headers in payloads - if payloads - and "text/plain" in headers.get('content-type', '')]) - or get_body_phash_simple(payloads)) +def get_body_phash(msg): + """ + Find the body payload-hash for this message. + """ + for part in msg.walk(): + if part.get_content_type() == "text/plain": + # XXX avoid hashing again + return get_hash(part.get_payload()) """ On getting the raw docs, we get also some of the headers to be able to -- cgit v1.2.3 From 751ace7534280187ecc83efb32e0cf9e18ac5bb2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 01:31:26 -0400 Subject: tests for mail.mail module: MessageCollection --- src/leap/mail/adaptors/soledad.py | 44 +++++++++++++++++++++++---------------- src/leap/mail/mail.py | 5 ++--- src/leap/mail/tests/test_mail.py | 18 +++++++++++++++- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index f0808af..522d2d3 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -290,7 +290,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): in the model as the `list_index`. :rtype: Deferred """ - # TODO + # TODO LIST (get_all) # [ ] extend support to indexes with n-ples # [ ] benchmark the cost of querying and returning indexes in a big # database. This might badly need pagination before being put to @@ -428,19 +428,28 @@ class MessageWrapper(object): content of the content-docs. """ if isinstance(mdoc, SoledadDocument): + mdoc_id = mdoc.doc_id mdoc = mdoc.content + else: + mdoc_id = None if not mdoc: mdoc = {} - self.mdoc = MetaMsgDocWrapper(**mdoc) + self.mdoc = MetaMsgDocWrapper(doc_id=mdoc_id, **mdoc) if isinstance(fdoc, SoledadDocument): + fdoc_id = fdoc.doc_id fdoc = fdoc.content - self.fdoc = FlagsDocWrapper(**fdoc) + else: + fdoc_id = None + self.fdoc = FlagsDocWrapper(doc_id=fdoc_id, **fdoc) self.fdoc.set_future_doc_id(self.mdoc.fdoc) if isinstance(hdoc, SoledadDocument): + hdoc_id = hdoc.doc_id hdoc = hdoc.content - self.hdoc = HeaderDocWrapper(**hdoc) + else: + hdoc_id = None + self.hdoc = HeaderDocWrapper(doc_id=hdoc_id, **hdoc) self.hdoc.set_future_doc_id(self.mdoc.hdoc) if cdocs is None: @@ -489,9 +498,15 @@ class MessageWrapper(object): return self.fdoc.update(store) def delete(self, store): + # TODO # Eventually this would have to do the duplicate search or send for the - # garbage collector. At least the fdoc can be unlinked. - raise NotImplementedError() + # garbage collector. At least mdoc and t the mdoc and fdoc can be + # unlinked. + d = [] + if self.mdoc.doc_id: + d.append(self.mdoc.delete(store)) + d.append(self.fdoc.delete(store)) + return defer.gatherResults(d) def copy(self, store, newmailbox): """ @@ -565,9 +580,6 @@ class MailboxWrapper(SoledadDocumentWrapper): closed = False subscribed = False - # I think we don't need to store this one. - # rw = True - class __meta__(object): index = "mbox" list_index = (indexes.TYPE_IDX, 'type_') @@ -717,9 +729,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return self.get_msg_from_docs( msg_class, mdoc, fdoc, hdoc, cdocs, uid=None) - def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, + def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): - metamsg_id = doc_id def wrap_meta_doc(doc): cls = MetaMsgDocWrapper @@ -740,8 +751,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return d def get_parts_doc_from_mdoc_id(): - mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0] - chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0] + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): return constants.FDOCID.format(mbox=mbox, chash=chash) @@ -753,22 +764,19 @@ class SoledadMailAdaptor(SoledadIndexMixin): fdoc_id = _get_fdoc_id_from_mdoc_id() hdoc_id = _get_hdoc_id_from_mdoc_id() + d_docs.append(store.get_doc(mdoc_id)) d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) d = defer.gatherResults(d_docs) return d - def add_mdoc_id_placeholder(docs_list): - return [None] + docs_list - if get_cdocs: - d = store.get_doc(metamsg_id) + d = store.get_doc(mdoc_id) d.addCallback(wrap_meta_doc) d.addCallback(get_part_docs_from_mdoc_wrapper) else: d = get_parts_doc_from_mdoc_id() - d.addCallback(add_mdoc_id_placeholder) d.addCallback(partial(self._get_msg_from_variable_doc_list, msg_class=MessageClass, uid=uid)) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 55e50f7..671642a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -420,13 +420,12 @@ class MessageCollection(object): """ wrapper = msg.get_wrapper() - def delete_mdoc_id(_): - # XXX does this work? + def delete_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( self.mbox_name, doc_id) d = wrapper.delete(self.store) - d.addCallback(delete_mdoc_id) + d.addCallback(delete_mdoc_id, wrapper) return d # TODO should add a delete-by-uid to collection? diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index cb97be5..d11df40 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -259,7 +259,23 @@ class MessageCollectionTestCase(unittest.TestCase, self.fail() def test_delete_msg(self): - self.fail() + d = self.add_msg_to_collection() + + def del_msg(collection): + def _delete_it(msg): + return collection.delete_msg(msg) + + d = collection.get_message_by_uid(1) + d.addCallback(_delete_it) + return d + + d.addCallback(lambda _: self.get_collection()) + d.addCallback(del_msg) + d.addCallback(self._test_delete_msg_cb) + return d + + def _test_delete_msg_cb(self, _): + return partial(self.assert_collection_count, expected=0) def test_update_flags(self): d = self.add_msg_to_collection() -- cgit v1.2.3 From 1b457bbe0eefa12d3e75b58247b53cc62aecc356 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 02:19:02 -0400 Subject: tests for mail.mail module: Account --- src/leap/mail/adaptors/soledad.py | 19 +++++---- src/leap/mail/mail.py | 46 ++++++++++++--------- src/leap/mail/mailbox_indexer.py | 2 +- src/leap/mail/tests/test_mail.py | 84 ++++++++++++++++++++++++++++++++------- 4 files changed, 110 insertions(+), 41 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 522d2d3..389307f 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -625,14 +625,11 @@ class SoledadIndexMixin(object): leap_assert(store, "Need a store") leap_assert_type(self.indexes, dict) - self._index_creation_deferreds = [] - def _on_indexes_created(ignored): - self.store_ready = True + self._index_creation_deferreds = [] def _create_index(name, expression): - d = store.create_index(name, *expression) - self._index_creation_deferreds.append(d) + return store.create_index(name, *expression) def _create_indexes(db_indexes): db_indexes = dict(db_indexes) @@ -640,7 +637,8 @@ class SoledadIndexMixin(object): for name, expression in self.indexes.items(): if name not in db_indexes: # The index does not yet exist. - _create_index(name, expression) + d = _create_index(name, expression) + self._index_creation_deferreds.append(d) continue if expression == db_indexes[name]: @@ -650,11 +648,16 @@ class SoledadIndexMixin(object): # we delete it and add the proper index expression. d1 = store.delete_index(name) d1.addCallback(lambda _: _create_index(name, expression)) + self._index_creation_deferreds.append(d1) - all_created = defer.gatherResults(self._index_creation_deferreds) + all_created = defer.gatherResults( + self._index_creation_deferreds, consumeErrors=True) all_created.addCallback(_on_indexes_created) return all_created + def _on_indexes_created(ignored): + self.store_ready = True + # Ask the database for currently existing indexes, and create them # if not found. d = store.list_indexes() @@ -832,9 +835,11 @@ class SoledadMailAdaptor(SoledadIndexMixin): have been updated. :rtype: defer.Deferred """ + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) return mbox_wrapper.update(store) def delete_mbox(self, store, mbox_wrapper): + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) return mbox_wrapper.delete(store) def get_all_mboxes(self, store): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 671642a..0c9b7a3 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -37,6 +37,9 @@ logger = logging.getLogger(name=__name__) # TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +# [ ] Change the doc_ids scheme for part-docs: use mailbox UID validity +# identifier, instead of name! (renames are broken!) +# [ ] Profile add_msg. def _get_mdoc_id(mbox, chash): """ @@ -360,7 +363,7 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_uid_next(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_name) # Manipulate messages @@ -464,8 +467,6 @@ class MessageCollection(object): return self.adaptor.update_msg(self.store, msg) -# TODO -------------------- split into account object? - class Account(object): """ Account is the top level abstraction to access collections of messages @@ -485,7 +486,6 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - mailboxes = None def __init__(self, store): self.store = store @@ -508,17 +508,17 @@ class Account(object): self._deferred_initialization.callback(None) d = self.adaptor.initialize_store(self.store) - d.addCallback(self.list_all_mailbox_names) + d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) - def callWhenReady(self, cb): - # XXX this could use adaptor.store_ready instead...?? + def callWhenReady(self, cb, *args, **kw): + # use adaptor.store_ready instead? if self._initialized: - cb(self) + cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb) + self._deferred_initialization.addCallback(cb, *args, **kw) return self._deferred_initialization # @@ -527,7 +527,7 @@ class Account(object): def list_all_mailbox_names(self): def filter_names(mboxes): - return [m.name for m in mboxes] + return [m.mbox for m in mboxes] d = self.get_all_mailboxes() d.addCallback(filter_names) @@ -540,35 +540,44 @@ class Account(object): def add_mailbox(self, name): def create_uid_table_cb(res): - d = self.mbox_uid.create_table(name) + d = self.mbox_indexer.create_table(name) d.addCallback(lambda _: res) return d - d = self.adaptor.__class__.get_or_create(name) + d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(create_uid_table_cb) return d def delete_mailbox(self, name): def delete_uid_table_cb(res): - d = self.mbox_uid.delete_table(name) + d = self.mbox_indexer.delete_table(name) d.addCallback(lambda _: res) return d - d = self.adaptor.delete_mbox(self.store) + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback( + lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) d.addCallback(delete_uid_table_cb) return d def rename_mailbox(self, oldname, newname): + # TODO incomplete/wrong!!! + # Should rename also ALL of the document ids that are pointing + # to the old mailbox!!! + + # TODO part-docs identifiers should have the UID_validity of the + # mailbox embedded, instead of the name! (so they can survive a rename) + def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update() + return wrapper.update(self.store) def rename_uid_table_cb(res): - d = self.mbox_uid.rename_table(oldname, newname) + d = self.mbox_indexer.rename_table(oldname, newname) d.addCallback(lambda _: res) return d - d = self.adaptor.__class__.get_or_create(oldname) + d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) d.addCallback(rename_uid_table_cb) return d @@ -585,7 +594,8 @@ class Account(object): self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) mboxwrapper_klass = self.adaptor.mboxwrapper_klass - d = mboxwrapper_klass.get_or_create(name) + #d = mboxwrapper_klass.get_or_create(name) + d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 1ceaec0..e5b813f 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -256,7 +256,7 @@ class MailboxIndexer(object): def increment(result): uid = _maybe_first_query_item(result) if uid is None: - return None + return 1 return uid + 1 sql = ("SELECT MAX(rowid) FROM {preffix}{name} " diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index d11df40..2c4b919 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -23,10 +23,8 @@ from functools import partial from email.parser import Parser from email.Utils import formatdate -from twisted.python import util - from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.mail import MessageCollection +from leap.mail.mail import MessageCollection, Account from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin @@ -57,7 +55,6 @@ def _get_msg_time(): return formatdate(timestamp) - class CollectionMixin(object): def get_collection(self, mbox_collection=True): @@ -86,6 +83,7 @@ class CollectionMixin(object): return d +# TODO profile add_msg. Why are these tests so SLOW??! class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): """ Tests for the Message class. @@ -256,7 +254,9 @@ class MessageCollectionTestCase(unittest.TestCase, return partial(self.assert_collection_count, expected=1) def test_coppy_msg(self): - self.fail() + # TODO ---- update when implementing messagecopier + # interface + self.fail("Not Yet Implemented") def test_delete_msg(self): d = self.add_msg_to_collection() @@ -298,27 +298,81 @@ class AccountTestCase(unittest.TestCase, SoledadTestMixin): """ Tests for the Account class. """ + def get_account(self): + store = self._soledad + return Account(store) def test_add_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_add_mailbox_cb) + return d + + def _test_add_mailbox_cb(self, mboxes): + expected = ['INBOX', 'TestMailbox'] + self.assertItemsEqual(mboxes, expected) def test_delete_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_delete_mailbox_cb) + return d + + def _test_delete_mailbox_cb(self, mboxes): + expected = [] + self.assertItemsEqual(mboxes, expected) def test_rename_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) + d = acc.callWhenReady(lambda _: acc.rename_mailbox( + "TestMailbox", "RenamedMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_rename_mailbox_cb) + return d - def test_list_all_mailbox_names(self): - self.fail() + def _test_rename_mailbox_cb(self, mboxes): + expected = ['INBOX', 'RenamedMailbox'] + self.assertItemsEqual(mboxes, expected) def test_get_all_mailboxes(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) + d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) + d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing2")) + d.addCallback(lambda _: acc.get_all_mailboxes()) + d.addCallback(self._test_get_all_mailboxes_cb) + return d - def test_get_collection_by_docs(self): - self.fail() + def _test_get_all_mailboxes_cb(self, mailboxes): + expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox", + "anotherthing", "anotherthing2"] + names = [m.mbox for m in mailboxes] + self.assertItemsEqual(names, expected) def test_get_collection_by_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) + d.addCallback(self._test_get_collection_by_mailbox_cb) + return d + + def _test_get_collection_by_mailbox_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def assert_uid_next_empty_collection(uid): + self.assertEqual(uid, 1) + d = collection.get_uid_next() + d.addCallback(assert_uid_next_empty_collection) + return d + + # XXX not yet implemented + + def test_get_collection_by_docs(self): + self.fail("Not Yet Implemented") def test_get_collection_by_tag(self): - self.fail() + self.fail("Not Yet Implemented") -- cgit v1.2.3 From c1fc9b52d8b577814e921d128357afdbd9278662 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 12 Jan 2015 20:47:29 -0400 Subject: Use mailbox uuids The previous implementation is naive, since it imposes a burden when renaming mailboxes. We're using uuids in the local uid tables instead, which is more cryptic but way more efficient. * receive mbox uuid instead of name * use mailbox uuid in identifiers --- src/leap/mail/adaptors/soledad.py | 29 ++-- .../mail/adaptors/tests/test_soledad_adaptor.py | 11 +- src/leap/mail/constants.py | 8 +- src/leap/mail/imap/mailbox.py | 12 +- src/leap/mail/imap/tests/test_imap.py | 26 ++- src/leap/mail/mail.py | 57 +++++-- src/leap/mail/mailbox_indexer.py | 101 ++++++------ src/leap/mail/tests/test_mailbox_indexer.py | 182 ++++++++++----------- 8 files changed, 235 insertions(+), 191 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 389307f..c5cfce0 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -338,7 +338,7 @@ class FlagsDocWrapper(SoledadDocumentWrapper): type_ = "flags" chash = "" - mbox = "inbox" + mbox_uuid = "" seen = False deleted = False recent = False @@ -350,11 +350,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): class __meta__(object): index = "mbox" - def set_mbox(self, mbox): + def set_mbox_uuid(self, mbox_uuid): # XXX raise error if already created, should use copy instead - new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash) + mbox_uuid = mbox_uuid.replace('-', '_') + new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash) self._future_doc_id = new_id - self.mbox = mbox + self.mbox_uuid = mbox_uuid class HeaderDocWrapper(SoledadDocumentWrapper): @@ -401,11 +402,12 @@ class MetaMsgDocWrapper(SoledadDocumentWrapper): hdoc = "" cdocs = [] - def set_mbox(self, mbox): + def set_mbox_uuid(self, mbox_uuid): # XXX raise error if already created, should use copy instead + mbox_uuid = mbox_uuid.replace('-', '_') chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] - new_id = constants.METAMSGID.format(mbox=mbox, chash=chash) - new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash) + new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash) self._future_doc_id = new_id self.fdoc = new_fdoc_id @@ -518,14 +520,15 @@ class MessageWrapper(object): # 4. return new wrapper (new meta too!) raise NotImplementedError() - def set_mbox(self, mbox): + def set_mbox_uuid(self, mbox_uuid): """ Set the mailbox for this wrapper. This method should only be used before the Documents for the MessageWrapper have been created, will raise otherwise. """ - self.mdoc.set_mbox(mbox) - self.fdoc.set_mbox(mbox) + mbox_uuid = mbox.uuid.replace('-', '_') + self.mdoc.set_mbox_uuid(mbox_uuid) + self.fdoc.set_mbox_uuid(mbox_uuid) def set_flags(self, flags): # TODO serialize the get + update @@ -574,6 +577,7 @@ class MailboxWrapper(SoledadDocumentWrapper): class model(models.SerializableModel): type_ = "mbox" mbox = INBOX_NAME + uuid = None flags = [] recent = [] created = 1 @@ -889,7 +893,10 @@ def _parse_msg(raw): def _build_meta_doc(chash, cdocs_phashes): _mdoc = MetaMsgDocWrapper() - _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash) + # FIXME passing the inbox name because we don't have the uuid at this + # point. + + _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash) _mdoc.hdoc = constants.HDOCID.format(chash=chash) _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] return _mdoc.serialize() diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 0cca5ef..7bdeef5 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -21,7 +21,6 @@ import os from functools import partial from twisted.internet import defer -from twisted.trial import unittest from leap.mail.adaptors import models from leap.mail.adaptors.soledad import SoledadDocumentWrapper @@ -62,7 +61,7 @@ class TestAdaptor(SoledadIndexMixin): 'by-type': ['type']} -class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): +class SoledadDocWrapperTestCase(SoledadTestMixin): """ Tests for the SoledadDocumentWrapper. """ @@ -284,7 +283,7 @@ class TestMessageClass(object): return self.wrapper -class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): +class SoledadMailAdaptorTestCase(SoledadTestMixin): """ Tests for the SoledadMailAdaptor. """ @@ -337,7 +336,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): hdoc="H-deadbeef", cdocs=["C-deadabad"]) fdoc = dict( - mbox="Foobox", + mbox_uuid="Foobox", flags=('\Seen', '\Nice'), tags=('Personal', 'TODO'), seen=False, deleted=False, @@ -355,7 +354,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): ('\Seen', '\Nice')) self.assertEqual(msg.wrapper.fdoc.tags, ('Personal', 'TODO')) - self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox") + self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox") self.assertEqual(msg.wrapper.hdoc.multi, False) self.assertEqual(msg.wrapper.hdoc.subject, "Test Msg") @@ -363,7 +362,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): "This is a test message") def test_get_msg_from_metamsg_doc_id(self): - # XXX complete-me! + # TODO complete-me! self.fail() def test_create_msg(self): diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index d76e652..4ef42cb 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -22,13 +22,13 @@ INBOX_NAME = "INBOX" # Regular expressions for the identifiers to be used in the Message Data Layer. -METAMSGID = "M-{mbox}-{chash}" -METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+" +METAMSGID = "M-{mbox_uuid}-{chash}" +METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+" METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" -FDOCID = "F-{mbox}-{chash}" -FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+" +FDOCID = "F-{mbox_uuid}-{chash}" +FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+" FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" HDOCID = "H-{chash}" diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index faeba9d..f2cbf75 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -236,6 +236,7 @@ class IMAPMailbox(object): :rtype: int """ + # TODO --- return the uid if it has it!!! d = self.collection.get_msg_by_uid(message) d.addCallback(lambda m: m.getUID()) return d @@ -357,7 +358,7 @@ class IMAPMailbox(object): reactor.callLater(0, self.notify_new) return x - d = self.collection.add_message(flags=flags, date=date) + d = self.collection.add_msg(message, flags=flags, date=date) d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d @@ -389,14 +390,15 @@ class IMAPMailbox(object): messages and number of recent messages. :rtype: Deferred """ - d_exists = self.getMessageCount() - d_recent = self.getRecentCount() + d_exists = defer.maybeDeferred(self.getMessageCount) + d_recent = defer.maybeDeferred(self.getRecentCount) d_list = [d_exists, d_recent] def log_num_msg(result): - exists, recent = result + exists, recent = tuple(result) logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( self.mbox_name, exists, recent)) + return result d = defer.gatherResults(d_list) d.addCallback(log_num_msg) @@ -654,7 +656,7 @@ class IMAPMailbox(object): return result def _get_unseen_deferred(self): - return self.getUnseenCount() + return defer.maybeDeferred(self.getUnseenCount) def __cb_signal_unread_to_ui(self, unseen): """ diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 5af499f..dbb823f 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -926,31 +926,39 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) - LeapIMAPServer.theAccount.addMailbox('root/subthing') + acc = self.server.theAccount + mailbox_name = "root_subthing" + + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def append(): return self.client.append( - 'root/subthing', - message, + mailbox_name, message, ('\\SEEN', '\\DELETED'), 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', ) - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestFullAppend, infile) + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - def _cbTestFullAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('root/subthing') - self.assertEqual(1, len(mb.messages)) + def print_mb(mb): + print "MB ----", mb + return mb + d.addCallback(print_mb) + d.addCallback(lambda mb: mb.collection.get_message_by_uid(1)) + return d.addCallback(self._cbTestFullAppend, infile) - msg = mb.messages.get_msg_by_uid(1) + def _cbTestFullAppend(self, msg, infile): + # TODO --- move to deferreds self.assertEqual( set(('\\Recent', '\\SEEN', '\\DELETED')), set(msg.getFlags())) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 0c9b7a3..b2caa33 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,6 +17,7 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import uuid import logging import StringIO @@ -283,6 +284,20 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + # TODO need to initialize count here because imap server does not + # expect a defered for the count. caller should return the deferred for + # prime_count (ie, initialize) when returning the collection + # TODO should increment and decrement when adding/deleting. + # TODO recent count should also be static. + + if not count: + count = 0 + self._count = count + + #def initialize(self): + #d = self.prime_count() + #return d + def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. @@ -297,6 +312,13 @@ class MessageCollection(object): return None return wrapper.mbox + @property + def mbox_uuid(self): + wrapper = getattr(self, "mbox_wrapper", None) + if not wrapper: + return None + return wrapper.mbox_uuid + def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -385,16 +407,16 @@ class MessageCollection(object): raise NotImplementedError() else: - mbox = self.mbox_name + mbox_id = self.mbox_uuid wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox(mbox) + wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( - self.mbox_name, doc_id) + self.mbox_uuid, doc_id) d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) @@ -410,7 +432,7 @@ class MessageCollection(object): def insert_copied_mdoc_id(wrapper): return self.mbox_indexer.insert_doc( - newmailbox, wrapper.mdoc.doc_id) + newmailbox_uuid, wrapper.mdoc.doc_id) wrapper = msg.get_wrapper() d = wrapper.copy(self.store, newmailbox) @@ -539,25 +561,32 @@ class Account(object): def add_mailbox(self, name): - def create_uid_table_cb(res): - d = self.mbox_indexer.create_table(name) - d.addCallback(lambda _: res) + def create_uuid(wrapper): + if not wrapper.uuid: + wrapper.uuid = uuid.uuid4() + return wrapper.update(self.store) + + def create_uid_table_cb(wrapper): + d = self.mbox_indexer.create_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(create_uuid) d.addCallback(create_uid_table_cb) return d def delete_mailbox(self, name): - def delete_uid_table_cb(res): - d = self.mbox_indexer.delete_table(name) - d.addCallback(lambda _: res) + + def delete_uid_table_cb(wrapper): + d = self.mbox_indexer.delete_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(delete_uid_table_cb) d.addCallback( lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) - d.addCallback(delete_uid_table_cb) return d def rename_mailbox(self, oldname, newname): @@ -572,14 +601,8 @@ class Account(object): wrapper.mbox = newname return wrapper.update(self.store) - def rename_uid_table_cb(res): - d = self.mbox_indexer.rename_table(oldname, newname) - d.addCallback(lambda _: res) - return d - d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) - d.addCallback(rename_uid_table_cb) return d # Get Collections diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index e5b813f..6155a7a 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -18,6 +18,7 @@ Local tables to store the message Unique Identifiers for a given mailbox. """ import re +import uuid from leap.mail.constants import METAMSGID_RE @@ -37,6 +38,25 @@ class WrongMetaDocIDError(Exception): pass +def sanitize(mailbox_id): + return mailbox_id.replace("-", "_") + + +def check_good_uuid(mailbox_id): + """ + Check that the passed mailbox identifier is a valid UUID. + :param mailbox_id: the uuid to check + :type mailbox_id: str + :return: None + :raises: AssertionError if a wrong uuid was passed. + """ + try: + uuid.UUID(str(mailbox_id)) + except (AttributeError, ValueError): + raise AssertionError( + "the mbox_id is not a valid uuid: %s" % mailbox_id) + + class MailboxIndexer(object): """ This class contains the commands needed to create, modify and alter the @@ -68,51 +88,33 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) - def create_table(self, mailbox): + def create_table(self, mailbox_id): """ Create the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox: the mailbox identifier. :type mailbox: str :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) sql = ("CREATE TABLE if not exists {preffix}{name}( " "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) - def delete_table(self, mailbox): + def delete_table(self, mailbox_id): """ Delete the UID table for a given mailbox. :param mailbox: the mailbox name :type mailbox: str :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=mailbox)) - return self._query(sql) - - def rename_table(self, oldmailbox, newmailbox): - """ - Delete the UID table for a given mailbox. - :param oldmailbox: the old mailbox name - :type oldmailbox: str - :param newmailbox: the new mailbox name - :type newmailbox: str - :rtype: Deferred - """ - assert oldmailbox - assert newmailbox - assert oldmailbox != newmailbox - sql = ("ALTER TABLE {preffix}{old} " - "RENAME TO {preffix}{new}".format( - preffix=self.table_preffix, - old=oldmailbox, new=newmailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) - def insert_doc(self, mailbox, doc_id): + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -128,10 +130,11 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert doc_id + mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -139,44 +142,44 @@ class MailboxIndexer(object): sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (doc_id,) sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=mailbox) + preffix=self.table_preffix, name=sanitize(mailbox_id)) d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) return d - def delete_doc_by_uid(self, mailbox, uid): + def delete_doc_by_uid(self, mailbox_id, uid): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the UID of the message. :type uid: int :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert uid sql = ("DELETE FROM {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (uid,) return self._query(sql, values) - def delete_doc_by_hash(self, mailbox, doc_id): + def delete_doc_by_hash(self, mailbox_id, doc_id): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. The doc_id must be in the format: - M++ + M-- - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str @@ -184,30 +187,32 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert doc_id sql = ("DELETE FROM {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (doc_id,) return self._query(sql, values) - def get_doc_id_from_uid(self, mailbox, uid): + def get_doc_id_from_uid(self, mailbox_id, uid): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ + check_good_uuid(mailbox_id) + def get_hash(result): return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (uid,) d = self._query(sql, values) d.addCallback(get_hash) @@ -218,7 +223,7 @@ class MailboxIndexer(object): # XXX dereference the range (n,*) raise NotImplementedError() - def count(self, mailbox): + def count(self, mailbox_id): """ Get the number of entries in the UID table for a given mailbox. @@ -227,16 +232,18 @@ class MailboxIndexer(object): :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ + check_good_uuid(mailbox_id) + def get_count(result): return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) return d - def get_next_uid(self, mailbox): + def get_next_uid(self, mailbox_id): """ Get the next integer beyond the highest UID count for a given mailbox. @@ -251,7 +258,7 @@ class MailboxIndexer(object): uid. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) def increment(result): uid = _maybe_first_query_item(result) @@ -261,7 +268,7 @@ class MailboxIndexer(object): sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=mailbox) + preffix=self.table_preffix, name=sanitize(mailbox_id)) d = self._query(sql) d.addCallback(increment) diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py index 47a3bdc..2edf1d8 100644 --- a/src/leap/mail/tests/test_mailbox_indexer.py +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -17,10 +17,9 @@ """ Tests for the mailbox_indexer module. """ +import uuid from functools import partial -from twisted.trial import unittest - from leap.mail import mailbox_indexer as mi from leap.mail.tests.common import SoledadTestMixin @@ -31,11 +30,13 @@ hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' -def fmt_hash(mailbox, hash): - return "M-" + mailbox + "-" + hash +def fmt_hash(mailbox_uuid, hash): + return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash + +mbox_id = str(uuid.uuid4()) -class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): +class MailboxIndexerTestCase(SoledadTestMixin): """ Tests for the MailboxUID class. """ @@ -57,17 +58,17 @@ class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): def select_uid_rows(self, mailbox): sql = "SELECT * FROM %s%s;" % ( - mi.MailboxIndexer.table_preffix, mailbox) + mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_')) d = self._soledad.raw_sqlcipher_query(sql) return d def test_create_table(self): def assert_table_created(tables): self.assertEqual( - tables, ["leapmail_uid_inbox"]) + tables, ["leapmail_uid_" + mbox_id.replace('-', '_')]) m_uid = self.get_mbox_uid() - d = m_uid.create_table('inbox') + d = m_uid.create_table(mbox_id) d.addCallback(self.list_mail_tables_cb) d.addCallback(assert_table_created) return d @@ -77,165 +78,162 @@ class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): self.assertEqual(tables, []) m_uid = self.get_mbox_uid() - d = m_uid.create_table('inbox') - d.addCallback(lambda _: m_uid.delete_table('inbox')) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.delete_table(mbox_id)) d.addCallback(self.list_mail_tables_cb) d.addCallback(assert_table_deleted) return d - def test_rename_table(self): - def assert_table_renamed(tables): - self.assertEqual( - tables, ["leapmail_uid_foomailbox"]) - - m_uid = self.get_mbox_uid() - d = m_uid.create_table('inbox') - d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) - d.addCallback(self.list_mail_tables_cb) - d.addCallback(assert_table_renamed) - return d + #def test_rename_table(self): + #def assert_table_renamed(tables): + #self.assertEqual( + #tables, ["leapmail_uid_foomailbox"]) +# + #m_uid = self.get_mbox_uid() + #d = m_uid.create_table('inbox') + #d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) + #d.addCallback(self.list_mail_tables_cb) + #d.addCallback(assert_table_renamed) + #return d def test_insert_doc(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) def assert_uid_rows(rows): expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] self.assertEquals(rows, expected) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) - d.addCallback(lambda _: self.select_uid_rows(mbox)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) d.addCallback(assert_uid_rows) return d def test_insert_doc_return(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' def assert_rowid(rowid, expected=None): self.assertEqual(rowid, expected) - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) d.addCallback(partial(assert_rowid, expected=1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) d.addCallback(partial(assert_rowid, expected=2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) d.addCallback(partial(assert_rowid, expected=3)) return d def test_delete_doc(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) def assert_uid_rows(rows): expected = [(4, h4), (5, h5)] self.assertEquals(rows, expected) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3)) - d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) d.addCallback(assert_uid_rows) return d def test_get_doc_id_from_uid(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' + #mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) + h1 = fmt_hash(mbox_id, hash_test0) def assert_doc_hash(res): self.assertEqual(res, h1) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1)) d.addCallback(assert_doc_hash) return d def test_count(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' + #mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) def assert_count_after_inserts(count): self.assertEquals(count, 5) - d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(lambda _: m_uid.count(mbox_id)) d.addCallback(assert_count_after_inserts) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) def assert_count_after_deletions(count): self.assertEquals(count, 3) - d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(lambda _: m_uid.count(mbox_id)) d.addCallback(assert_count_after_deletions) return d def test_get_next_uid(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' + #mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) def assert_next_uid(result, expected=1): self.assertEquals(result, expected) - d.addCallback(lambda _: m_uid.get_next_uid(mbox)) + d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) d.addCallback(partial(assert_next_uid, expected=6)) return d -- cgit v1.2.3 From 68500fb15dbb7531eeb397ccee2c160d71284d97 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 7 Jan 2015 12:12:24 -0400 Subject: Complete IMAP implementation, update tests --- src/leap/mail/adaptors/soledad.py | 183 +++++-- src/leap/mail/adaptors/soledad_indexes.py | 15 +- .../mail/adaptors/tests/test_soledad_adaptor.py | 7 +- src/leap/mail/imap/account.py | 174 ++++--- src/leap/mail/imap/mailbox.py | 116 +++-- src/leap/mail/imap/messages.py | 13 +- src/leap/mail/imap/server.py | 30 +- src/leap/mail/imap/service/imap.py | 98 +--- src/leap/mail/imap/tests/leap_tests_imap.zsh | 178 ------- src/leap/mail/imap/tests/stress_tests_imap.zsh | 178 +++++++ src/leap/mail/imap/tests/test_imap.py | 539 +++++++++++---------- src/leap/mail/imap/tests/utils.py | 212 +++----- src/leap/mail/mail.py | 114 ++++- src/leap/mail/mailbox_indexer.py | 70 ++- src/leap/mail/tests/common.py | 17 +- src/leap/mail/tests/test_mailbox_indexer.py | 41 +- 16 files changed, 1098 insertions(+), 887 deletions(-) delete mode 100755 src/leap/mail/imap/tests/leap_tests_imap.zsh create mode 100755 src/leap/mail/imap/tests/stress_tests_imap.zsh diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index c5cfce0..d99f677 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -23,7 +23,6 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer -from twisted.python import util from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -56,6 +55,14 @@ class DuplicatedDocumentError(Exception): pass +def cleanup_deferred_locks(): + """ + Need to use this from within trial to cleanup the reactor before + each run. + """ + SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock) + + class SoledadDocumentWrapper(models.DocumentWrapper): """ A Wrapper object that can be manipulated, passed around, and serialized in @@ -526,7 +533,7 @@ class MessageWrapper(object): This method should only be used before the Documents for the MessageWrapper have been created, will raise otherwise. """ - mbox_uuid = mbox.uuid.replace('-', '_') + mbox_uuid = mbox_uuid.replace('-', '_') self.mdoc.set_mbox_uuid(mbox_uuid) self.fdoc.set_mbox_uuid(mbox_uuid) @@ -536,6 +543,9 @@ class MessageWrapper(object): flags = tuple() leap_assert_type(flags, tuple) self.fdoc.flags = list(flags) + self.fdoc.deleted = "\\Deleted" in flags + self.fdoc.seen = "\\Seen" in flags + self.fdoc.recent = "\\Recent" in flags def set_tags(self, tags): # TODO serialize the get + update @@ -593,26 +603,30 @@ class MailboxWrapper(SoledadDocumentWrapper): # Soledad Adaptor # -# TODO make this an interface? class SoledadIndexMixin(object): """ - this will need a class attribute `indexes`, that is a dictionary containing + This will need a class attribute `indexes`, that is a dictionary containing the index definitions for the underlying u1db store underlying soledad. It needs to be in the following format: {'index-name': ['field1', 'field2']} + + You can also add a class attribute `wait_for_indexes` to any class + inheriting from this Mixin, that should be a list of strings representing + the methods that need to wait until the indexes have been initialized + before being able to work properly. """ + # TODO move this mixin to soledad itself + # so that each application can pass a set of indexes for their data model. + # TODO could have a wrapper class for indexes, supporting introspection # and __getattr__ - indexes = {} - store_ready = False - _index_creation_deferreds = [] + # TODO make this an interface? - # TODO we might want to move this logic to soledad itself - # so that each application can pass a set of indexes for their data model. - # TODO check also the decorator used in keymanager for waiting for indexes - # to be ready. + indexes = {} + wait_for_indexes = [] + store_ready = False def initialize_store(self, store): """ @@ -626,47 +640,81 @@ class SoledadIndexMixin(object): # TODO I think we *should* get another deferredLock in here, but # global to the soledad namespace, to protect from several points # initializing soledad indexes at the same time. + self._wait_for_indexes() - leap_assert(store, "Need a store") - leap_assert_type(self.indexes, dict) + d = self._init_indexes(store) + d.addCallback(self._restore_waiting_methods) + return d - self._index_creation_deferreds = [] + def _init_indexes(self, store): + """ + Initialize the database indexes. + """ + leap_assert(store, "Cannot init indexes with null soledad") + leap_assert_type(self.indexes, dict) def _create_index(name, expression): return store.create_index(name, *expression) - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - + def init_idexes(indexes): + deferreds = [] + db_indexes = dict(indexes) + # Loop through the indexes we expect to find. for name, expression in self.indexes.items(): if name not in db_indexes: # The index does not yet exist. d = _create_index(name, expression) - self._index_creation_deferreds.append(d) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = store.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - self._index_creation_deferreds.append(d1) - - all_created = defer.gatherResults( - self._index_creation_deferreds, consumeErrors=True) - all_created.addCallback(_on_indexes_created) - return all_created - - def _on_indexes_created(ignored): + deferreds.append(d) + elif expression != db_indexes[name]: + # The index exists but the definition is not what expected, + # so we delete it and add the proper index expression. + d = store.delete_index(name) + d.addCallback( + lambda _: _create_index(name, *expression)) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def store_ready(whatever): self.store_ready = True + return whatever - # Ask the database for currently existing indexes, and create them - # if not found. - d = store.list_indexes() - d.addCallback(_create_indexes) - return d + self.deferred_indexes = store.list_indexes() + self.deferred_indexes.addCallback(init_idexes) + self.deferred_indexes.addCallback(store_ready) + return self.deferred_indexes + + def _wait_for_indexes(self): + """ + Make the marked methods to wait for the indexes to be ready. + Heavily based on + http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/ + + :param methods: methods that need to wait for the indexes to be ready + :type methods: tuple(str) + """ + leap_assert_type(self.wait_for_indexes, list) + methods = self.wait_for_indexes + + self.waiting = [] + self.stored = {} + + def makeWrapper(method): + def wrapper(*args, **kw): + d = defer.Deferred() + d.addCallback(lambda _: self.stored[method](*args, **kw)) + self.waiting.append(d) + return d + return wrapper + + for method in methods: + self.stored[method] = getattr(self, method) + setattr(self, method, makeWrapper(method)) + + def _restore_waiting_methods(self, _): + for method in self.stored: + setattr(self, method, self.stored[method]) + for d in self.waiting: + d.callback(None) class SoledadMailAdaptor(SoledadIndexMixin): @@ -675,8 +723,18 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] + mboxwrapper_klass = MailboxWrapper + def __init__(self): + SoledadIndexMixin.__init__(self) + + mboxwrapper_klass = MailboxWrapper + + def __init__(self): + SoledadIndexMixin.__init__(self) + # Message handling def get_msg_from_string(self, MessageClass, raw_msg): @@ -762,10 +820,10 @@ class SoledadMailAdaptor(SoledadIndexMixin): chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox=mbox, chash=chash) + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) def _get_hdoc_id_from_mdoc_id(): - return constants.HDOCID.format(mbox=mbox, chash=chash) + return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) d_docs = [] fdoc_id = _get_fdoc_id_from_mdoc_id() @@ -816,6 +874,47 @@ class SoledadMailAdaptor(SoledadIndexMixin): wrapper = msg.get_wrapper() return wrapper.update(store) + # batch deletion + + def del_all_flagged_messages(self, store, mbox_uuid): + """ + Delete all messages flagged as deleted. + """ + def err(f): + print "ERROR GETTING FROM INDEX" + f.printTraceback() + + def delete_fdoc_and_mdoc_flagged(fdocs): + # low level here, not using the wrappers... + # get meta doc ids from the flag doc ids + fdoc_ids = [doc.doc_id for doc in fdocs] + mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) + + def delete_all_docs(mdocs, fdocs): + mdocs = list(mdocs) + doc_ids = [m.doc_id for m in mdocs] + _d = [] + docs = mdocs + fdocs + for doc in docs: + _d.append(store.delete_doc(doc)) + d = defer.gatherResults(_d) + # return the mdocs ids only + d.addCallback(lambda _: doc_ids) + return d + + d = store.get_docs(mdoc_ids) + d.addCallback(delete_all_docs, fdocs) + d.addErrback(err) + return d + + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + deleted_index = indexes.TYPE_MBOX_DEL_IDX + + d = store.get_from_index(deleted_index, type_, uuid, "1") + d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) + return d + # Mailbox handling def get_or_create_mbox(self, store, name): diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py index f3e990d..856dfb4 100644 --- a/src/leap/mail/adaptors/soledad_indexes.py +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -25,6 +25,7 @@ Soledad Indexes for Mail Documents. TYPE = "type" MBOX = "mbox" +MBOX_UUID = "mbox_uuid" FLAGS = "flags" HEADERS = "head" CONTENT = "cnt" @@ -46,7 +47,7 @@ UID = "uid" TYPE_IDX = 'by-type' TYPE_MBOX_IDX = 'by-type-and-mbox' -#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' +TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MSGID_IDX = 'by-type-and-message-id' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' @@ -62,9 +63,6 @@ TYPE_P_HASH_IDX = 'by-type-and-payloadhash' JUST_MAIL_IDX = "just-mail" JUST_MAIL_COMPAT_IDX = "just-mail-compat" -# Tomas created the `recent and seen index`, but the semantic is not too -# correct since the recent flag is volatile --- XXX review and delete. -#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' # TODO # it would be nice to measure the cost of indexing @@ -77,6 +75,7 @@ MAIL_INDEXES = { # generic TYPE_IDX: [TYPE], TYPE_MBOX_IDX: [TYPE, MBOX], + TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID], # XXX deprecate 0.4.0 # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], @@ -97,11 +96,9 @@ MAIL_INDEXES = { TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], # messages - TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'], - #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX, - #'bool(recent)', 'bool(seen)'], + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], # incoming queue JUST_MAIL_IDX: [INCOMING_KEY, diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 7bdeef5..3dc79fe 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -276,8 +276,9 @@ HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): - def __init__(self, wrapper): + def __init__(self, wrapper, uid): self.wrapper = wrapper + self.uid = uid def get_wrapper(self): return self.wrapper @@ -322,9 +323,9 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): self.assertTrue(msg.wrapper.cdocs is not None) self.assertEquals(len(msg.wrapper.cdocs), 1) self.assertEquals(msg.wrapper.fdoc.chash, chash) - self.assertEquals(msg.wrapper.fdoc.size, 3834) + self.assertEquals(msg.wrapper.fdoc.size, 3837) self.assertEquals(msg.wrapper.hdoc.chash, chash) - self.assertEqual(msg.wrapper.hdoc.headers['subject'], + self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'], subject) self.assertEqual(msg.wrapper.hdoc.subject, subject) self.assertEqual(msg.wrapper.cdocs[1].phash, phash) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0baf078..dfc0d62 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -63,7 +63,7 @@ class IMAPAccount(object): selected = None closed = False - def __init__(self, user_id, store): + def __init__(self, user_id, store, d=None): """ Keeps track of the mailboxes and subscriptions handled by this account. @@ -72,21 +72,31 @@ class IMAPAccount(object): :param store: a Soledad instance. :type store: Soledad + + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred """ leap_assert(store, "Need a store instance to initialize") leap_assert_type(store, Soledad) # TODO assert too that the name matches the user/uuid with which - # soledad has been initialized. + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. + self.user_id = user_id - self.account = Account(store) + self.account = Account(store, ready_cb=lambda: d.callback(self)) def _return_mailbox_from_collection(self, collection, readwrite=1): if collection is None: return None - return IMAPMailbox(collection, rw=readwrite) + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + + def callWhenReady(self, cb, *args, **kw): + d = self.account.callWhenReady(cb, *args, **kw) + return d - # XXX Where's this used from? -- self.delete... def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -102,11 +112,12 @@ class IMAPAccount(object): def check_it_exists(mailboxes): if name not in mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) + return True d = self.account.list_all_mailbox_names() d.addCallback(check_it_exists) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) - d.addCallbacK(self._return_mailbox_from_collection) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) return d # @@ -130,12 +141,9 @@ class IMAPAccount(object): """ name = normalize_mailbox(name) + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... leap_assert(name, "Need a mailbox name to create a mailbox") - - def check_it_does_not_exist(mailboxes): - if name in mailboxes: - raise imap4.MailboxCollision(repr(name)) - if creation_ts is None: # by default, we pass an int value # taken from the current time @@ -143,14 +151,20 @@ class IMAPAccount(object): # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes + def set_mbox_creation_ts(collection): - d = collection.set_mbox_attr("created") + d = collection.set_mbox_attr("created", creation_ts) d.addCallback(lambda _: collection) return d d = self.account.list_all_mailbox_names() d.addCallback(check_it_does_not_exist) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallback(lambda _: self.account.add_mailbox(name)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) d.addCallback(set_mbox_creation_ts) d.addCallback(self._return_mailbox_from_collection) return d @@ -172,39 +186,40 @@ class IMAPAccount(object): :raise MailboxException: Raised if this mailbox cannot be added. """ - # TODO raise MailboxException paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] sep = '/' + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + for accum in range(1, len(paths)): - try: - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - subs.append(d) - # XXX should this be handled by the deferred? - except imap4.MailboxCollision: - pass - try: - df = self.addMailbox(sep.join(paths)) - except imap4.MailboxCollision: + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): - df = defer.succeed(False) + return defer.succeed(False) else: - df = defer.succeed(True) - finally: - subs.append(df) + return defer.succeed(True) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) def all_good(result): return all(result) if subs: - d1 = defer.gatherResults(subs, consumeErrors=True) + d1 = defer.gatherResults(subs) d1.addCallback(all_good) + return d1 else: - d1 = defer.succeed(False) - return d1 + return defer.succeed(False) def select(self, name, readwrite=1): """ @@ -216,7 +231,7 @@ class IMAPAccount(object): :param readwrite: 1 for readwrite permissions. :type readwrite: int - :rtype: SoledadMailbox + :rtype: IMAPMailbox """ name = normalize_mailbox(name) @@ -245,9 +260,6 @@ class IMAPAccount(object): """ Deletes a mailbox. - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - :param name: the mailbox to be deleted :type name: str @@ -258,10 +270,10 @@ class IMAPAccount(object): :rtype: Deferred """ name = normalize_mailbox(name) - _mboxes = [] + _mboxes = None def check_it_exists(mailboxes): - # FIXME works? -- pass variable ref to outer scope + global _mboxes _mboxes = mailboxes if name not in mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -274,6 +286,7 @@ class IMAPAccount(object): return mbox.destroy() def check_can_be_deleted(mbox): + global _mboxes # See if this box is flagged \Noselect mbox_flags = mbox.getFlags() if MessageFlags.NOSELECT_FLAG in mbox_flags: @@ -317,29 +330,27 @@ class IMAPAccount(object): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) - # FIXME check that scope works (test) - _mboxes = [] - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox(repr(oldname)) - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + def rename_inferiors(inferiors_result): + inferiors, mailboxes = inferiors_result + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] - for (old, new) in inferiors: - if new in _mboxes: - raise imap4.MailboxCollision(repr(new)) + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) - rename_deferreds = [] + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) - for (old, new) in inferiors: - d = self.account.rename_mailbox(old, new) - rename_deferreds.append(d) + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 - d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - return d1 + d = self._inferiorNames(oldname) + d.addCallback(rename_inferiors) + return d - # FIXME use deferreds (list_all_mailbox_names, etc) def _inferiorNames(self, name): """ Return hierarchically inferior mailboxes. @@ -348,13 +359,17 @@ class IMAPAccount(object): :rtype: list """ # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d - # TODO use mail.Account.list_mailboxes def listMailboxes(self, ref, wildcard): """ List the mailboxes. @@ -371,11 +386,21 @@ class IMAPAccount(object): :param wildcard: mailbox name with possible wildcards :type wildcard: str """ - # XXX use wildcard in index query - # TODO get deferreds wildcard = imap4.wildcardToRegexp(wildcard, '/') - ref = self._inferiorNames(normalize_mailbox(ref)) - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) + + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d # # The rest of the methods are specific for leap.mail.imap.account.Account @@ -393,7 +418,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def get_subscribed(mbox): - return mbox.get_mbox_attr("subscribed") + return mbox.collection.get_mbox_attr("subscribed") d = self.getMailbox(name) d.addCallback(get_subscribed) @@ -410,7 +435,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_subscribed(mbox): - return mbox.set_mbox_attr("subscribed", True) + return mbox.collection.set_mbox_attr("subscribed", True) d = self.getMailbox(name) d.addCallback(set_subscribed) @@ -427,16 +452,19 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_unsubscribed(mbox): - return mbox.set_mbox_attr("subscribed", False) + return mbox.collection.set_mbox_attr("subscribed", False) d = self.getMailbox(name) d.addCallback(set_unsubscribed) return d - # TODO -- get__all_mboxes, return tuple - # with ... name? and subscribed bool... def getSubscriptions(self): - raise NotImplementedError() + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] + + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d # # INamespacePresenter diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index f2cbf75..e1eb6bf 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -21,9 +21,11 @@ import re import logging import StringIO import cStringIO +import time import os from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -36,6 +38,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage logger = logging.getLogger(__name__) @@ -88,7 +91,7 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - imap4.ICloseableMailbox, + #imap4.ICloseableMailbox, imap4.ISearchableMailbox, imap4.IMessageCopier) @@ -105,8 +108,7 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. + SoledadMailbox constructor. :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -115,6 +117,7 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw + self.closed = False self._uidvalidity = None self.collection = collection @@ -139,6 +142,11 @@ class IMAPMailbox(object): """ return self._listeners[self.mbox_name] + def get_imap_message(self, message): + msg = IMAPMessage(message) + msg.store = self.collection.store + return msg + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. @@ -308,17 +316,24 @@ class IMAPMailbox(object): :type names: iter """ r = {} + maybe = defer.maybeDeferred if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() + r[self.CMD_MSG] = maybe(self.getMessageCount) if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() + r[self.CMD_RECENT] = maybe(self.getRecentCount) if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getUIDNext() + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d def addMessage(self, message, flags, date=None): """ @@ -341,11 +356,15 @@ class IMAPMailbox(object): # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) + if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) + if date is None: + date = formatdate(time.time()) + # if PROFILE_CMD: # do_profile_cmd(d, "APPEND") @@ -419,10 +438,11 @@ class IMAPMailbox(object): self.setFlags((MessageFlags.NOSELECT_FLAG,)) def remove_mbox(_): - # FIXME collection does not have a delete_mbox method, - # it's in account. - # XXX should take care of deleting the uid table too. - return self.collection.delete_mbox(self.mbox_name) + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d d = self.deleteAllDocs() d.addCallback(remove_mbox) @@ -431,13 +451,14 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - def close(self): - """ - Expunge and mark as closed - """ - d = self.expunge() - d.addCallback(self._close_cb) - return d + # TODO server already calls expunge for closing + #def close(self): + #""" + #Expunge and mark as closed + #""" + #d = self.expunge() + #d.addCallback(self._close_cb) + #return d def expunge(self): """ @@ -445,10 +466,7 @@ class IMAPMailbox(object): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - d = defer.Deferred() - # FIXME actually broken. - # Iterate through index, and do a expunge. - return d + return self.collection.delete_all_flagged() # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): @@ -465,12 +483,12 @@ class IMAPMailbox(object): except TypeError: # looks like we cannot iterate try: + # XXX fixme, does not exist messages_asked.last = self.last_uid except ValueError: pass return messages_asked - # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -480,10 +498,16 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: set """ - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - return seq_messg + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(sequence): + set_asked = set(messages_asked) + set_exist = set(sequence) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d def fetch(self, messages_asked, uid): """ @@ -509,26 +533,41 @@ class IMAPMailbox(object): sequence = False # sequence = True if uid == 0 else False + getmsg = self.collection.get_message_by_uid messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - getmsg = self.collection.get_msg_by_uid + d_sequence = self._filter_msg_seq(messages_asked) + + def get_imap_messages_for_sequence(sequence): + def _zip_msgid(messages): + return zip( + list(sequence), + map(self.get_imap_message, messages)) + + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in sequence: + d_msg.append(getmsg(msgid)) + + d = defer.gatherResults(d_msg) + d.addCallback(_zip_msgid) + return d # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError + else: - got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) - result = ((msgid, msg) for msgid, msg in got_msg - if msg is not None) - reactor.callLater(0, self.unset_recent_flags, seq_messg) + d_sequence.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - - return result + return d_sequence def fetch_flags(self, messages_asked, uid): """ @@ -755,6 +794,7 @@ class IMAPMailbox(object): # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on @@ -821,7 +861,7 @@ class IMAPMailbox(object): Representation string for this mailbox. """ return u"" % ( - self.mbox_name, self.messages.count()) + self.mbox_name, self.collection.count()) _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 883da35..9b00162 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -18,18 +18,12 @@ IMAPMessage and IMAPMessageCollection. """ import logging -# import StringIO from twisted.mail import imap4 from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset - from leap.mail.utils import find_charset -from leap.mail.imap.messageparts import MessagePart -# from leap.mail.imap.messagepargs import MessagePartDoc logger = logging.getLogger(__name__) @@ -116,13 +110,17 @@ class IMAPMessage(object): # IMessagePart # - def getBodyFile(self): + def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading :rtype: StringIO """ + if store is None: + store = self.store + return self.message.get_body_file(store) + # TODO refactor with getBodyFile in MessagePart #body = bdoc_content.get(self.RAW_KEY, "") @@ -141,7 +139,6 @@ class IMAPMessage(object): #finally: #return write_fd(body) - return self.message.get_body_file() def getSize(self): """ diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index cf0ba74..b4f320a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Leap IMAP4 Server Implementation. +LEAP IMAP4 Server Implementation. """ from copy import copy @@ -36,9 +36,9 @@ from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile -class LeapIMAPServer(imap4.IMAP4Server): +class LEAPIMAPServer(imap4.IMAP4Server): """ - An IMAP4 Server with mailboxes backed by soledad + An IMAP4 Server with a LEAP Storage Backend. """ def __init__(self, *args, **kwargs): # pop extraneous arguments @@ -51,7 +51,6 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid - self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -146,12 +145,15 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ - self.reactor.callFromThread(self.mbox.notify_new) + reactor.callFromThread(self.mbox.notify_new) def _cbSelectWork(self, mbox, cmdName, tag): """ - Callback for selectWork, patched to avoid conformance errors due to - incomplete UIDVALIDITY line. + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count """ if mbox is None: self.sendNegativeResponse(tag, 'No such mailbox') @@ -161,9 +163,9 @@ class LeapIMAPServer(imap4.IMAP4Server): return flags = mbox.getFlags() + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') - self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding @@ -353,12 +355,9 @@ class LeapIMAPServer(imap4.IMAP4Server): self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) # -------------------- end isSubscribed patch ----------- - # TODO ---- - # subscribe method had also to be changed to accomodate - # deferred - # Revert to regular methods as soon as we implement non-deferred memory - # cache. + # TODO subscribe method had also to be changed to accomodate deferred def do_SUBSCRIBE(self, tag, name): + print "DOING SUBSCRIBE" name = self._parseMbox(name) def _subscribeCb(_): @@ -421,7 +420,8 @@ class LeapIMAPServer(imap4.IMAP4Server): def _renameEb(failure): m = failure.value - print "rename failure!" + print "SERVER rename failure!" + print m if failure.check(TypeError): self.sendBadResponse(tag, 'Invalid command syntax') elif failure.check(imap4.MailboxException): @@ -479,7 +479,7 @@ class LeapIMAPServer(imap4.IMAP4Server): if failure.check(imap4.MailboxException): self.sendNegativeResponse(tag, str(m)) else: - print "other error" + print "SERVER: other error" log.err() self.sendBadResponse( tag, diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 10ba32a..5d88a79 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # imap.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,15 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Imap service initialization +IMAP service initialization """ import logging import os -import time -from twisted.internet import defer, threads -from twisted.internet.protocol import ServerFactory +from twisted.internet import reactor from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory from twisted.mail import imap4 from twisted.python import log @@ -32,42 +31,14 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.account import IMAPAccount from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.server import LEAPIMAPServer from leap.soledad.client import Soledad -# The default port in which imap service will run -IMAP_PORT = 1984 - -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -###################################################### -# Temporary workaround for RecursionLimit when using -# qt4reactor. Do remove when we move to poll or select -# reactor, which do not show those problems. See #4974 -import resource -import sys - -try: - sys.setrecursionlimit(10**7) -except Exception: - print "Error setting recursion limit" -try: - # Increase max stack size from 8MB to 256MB - resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) -except Exception: - print "Error setting stack size" - -###################################################### - DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole @@ -81,6 +52,13 @@ if DO_PROFILE: pr = cProfile.Profile() pr.enable() +# The default port in which imap service will run +IMAP_PORT = 1984 + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + class IMAPAuthRealm(object): """ @@ -114,12 +92,8 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad - self._memstore = MemoryStore( - permanent_store=SoledadStore(soledad)) - theAccount = SoledadBackedAccount( - uuid, soledad=soledad, - memstore=self._memstore) + theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount # XXX how to pass the store along? @@ -131,7 +105,8 @@ class LeapIMAPFactory(ServerFactory): :param addr: remote ip address :type addr: str """ - imapProtocol = LeapIMAPServer( + # XXX addr not used??! + imapProtocol = LEAPIMAPServer( uuid=self._uuid, userid=self._userid, soledad=self._soledad) @@ -139,41 +114,18 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol - def doStop(self, cv=None): + def doStop(self): """ Stops imap service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition - :return: a Deferred that stops and flushes the in memory store data to - disk in another thread. - :rtype: Deferred """ + # TODO should wait for all the pending deferreds, + # the twisted way! if DO_PROFILE: log.msg("Stopping PROFILING") pr.disable() pr.dump_stats(PROFILE_DAT) - ServerFactory.doStop(self) - - if cv is not None: - def _stop_imap_cb(): - logger.debug('Stopping in memory store.') - self._memstore.stop_and_flush() - while not self._memstore.producer.is_queue_empty(): - logger.debug('Waiting for queue to be empty.') - # TODO use a gatherResults over the new/dirty - # deferred list, - # as in memorystore's expunge() method. - time.sleep(1) - # notify that service has stopped - logger.debug('Notifying that service has stopped.') - cv.acquire() - cv.notify() - cv.release() - - return threads.deferToThread(_stop_imap_cb) + return ServerFactory.doStop(self) def run_service(*args, **kwargs): @@ -185,11 +137,6 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ - from twisted.internet import reactor - # it looks like qtreactor does not honor this, - # but other reactors should. - reactor.suggestThreadPoolSize(20) - leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -201,13 +148,14 @@ def run_service(*args, **kwargs): leap_check(userid is not None, "need an user id") offline = kwargs.get('offline', False) - uuid = soledad._get_uuid() + uuid = soledad.uuid factory = LeapIMAPFactory(uuid, userid, soledad) try: tport = reactor.listenTCP(port, factory, interface="localhost") if not offline: + # FIXME --- update after meskio's work fetcher = LeapIncomingMail( keymanager, soledad, @@ -236,6 +184,8 @@ def run_service(*args, **kwargs): interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature return fetcher, tport, factory # not ok, signal error. diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh deleted file mode 100755 index 544faca..0000000 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/zsh -# BATCH STRESS TEST FOR IMAP ---------------------- -# http://imgs.xkcd.com/comics/science.jpg -# -# Run imaptest against a LEAP IMAP server -# for a fixed period of time, and collect output. -# -# Author: Kali Kaneko -# Date: 2014 01 26 -# -# To run, you need to have `imaptest` in your path. -# See: -# http://www.imapwiki.org/ImapTest/Installation -# -# For the tests, I'm using a 10MB file sample that -# can be downloaded from: -# http://www.dovecot.org/tmp/dovecot-crlf -# -# Want to contribute to benchmarking? -# -# 1. Create a pristine account in a bitmask provider. -# -# 2. Launch your bitmask client, with different flags -# if you desire. -# -# For example to try the nosync flag in sqlite: -# -# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log -# -# 3. Run at several points in time (ie: just after -# launching the bitmask client. one minute after, -# ten minutes after) -# -# mkdir data -# cd data -# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log -# -# 4. Submit your results to: kali at leap dot se -# together with the logs of the bitmask run. -# -# Please provide also details about your system, and -# the type of hard disk setup you are running against. -# - -# ------------------------------------------------ -# Edit these variables if you are too lazy to pass -# the user and mbox as parameters. Like me. - -USER="test_f14@dev.bitmask.net" -MBOX="~/leap/imaptest/data/dovecot-crlf" - -HOST="localhost" -PORT="1984" - -# in case you have it aliased -GREP="/bin/grep" -IMAPTEST="imaptest" - -# ----------------------------------------------- -# -# These should be kept constant across benchmarking -# runs across different machines, for comparability. - -DURATION=200 -NUM_MSG=200 - - -# TODO add another function, and a cli flag, to be able -# to take several aggretates spaced in time, along a period -# of several minutes. - -imaptest_cmd() { - stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ - port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ - no_pipelining 2>/dev/null -} - -stress_imap() { - mkfifo imap_pipe - cat imap_pipe | tee output & - imaptest_cmd >> imap_pipe -} - -wait_and_kill() { - while : - do - sleep $DURATION - pkill -2 imaptest - rm imap_pipe - break - done -} - -print_results() { - sleep 1 - echo - echo - echo "AGGREGATED RESULTS" - echo "----------------------" - echo "\tavg\tstdev" - $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ - gawk ' -function avg(data, count) { - sum=0; - for( x=0; x <= count-1; x++) { - sum += data[x]; - } - return sum/count; -} -function std_dev(data, count) { - sum=0; - for( x=0; x <= count-1; x++) { - sum += data[x]; - } - average = sum/count; - - sumsq=0; - for( x=0; x <= count-1; x++) { - sumsq += (data[x] - average)^2; - } - return sqrt(sumsq/count); -} -BEGIN { - cnt = 0 -} END { - -printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); -printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); -printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); -printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); -printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); -printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); -printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); -printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); -printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); -printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); -printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); - -print "" -print "TOT samples", NR; -} -{ - it = cnt++; - array[1][it] = $1; - array[2][it] = $2; - array[3][it] = $3; - array[4][it] = $4; - array[5][it] = $5; - array[6][it] = $6; - array[7][it] = $7; - array[8][it] = $8; - array[9][it] = $9; - array[10][it] = $10; - array[11][it] = $11; -}' -} - - -{ test $1 = "--help" } && { - echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" - exit 0 -} - -# If the first parameter is passed, take it as the user -{ test $1 } && { - USER=$1 -} - -# If the second parameter is passed, take it as the mbox -{ test $2 } && { - MBOX=$2 -} - -echo "[+] LEAP IMAP TESTS" -echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" -wait_and_kill & -stress_imap -print_results diff --git a/src/leap/mail/imap/tests/stress_tests_imap.zsh b/src/leap/mail/imap/tests/stress_tests_imap.zsh new file mode 100755 index 0000000..544faca --- /dev/null +++ b/src/leap/mail/imap/tests/stress_tests_imap.zsh @@ -0,0 +1,178 @@ +#!/bin/zsh +# BATCH STRESS TEST FOR IMAP ---------------------- +# http://imgs.xkcd.com/comics/science.jpg +# +# Run imaptest against a LEAP IMAP server +# for a fixed period of time, and collect output. +# +# Author: Kali Kaneko +# Date: 2014 01 26 +# +# To run, you need to have `imaptest` in your path. +# See: +# http://www.imapwiki.org/ImapTest/Installation +# +# For the tests, I'm using a 10MB file sample that +# can be downloaded from: +# http://www.dovecot.org/tmp/dovecot-crlf +# +# Want to contribute to benchmarking? +# +# 1. Create a pristine account in a bitmask provider. +# +# 2. Launch your bitmask client, with different flags +# if you desire. +# +# For example to try the nosync flag in sqlite: +# +# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log +# +# 3. Run at several points in time (ie: just after +# launching the bitmask client. one minute after, +# ten minutes after) +# +# mkdir data +# cd data +# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log +# +# 4. Submit your results to: kali at leap dot se +# together with the logs of the bitmask run. +# +# Please provide also details about your system, and +# the type of hard disk setup you are running against. +# + +# ------------------------------------------------ +# Edit these variables if you are too lazy to pass +# the user and mbox as parameters. Like me. + +USER="test_f14@dev.bitmask.net" +MBOX="~/leap/imaptest/data/dovecot-crlf" + +HOST="localhost" +PORT="1984" + +# in case you have it aliased +GREP="/bin/grep" +IMAPTEST="imaptest" + +# ----------------------------------------------- +# +# These should be kept constant across benchmarking +# runs across different machines, for comparability. + +DURATION=200 +NUM_MSG=200 + + +# TODO add another function, and a cli flag, to be able +# to take several aggretates spaced in time, along a period +# of several minutes. + +imaptest_cmd() { + stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ + port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ + no_pipelining 2>/dev/null +} + +stress_imap() { + mkfifo imap_pipe + cat imap_pipe | tee output & + imaptest_cmd >> imap_pipe +} + +wait_and_kill() { + while : + do + sleep $DURATION + pkill -2 imaptest + rm imap_pipe + break + done +} + +print_results() { + sleep 1 + echo + echo + echo "AGGREGATED RESULTS" + echo "----------------------" + echo "\tavg\tstdev" + $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ + gawk ' +function avg(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + return sum/count; +} +function std_dev(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + average = sum/count; + + sumsq=0; + for( x=0; x <= count-1; x++) { + sumsq += (data[x] - average)^2; + } + return sqrt(sumsq/count); +} +BEGIN { + cnt = 0 +} END { + +printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); +printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); +printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); +printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); +printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); +printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); +printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); +printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); +printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); +printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); +printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); + +print "" +print "TOT samples", NR; +} +{ + it = cnt++; + array[1][it] = $1; + array[2][it] = $2; + array[3][it] = $3; + array[4][it] = $4; + array[5][it] = $5; + array[6][it] = $6; + array[7][it] = $7; + array[8][it] = $8; + array[9][it] = $9; + array[10][it] = $10; + array[11][it] = $11; +}' +} + + +{ test $1 = "--help" } && { + echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" + exit 0 +} + +# If the first parameter is passed, take it as the user +{ test $1 } && { + USER=$1 +} + +# If the second parameter is passed, take it as the mbox +{ test $2 } && { + MBOX=$2 +} + +echo "[+] LEAP IMAP TESTS" +echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" +wait_and_kill & +stress_imap +print_results diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dbb823f..d7fcdce 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -17,7 +17,7 @@ """ Test case for leap.email.imap.server TestCases taken from twisted tests and modified to make them work -against SoledadBackedAccount. +against our implementation of the IMAPAccount. @authors: Kali Kaneko, XXX add authors from the original twisted tests. @@ -32,19 +32,13 @@ import types from twisted.mail import imap4 from twisted.internet import defer -from twisted.trial import unittest from twisted.python import util from twisted.python import failure from twisted import cred - -# import u1db - -from leap.mail.imap.mailbox import SoledadMailbox -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import IMAPMessageCollection from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -81,7 +75,7 @@ class TestRealm: # TestCases # -class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): +class MessageCollectionTestCase(IMAP4HelperMixin): """ Tests for the MessageCollection class """ @@ -95,10 +89,9 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ super(MessageCollectionTestCase, self).setUp() - # TODO deprecate memstore - memstore = MemoryStore() - self.messages = MessageCollection("testmbox%s" % (self.count,), - self._soledad, memstore=memstore) + # FIXME --- update initialization + self.messages = IMAPMessageCollection( + "testmbox%s" % (self.count,), self._soledad) MessageCollectionTestCase.count += 1 def tearDown(self): @@ -207,23 +200,18 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): add_1().addCallback(lambda ignored: self.assertEqual( mc.count(), 3)) - # XXX this has to be redone to fit memstore ------------# - #newmsg = mc._get_empty_doc() - #newmsg['mailbox'] = "mailbox/foo" - #mc._soledad.create_doc(newmsg) - #self.assertEqual(mc.count(), 3) - #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + +# DEBUG --- +#from twisted.internet.base import DelayedCall +#DelayedCall.debug = True -class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): - # TODO this currently will use a memory-only store. - # create a different one for testing soledad sync. +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ - Tests for the generic behavior of the LeapIMAP4Server + Tests for the generic behavior of the LEAPIMAP4Server which, right now, it's just implemented in this test file as - LeapIMAPServer. We will move the implementation, together with + LEAPIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -243,6 +231,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') + acc = self.server.theAccount def cb(): self.result.append(1) @@ -267,24 +256,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1 = self.connected.addCallback(strip(login)) d1.addCallback(strip(create)) d2 = self.loopback() - d = defer.gatherResults([d1, d2]) + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestCreate, succeed, fail) - def _cbTestCreate(self, ignored, succeed, fail): + def _cbTestCreate(self, mailboxes, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = LeapIMAPServer.theAccount.mailboxes - answers = ([u'INBOX', u'testbox', u'test/box', u'test', u'test/box/box', 'foobox']) - self.assertEqual(sorted(mboxes), sorted([a for a in answers])) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - acc = LeapIMAPServer.theAccount - d0 = lambda: acc.addMailbox('test-delete/me') + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -292,15 +280,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def delete(): return self.client.delete('test-delete/me') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback( - lambda _: self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, ['INBOX'])) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) return d def testIllegalInboxDelete(self): @@ -359,29 +349,34 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - acc = LeapIMAPServer.theAccount - d_del0 = lambda: acc.addMailbox('delete') - d_del1 = lambda: acc.addMailbox('delete/me') - - def set_noselect_flag(): - mbox = acc.getMailbox('delete') - mbox.setFlags((r'\Noselect',)) + acc = self.server.theAccount def login(): return self.client.login(TEST_USER, TEST_PASSWD) - def delete(): + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): return self.client.delete('delete') def deleteFailed(failure): self.failure = failure self.failure = None + d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d_del0)) - d1.addCallback(strip(d_del1)) - d1.addCallback(strip(set_noselect_flag)) - d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -393,11 +388,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly def testRename(self): """ Test whether we can rename a mailbox """ - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -405,16 +404,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, - ['INBOX', 'newname'])) + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) return d def testIllegalInboxRename(self): @@ -448,7 +447,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - acc = LeapIMAPServer.theAccount + acc = LEAPIMAPServer.theAccount dc1 = lambda: acc.create('oldmbox/m1') dc2 = lambda: acc.create('oldmbox/m2') @@ -468,7 +467,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, ignored): - mboxes = LeapIMAPServer.theAccount.mailboxes + mboxes = LEAPIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] self.assertEqual(sorted(mboxes), sorted([s for s in expected])) @@ -476,6 +475,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can mark a mailbox as subscribed to """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -483,9 +487,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.subscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return self.server.theAccount.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -500,7 +505,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can unsubscribe from a set of mailboxes """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) dc1 = lambda: acc.subscribe('this/mbox') dc2 = lambda: acc.subscribe('that/mbox') @@ -512,9 +522,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.unsubscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return acc.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallback(strip(dc1)) d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) @@ -531,10 +542,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to select a mailbox """ - acc = self.server.theAccount - d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + mbox_name = "TESTMAILBOXSELECT" self.selectedArgs = None + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -542,30 +557,26 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def selected(args): self.selectedArgs = args self._cbStopClient(None) - d = self.client.select('TESTMAILBOX-SELECT') + d = self.client.select(mbox_name) d.addCallback(selected) return d - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(select)) - d1.addErrback(self._ebGeneral) + #d1.addErrback(self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d def _cbTestSelect(self, ignored): - mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. + self.assertTrue(self.selectedArgs is not None) self.assertEqual(self.selectedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': True @@ -583,13 +594,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): caps.update(c) self.server.transport.loseConnection() return self.client.getCapabilities().addCallback(gotCaps) - d1 = self.connected.addCallback( + + d1 = self.connected + d1.addCallback( strip(getCaps)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, 'IDLE': None} - - return d.addCallback(lambda _: self.assertEqual(expected, caps)) + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d def testCapabilityWithAuth(self): caps = {} @@ -610,7 +624,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'IDLE': None, 'LITERAL+': None, 'AUTH': ['CRAM-MD5']} - return d.addCallback(lambda _: self.assertEqual(expCap, caps)) + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d # # authentication @@ -658,7 +673,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') def testFailedLogin(self): @@ -696,7 +710,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # @@ -743,11 +756,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): for details. """ # TODO implement the IMAP4ClientExamineTests testcase. - - self.server.theAccount.addMailbox('test-mailbox-e', - creation_ts=42) + mbox_name = "test_mailbox_e" + acc = self.server.theAccount self.examinedArgs = None + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -755,11 +770,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def examined(args): self.examinedArgs = args self._cbStopClient(None) - d = self.client.examine('test-mailbox-e') + d = self.client.examine(mbox_name) d.addCallback(examined) return d - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(examine)) d1.addErrback(self._ebGeneral) d2 = self.loopback() @@ -767,27 +783,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('test-mailbox-e') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. self.assertEqual(self.examinedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) def _listSetup(self, f, f2=None): - acc = LeapIMAPServer.theAccount - dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) - dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) - dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) + acc = self.server.theAccount + + dc1 = lambda: acc.addMailbox('root_subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root_another_thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non_root_subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -816,12 +824,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ def list(): return self.client.list('root', '%') + d = self._listSetup(list) d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), - (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") ]) )) return d @@ -830,28 +839,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def subs_mailbox(): # why not client.subscribe instead? - return acc.subscribe('root/subthing') + return acc.subscribe('root_subthing') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) + [(IMAPMailbox.init_flags, "/", "root_subthing")]) return d def testStatus(self): """ Test Status command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def add_mailbox(): - return acc.addMailbox('root/subthings') + return acc.addMailbox('root_subthings') # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -861,7 +870,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def status(): return self.client.status( - 'root/subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') def statused(result): self.statused = result @@ -927,7 +936,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) acc = self.server.theAccount - mailbox_name = "root_subthing" + mailbox_name = "root/subthing" def add_mailbox(): return acc.addMailbox(mailbox_name) @@ -948,42 +957,62 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - def print_mb(mb): - print "MB ----", mb - return mb - d.addCallback(print_mb) - d.addCallback(lambda mb: mb.collection.get_message_by_uid(1)) + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback(self._cbTestFullAppend, infile) - def _cbTestFullAppend(self, msg, infile): - # TODO --- move to deferreds - self.assertEqual( - set(('\\Recent', '\\SEEN', '\\DELETED')), - set(msg.getFlags())) + def _cbTestFullAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = dict(parsed.items()) - self.assertEqual( - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - msg.getInternalDate()) + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) - parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - headers = dict(parsed.items()) - self.assertEqual( - body, - msg.getBodyFile().read()) - gotheaders = msg.getHeaders(True) + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(expected_headers, headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) - self.assertItemsEqual( - headers, gotheaders) + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d def testPartialAppend(self): """ Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -998,34 +1027,46 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): (), self.client._IMAP4Client__cbContinueAppend, message ) ) - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback( self._cbTestPartialAppend, infile) - def _cbTestPartialAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - self.assertEqual(1, len(mb.messages)) - msg = mb.messages.get_msg_by_uid(1) - self.assertEqual( - set(('\\SEEN', '\\Recent')), - set(msg.getFlags()) - ) + def _cbTestPartialAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - self.assertEqual( - body, - msg.getBodyFile().read()) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d def testCheck(self): """ Test check command """ - LeapIMAPServer.theAccount.addMailbox('root/subthing') + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1036,98 +1077,106 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def check(): return self.client.check() - d = self.connected.addCallback(strip(login)) + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(check), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) - return self.loopback() - - # Okay, that was fun - - def testClose(self): - """ - Test closing the mailbox. We expect to get deleted all messages flagged - as such. - """ - acc = self.server.theAccount - name = 'mailbox-close' - - d0 = lambda: acc.addMailbox(name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select(name) - - def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) - - def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) - return d - - def close(): - return self.client.close() - - d = self.connected.addCallback(strip(login)) - d.addCallback(strip(d0)) - d.addCallbacks(strip(select), self._ebGeneral) - d.addCallback(strip(get_mailbox)) - d.addCallbacks(strip(add_messages), self._ebGeneral) - d.addCallbacks(strip(close), self._ebGeneral) - d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - - def _cbTestClose(self, ignored): - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - self.assertTrue(msg is not None) - - self.assertEqual( - dict(msg.hdoc.content)['subject'], - 'Message 2') - self.failUnless(self.mailbox.closed) + return defer.gatherResults([d, d2]) + + # Okay, that was much fun indeed + + # skipping close test: we just need expunge for now. + #def testClose(self): + #""" + #Test closing the mailbox. We expect to get deleted all messages flagged + #as such. + #""" + #acc = self.server.theAccount + #mailbox_name = 'mailbox-close' +# + #def add_mailbox(): + #return acc.addMailbox(mailbox_name) +# + #def login(): + #return self.client.login(TEST_USER, TEST_PASSWD) +# + #def select(): + #return self.client.select(mailbox_name) +# + #def get_mailbox(): + #def _save_mbox(mailbox): + #self.mailbox = mailbox + #d = self.server.theAccount.getMailbox(mailbox_name) + #d.addCallback(_save_mbox) + #return d +# + #def add_messages(): + #d1 = self.mailbox.addMessage( + #'test 1', flags=('\\Deleted', 'AnotherFlag')) + #d2 = self.mailbox.addMessage( + #'test 2', flags=('AnotherFlag',)) + #d3 = self.mailbox.addMessage( + #'test 3', flags=('\\Deleted',)) + #d = defer.gatherResults([d1, d2, d3]) + #return d +# + #def close(): + #return self.client.close() +# + #d = self.connected.addCallback(strip(add_mailbox)) + #d.addCallback(strip(login)) + #d.addCallbacks(strip(select), self._ebGeneral) + #d.addCallback(strip(get_mailbox)) + #d.addCallbacks(strip(add_messages), self._ebGeneral) + #d.addCallbacks(strip(close), self._ebGeneral) + #d.addCallbacks(self._cbStopClient, self._ebGeneral) + #d2 = self.loopback() + #d1 = defer.gatherResults([d, d2]) + #d1.addCallback(lambda _: self.mailbox.getMessageCount()) + #d1.addCallback(self._cbTestClose) + #return d1 +# + #def _cbTestClose(self, count): + # TODO is this correct? count should not take into account those + # flagged as deleted??? + #self.assertEqual(count, 1) + # TODO --- assert flags are those of the message #2 + #self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ acc = self.server.theAccount - name = 'mailbox-expunge' + mailbox_name = 'mailboxexpunge' - d0 = lambda: acc.addMailbox(name) + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def select(): - return self.client.select('mailbox-expunge') + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag')) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',))) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',))) return d def expunge(): @@ -1138,49 +1187,53 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = results self.results = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) - d1.addCallbacks(strip(select), self._ebGeneral) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.mailbox.getMessageCount()) return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored): + def _cbTestExpunge(self, count): # we only left 1 mssage with no deleted flag - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - - msg = list(self.mailbox.messages)[0] - self.assertTrue(msg is not None) - - self.assertEqual( - msg.hdoc.content['subject'], - 'Message 2') - + self.assertEqual(count, 1) # the uids of the deleted messages self.assertItemsEqual(self.results, [1, 3]) -class AccountTestCase(IMAP4HelperMixin, unittest.TestCase): +class AccountTestCase(IMAP4HelperMixin): """ Test the Account. """ def _create_empty_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('') + return self.server.theAccount.addMailbox('') def _create_one_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('one') + return self.server.theAccount.addMailbox('one') def test_illegalMailboxCreate(self): - self.assertRaises(AssertionError, self._create_empty_mailbox) + # FIXME --- account.addMailbox needs to raise a failure, + # not the direct exception. + self.stashed = None + + def stash(result): + self.stashed = result + + d = self._create_empty_mailbox() + d.addBoth(stash) + d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, + failure.Failure))) + return d + #self.assertRaises(AssertionError, self._create_empty_mailbox) -class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): """ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 920eeb0..5708787 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -1,30 +1,44 @@ -import os -import tempfile -import shutil - +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Common utilities for testing Soledad IMAP Server. +""" from email import parser from mock import Mock from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback +from twisted.python import log -from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.soledad.client import Soledad +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.tests.common import SoledadTestMixin TEST_USER = "testuser@leap.se" TEST_PASSWD = "1234" + # # Simple IMAP4 Client for testing # - class SimpleClient(imap4.IMAP4Client): - """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -51,160 +65,57 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() -# XXX move to common helper -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - - _soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return _soledad - - -# XXX this is not properly a mixin, since helper already inherits from -# uniittest.Testcase -class IMAP4HelperMixin(BaseLeapTest): +class IMAP4HelperMixin(SoledadTestMixin): """ MixIn containing several utilities to be shared across different TestCases """ - serverCTX = None clientCTX = None - # setUpClass cannot be a classmethod in trial, see: - # https://twistedmatrix.com/trac/ticket/1870 - def setUp(self): - """ - Setup method for each test. - - Initializes and run a LEAP IMAP4 Server. - """ - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - UUID = 'deadbeef', - USERID = TEST_USER - memstore = MemoryStore() - ########### + soledad_adaptor.cleanup_deferred_locks() - d_server_ready = defer.Deferred() + UUID = 'deadbeef', + USERID = TEST_USER - self.server = LeapIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - soledad=self._soledad) + def setup_server(account): + self.server = LEAPIMAPServer( + uuid=UUID, userid=USERID, + contextFactory=self.serverCTX, + soledad=self._soledad) + self.server.theAccount = account - self.client = SimpleClient( - d_server_ready, contextFactory=self.clientCTX) + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready - theAccount = SoledadBackedAccount( - USERID, - soledad=self._soledad, - memstore=memstore) - d_account_ready = theAccount.callWhenReady(lambda r: None) - LeapIMAPServer.theAccount = theAccount + def setup_account(_): + self.parser = parser.Parser() - self.connected = defer.gatherResults( - [d_server_ready, d_account_ready]) + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + self._soledad.sync = Mock() - # XXX FIXME -------------------------------------------- - # XXX this needs to be done differently, - # have to be hooked on initialization callback instead. - # in case we get something from previous tests... - #for mb in self.server.theAccount.mailboxes: - #self.server.theAccount.delete(mb) + d = defer.Deferred() + self.acc = IMAPAccount(USERID, self._soledad, d=d) + return d - # email parser - self.parser = parser.Parser() + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d def tearDown(self): - """ - tearDown method called after each test. - """ - try: - self._soledad.close() - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - def populateMessages(self): - """ - Populates soledad instance with several simple messages - """ - # XXX we should encapsulate this thru SoledadBackedAccount - # instead. - - # XXX we also should put this in a mailbox! - - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") - # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") - - def delete_all_docs(self): - """ - Deletes all the docs in the testing instance of the - SoledadBackedAccount. - """ - self.server.theAccount.deleteAllMessages( - iknowhatiamdoing=True) + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected def _cbStopClient(self, ignore): self.client.transport.loseConnection() @@ -212,11 +123,8 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - # can we do something similar? - # I guess this was ok with trial, but not in noseland... - # log.err(failure, "Problem with %r" % (self.function,)) - raise failure.value - # failure.trap(Exception) + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) def loopback(self): return loopback.loopbackAsync(self.server, self.client) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b2caa33..8137f97 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -172,7 +172,7 @@ class Message(object): :return: An RFC822-formatted date string. :rtype: str """ - return self._wrapper.fdoc.date + return self._wrapper.hdoc.date # imap.IMessageParts @@ -271,9 +271,10 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, + count=None): """ - Constructore for a MessageCollection. + Constructor for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -317,7 +318,7 @@ class MessageCollection(object): wrapper = getattr(self, "mbox_wrapper", None) if not wrapper: return None - return wrapper.mbox_uuid + return wrapper.uuid def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -364,10 +365,18 @@ class MessageCollection(object): self.messageklass, self.store, doc_id, uid=uid, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) d.addCallback(get_msg_from_mdoc_id) return d + # TODO deprecate ??? --- + def _prime_count(self): + def update_count(count): + self._count = count + d = self.mbox_indexer.count(self.mbox_name) + d.addCallback(update_count) + return d + def count(self): """ Count the messages in this collection. @@ -376,7 +385,17 @@ class MessageCollection(object): """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_name) + + d = self.mbox_indexer.count(self.mbox_uuid) + return d + + def count_recent(self): + # FIXME HACK + return 0 + + def count_unseen(self): + # FIXME hack + return 0 def get_uid_next(self): """ @@ -385,7 +404,13 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_next_uid(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_uuid) + + def all_uid_iter(self): + """ + Iterator through all the uids for this collection. + """ + return self.mbox_indexer.all_uid_iter(self.mbox_uuid) # Manipulate messages @@ -397,6 +422,7 @@ class MessageCollection(object): flags = tuple() if not tags: tags = tuple() + leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -408,10 +434,10 @@ class MessageCollection(object): else: mbox_id = self.mbox_uuid + wrapper.set_mbox_uuid(mbox_id) wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id @@ -420,6 +446,8 @@ class MessageCollection(object): d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) + d.addErrback(lambda f: f.printTraceback()) + return d def copy_msg(self, msg, newmailbox): @@ -453,8 +481,47 @@ class MessageCollection(object): d.addCallback(delete_mdoc_id, wrapper) return d + def delete_all_flagged(self): + """ + Delete all messages flagged as \\Deleted. + Used from IMAPMailbox.expunge() + """ + def get_uid_list(hashes): + d = [] + for h in hashes: + d.append(self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, h)) + return defer.gatherResults(d), hashes + + def delete_uid_entries((uids, hashes)): + d = [] + for h in hashes: + d.append(self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, h)) + return defer.gatherResults(d).addCallback( + lambda _: uids) + + mdocs_deleted = self.adaptor.del_all_flagged_messages( + self.store, self.mbox_uuid) + mdocs_deleted.addCallback(get_uid_list) + mdocs_deleted.addCallback(delete_uid_entries) + return mdocs_deleted + # TODO should add a delete-by-uid to collection? + def delete_all_docs(self): + def del_all_uid(uid_list): + deferreds = [] + for uid in uid_list: + d = self.get_message_by_uid(uid) + d.addCallback(lambda msg: msg.delete()) + deferreds.append(d) + return defer.gatherResults(deferreds) + + d = self.all_uid_iter() + d.addCallback(del_all_uid) + return d + def _update_flags_or_tags(self, old, new, mode): if mode == Flagsmode.APPEND: final = list((set(tuple(old) + new))) @@ -509,45 +576,49 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - def __init__(self, store): + def __init__(self, store, ready_cb=None): self.store = store self.adaptor = self.adaptor_class() self.mbox_indexer = MailboxIndexer(self.store) + self.deferred_initialization = defer.Deferred() self._initialized = False - self._deferred_initialization = defer.Deferred() + self._ready_cb = ready_cb - self._initialize_storage() + self._init_d = self._initialize_storage() def _initialize_storage(self): def add_mailbox_if_none(mboxes): if not mboxes: - self.add_mailbox(INBOX_NAME) + return self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True - self._deferred_initialization.callback(None) + self.deferred_initialization.callback(None) + if self._ready_cb is not None: + self._ready_cb() d = self.adaptor.initialize_store(self.store) d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) + return d def callWhenReady(self, cb, *args, **kw): - # use adaptor.store_ready instead? if self._initialized: cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb, *args, **kw) - return self._deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts # def list_all_mailbox_names(self): + def filter_names(mboxes): return [m.mbox for m in mboxes] @@ -563,8 +634,11 @@ class Account(object): def create_uuid(wrapper): if not wrapper.uuid: - wrapper.uuid = uuid.uuid4() - return wrapper.update(self.store) + wrapper.uuid = str(uuid.uuid4()) + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + return wrapper def create_uid_table_cb(wrapper): d = self.mbox_indexer.create_table(wrapper.uuid) @@ -599,7 +673,7 @@ class Account(object): def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update(self.store) + return wrapper, wrapper.update(self.store) d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) @@ -616,8 +690,6 @@ class Account(object): return MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - mboxwrapper_klass = self.adaptor.mboxwrapper_klass - #d = mboxwrapper_klass.get_or_create(name) d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 6155a7a..22e57d4 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -114,6 +114,24 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=sanitize(oldmailbox), new=sanitize(newmailbox))) + return self._query(sql) + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -134,7 +152,7 @@ class MailboxIndexer(object): assert doc_id mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -148,9 +166,11 @@ class MailboxIndexer(object): sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) + d.addErrback(lambda f: f.printTraceback()) return d def delete_doc_by_uid(self, mailbox_id, uid): @@ -199,13 +219,14 @@ class MailboxIndexer(object): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox uuid + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) @@ -218,7 +239,24 @@ class MailboxIndexer(object): d.addCallback(get_hash) return d - def get_doc_ids_from_uids(self, mailbox, uids): + def get_uid_from_doc_id(self, mailbox_id, doc_id): + check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') + + def get_uid(result): + return _maybe_first_query_item(result) + + sql = ("SELECT uid from {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_id))) + values = (doc_id,) + d = self._query(sql, values) + d.addCallback(get_uid) + return d + + + + def get_doc_ids_from_uids(self, mailbox_id, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() @@ -227,8 +265,8 @@ class MailboxIndexer(object): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox: the mailbox name - :type mailbox: str + :param mailbox_id: the mailbox uuid + :type mailbox_id: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ @@ -241,6 +279,7 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) + d.addErrback(lambda _: 0) return d def get_next_uid(self, mailbox_id): @@ -252,7 +291,7 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. @@ -273,3 +312,22 @@ class MailboxIndexer(object): d = self._query(sql) d.addCallback(increment) return d + + def all_uid_iter(self, mailbox_id): + """ + Get a sequence of all the uids in this mailbox. + + :param mailbox_id: the mailbox uuid + :type mailbox_id: str + """ + check_good_uuid(mailbox_id) + + sql = ("SELECT uid from {preffix}{name} ").format( + preffix=self.table_preffix, name=sanitize(mailbox_id)) + + def get_results(result): + return [x[0] for x in result] + + d = self._query(sql) + d.addCallback(get_results) + return d diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py index fefa7ee..a411b2d 100644 --- a/src/leap/mail/tests/common.py +++ b/src/leap/mail/tests/common.py @@ -21,6 +21,9 @@ import os import shutil import tempfile +from twisted.internet import defer +from twisted.trial import unittest + from leap.common.testing.basetest import BaseLeapTest from leap.soledad.client import Soledad @@ -60,7 +63,7 @@ def _initialize_soledad(email, gnupg_home, tempdir): return soledad -class SoledadTestMixin(BaseLeapTest): +class SoledadTestMixin(unittest.TestCase, BaseLeapTest): """ It is **VERY** important that this base is added *AFTER* unittest.TestCase """ @@ -68,15 +71,7 @@ class SoledadTestMixin(BaseLeapTest): def setUp(self): self.results = [] - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir + self.setUpEnv() # Soledad: config info self.gnupg_home = "%s/gnupg" % self.tempdir @@ -88,6 +83,8 @@ class SoledadTestMixin(BaseLeapTest): self.gnupg_home, self.tempdir) + return defer.succeed(True) + def tearDown(self): """ tearDown method called after each test. diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py index 2edf1d8..b82fd2d 100644 --- a/src/leap/mail/tests/test_mailbox_indexer.py +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -84,18 +84,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): d.addCallback(assert_table_deleted) return d - #def test_rename_table(self): - #def assert_table_renamed(tables): - #self.assertEqual( - #tables, ["leapmail_uid_foomailbox"]) -# - #m_uid = self.get_mbox_uid() - #d = m_uid.create_table('inbox') - #d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) - #d.addCallback(self.list_mail_tables_cb) - #d.addCallback(assert_table_renamed) - #return d - def test_insert_doc(self): m_uid = self.get_mbox_uid() @@ -168,7 +156,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_get_doc_id_from_uid(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) @@ -183,7 +170,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_count(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) h2 = fmt_hash(mbox_id, hash_test1) @@ -216,7 +202,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_get_next_uid(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) h2 = fmt_hash(mbox_id, hash_test1) @@ -237,3 +222,29 @@ class MailboxIndexerTestCase(SoledadTestMixin): d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) d.addCallback(partial(assert_next_uid, expected=6)) return d + + def test_all_uid_iter(self): + + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4)) + + def assert_all_uid(result, expected=[2, 3, 5]): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id)) + d.addCallback(partial(assert_all_uid)) + return d -- cgit v1.2.3 From 16f7099a8c7c1dd31f1b8bab78cc6554a01d63c7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 14 Jan 2015 01:09:19 -0400 Subject: patch cbSelect to accept deferreds for count* --- src/leap/mail/imap/account.py | 8 +++++--- src/leap/mail/imap/server.py | 14 ++++++++++++-- src/leap/mail/imap/tests/test_imap.py | 20 +++++++++++--------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index dfc0d62..8a6e87e 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -330,8 +330,7 @@ class IMAPAccount(object): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) - def rename_inferiors(inferiors_result): - inferiors, mailboxes = inferiors_result + def rename_inferiors((inferiors, mailboxes)): rename_deferreds = [] inferiors = [ (o, o.replace(oldname, newname, 1)) for o in inferiors] @@ -347,7 +346,10 @@ class IMAPAccount(object): d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) return d1 - d = self._inferiorNames(oldname) + d1 = self._inferiorNames(oldname) + d2 = self.account.list_all_mailbox_names() + + d = defer.gatherResults([d1, d2]) d.addCallback(rename_inferiors) return d diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b4f320a..32c921d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -162,10 +162,20 @@ class LEAPIMAPServer(imap4.IMAP4Server): self.sendNegativeResponse(tag, 'Mailbox cannot be selected') return + d1 = defer.maybeDeferred(mbox.getMessageCount) + d2 = defer.maybeDeferred(mbox.getRecentCount) + return defer.gatherResults([d1, d2]).addCallback( + self.__cbSelectWork, mbox, cmdName, tag) + + def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag): flags = mbox.getFlags() self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) - self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') - self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') + + # Patched ------------------------------------------------------- + # accept deferreds for the count + self.sendUntaggedResponse(str(msg_count) + ' EXISTS') + self.sendUntaggedResponse(str(recent_count) + ' RECENT') + # ---------------------------------------------------------------- # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index d7fcdce..6be41cd 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -447,9 +447,12 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ Try to rename hierarchical mailboxes """ - acc = LEAPIMAPServer.theAccount - dc1 = lambda: acc.create('oldmbox/m1') - dc2 = lambda: acc.create('oldmbox/m2') + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('oldmbox/m1'), + acc.addMailbox('oldmbox/m2')]) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -457,19 +460,18 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(dc1)) - d1.addCallback(strip(dc2)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestHierarchicalRename) - def _cbTestHierarchicalRename(self, ignored): - mboxes = LEAPIMAPServer.theAccount.mailboxes + def _cbTestHierarchicalRename(self, mailboxes): expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] - self.assertEqual(sorted(mboxes), sorted([s for s in expected])) + self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) def testSubscribe(self): """ -- cgit v1.2.3 From c13e17114cfb58acc4911ed98c367faf47717ec0 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 14 Jan 2015 13:50:02 -0600 Subject: Refactor fetch into leap.mail.incoming IService --- src/leap/mail/imap/account.py | 2 +- src/leap/mail/imap/fetch.py | 749 -------------------- src/leap/mail/imap/service/imap.py | 30 +- src/leap/mail/imap/tests/test_incoming_mail.py | 193 ------ src/leap/mail/incoming/__init__.py | 0 src/leap/mail/incoming/service.py | 760 +++++++++++++++++++++ src/leap/mail/incoming/tests/__init__.py | 0 src/leap/mail/incoming/tests/test_incoming_mail.py | 202 ++++++ src/leap/mail/mail.py | 17 +- 9 files changed, 971 insertions(+), 982 deletions(-) delete mode 100644 src/leap/mail/imap/fetch.py delete mode 100644 src/leap/mail/imap/tests/test_incoming_mail.py create mode 100644 src/leap/mail/incoming/__init__.py create mode 100644 src/leap/mail/incoming/service.py create mode 100644 src/leap/mail/incoming/tests/__init__.py create mode 100644 src/leap/mail/incoming/tests/test_incoming_mail.py diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 8a6e87e..146d066 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -63,7 +63,7 @@ class IMAPAccount(object): selected = None closed = False - def __init__(self, user_id, store, d=None): + def __init__(self, user_id, store, d=defer.Deferred()): """ Keeps track of the mailboxes and subscriptions handled by this account. diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py deleted file mode 100644 index dbc726a..0000000 --- a/src/leap/mail/imap/fetch.py +++ /dev/null @@ -1,749 +0,0 @@ -# -*- coding: utf-8 -*- -# fetch.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Incoming mail fetcher. -""" -import copy -import logging -import shlex -import threading -import time -import traceback -import warnings - -from email.parser import Parser -from email.generator import Generator -from email.utils import parseaddr -from StringIO import StringIO -from urlparse import urlparse - -from twisted.python import log -from twisted.internet import defer, reactor -from twisted.internet.task import LoopingCall -from twisted.internet.task import deferLater -from u1db import errors as u1db_errors - -from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type -from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING -from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING -from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED -from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY -from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN -from leap.common.mail import get_email_charset -from leap.keymanager import errors as keymanager_errors -from leap.keymanager.openpgp import OpenPGPKey -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.fields import fields -from leap.mail.utils import json_loads, empty, first -from leap.soledad.client import Soledad -from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY -from leap.soledad.common.errors import InvalidAuthTokenError - - -logger = logging.getLogger(__name__) - -MULTIPART_ENCRYPTED = "multipart/encrypted" -MULTIPART_SIGNED = "multipart/signed" -PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" -PGP_END = "-----END PGP MESSAGE-----" - - -class MalformedMessage(Exception): - """ - Raised when a given message is not well formed. - """ - pass - - -class LeapIncomingMail(object): - """ - Fetches and process mail from the incoming pool. - - This object has public methods start_loop and stop that will - actually initiate a LoopingCall with check_period recurrency. - The LoopingCall itself will invoke the fetch method each time - that the check_period expires. - - This loop will sync the soledad db with the remote server and - process all the documents found tagged as incoming mail. - """ - - RECENT_FLAG = "\\Recent" - CONTENT_KEY = "content" - - LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' - """ - Header added to messages when they are decrypted by the IMAP fetcher, - which states the validity of an eventual signature that might be included - in the encrypted blob. - """ - LEAP_SIGNATURE_VALID = 'valid' - LEAP_SIGNATURE_INVALID = 'invalid' - LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' - - fetching_lock = threading.Lock() - - def __init__(self, keymanager, soledad, imap_account, - check_period, userid): - - """ - Initialize LeapIncomingMail.. - - :param keymanager: a keymanager instance - :type keymanager: keymanager.KeyManager - - :param soledad: a soledad instance - :type soledad: Soledad - - :param imap_account: the account to fetch periodically - :type imap_account: SoledadBackedAccount - - :param check_period: the period to fetch new mail, in seconds. - :type check_period: int - """ - - leap_assert(keymanager, "need a keymanager to initialize") - leap_assert_type(soledad, Soledad) - leap_assert(check_period, "need a period to check incoming mail") - leap_assert_type(check_period, int) - leap_assert(userid, "need a userid to initialize") - - self._keymanager = keymanager - self._soledad = soledad - self.imapAccount = imap_account - self._inbox = self.imapAccount.getMailbox('inbox') - self._userid = userid - - self._loop = None - self._check_period = check_period - - # initialize a mail parser only once - self._parser = Parser() - - # - # Public API: fetch, start_loop, stop. - # - - def fetch(self): - """ - Fetch incoming mail, to be called periodically. - - Calls a deferred that will execute the fetch callback - in a separate thread - """ - def syncSoledadCallback(result): - # FIXME this needs a matching change in mx!!! - # --> need to add ERROR_DECRYPTING_KEY = False - # as default. - try: - doclist = self._soledad.get_from_index( - fields.JUST_MAIL_IDX, "*", "0") - except u1db_errors.InvalidGlobbing: - # It looks like we are a dealing with an outdated - # mx. Fallback to the version of the index - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - doclist = self._soledad.get_from_index( - fields.JUST_MAIL_COMPAT_IDX, "*") - return self._process_doclist(doclist) - - logger.debug("fetching mail for: %s %s" % ( - self._soledad.uuid, self._userid)) - if not self.fetching_lock.locked(): - d1 = self._sync_soledad() - d = defer.gatherResults([d1], consumeErrors=True) - d.addCallbacks(syncSoledadCallback, self._errback) - d.addCallbacks(self._signal_fetch_to_ui, self._errback) - return d - else: - logger.debug("Already fetching mail.") - - def start_loop(self): - """ - Starts a loop to fetch mail. - """ - if self._loop is None: - self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) - else: - logger.warning("Tried to start an already running fetching loop.") - - def stop(self): - # XXX change the name to stop_loop, for consistency. - """ - Stops the loop that fetches mail. - """ - if self._loop and self._loop.running is True: - self._loop.stop() - self._loop = None - - # - # Private methods. - # - - # synchronize incoming mail - - def _errback(self, failure): - logger.exception(failure.value) - traceback.print_exc() - - @deferred_to_thread - def _sync_soledad(self): - """ - Synchronize with remote soledad. - - :returns: a list of LeapDocuments, or None. - :rtype: iterable or None - """ - with self.fetching_lock: - try: - log.msg('FETCH: syncing soledad...') - self._soledad.sync() - log.msg('FETCH soledad SYNCED.') - except InvalidAuthTokenError: - # if the token is invalid, send an event so the GUI can - # disable mail and show an error message. - leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) - - def _signal_fetch_to_ui(self, doclist): - """ - Send leap events to ui. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: doclist - :rtype: iterable - """ - doclist = first(doclist) # gatherResults pass us a list - if doclist: - fetched_ts = time.mktime(time.gmtime()) - num_mails = len(doclist) if doclist is not None else 0 - if num_mails != 0: - log.msg("there are %s mails" % (num_mails,)) - leap_events.signal( - IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) - return doclist - - def _signal_unread_to_ui(self, *args): - """ - Sends unread event to ui. - """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) - - # process incoming mail. - - def _process_doclist(self, doclist): - """ - Iterates through the doclist, checks if each doc - looks like a message, and yields a deferred that will decrypt and - process the message. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: a list of deferreds for individual messages. - """ - log.msg('processing doclist') - if not doclist: - logger.debug("no docs found") - return - num_mails = len(doclist) - - deferreds = [] - for index, doc in enumerate(doclist): - logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - leap_events.signal( - IMAP_MSG_PROCESSING, str(index), str(num_mails)) - - keys = doc.content.keys() - - # TODO Compatibility check with the index in pre-0.6 mx - # that does not write the ERROR_DECRYPTING_KEY - # This should be removed in 0.7 - - has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) - if has_errors is None: - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - - if has_errors: - logger.debug("skipping msg with decrypting errors...") - elif self._is_msg(keys): - d = self._decrypt_doc(doc) - d.addCallback(self._extract_keys) - d.addCallbacks(self._add_message_locally, self._errback) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - # - # operations on individual messages - # - - @deferred_to_thread - def _decrypt_doc(self, doc): - """ - Decrypt the contents of a document. - - :param doc: A document containing an encrypted message. - :type doc: SoledadDocument - - :return: A Deferred that will be fired with the document and the - decrypted message. - :rtype: SoledadDocument, str - """ - log.msg('decrypting msg') - - def process_decrypted(res): - if isinstance(res, tuple): - decrdata, _ = res - success = True - else: - decrdata = "" - success = False - - leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - - data = self._process_decrypted_doc(doc, decrdata) - return doc, data - - d = self._keymanager.decrypt( - doc.content[ENC_JSON_KEY], - self._userid, OpenPGPKey) - d.addErrback(self._errback) - d.addCallback(process_decrypted) - return d - - def _process_decrypted_doc(self, doc, data): - """ - Process a document containing a succesfully decrypted message. - - :param doc: the incoming message - :type doc: SoledadDocument - :param data: the json-encoded, decrypted content of the incoming - message - :type data: str - - :return: the processed data. - :rtype: str - """ - log.msg('processing decrypted doc') - - # XXX turn this into an errBack for each one of - # the deferreds that would process an individual document - try: - msg = json_loads(data) - except UnicodeError as exc: - logger.error("Error while decrypting %s" % (doc.doc_id,)) - logger.exception(exc) - - # we flag the message as "with decrypting errors", - # to avoid further decryption attempts during sync - # cycles until we're prepared to deal with that. - # What is the same, when Ivan deals with it... - # A new decrypting attempt event could be triggered by a - # future a library upgrade, or a cli flag to the client, - # we just `defer` that for now... :) - doc.content[fields.ERROR_DECRYPTING_KEY] = True - deferLater(reactor, 0, self._update_incoming_message, doc) - - # FIXME this is just a dirty hack to delay the proper - # deferred organization here... - # and remember, boys, do not do this at home. - return [] - - if not isinstance(msg, dict): - defer.returnValue(False) - if not msg.get(fields.INCOMING_KEY, False): - defer.returnValue(False) - - # ok, this is an incoming message - rawmsg = msg.get(self.CONTENT_KEY, None) - if not rawmsg: - return "" - return self._maybe_decrypt_msg(rawmsg) - - @deferred_to_thread - def _update_incoming_message(self, doc): - """ - Do a put for a soledad document. This probably has been called only - in the case that we've needed to update the ERROR_DECRYPTING_KEY - flag in an incoming message, to get it out of the decrypting queue. - - :param doc: the SoledadDocument to update - :type doc: SoledadDocument - """ - log.msg("Updating SoledadDoc %s" % (doc.doc_id)) - self._soledad.put_doc(doc) - - @deferred_to_thread - def _delete_incoming_message(self, doc): - """ - Delete document. - - :param doc: the SoledadDocument to delete - :type doc: SoledadDocument - """ - log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) - self._soledad.delete_doc(doc) - - def _maybe_decrypt_msg(self, data): - """ - Tries to decrypt a gpg message if data looks like one. - - :param data: the text to be decrypted. - :type data: str - :return: data, possibly decrypted. - :rtype: str - """ - leap_assert_type(data, str) - log.msg('maybe decrypting doc') - - # parse the original message - encoding = get_email_charset(data) - msg = self._parser.parsestr(data) - - fromHeader = msg.get('from', None) - senderAddress = None - if (fromHeader is not None - and (msg.get_content_type() == MULTIPART_ENCRYPTED - or msg.get_content_type() == MULTIPART_SIGNED)): - senderAddress = parseaddr(fromHeader) - - def add_leap_header(decrmsg, signkey): - if (senderAddress is None or - isinstance(signkey, keymanager_errors.KeyNotFound)): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_COULD_NOT_VERIFY) - elif isinstance(signkey, keymanager_errors.InvalidSignature): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_INVALID) - else: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_VALID, - pubkey=signkey.key_id) - return decrmsg.as_string() - - if msg.get_content_type() == MULTIPART_ENCRYPTED: - d = self._decrypt_multipart_encrypted_msg( - msg, encoding, senderAddress) - else: - d = self._maybe_decrypt_inline_encrypted_msg( - msg, encoding, senderAddress) - d.addCallback(add_leap_header) - return d - - def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): - """ - Decrypt a message with content-type 'multipart/encrypted'. - - :param msg: The original encrypted message. - :type msg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderAddress: The email address of the sender of the message. - :type senderAddress: str - - :return: A Deferred that will be fired with a tuple containing a - decrypted Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature or KeyNotFound. - :rtype: Deferred - """ - log.msg('decrypting multipart encrypted msg') - msg = copy.deepcopy(msg) - self._msg_multipart_sanity_check(msg) - - # parse message and get encrypted content - pgpencmsg = msg.get_payload()[1] - encdata = pgpencmsg.get_payload() - - # decrypt or fail gracefully - def build_msg(res): - decrdata, signkey = res - - decrmsg = self._parser.parsestr(decrdata) - # remove original message's multipart/encrypted content-type - del(msg['content-type']) - - # replace headers back in original message - for hkey, hval in decrmsg.items(): - try: - # this will raise KeyError if header is not present - msg.replace_header(hkey, hval) - except KeyError: - msg[hkey] = hval - - # all ok, replace payload by unencrypted payload - msg.set_payload(decrmsg.get_payload()) - return (msg, signkey) - - d = self._keymanager.decrypt( - encdata, self._userid, OpenPGPKey, - verify=senderAddress) - d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) - return d - - def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, - senderAddress): - """ - Possibly decrypt an inline OpenPGP encrypted message. - - :param origmsg: The original, possibly encrypted message. - :type origmsg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderAddress: The email address of the sender of the message. - :type senderAddress: str - - :return: A Deferred that will be fired with a tuple containing a - decrypted Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature or KeyNotFound. - :rtype: Deferred - """ - log.msg('maybe decrypting inline encrypted msg') - # serialize the original message - buf = StringIO() - g = Generator(buf) - g.flatten(origmsg) - data = buf.getvalue() - - def decrypted_data(res): - decrdata, signkey = res - return data.replace(pgp_message, decrdata), signkey - - def encode_and_return(res): - data, signkey = res - if isinstance(data, unicode): - data = data.encode(encoding, 'replace') - return (self._parser.parsestr(data), signkey) - - # handle exactly one inline PGP message - if PGP_BEGIN in data: - begin = data.find(PGP_BEGIN) - end = data.find(PGP_END) - pgp_message = data[begin:end + len(PGP_END)] - d = self._keymanager.decrypt( - pgp_message, self._userid, OpenPGPKey, - verify=senderAddress) - d.addCallbacks(decrypted_data, self._decryption_error, - errbackArgs=(data,)) - else: - d = defer.succeed((data, None)) - d.addCallback(encode_and_return) - return d - - def _decryption_error(self, failure, msg): - """ - Check for known decryption errors - """ - if failure.check(keymanager_errors.DecryptError): - logger.warning('Failed to decrypt encrypted message (%s). ' - 'Storing message without modifications.' - % str(failure.value)) - return (msg, None) - elif failure.check(keymanager_errors.KeyNotFound): - logger.error('Failed to find private key for decryption (%s). ' - 'Storing message without modifications.' - % str(failure.value)) - return (msg, None) - else: - return failure - - def _extract_keys(self, msgtuple): - """ - Retrieve attached keys to the mesage and parse message headers for an - *OpenPGP* header as described on the `IETF draft - ` - only urls with https and the same hostname than the email are supported - for security reasons. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - - :return: A Deferred that will be fired with msgtuple when key - extraction finishes - :rtype: Deferred - """ - OpenPGP_HEADER = 'OpenPGP' - doc, data = msgtuple - - # XXX the parsing of the message is done in mailbox.addMessage, maybe - # we should do it in this module so we don't need to parse it again - # here - msg = self._parser.parsestr(data) - _, fromAddress = parseaddr(msg['from']) - - header = msg.get(OpenPGP_HEADER, None) - dh = defer.success() - if header is not None: - dh = self._extract_openpgp_header(header, fromAddress) - - da = defer.success() - if msg.is_multipart(): - da = self._extract_attached_key(msg.get_payload(), fromAddress) - - d = defer.gatherResults([dh, da]) - d.addCallback(lambda _: msgtuple) - return d - - def _extract_openpgp_header(self, header, address): - """ - Import keys from the OpenPGP header - - :param header: OpenPGP header string - :type header: str - :param address: email address in the from header - :type address: str - - :return: A Deferred that will be fired when header extraction is done - :rtype: Deferred - """ - d = defer.success() - fields = dict([f.strip(' ').split('=') for f in header.split(';')]) - if 'url' in fields: - url = shlex.split(fields['url'])[0] # remove quotations - urlparts = urlparse(url) - addressHostname = address.split('@')[1] - if (urlparts.scheme == 'https' - and urlparts.hostname == addressHostname): - def fetch_error(failure): - if failure.check(keymanager_errors.KeyNotFound): - logger.warning("Url from OpenPGP header %s failed" - % (url,)) - elif failure.check(keymanager_errors.KeyAttributesDiffer): - logger.warning("Key from OpenPGP header url %s didn't " - "match the from address %s" - % (url, address)) - else: - return failure - - d = self._keymanager.fetch_key(address, url, OpenPGPKey) - d.addCallback( - lambda _: - logger.info("Imported key from header %s" % (url,))) - d.addErrback(fetch_error) - else: - logger.debug("No valid url on OpenPGP header %s" % (url,)) - else: - logger.debug("There is no url on the OpenPGP header: %s" - % (header,)) - return d - - def _extract_attached_key(self, attachments, address): - """ - Import keys from the attachments - - :param attachments: email attachment list - :type attachments: list(email.Message) - :param address: email address in the from header - :type address: str - - :return: A Deferred that will be fired when all the keys are stored - :rtype: Deferred - """ - MIME_KEY = "application/pgp-keys" - - deferreds = [] - for attachment in attachments: - if MIME_KEY == attachment.get_content_type(): - logger.debug("Add key from attachment") - d = self._keymanager.put_raw_key( - attachment.get_payload(), - OpenPGPKey, - address=address) - deferreds.append(d) - return defer.gatherResults(deferreds) - - def _add_message_locally(self, msgtuple): - """ - Adds a message to local inbox and delete it from the incoming db - in soledad. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - - :return: A Deferred that will be fired when the messages is stored - :rtype: Defferred - """ - doc, data = msgtuple - log.msg('adding message %s to local db' % (doc.doc_id,)) - - if isinstance(data, list): - if empty(data): - return False - data = data[0] - - def msgSavedCallback(result): - if not empty(result): - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - deferLater(reactor, 0, self._delete_incoming_message, doc) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) - - d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), - notify_on_disk=True) - d.addCallbacks(msgSavedCallback, self._errback) - return d - - # - # helpers - # - - def _msg_multipart_sanity_check(self, msg): - """ - Performs a sanity check against a multipart encrypted msg - - :param msg: The original encrypted message. - :type msg: Message - """ - # sanity check - payload = msg.get_payload() - if len(payload) != 2: - raise MalformedMessage( - 'Multipart/encrypted messages should have exactly 2 body ' - 'parts (instead of %d).' % len(payload)) - if payload[0].get_content_type() != 'application/pgp-encrypted': - raise MalformedMessage( - "Multipart/encrypted messages' first body part should " - "have content type equal to 'application/pgp-encrypted' " - "(instead of %s)." % payload[0].get_content_type()) - if payload[1].get_content_type() != 'application/octet-stream': - raise MalformedMessage( - "Multipart/encrypted messages' second body part should " - "have content type equal to 'octet-stream' (instead of " - "%s)." % payload[1].get_content_type()) - - def _is_msg(self, keys): - """ - Checks if the keys of a dictionary match the signature - of the document type we use for messages. - - :param keys: iterable containing the strings to match. - :type keys: iterable of strings. - :rtype: bool - """ - return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 5d88a79..93e4d62 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -30,10 +30,9 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check -from leap.keymanager import KeyManager from leap.mail.imap.account import IMAPAccount -from leap.mail.imap.fetch import LeapIncomingMail from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.incoming import IncomingMail from leap.soledad.client import Soledad from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED @@ -55,10 +54,6 @@ if DO_PROFILE: # The default port in which imap service will run IMAP_PORT = 1984 -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - class IMAPAuthRealm(object): """ @@ -132,21 +127,16 @@ def run_service(*args, **kwargs): """ Main entry point to run the service from the client. - :returns: the LoopingCall instance that will have to be stoppped - before shutting down the client, the port as returned by - the reactor when starts listening, and the factory for - the protocol. + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. """ leap_assert(len(args) == 2) - soledad, keymanager = args + soledad = args leap_assert_type(soledad, Soledad) - leap_assert_type(keymanager, KeyManager) port = kwargs.get('port', IMAP_PORT) - check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") - offline = kwargs.get('offline', False) uuid = soledad.uuid factory = LeapIMAPFactory(uuid, userid, soledad) @@ -154,16 +144,6 @@ def run_service(*args, **kwargs): try: tport = reactor.listenTCP(port, factory, interface="localhost") - if not offline: - # FIXME --- update after meskio's work - fetcher = LeapIncomingMail( - keymanager, - soledad, - factory.theAccount, - check_period, - userid) - else: - fetcher = None except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) @@ -186,7 +166,7 @@ def run_service(*args, **kwargs): leap_events.signal(IMAP_SERVICE_STARTED, str(port)) # FIXME -- change service signature - return fetcher, tport, factory + return tport, factory # not ok, signal error. leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py deleted file mode 100644 index 03c0164..0000000 --- a/src/leap/mail/imap/tests/test_incoming_mail.py +++ /dev/null @@ -1,193 +0,0 @@ -# -*- coding: utf-8 -*- -# test_imap.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test case for leap.email.imap.fetch - -@authors: Ruben Pollan, - -@license: GPLv3, see included LICENSE file -""" - -import json - -from email.mime.application import MIMEApplication -from email.mime.multipart import MIMEMultipart -from email.parser import Parser -from mock import Mock - -from leap.keymanager.openpgp import OpenPGPKey -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.fields import fields -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.service.imap import INCOMING_CHECK_PERIOD -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import ( - EncryptionSchemes, - ENC_JSON_KEY, - ENC_SCHEME_KEY, -) - - -class LeapIncomingMailTestCase(TestCaseWithKeyManager): - """ - Tests for the incoming mail parser - """ - NICKSERVER = "http://domain" - FROM_ADDRESS = "test@somedomain.com" - BODY = """ -Governments of the Industrial World, you weary giants of flesh and steel, I -come from Cyberspace, the new home of Mind. On behalf of the future, I ask -you of the past to leave us alone. You are not welcome among us. You have -no sovereignty where we gather. - """ - EMAIL = """from: Test from SomeDomain <%(from)s> -to: %(to)s -subject: independence of cyberspace - -%(body)s - """ % { - "from": FROM_ADDRESS, - "to": ADDRESS, - "body": BODY - } - - def setUp(self): - super(LeapIncomingMailTestCase, self).setUp() - - # Soledad sync makes trial block forever. The sync it's mocked to fix - # this problem. _mock_soledad_get_from_index can be used from the tests - # to provide documents. - self._soledad.sync = Mock() - - memstore = MemoryStore() - theAccount = SoledadBackedAccount( - ADDRESS, - soledad=self._soledad, - memstore=memstore) - self.fetcher = LeapIncomingMail( - self._km, - self._soledad, - theAccount, - INCOMING_CHECK_PERIOD, - ADDRESS) - - def tearDown(self): - del self.fetcher - super(LeapIncomingMailTestCase, self).tearDown() - - def testExtractOpenPGPHeader(self): - """ - Test the OpenPGP header key extraction - """ - KEYURL = "https://somedomain.com/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = Parser().parsestr(self.EMAIL) - message.add_header("OpenPGP", OpenPGP) - email = self._create_incoming_email(message.as_string()) - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) - self.fetcher._keymanager.fetch_key = Mock() - - def fetch_key_called(ret): - self.fetcher._keymanager.fetch_key.assert_called_once_with( - self.FROM_ADDRESS, KEYURL, OpenPGPKey) - - d = self.fetcher.fetch() - d.addCallback(fetch_key_called) - return d - - def testExtractOpenPGPHeaderInvalidUrl(self): - """ - Test the OpenPGP header key extraction - """ - KEYURL = "https://someotherdomain.com/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = Parser().parsestr(self.EMAIL) - message.add_header("OpenPGP", OpenPGP) - email = self._create_incoming_email(message.as_string()) - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) - self.fetcher._keymanager.fetch_key = Mock() - - def fetch_key_called(ret): - self.assertFalse(self.fetcher._keymanager.fetch_key.called) - - d = self.fetcher.fetch() - d.addCallback(fetch_key_called) - return d - - def testExtractAttachedKey(self): - """ - Test the OpenPGP header key extraction - """ - KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." - - message = MIMEMultipart() - message.add_header("from", self.FROM_ADDRESS) - key = MIMEApplication("", "pgp-keys") - key.set_payload(KEY) - message.attach(key) - - def put_raw_key_called(ret): - self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, OpenPGPKey, address=self.FROM_ADDRESS) - - d = self.mock_fetch(message.as_string()) - d.addCallback(put_raw_key_called) - return d - - def _mock_fetch(self, message): - self.fetcher._keymanager.fetch_key = Mock() - d = self._create_incoming_email(message) - d.addCallback( - lambda email: - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) - d.addCallback(lambda _: self.fetcher.fetch()) - return d - - def _create_incoming_email(self, email_str): - email = SoledadDocument() - data = json.dumps( - {"incoming": True, "content": email_str}, - ensure_ascii=False) - - def set_email_content(pubkey): - email.content = { - fields.INCOMING_KEY: True, - fields.ERROR_DECRYPTING_KEY: False, - ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, - ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) - } - return email - - d = self._km.get_key(ADDRESS, OpenPGPKey) - d.addCallback(set_email_content) - return d - - def _mock_soledad_get_from_index(self, index_name, value): - get_from_index = self._soledad.get_from_index - - def soledad_mock(idx_name, *key_values): - if index_name == idx_name: - return value - return get_from_index(idx_name, *key_values) - self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/src/leap/mail/incoming/__init__.py b/src/leap/mail/incoming/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py new file mode 100644 index 0000000..e52c727 --- /dev/null +++ b/src/leap/mail/incoming/service.py @@ -0,0 +1,760 @@ +# -*- coding: utf-8 -*- +# service.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Incoming mail fetcher. +""" +import copy +import logging +import shlex +import threading +import time +import traceback +import warnings + +from email.parser import Parser +from email.generator import Generator +from email.utils import parseaddr +from StringIO import StringIO +from urlparse import urlparse + +from twisted.application.service import Service +from twisted.python import log +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall +from twisted.internet.task import deferLater +from u1db import errors as u1db_errors + +from leap.common import events as leap_events +from leap.common.check import leap_assert, leap_assert_type +from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING +from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING +from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED +from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY +from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN +from leap.common.mail import get_email_charset +from leap.keymanager import errors as keymanager_errors +from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.decorators import deferred_to_thread +from leap.mail.utils import json_loads, empty, first +from leap.soledad.client import Soledad +from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY +from leap.soledad.common.errors import InvalidAuthTokenError + + +logger = logging.getLogger(__name__) + +MULTIPART_ENCRYPTED = "multipart/encrypted" +MULTIPART_SIGNED = "multipart/signed" +PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" +PGP_END = "-----END PGP MESSAGE-----" + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + + +class MalformedMessage(Exception): + """ + Raised when a given message is not well formed. + """ + pass + + +class IncomingMail(Service): + """ + Fetches and process mail from the incoming pool. + + This object implements IService interface, has public methods + startService and stopService that will actually initiate a + LoopingCall with check_period recurrency. + The LoopingCall itself will invoke the fetch method each time + that the check_period expires. + + This loop will sync the soledad db with the remote server and + process all the documents found tagged as incoming mail. + """ + + name = "IncomingMail" + + RECENT_FLAG = "\\Recent" + CONTENT_KEY = "content" + + LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' + """ + Header added to messages when they are decrypted by the IMAP fetcher, + which states the validity of an eventual signature that might be included + in the encrypted blob. + """ + LEAP_SIGNATURE_VALID = 'valid' + LEAP_SIGNATURE_INVALID = 'invalid' + LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' + + fetching_lock = threading.Lock() + + def __init__(self, keymanager, soledad, inbox, userid, + check_period=INCOMING_CHECK_PERIOD): + + """ + Initialize IncomingMail.. + + :param keymanager: a keymanager instance + :type keymanager: keymanager.KeyManager + + :param soledad: a soledad instance + :type soledad: Soledad + + :param inbox: the inbox where the new emails will be stored + :type inbox: IMAPMailbox + + :param check_period: the period to fetch new mail, in seconds. + :type check_period: int + """ + + leap_assert(keymanager, "need a keymanager to initialize") + leap_assert_type(soledad, Soledad) + leap_assert(check_period, "need a period to check incoming mail") + leap_assert_type(check_period, int) + leap_assert(userid, "need a userid to initialize") + + self._keymanager = keymanager + self._soledad = soledad + self._inbox = inbox + self._userid = userid + + self._loop = None + self._check_period = check_period + + # initialize a mail parser only once + self._parser = Parser() + + # + # Public API: fetch, start_loop, stop. + # + + def fetch(self): + """ + Fetch incoming mail, to be called periodically. + + Calls a deferred that will execute the fetch callback + in a separate thread + """ + def mail_compat(failure): + if failure.check(u1db_errors.InvalidGlobbing): + # It looks like we are a dealing with an outdated + # mx. Fallback to the version of the index + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + return self._soledad.get_from_index( + fields.JUST_MAIL_COMPAT_IDX, "*") + return failure + + def syncSoledadCallback(_): + d = self._soledad.get_from_index( + fields.JUST_MAIL_IDX, "*", "0") + d.addErrback(mail_compat) + d.addCallback(self._process_doclist) + return d + + logger.debug("fetching mail for: %s %s" % ( + self._soledad.uuid, self._userid)) + if not self.fetching_lock.locked(): + d1 = self._sync_soledad() + d = defer.gatherResults([d1], consumeErrors=True) + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + return d + else: + logger.debug("Already fetching mail.") + + def startService(self): + """ + Starts a loop to fetch mail. + """ + Service.startService(self) + if self._loop is None: + self._loop = LoopingCall(self.fetch) + self._loop.start(self._check_period) + else: + logger.warning("Tried to start an already running fetching loop.") + + def stopService(self): + """ + Stops the loop that fetches mail. + """ + if self._loop and self._loop.running is True: + self._loop.stop() + self._loop = None + Service.stopService(self) + + # + # Private methods. + # + + # synchronize incoming mail + + def _errback(self, failure): + logger.exception(failure.value) + traceback.print_exc() + + @deferred_to_thread + def _sync_soledad(self): + """ + Synchronize with remote soledad. + + :returns: a list of LeapDocuments, or None. + :rtype: iterable or None + """ + with self.fetching_lock: + try: + log.msg('FETCH: syncing soledad...') + self._soledad.sync() + log.msg('FETCH soledad SYNCED.') + except InvalidAuthTokenError: + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) + + def _signal_fetch_to_ui(self, doclist): + """ + Send leap events to ui. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: doclist + :rtype: iterable + """ + doclist = first(doclist) # gatherResults pass us a list + if doclist: + fetched_ts = time.mktime(time.gmtime()) + num_mails = len(doclist) if doclist is not None else 0 + if num_mails != 0: + log.msg("there are %s mails" % (num_mails,)) + leap_events.signal( + IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) + return doclist + + def _signal_unread_to_ui(self, *args): + """ + Sends unread event to ui. + """ + leap_events.signal( + IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) + + # process incoming mail. + + def _process_doclist(self, doclist): + """ + Iterates through the doclist, checks if each doc + looks like a message, and yields a deferred that will decrypt and + process the message. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: a list of deferreds for individual messages. + """ + log.msg('processing doclist') + if not doclist: + logger.debug("no docs found") + return + num_mails = len(doclist) + + deferreds = [] + for index, doc in enumerate(doclist): + logger.debug("processing doc %d of %d" % (index + 1, num_mails)) + leap_events.signal( + IMAP_MSG_PROCESSING, str(index), str(num_mails)) + + keys = doc.content.keys() + + # TODO Compatibility check with the index in pre-0.6 mx + # that does not write the ERROR_DECRYPTING_KEY + # This should be removed in 0.7 + + has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) + if has_errors is None: + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + + if has_errors: + logger.debug("skipping msg with decrypting errors...") + elif self._is_msg(keys): + d = self._decrypt_doc(doc) + d.addCallback(self._extract_keys) + d.addCallbacks(self._add_message_locally, self._errback) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + # + # operations on individual messages + # + + #FIXME: @deferred_to_thread + def _decrypt_doc(self, doc): + """ + Decrypt the contents of a document. + + :param doc: A document containing an encrypted message. + :type doc: SoledadDocument + + :return: A Deferred that will be fired with the document and the + decrypted message. + :rtype: SoledadDocument, str + """ + log.msg('decrypting msg') + + def process_decrypted(res): + if isinstance(res, tuple): + decrdata, _ = res + success = True + else: + decrdata = "" + success = False + + leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") + return self._process_decrypted_doc(doc, decrdata) + + d = self._keymanager.decrypt( + doc.content[ENC_JSON_KEY], + self._userid, OpenPGPKey) + d.addErrback(self._errback) + d.addCallback(process_decrypted) + d.addCallback(lambda data: (doc, data)) + return d + + def _process_decrypted_doc(self, doc, data): + """ + Process a document containing a succesfully decrypted message. + + :param doc: the incoming message + :type doc: SoledadDocument + :param data: the json-encoded, decrypted content of the incoming + message + :type data: str + + :return: a Deferred that will be fired with an str of the proccessed + data. + :rtype: Deferred + """ + log.msg('processing decrypted doc') + + # XXX turn this into an errBack for each one of + # the deferreds that would process an individual document + try: + msg = json_loads(data) + except UnicodeError as exc: + logger.error("Error while decrypting %s" % (doc.doc_id,)) + logger.exception(exc) + + # we flag the message as "with decrypting errors", + # to avoid further decryption attempts during sync + # cycles until we're prepared to deal with that. + # What is the same, when Ivan deals with it... + # A new decrypting attempt event could be triggered by a + # future a library upgrade, or a cli flag to the client, + # we just `defer` that for now... :) + doc.content[fields.ERROR_DECRYPTING_KEY] = True + deferLater(reactor, 0, self._update_incoming_message, doc) + + # FIXME this is just a dirty hack to delay the proper + # deferred organization here... + # and remember, boys, do not do this at home. + return [] + + if not isinstance(msg, dict): + defer.returnValue(False) + if not msg.get(fields.INCOMING_KEY, False): + defer.returnValue(False) + + # ok, this is an incoming message + rawmsg = msg.get(self.CONTENT_KEY, None) + if not rawmsg: + return "" + return self._maybe_decrypt_msg(rawmsg) + + @deferred_to_thread + def _update_incoming_message(self, doc): + """ + Do a put for a soledad document. This probably has been called only + in the case that we've needed to update the ERROR_DECRYPTING_KEY + flag in an incoming message, to get it out of the decrypting queue. + + :param doc: the SoledadDocument to update + :type doc: SoledadDocument + """ + log.msg("Updating SoledadDoc %s" % (doc.doc_id)) + self._soledad.put_doc(doc) + + @deferred_to_thread + def _delete_incoming_message(self, doc): + """ + Delete document. + + :param doc: the SoledadDocument to delete + :type doc: SoledadDocument + """ + log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) + self._soledad.delete_doc(doc) + + def _maybe_decrypt_msg(self, data): + """ + Tries to decrypt a gpg message if data looks like one. + + :param data: the text to be decrypted. + :type data: str + + :return: a Deferred that will be fired with an str of data, possibly + decrypted. + :rtype: Deferred + """ + leap_assert_type(data, str) + log.msg('maybe decrypting doc') + + # parse the original message + encoding = get_email_charset(data) + msg = self._parser.parsestr(data) + + fromHeader = msg.get('from', None) + senderAddress = None + if (fromHeader is not None + and (msg.get_content_type() == MULTIPART_ENCRYPTED + or msg.get_content_type() == MULTIPART_SIGNED)): + senderAddress = parseaddr(fromHeader) + + def add_leap_header(ret): + decrmsg, signkey = ret + if (senderAddress is None or + isinstance(signkey, keymanager_errors.KeyNotFound)): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_COULD_NOT_VERIFY) + elif isinstance(signkey, keymanager_errors.InvalidSignature): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_INVALID) + else: + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_VALID, + pubkey=signkey.key_id) + return decrmsg.as_string() + + if msg.get_content_type() == MULTIPART_ENCRYPTED: + d = self._decrypt_multipart_encrypted_msg( + msg, encoding, senderAddress) + else: + d = self._maybe_decrypt_inline_encrypted_msg( + msg, encoding, senderAddress) + d.addCallback(add_leap_header) + return d + + def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): + """ + Decrypt a message with content-type 'multipart/encrypted'. + + :param msg: The original encrypted message. + :type msg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('decrypting multipart encrypted msg') + msg = copy.deepcopy(msg) + self._msg_multipart_sanity_check(msg) + + # parse message and get encrypted content + pgpencmsg = msg.get_payload()[1] + encdata = pgpencmsg.get_payload() + + # decrypt or fail gracefully + def build_msg(res): + decrdata, signkey = res + + decrmsg = self._parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(msg['content-type']) + + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + msg.replace_header(hkey, hval) + except KeyError: + msg[hkey] = hval + + # all ok, replace payload by unencrypted payload + msg.set_payload(decrmsg.get_payload()) + return (msg, signkey) + + d = self._keymanager.decrypt( + encdata, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) + return d + + def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, + senderAddress): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('maybe decrypting inline encrypted msg') + # serialize the original message + buf = StringIO() + g = Generator(buf) + g.flatten(origmsg) + data = buf.getvalue() + + def decrypted_data(res): + decrdata, signkey = res + return data.replace(pgp_message, decrdata), signkey + + def encode_and_return(res): + data, signkey = res + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return (self._parser.parsestr(data), signkey) + + # handle exactly one inline PGP message + if PGP_BEGIN in data: + begin = data.find(PGP_BEGIN) + end = data.find(PGP_END) + pgp_message = data[begin:end + len(PGP_END)] + d = self._keymanager.decrypt( + pgp_message, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(decrypted_data, self._decryption_error, + errbackArgs=(data,)) + else: + d = defer.succeed((data, None)) + d.addCallback(encode_and_return) + return d + + def _decryption_error(self, failure, msg): + """ + Check for known decryption errors + """ + if failure.check(keymanager_errors.DecryptError): + logger.warning('Failed to decrypt encrypted message (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + elif failure.check(keymanager_errors.KeyNotFound): + logger.error('Failed to find private key for decryption (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + else: + return failure + + def _extract_keys(self, msgtuple): + """ + Retrieve attached keys to the mesage and parse message headers for an + *OpenPGP* header as described on the `IETF draft + ` + only urls with https and the same hostname than the email are supported + for security reasons. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired with msgtuple when key + extraction finishes + :rtype: Deferred + """ + OpenPGP_HEADER = 'OpenPGP' + doc, data = msgtuple + + # XXX the parsing of the message is done in mailbox.addMessage, maybe + # we should do it in this module so we don't need to parse it again + # here + msg = self._parser.parsestr(data) + _, fromAddress = parseaddr(msg['from']) + + header = msg.get(OpenPGP_HEADER, None) + dh = defer.succeed(None) + if header is not None: + dh = self._extract_openpgp_header(header, fromAddress) + + da = defer.succeed(None) + if msg.is_multipart(): + da = self._extract_attached_key(msg.get_payload(), fromAddress) + + d = defer.gatherResults([dh, da]) + d.addCallback(lambda _: msgtuple) + return d + + def _extract_openpgp_header(self, header, address): + """ + Import keys from the OpenPGP header + + :param header: OpenPGP header string + :type header: str + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when header extraction is done + :rtype: Deferred + """ + d = defer.succeed(None) + fields = dict([f.strip(' ').split('=') for f in header.split(';')]) + if 'url' in fields: + url = shlex.split(fields['url'])[0] # remove quotations + urlparts = urlparse(url) + addressHostname = address.split('@')[1] + if (urlparts.scheme == 'https' + and urlparts.hostname == addressHostname): + def fetch_error(failure): + if failure.check(keymanager_errors.KeyNotFound): + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + elif failure.check(keymanager_errors.KeyAttributesDiffer): + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, address)) + else: + return failure + + d = self._keymanager.fetch_key(address, url, OpenPGPKey) + d.addCallback( + lambda _: + logger.info("Imported key from header %s" % (url,))) + d.addErrback(fetch_error) + else: + logger.debug("No valid url on OpenPGP header %s" % (url,)) + else: + logger.debug("There is no url on the OpenPGP header: %s" + % (header,)) + return d + + def _extract_attached_key(self, attachments, address): + """ + Import keys from the attachments + + :param attachments: email attachment list + :type attachments: list(email.Message) + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when all the keys are stored + :rtype: Deferred + """ + MIME_KEY = "application/pgp-keys" + + deferreds = [] + for attachment in attachments: + if MIME_KEY == attachment.get_content_type(): + logger.debug("Add key from attachment") + d = self._keymanager.put_raw_key( + attachment.get_payload(), + OpenPGPKey, + address=address) + deferreds.append(d) + return defer.gatherResults(deferreds) + + def _add_message_locally(self, msgtuple): + """ + Adds a message to local inbox and delete it from the incoming db + in soledad. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired when the messages is stored + :rtype: Defferred + """ + doc, data = msgtuple + log.msg('adding message %s to local db' % (doc.doc_id,)) + + if isinstance(data, list): + if empty(data): + return False + data = data[0] + + def msgSavedCallback(result): + if not empty(result): + leap_events.signal(IMAP_MSG_SAVED_LOCALLY) + deferLater(reactor, 0, self._delete_incoming_message, doc) + leap_events.signal(IMAP_MSG_DELETED_INCOMING) + + d = self._inbox.addMessage(data, (self.RECENT_FLAG,)) + d.addCallbacks(msgSavedCallback, self._errback) + return d + + # + # helpers + # + + def _msg_multipart_sanity_check(self, msg): + """ + Performs a sanity check against a multipart encrypted msg + + :param msg: The original encrypted message. + :type msg: Message + """ + # sanity check + payload = msg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/incoming/tests/__init__.py b/src/leap/mail/incoming/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py new file mode 100644 index 0000000..bf95b1d --- /dev/null +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# test_incoming_mail.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test case for leap.mail.incoming.service + +@authors: Ruben Pollan, + +@license: GPLv3, see included LICENSE file +""" + +import json + +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.parser import Parser +from mock import Mock +from twisted.internet import defer + +from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.constants import INBOX_NAME +from leap.mail.imap.account import IMAPAccount +from leap.mail.incoming.service import IncomingMail +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, +) +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + + +class IncomingMailTestCase(TestCaseWithKeyManager): + """ + Tests for the incoming mail parser + """ + NICKSERVER = "http://domain" + FROM_ADDRESS = "test@somedomain.com" + BODY = """ +Governments of the Industrial World, you weary giants of flesh and steel, I +come from Cyberspace, the new home of Mind. On behalf of the future, I ask +you of the past to leave us alone. You are not welcome among us. You have +no sovereignty where we gather. + """ + EMAIL = """from: Test from SomeDomain <%(from)s> +to: %(to)s +subject: independence of cyberspace + +%(body)s + """ % { + "from": FROM_ADDRESS, + "to": ADDRESS, + "body": BODY + } + + def setUp(self): + def getInbox(_): + theAccount = IMAPAccount(ADDRESS, self._soledad) + return theAccount.callWhenReady( + lambda _: theAccount.getMailbox(INBOX_NAME)) + + def setUpFetcher(inbox): + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + self._soledad.sync = Mock() + + self.fetcher = IncomingMail( + self._km, + self._soledad, + inbox, + ADDRESS) + + d = super(IncomingMailTestCase, self).setUp() + d.addCallback(getInbox) + d.addCallback(setUpFetcher) + return d + + def tearDown(self): + del self.fetcher + return super(IncomingMailTestCase, self).tearDown() + + def testExtractOpenPGPHeader(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://somedomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock( + return_value=defer.succeed(None)) + + def fetch_key_called(ret): + self.fetcher._keymanager.fetch_key.assert_called_once_with( + self.FROM_ADDRESS, KEYURL, OpenPGPKey) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractOpenPGPHeaderInvalidUrl(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://someotherdomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock() + + def fetch_key_called(ret): + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractAttachedKey(self): + """ + Test the OpenPGP header key extraction + """ + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", self.FROM_ADDRESS) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.succeed(None)) + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, OpenPGPKey, address=self.FROM_ADDRESS) + + d = self._mock_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def _mock_fetch(self, message): + self.fetcher._keymanager.fetch_key = Mock() + d = self._create_incoming_email(message) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + return d + + def _create_incoming_email(self, email_str): + email = SoledadDocument() + data = json.dumps( + {"incoming": True, "content": email_str}, + ensure_ascii=False) + + def set_email_content(encr_data): + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: encr_data + } + return email + d = self._km.encrypt(data, ADDRESS, OpenPGPKey, fetch_remote=False) + d.addCallback(set_email_content) + return d + + def _mock_soledad_get_from_index(self, index_name, value): + get_from_index = self._soledad.get_from_index + + def soledad_mock(idx_name, *key_values): + if index_name == idx_name: + return defer.succeed(value) + return get_from_index(idx_name, *key_values) + self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8137f97..cb37d25 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -414,15 +414,10 @@ class MessageCollection(object): # Manipulate messages - def add_msg(self, raw_msg, flags=None, tags=None, date=None): + def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date=""): """ Add a message to this collection. """ - if not flags: - flags = tuple() - if not tags: - tags = tuple() - leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -582,7 +577,6 @@ class Account(object): self.mbox_indexer = MailboxIndexer(self.store) self.deferred_initialization = defer.Deferred() - self._initialized = False self._ready_cb = ready_cb self._init_d = self._initialize_storage() @@ -594,7 +588,6 @@ class Account(object): return self.add_mailbox(INBOX_NAME) def finish_initialization(result): - self._initialized = True self.deferred_initialization.callback(None) if self._ready_cb is not None: self._ready_cb() @@ -606,12 +599,8 @@ class Account(object): return d def callWhenReady(self, cb, *args, **kw): - if self._initialized: - cb(self, *args, **kw) - return defer.succeed(None) - else: - self.deferred_initialization.addCallback(cb, *args, **kw) - return self.deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts -- cgit v1.2.3 From 22d2bf496de91690bbced08fbb2e24e6f5040779 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 15 Jan 2015 15:50:20 -0400 Subject: update mail/imap tests --- src/leap/mail/mail.py | 6 ++---- src/leap/mail/tests/test_mail.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index cb37d25..8629d0e 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -285,6 +285,8 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + # TODO --- review this count shit. I think it's better to patch server + # to accept deferreds. # TODO need to initialize count here because imap server does not # expect a defered for the count. caller should return the deferred for # prime_count (ie, initialize) when returning the collection @@ -295,10 +297,6 @@ class MessageCollection(object): count = 0 self._count = count - #def initialize(self): - #d = self.prime_count() - #return d - def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 2c4b919..9bc553f 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -17,8 +17,10 @@ """ Tests for the mail module. """ -import time import os +import time +import uuid + from functools import partial from email.parser import Parser from email.Utils import formatdate @@ -28,9 +30,6 @@ from leap.mail.mail import MessageCollection, Account from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin -# from twisted.internet import defer -from twisted.trial import unittest - HERE = os.path.split(os.path.abspath(__file__))[0] @@ -67,24 +66,26 @@ class CollectionMixin(object): if mbox_collection: mbox_indexer = MailboxIndexer(store) mbox_name = "TestMbox" + mbox_uuid = str(uuid.uuid4()) else: mbox_indexer = mbox_name = None def get_collection_from_mbox_wrapper(wrapper): + wrapper.uuid = mbox_uuid return MessageCollection( adaptor, store, mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) d = adaptor.initialize_store(store) if mbox_collection: - d.addCallback(lambda _: mbox_indexer.create_table(mbox_name)) + d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) d.addCallback(get_collection_from_mbox_wrapper) return d # TODO profile add_msg. Why are these tests so SLOW??! -class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): +class MessageTestCase(SoledadTestMixin, CollectionMixin): """ Tests for the Message class. """ @@ -204,8 +205,7 @@ class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): self.assertEquals(msg.get_tags(), self.msg_tags) -class MessageCollectionTestCase(unittest.TestCase, - SoledadTestMixin, CollectionMixin): +class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): """ Tests for the MessageCollection class. """ @@ -294,7 +294,7 @@ class MessageCollectionTestCase(unittest.TestCase, pass -class AccountTestCase(unittest.TestCase, SoledadTestMixin): +class AccountTestCase(SoledadTestMixin): """ Tests for the Account class. """ -- cgit v1.2.3 From 9c593889be5bd307b429d160f55a6d83b4c660f1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 20:26:37 -0400 Subject: return the deferred from the incoming.startService() call --- src/leap/mail/incoming/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index e52c727..71c356f 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -190,7 +190,7 @@ class IncomingMail(Service): Service.startService(self) if self._loop is None: self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) + return self._loop.start(self._check_period) else: logger.warning("Tried to start an already running fetching loop.") -- cgit v1.2.3 From aeccc5f0c8c5b9da38f24dae0bb0cebe8a64c14d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 20:27:24 -0400 Subject: tests: add link related to trial block with sync --- src/leap/mail/imap/tests/utils.py | 1 + src/leap/mail/incoming/tests/test_incoming_mail.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 5708787..83c3f29 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -99,6 +99,7 @@ class IMAP4HelperMixin(SoledadTestMixin): # Soledad sync makes trial block forever. The sync it's mocked to # fix this problem. _mock_soledad_get_from_index can be used from # the tests to provide documents. + # TODO see here, possibly related? -- http://www.pythoneye.com/83_20424875/ self._soledad.sync = Mock() d = defer.Deferred() diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index bf95b1d..0745ee0 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -80,6 +80,7 @@ subject: independence of cyberspace # Soledad sync makes trial block forever. The sync it's mocked to # fix this problem. _mock_soledad_get_from_index can be used from # the tests to provide documents. + # TODO ---- see here http://www.pythoneye.com/83_20424875/ self._soledad.sync = Mock() self.fetcher = IncomingMail( -- cgit v1.2.3 From b8a2b00b379588de262ab30ccddba4e6d89e1a36 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 01:20:42 -0400 Subject: bug: fix empty definition; remove threading use this fixes a bug by which incoming service was not deleting the message from incoming after correclty saving all the message subparts into soledad. --- src/leap/mail/incoming/service.py | 66 +++++++++++++++++++-------------------- src/leap/mail/utils.py | 3 ++ 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 71c356f..0b2f7c2 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -20,7 +20,6 @@ Incoming mail fetcher. import copy import logging import shlex -import threading import time import traceback import warnings @@ -51,7 +50,6 @@ from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey from leap.mail.adaptors import soledad_indexes as fields -from leap.mail.decorators import deferred_to_thread from leap.mail.utils import json_loads, empty, first from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -90,6 +88,7 @@ class IncomingMail(Service): This loop will sync the soledad db with the remote server and process all the documents found tagged as incoming mail. """ + # TODO implements IService? name = "IncomingMail" @@ -106,8 +105,6 @@ class IncomingMail(Service): LEAP_SIGNATURE_INVALID = 'invalid' LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' - fetching_lock = threading.Lock() - def __init__(self, keymanager, soledad, inbox, userid, check_period=INCOMING_CHECK_PERIOD): @@ -174,14 +171,10 @@ class IncomingMail(Service): logger.debug("fetching mail for: %s %s" % ( self._soledad.uuid, self._userid)) - if not self.fetching_lock.locked(): - d1 = self._sync_soledad() - d = defer.gatherResults([d1], consumeErrors=True) - d.addCallbacks(syncSoledadCallback, self._errback) - d.addCallbacks(self._signal_fetch_to_ui, self._errback) - return d - else: - logger.debug("Already fetching mail.") + d = self._sync_soledad() + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + return d def startService(self): """ @@ -213,7 +206,6 @@ class IncomingMail(Service): logger.exception(failure.value) traceback.print_exc() - @deferred_to_thread def _sync_soledad(self): """ Synchronize with remote soledad. @@ -221,15 +213,21 @@ class IncomingMail(Service): :returns: a list of LeapDocuments, or None. :rtype: iterable or None """ - with self.fetching_lock: - try: - log.msg('FETCH: syncing soledad...') - self._soledad.sync() - log.msg('FETCH soledad SYNCED.') - except InvalidAuthTokenError: - # if the token is invalid, send an event so the GUI can - # disable mail and show an error message. - leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) + def _log_synced(result): + log.msg('FETCH soledad SYNCED.') + print "Result: ", result + return result + try: + log.msg('FETCH: syncing soledad...') + d = self._soledad.sync() + d.addCallback(_log_synced) + return d + # TODO is this still raised? or should we do failure.trap + # instead? + except InvalidAuthTokenError: + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) def _signal_fetch_to_ui(self, doclist): """ @@ -305,7 +303,6 @@ class IncomingMail(Service): # operations on individual messages # - #FIXME: @deferred_to_thread def _decrypt_doc(self, doc): """ Decrypt the contents of a document. @@ -388,7 +385,6 @@ class IncomingMail(Service): return "" return self._maybe_decrypt_msg(rawmsg) - @deferred_to_thread def _update_incoming_message(self, doc): """ Do a put for a soledad document. This probably has been called only @@ -398,10 +394,9 @@ class IncomingMail(Service): :param doc: the SoledadDocument to update :type doc: SoledadDocument """ - log.msg("Updating SoledadDoc %s" % (doc.doc_id)) - self._soledad.put_doc(doc) + log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id)) + return self._soledad.put_doc(doc) - @deferred_to_thread def _delete_incoming_message(self, doc): """ Delete document. @@ -409,8 +404,9 @@ class IncomingMail(Service): :param doc: the SoledadDocument to delete :type doc: SoledadDocument """ + print "DELETING INCOMING MESSAGE" log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) - self._soledad.delete_doc(doc) + return self._soledad.delete_doc(doc) def _maybe_decrypt_msg(self, data): """ @@ -705,16 +701,18 @@ class IncomingMail(Service): doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) - if isinstance(data, list): - if empty(data): - return False - data = data[0] + #if isinstance(data, list): + #if empty(data): + #return False + #data = data[0] def msgSavedCallback(result): if not empty(result): leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - deferLater(reactor, 0, self._delete_incoming_message, doc) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) + print "DEFERRING THE DELETION ----->" + return self._delete_incoming_message(doc) + # TODO add notification as a callback + #leap_events.signal(IMAP_MSG_DELETED_INCOMING) d = self._inbox.addMessage(data, (self.RECENT_FLAG,)) d.addCallbacks(msgSavedCallback, self._errback) diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py index 457097b..8e51024 100644 --- a/src/leap/mail/utils.py +++ b/src/leap/mail/utils.py @@ -45,9 +45,12 @@ def first(things): def empty(thing): """ Return True if a thing is None or its length is zero. + If thing is a number (int, float, long), return False. """ if thing is None: return True + if isinstance(thing, (int, float, long)): + return False if isinstance(thing, SoledadDocument): thing = thing.content try: -- cgit v1.2.3 From 8c65f09a16e4e00452dffa7d72771d9fac21c9c0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 13:48:21 -0400 Subject: imap: complete FETCH implementation --- src/leap/mail/adaptors/soledad.py | 38 +++++++++++++--- src/leap/mail/imap/mailbox.py | 95 ++++++++++++++++++++++++++------------- src/leap/mail/imap/messages.py | 23 +++++++++- src/leap/mail/imap/server.py | 3 ++ src/leap/mail/mail.py | 49 ++++++++++++++++---- src/leap/mail/mailbox_indexer.py | 14 +++++- 6 files changed, 171 insertions(+), 51 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index d99f677..46dbe4c 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -364,6 +364,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): self._future_doc_id = new_id self.mbox_uuid = mbox_uuid + def get_flags(self): + """ + Get the flags for this message (as a tuple of strings, not unicode). + """ + return map(str, self.flags) + class HeaderDocWrapper(SoledadDocumentWrapper): @@ -727,11 +733,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): mboxwrapper_klass = MailboxWrapper - def __init__(self): - SoledadIndexMixin.__init__(self) - - mboxwrapper_klass = MailboxWrapper - def __init__(self): SoledadIndexMixin.__init__(self) @@ -792,7 +793,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): fdoc, hdoc = doc_list[:3] cdocs = dict(enumerate(doc_list[3:], 1)) return self.get_msg_from_docs( - msg_class, mdoc, fdoc, hdoc, cdocs, uid=None) + msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): @@ -847,6 +848,30 @@ class SoledadMailAdaptor(SoledadIndexMixin): msg_class=MessageClass, uid=uid)) return d + def get_flags_from_mdoc_id(self, store, mdoc_id): + """ + # XXX stuff here... + """ + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) + + fdoc_id = _get_fdoc_id_from_mdoc_id() + + def wrap_fdoc(doc): + cls = FlagsDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_flags(fdoc_wrapper): + return fdoc_wrapper.get_flags() + + d = store.get_doc(fdoc_id) + d.addCallback(wrap_fdoc) + d.addCallback(get_flags) + return d + def create_msg(self, store, msg): """ :param store: an instance of soledad, or anything that behaves alike @@ -881,7 +906,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): Delete all messages flagged as deleted. """ def err(f): - print "ERROR GETTING FROM INDEX" f.printTraceback() def delete_fdoc_and_mdoc_flagged(fdocs): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index e1eb6bf..a000133 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -143,9 +143,9 @@ class IMAPMailbox(object): return self._listeners[self.mbox_name] def get_imap_message(self, message): - msg = IMAPMessage(message) - msg.store = self.collection.store - return msg + d = defer.Deferred() + IMAPMessage(message, store=self.collection.store, d=d) + return d # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size @@ -468,7 +468,6 @@ class IMAPMailbox(object): raise imap4.ReadOnlyMailbox return self.collection.delete_all_flagged() - # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): """ Put an upper bound to a messages sequence if this is open. @@ -477,16 +476,18 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: MessageSet """ + def set_last(last_uid): + messages_asked.last = last_uid + return messages_asked + if not messages_asked.last: try: iter(messages_asked) except TypeError: # looks like we cannot iterate - try: - # XXX fixme, does not exist - messages_asked.last = self.last_uid - except ValueError: - pass + d = self.collection.get_last_uid() + d.addCallback(set_last) + return d return messages_asked def _filter_msg_seq(self, messages_asked): @@ -524,50 +525,64 @@ class IMAPMailbox(object): otherwise. :type uid: bool - :rtype: deferred + :rtype: deferred with a generator that yields... """ # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we # switch to content-hash based index + local UID table. - sequence = False - # sequence = True if uid == 0 else False + is_sequence = True if uid == 0 else False getmsg = self.collection.get_message_by_uid + getimapmsg = self.get_imap_message - messages_asked = self._bound_seq(messages_asked) - d_sequence = self._filter_msg_seq(messages_asked) + def get_imap_messages_for_sequence(msg_sequence): + + def _get_imap_msg(messages): + d_imapmsg = [] + for msg in messages: + d_imapmsg.append(getimapmsg(msg)) + return defer.gatherResults(d_imapmsg) - def get_imap_messages_for_sequence(sequence): - def _zip_msgid(messages): - return zip( - list(sequence), - map(self.get_imap_message, messages)) + def _zip_msgid(imap_messages): + zipped = zip( + list(msg_sequence), imap_messages) + return (item for item in zipped) def _unset_recent(sequence): reactor.callLater(0, self.unset_recent_flags, sequence) return sequence d_msg = [] - for msgid in sequence: + for msgid in msg_sequence: d_msg.append(getmsg(msgid)) d = defer.gatherResults(d_msg) + d.addCallback(_get_imap_msg) d.addCallback(_zip_msgid) return d # for sequence numbers (uid = 0) - if sequence: + if is_sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: - d_sequence.addCallback(get_imap_messages_for_sequence) + d = self._get_sequence_of_messages(messages_asked) + d.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - return d_sequence + return d + + def _get_sequence_of_messages(self, messages_asked): + def get_sequence(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = defer.maybeDeferred(self._bound_seq, messages_asked) + d.addCallback(get_sequence) + return d def fetch_flags(self, messages_asked, uid): """ @@ -611,8 +626,8 @@ class IMAPMailbox(object): :param d: deferred whose callback will be called with result. :type d: Deferred - :rtype: A tuple of two-tuples of message sequence numbers and - flagsPart + :rtype: A generator that yields two-tuples of message sequence numbers + and flagsPart """ class flagsPart(object): def __init__(self, uid, flags): @@ -625,14 +640,30 @@ class IMAPMailbox(object): def getFlags(self): return map(str, self.flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) + def pack_flags(result): + #if result is None: + #print "No result" + #return + _uid, _flags = result + return _uid, flagsPart(_uid, _flags) - # FIXME use deferreds here - all_flags = self.collection.get_all_flags(self.mbox_name) - result = ((msgid, flagsPart( - msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - d.callback(result) + def get_flags_for_seq(sequence): + d_all_flags = [] + for msgid in sequence: + d_flags_per_uid = self.collection.get_flags_by_uid(msgid) + d_flags_per_uid.addCallback(pack_flags) + d_all_flags.append(d_flags_per_uid) + gotflags = defer.gatherResults(d_all_flags) + gotflags.addCallback(get_uid_flag_generator) + return gotflags + + def get_uid_flag_generator(result): + generator = (item for item in result) + d.callback(generator) + + d_seq = self._get_sequence_of_messages(messages_asked) + d_seq.addCallback(get_flags_for_seq) + return d_seq def fetch_headers(self, messages_asked, uid): """ diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 9b00162..d4b5d1f 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -19,6 +19,7 @@ IMAPMessage and IMAPMessageCollection. """ import logging from twisted.mail import imap4 +from twisted.internet import defer from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -40,11 +41,17 @@ class IMAPMessage(object): implements(imap4.IMessage) - def __init__(self, message): + def __init__(self, message, prefetch_body=True, + store=None, d=defer.Deferred()): """ Initializes a LeapMessage. """ self.message = message + self.__body_fd = None + self.store = store + if prefetch_body: + gotbody = self.__prefetch_body_file() + gotbody.addCallback(lambda _: d.callback(self)) # IMessage implementation @@ -109,14 +116,26 @@ class IMAPMessage(object): # # IMessagePart # + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading - :rtype: StringIO + :rtype: a deferred that will fire with a StringIO object. """ + if self.__body_fd is not None: + fd = self.__body_fd + fd.seek(0) + return fd + if store is None: store = self.store return self.message.get_body_file(store) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 32c921d..38a3fd4 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -112,6 +112,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): ebFetch = self._IMAP4Server__ebFetch if len(query) == 1 and str(query[0]) == "flags": + print ">>>>>>>>> fetching flags" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -121,6 +122,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): ).addErrback(ebFetch, tag) elif len(query) == 1 and str(query[0]) == "rfc822.header": + print ">>>>>>>> fetching headers" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -129,6 +131,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): cbFetch, tag, query, uid ).addErrback(ebFetch, tag) else: + print ">>>>>>> Fetching other" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8629d0e..976df5a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -155,7 +155,7 @@ class Message(object): Get flags for this message. :rtype: tuple """ - return tuple(self._wrapper.fdoc.flags) + return self._wrapper.fdoc.get_flags() def get_internal_date(self): """ @@ -184,6 +184,7 @@ class Message(object): def get_body_file(self, store): """ + Get a file descriptor with the body content. """ def write_and_rewind_if_found(cdoc): if not cdoc: @@ -367,14 +368,36 @@ class MessageCollection(object): d.addCallback(get_msg_from_mdoc_id) return d - # TODO deprecate ??? --- - def _prime_count(self): - def update_count(count): - self._count = count - d = self.mbox_indexer.count(self.mbox_name) - d.addCallback(update_count) + def get_flags_by_uid(self, uid, absolute=True): + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_flags_from_mdoc_id(doc_id): + if doc_id is None: # XXX needed? or bug? + return None + return self.adaptor.get_flags_from_mdoc_id( + self.store, doc_id) + + def wrap_in_tuple(flags): + return (uid, flags) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) + d.addCallback(get_flags_from_mdoc_id) + d.addCallback(wrap_in_tuple) return d + # TODO ------------------------------ FIXME FIXME FIXME implement this! + def set_flags(self, *args, **kw): + pass + + # TODO deprecate ??? --- + #def _prime_count(self): + #def update_count(count): + #self._count = count + #d = self.mbox_indexer.count(self.mbox_name) + #d.addCallback(update_count) + #return d + def count(self): """ Count the messages in this collection. @@ -389,11 +412,13 @@ class MessageCollection(object): def count_recent(self): # FIXME HACK - return 0 + # TODO ------------------------ implement this + return 3 def count_unseen(self): # FIXME hack - return 0 + # TODO ------------------------ implement this + return 3 def get_uid_next(self): """ @@ -404,6 +429,12 @@ class MessageCollection(object): """ return self.mbox_indexer.get_next_uid(self.mbox_uuid) + def get_last_uid(self): + """ + Get the last UID for this mailbox. + """ + return self.mbox_indexer.get_last_uid(self.mbox_uuid) + def all_uid_iter(self): """ Iterator through all the uids for this collection. diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 22e57d4..43a1f60 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -305,12 +305,24 @@ class MailboxIndexer(object): return 1 return uid + 1 + d = self.get_last_uid(mailbox_id) + d.addCallback(increment) + return d + + def get_last_uid(self, mailbox_id): + """ + Get the highest UID for a given mailbox. + """ + check_good_uuid(mailbox_id) sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + def getit(result): + return _maybe_first_query_item(result) + d = self._query(sql) - d.addCallback(increment) + d.addCallback(getit) return d def all_uid_iter(self, mailbox_id): -- cgit v1.2.3 From d327d8416475493864b1e0922b51812be566e028 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 21 Jan 2015 00:50:27 -0400 Subject: imap: implement setting of message flags --- src/leap/mail/imap/mailbox.py | 51 +++++++++++++++++++++++------------------- src/leap/mail/imap/messages.py | 23 ------------------- src/leap/mail/imap/server.py | 6 ----- src/leap/mail/mail.py | 38 +++++++++---------------------- 4 files changed, 39 insertions(+), 79 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index a000133..58fc514 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -231,7 +231,7 @@ class IMAPMailbox(object): """ return self.collection.get_mbox_attr("created") - def getUID(self, message): + def getUID(self, message_number): """ Return the UID of a message in the mailbox @@ -239,15 +239,15 @@ class IMAPMailbox(object): but in the future will be useful to get absolute UIDs from message sequence numbers. - :param message: the message uid + :param message: the message sequence number. :type message: int :rtype: int + :return: the UID of the message. """ - # TODO --- return the uid if it has it!!! - d = self.collection.get_msg_by_uid(message) - d.addCallback(lambda m: m.getUID()) - return d + # TODO support relative sequences. The (imap) message should + # receive a sequence number attribute: a deferred is not expected + return message_number def getUIDNext(self): """ @@ -451,15 +451,6 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - # TODO server already calls expunge for closing - #def close(self): - #""" - #Expunge and mark as closed - #""" - #d = self.expunge() - #d.addCallback(self._close_cb) - #return d - def expunge(self): """ Remove all messages flagged \\Deleted @@ -641,9 +632,6 @@ class IMAPMailbox(object): return map(str, self.flags) def pack_flags(result): - #if result is None: - #print "No result" - #return _uid, _flags = result return _uid, flagsPart(_uid, _flags) @@ -790,14 +778,31 @@ class IMAPMailbox(object): :type observer: deferred """ # XXX implement also sequence (uid = 0) - # XXX we should prevent client from setting Recent flag? + # TODO we should prevent client from setting Recent flag leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - self.collection.set_flags( - self.mbox_name, seq_messg, flags, mode, observer) + + def set_flags_for_seq(sequence): + + def return_result_dict(list_of_flags): + result = dict(zip(list(sequence), list_of_flags)) + observer.callback(result) + return result + + d_all_set = [] + for msgid in sequence: + d = self.collection.get_message_by_uid(msgid) + d.addCallback(lambda msg: self.collection.update_flags( + msg, flags, mode)) + d_all_set.append(d) + got_flags_setted = defer.gatherResults(d_all_set) + got_flags_setted.addCallback(return_result_dict) + return got_flags_setted + + d_seq = self._get_sequence_of_messages(messages_asked) + d_seq.addCallback(set_flags_for_seq) + return d_seq # ISearchableMailbox diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index d4b5d1f..df50323 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -73,29 +73,6 @@ class IMAPMessage(object): """ return self.message.get_flags() - # setFlags not in the interface spec but we use it with store command. - - # XXX if we can move it to a collection method, we don't need to pass - # collection to the IMAPMessage - - # lookup method? IMAPMailbox? - - #def setFlags(self, flags, mode): - #""" - #Sets the flags for this message -# - #:param flags: the flags to update in the message. - #:type flags: tuple of str - #:param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - #:type mode: int - #""" - #leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - # XXX - # return new flags - # map to str - #self.message.set_flags(flags, mode) - #self.collection.update_flags(self.message, flags, mode) - def getInternalDate(self): """ Retrieve the date internally associated with this message diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 38a3fd4..f294f42 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -112,7 +112,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): ebFetch = self._IMAP4Server__ebFetch if len(query) == 1 and str(query[0]) == "flags": - print ">>>>>>>>> fetching flags" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -122,7 +121,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): ).addErrback(ebFetch, tag) elif len(query) == 1 and str(query[0]) == "rfc822.header": - print ">>>>>>>> fetching headers" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -131,7 +129,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): cbFetch, tag, query, uid ).addErrback(ebFetch, tag) else: - print ">>>>>>> Fetching other" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -370,7 +367,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): # TODO subscribe method had also to be changed to accomodate deferred def do_SUBSCRIBE(self, tag, name): - print "DOING SUBSCRIBE" name = self._parseMbox(name) def _subscribeCb(_): @@ -433,8 +429,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): def _renameEb(failure): m = failure.value - print "SERVER rename failure!" - print m if failure.check(TypeError): self.sendBadResponse(tag, 'Invalid command syntax') elif failure.check(imap4.MailboxException): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 976df5a..9b7a562 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -272,8 +272,7 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, - count=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ Constructor for a MessageCollection. """ @@ -288,15 +287,6 @@ class MessageCollection(object): # TODO --- review this count shit. I think it's better to patch server # to accept deferreds. - # TODO need to initialize count here because imap server does not - # expect a defered for the count. caller should return the deferred for - # prime_count (ie, initialize) when returning the collection - # TODO should increment and decrement when adding/deleting. - # TODO recent count should also be static. - - if not count: - count = 0 - self._count = count def is_mailbox_collection(self): """ @@ -386,18 +376,6 @@ class MessageCollection(object): d.addCallback(wrap_in_tuple) return d - # TODO ------------------------------ FIXME FIXME FIXME implement this! - def set_flags(self, *args, **kw): - pass - - # TODO deprecate ??? --- - #def _prime_count(self): - #def update_count(count): - #self._count = count - #d = self.mbox_indexer.count(self.mbox_name) - #d.addCallback(update_count) - #return d - def count(self): """ Count the messages in this collection. @@ -479,6 +457,7 @@ class MessageCollection(object): Copy the message to another collection. (it only makes sense for mailbox collections) """ + # TODO currently broken ------------------FIXME- if not self.is_mailbox_collection(): raise NotImplementedError() @@ -555,19 +534,21 @@ class MessageCollection(object): final = new return final - def udpate_flags(self, msg, flags, mode): + def update_flags(self, msg, flags, mode): """ Update flags for a given message. """ wrapper = msg.get_wrapper() current = wrapper.fdoc.flags - newflags = self._update_flags_or_tags(current, flags, mode) + newflags = map(str, self._update_flags_or_tags(current, flags, mode)) wrapper.fdoc.flags = newflags wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags - return self.adaptor.update_msg(self.store, msg) + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(lambda _: newflags) + return d def update_tags(self, msg, tags, mode): """ @@ -576,8 +557,11 @@ class MessageCollection(object): wrapper = msg.get_wrapper() current = wrapper.fdoc.tags newtags = self._update_flags_or_tags(current, tags, mode) + wrapper.fdoc.tags = newtags - return self.adaptor.update_msg(self.store, msg) + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(newtags) + return d class Account(object): -- cgit v1.2.3 From 2e0fc3400fa55195b542a11e644f5be36b8ad659 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 21 Jan 2015 11:52:09 -0400 Subject: rename confusing attribute for account --- src/leap/mail/imap/account.py | 11 ++++++++++- src/leap/mail/imap/server.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 146d066..0cf583b 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -61,7 +61,7 @@ class IMAPAccount(object): implements(imap4.IAccount, imap4.INamespacePresenter) selected = None - closed = False + session_ended = False def __init__(self, user_id, store, d=defer.Deferred()): """ @@ -92,6 +92,15 @@ class IMAPAccount(object): return None mbox = IMAPMailbox(collection, rw=readwrite) return mbox + def end_session(self): + """ + Used to mark when the session has closed, and we should not allow any + more commands from the client. + + Right now it's called from the client backend. + """ + # TODO move its use to the service shutdown in leap.mail + self.session_ended = True def callWhenReady(self, cb, *args, **kw): d = self.account.callWhenReady(cb, *args, **kw) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index f294f42..23ddefc 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -67,7 +67,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ - if self.theAccount.closed is True and self.state != "unauth": + if self.theAccount.session_ended is True and self.state != "unauth": log.msg("Closing the session. State: unauth") self.state = "unauth" -- cgit v1.2.3 From aab87b503184619b5637d6b326ec7717e34dfb99 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:20:37 -0400 Subject: lots of little fixes after meskio's review mostly having to do with poor, missing or outdated documentation, naming of confusing things and reordering of code blocks for improved readability. --- src/leap/mail/adaptors/soledad.py | 85 ++++++++++------------ src/leap/mail/imap/account.py | 82 ++++++++++++---------- src/leap/mail/imap/mailbox.py | 43 +++++++----- src/leap/mail/imap/messages.py | 28 +++++++- src/leap/mail/imap/server.py | 10 ++- src/leap/mail/imap/service/imap.py | 20 +++--- src/leap/mail/imap/tests/test_imap.py | 61 +--------------- src/leap/mail/mail.py | 70 ++++++++++++------- src/leap/mail/mailbox_indexer.py | 128 ++++++++++++++-------------------- src/leap/mail/walk.py | 4 +- 10 files changed, 255 insertions(+), 276 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 46dbe4c..9f0bb30 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -19,7 +19,6 @@ Soledadad MailAdaptor module. import re from collections import defaultdict from email import message_from_string -from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer @@ -72,6 +71,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): deletion. """ # TODO we could also use a _dirty flag (in models) + # TODO add a get_count() method ??? -- that is extended over u1db. # We keep a dictionary with DeferredLocks, that will be # unique to every subclass of SoledadDocumentWrapper. @@ -86,10 +86,9 @@ class SoledadDocumentWrapper(models.DocumentWrapper): """ return cls._k_locks[cls.__name__] - def __init__(self, **kwargs): - doc_id = kwargs.pop('doc_id', None) + def __init__(self, doc_id=None, future_doc_id=None, **kwargs): self._doc_id = doc_id - self._future_doc_id = kwargs.pop('future_doc_id', None) + self._future_doc_id = future_doc_id self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs) @@ -123,7 +122,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_doc_id(doc): self._doc_id = doc.doc_id - self._future_doc_id = None + self.set_future_doc_id(None) return doc if self.future_doc_id is None: @@ -201,6 +200,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): @classmethod def _get_or_create(cls, store, index, value): + # TODO shorten this method. assert store is not None assert index is not None assert value is not None @@ -211,6 +211,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): except AttributeError: raise RuntimeError("The model is badly defined") + # TODO separate into another method? def try_to_get_doc_from_index(indexes): values = [] idx_def = dict(indexes)[index] @@ -323,9 +324,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper): d.addCallback(wrap_docs) return d - # TODO - # [ ] get_count() ??? - def __repr__(self): try: idx = getattr(self, self.model.__meta__.index) @@ -442,29 +440,23 @@ class MessageWrapper(object): integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ - if isinstance(mdoc, SoledadDocument): - mdoc_id = mdoc.doc_id - mdoc = mdoc.content - else: - mdoc_id = None - if not mdoc: - mdoc = {} - self.mdoc = MetaMsgDocWrapper(doc_id=mdoc_id, **mdoc) - - if isinstance(fdoc, SoledadDocument): - fdoc_id = fdoc.doc_id - fdoc = fdoc.content - else: - fdoc_id = None - self.fdoc = FlagsDocWrapper(doc_id=fdoc_id, **fdoc) + + def get_doc_wrapper(doc, cls): + if isinstance(doc, SoledadDocument): + doc_id = doc.doc_id + doc = doc.content + else: + doc_id = None + if not doc: + doc = {} + return cls(doc_id=doc_id, **doc) + + self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper) + + self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper) self.fdoc.set_future_doc_id(self.mdoc.fdoc) - if isinstance(hdoc, SoledadDocument): - hdoc_id = hdoc.doc_id - hdoc = hdoc.content - else: - hdoc_id = None - self.hdoc = HeaderDocWrapper(doc_id=hdoc_id, **hdoc) + self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper) self.hdoc.set_future_doc_id(self.mdoc.hdoc) if cdocs is None: @@ -489,10 +481,6 @@ class MessageWrapper(object): "Cannot create: fdoc has a doc_id") # TODO check that the doc_ids in the mdoc are coherent - # TODO I think we need to tolerate the no hdoc.doc_id case, for when we - # are doing a copy to another mailbox. - # leap_assert(self.hdoc.doc_id is None, - # "Cannot create: hdoc has a doc_id") d = [] d.append(self.mdoc.create(store)) d.append(self.fdoc.create(store)) @@ -566,8 +554,9 @@ class MessageWrapper(object): def get_subpart_dict(self, index): """ - :param index: index, 1-indexed + :param index: the part to lookup, 1-indexed :type index: int + :rtype: dict """ return self.hdoc.part_map[str(index)] @@ -785,16 +774,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): assert(MessageClass is not None) return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) - def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): - if len(doc_list) == 3: - mdoc, fdoc, hdoc = doc_list - cdocs = None - elif len(doc_list) > 3: - fdoc, hdoc = doc_list[:3] - cdocs = dict(enumerate(doc_list[3:], 1)) - return self.get_msg_from_docs( - msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) - def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): @@ -844,10 +823,21 @@ class SoledadMailAdaptor(SoledadIndexMixin): else: d = get_parts_doc_from_mdoc_id() - d.addCallback(partial(self._get_msg_from_variable_doc_list, - msg_class=MessageClass, uid=uid)) + d.addCallback(self._get_msg_from_variable_doc_list, + msg_class=MessageClass, uid=uid) return d + def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): + if len(doc_list) == 3: + mdoc, fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 3: + # XXX is this case used? + mdoc, fdoc, hdoc = doc_list[:3] + cdocs = dict(enumerate(doc_list[3:], 1)) + return self.get_msg_from_docs( + msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) + def get_flags_from_mdoc_id(self, store, mdoc_id): """ # XXX stuff here... @@ -875,7 +865,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): def create_msg(self, store, msg): """ :param store: an instance of soledad, or anything that behaves alike - :type store: :param msg: a Message object. :return: a Deferred that is fired when all the underlying documents @@ -889,8 +878,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ :param msg: a Message object. :param store: an instance of soledad, or anything that behaves alike - :type store: - :param msg: a Message object. :return: a Deferred that is fired when all the underlying documents have been updated (actually, it's only the fdoc that's allowed to update). diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0cf583b..38df845 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -49,9 +49,6 @@ if PROFILE_CMD: # Soledad IMAP Account ####################################### -# XXX watchout, account needs to be ready... so we should maybe return -# a deferred to the IMAP service when it's initialized - class IMAPAccount(object): """ An implementation of an imap4 Account @@ -67,8 +64,14 @@ class IMAPAccount(object): """ Keeps track of the mailboxes and subscriptions handled by this account. - :param account: The name of the account (user id). - :type account: str + The account is not ready to be used, since the store needs to be + initialized and we also need to do some initialization routines. + You can either pass a deferred to this constructor, or use + `callWhenReady` method. + + :param user_id: The name of the account (user id, in the form + user@provider). + :type user_id: str :param store: a Soledad instance. :type store: Soledad @@ -87,11 +90,6 @@ class IMAPAccount(object): self.user_id = user_id self.account = Account(store, ready_cb=lambda: d.callback(self)) - def _return_mailbox_from_collection(self, collection, readwrite=1): - if collection is None: - return None - mbox = IMAPMailbox(collection, rw=readwrite) - return mbox def end_session(self): """ Used to mark when the session has closed, and we should not allow any @@ -103,6 +101,12 @@ class IMAPAccount(object): self.session_ended = True def callWhenReady(self, cb, *args, **kw): + """ + Execute callback when the account is ready to be used. + XXX note that this callback will be called with a first ignored + parameter. + """ + # TODO ignore the first parameter and change tests accordingly. d = self.account.callWhenReady(cb, *args, **kw) return d @@ -129,6 +133,12 @@ class IMAPAccount(object): d.addCallback(self._return_mailbox_from_collection) return d + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + # # IAccount # @@ -146,7 +156,7 @@ class IMAPAccount(object): :type creation_ts: int :returns: a Deferred that will contain the document if successful. - :rtype: bool + :rtype: defer.Deferred """ name = normalize_mailbox(name) @@ -190,25 +200,15 @@ class IMAPAccount(object): :return: A deferred that will fire with a true value if the creation - succeeds. + succeeds. The deferred might fail with a MailboxException + if the mailbox cannot be added. :rtype: Deferred - :raise MailboxException: Raised if this mailbox cannot be added. """ - paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] - sep = '/' - def pass_on_collision(failure): failure.trap(imap4.MailboxCollision) return True - for accum in range(1, len(paths)): - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - d.addErrback(pass_on_collision) - subs.append(d) - def handle_collision(failure): failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): @@ -216,19 +216,26 @@ class IMAPAccount(object): else: return defer.succeed(True) + def all_good(result): + return all(result) + + paths = filter(None, normalize_mailbox(pathspec).split('/')) + subs = [] + sep = '/' + + for accum in range(1, len(paths)): + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + df = self.addMailbox(sep.join(paths)) df.addErrback(handle_collision) subs.append(df) - def all_good(result): - return all(result) - - if subs: - d1 = defer.gatherResults(subs) - d1.addCallback(all_good) - return d1 - else: - return defer.succeed(False) + d1 = defer.gatherResults(subs) + d1.addCallback(all_good) + return d1 def select(self, name, readwrite=1): """ @@ -285,8 +292,7 @@ class IMAPAccount(object): global _mboxes _mboxes = mailboxes if name not in mailboxes: - err = imap4.MailboxException("No such mailbox: %r" % name) - return defer.fail(err) + raise imap4.MailboxException("No such mailbox: %r" % name) def get_mailbox(_): return self.getMailbox(name) @@ -303,10 +309,9 @@ class IMAPAccount(object): # as part of their root. for others in _mboxes: if others != name and others.startswith(name): - err = imap4.MailboxException( + raise imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") - return defer.fail(err) return mbox d = self.account.list_all_mailbox_names() @@ -324,8 +329,6 @@ class IMAPAccount(object): # if self._inferiorNames(name) > 1: # self._index.removeMailbox(name) - # TODO use mail.Account.rename_mailbox - # TODO finish conversion to deferreds def rename(self, oldname, newname): """ Renames a mailbox. @@ -460,6 +463,9 @@ class IMAPAccount(object): :type name: str :rtype: Deferred """ + # TODO should raise MailboxException if attempted to unsubscribe + # from a mailbox that is not currently subscribed. + # TODO factor out with subscribe method. name = normalize_mailbox(name) def set_unsubscribed(mbox): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 58fc514..52f4dd5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -91,8 +91,9 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - #imap4.ICloseableMailbox, imap4.ISearchableMailbox, + # XXX I think we do not need to implement CloseableMailbox, do we? + # imap4.ICloseableMailbox imap4.IMessageCopier) init_flags = INIT_FLAGS @@ -108,8 +109,6 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. - :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -117,14 +116,10 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw - self.closed = False self._uidvalidity = None self.collection = collection - if not self.getFlags(): - self.setFlags(self.init_flags) - @property def mbox_name(self): return self.collection.mbox_name @@ -201,6 +196,7 @@ class IMAPMailbox(object): "flags expected to be a tuple") return self.collection.set_mbox_attr("flags", flags) + # TODO - not used? @property def is_closed(self): """ @@ -211,6 +207,7 @@ class IMAPMailbox(object): """ return self.collection.get_mbox_attr("closed") + # TODO - not used? def set_closed(self, closed): """ Set the closed attribute for this mailbox. @@ -448,9 +445,6 @@ class IMAPMailbox(object): d.addCallback(remove_mbox) return d - def _close_cb(self, result): - self.closed = True - def expunge(self): """ Remove all messages flagged \\Deleted @@ -555,24 +549,23 @@ class IMAPMailbox(object): # for sequence numbers (uid = 0) if is_sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: - d = self._get_sequence_of_messages(messages_asked) + d = self._get_messages_range(messages_asked) d.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) return d - def _get_sequence_of_messages(self, messages_asked): - def get_sequence(messages_asked): + def _get_messages_range(self, messages_asked): + def get_range(messages_asked): return self._filter_msg_seq(messages_asked) d = defer.maybeDeferred(self._bound_seq, messages_asked) - d.addCallback(get_sequence) + d.addCallback(get_range) return d def fetch_flags(self, messages_asked, uid): @@ -599,6 +592,10 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError + d = defer.Deferred() reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: @@ -649,7 +646,7 @@ class IMAPMailbox(object): generator = (item for item in result) d.callback(generator) - d_seq = self._get_sequence_of_messages(messages_asked) + d_seq = self._get_messages_range(messages_asked) d_seq.addCallback(get_flags_for_seq) return d_seq @@ -677,7 +674,11 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ + # TODO implement sequences # TODO how often is thunderbird doing this? + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError class headersPart(object): def __init__(self, uid, headers): @@ -753,6 +754,12 @@ class IMAPMailbox(object): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ + # TODO implement sequences + # TODO how often is thunderbird doing this? + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError + if not self.isWriteable(): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox @@ -777,7 +784,7 @@ class IMAPMailbox(object): done. :type observer: deferred """ - # XXX implement also sequence (uid = 0) + # TODO implement also sequence (uid = 0) # TODO we should prevent client from setting Recent flag leap_assert(not isinstance(flags, basestring), "flags cannot be a string") @@ -800,7 +807,7 @@ class IMAPMailbox(object): got_flags_setted.addCallback(return_result_dict) return got_flags_setted - d_seq = self._get_sequence_of_messages(messages_asked) + d_seq = self._get_messages_range(messages_asked) d_seq.addCallback(set_flags_for_seq) return d_seq diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index df50323..8f4c953 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -36,16 +36,38 @@ logger = logging.getLogger(__name__) class IMAPMessage(object): """ - The main representation of a message. + The main representation of a message as seen by the IMAP Server. + This class implements the semantics specific to IMAP specification. """ - implements(imap4.IMessage) def __init__(self, message, prefetch_body=True, store=None, d=defer.Deferred()): """ - Initializes a LeapMessage. + Get an IMAPMessage. A mail.Message is needed, since many of the methods + are proxied to that object. + + + If you do not need to prefetch the body of the message, you can set + `prefetch_body` to False, but the current imap server implementation + expect the getBodyFile method to return inmediately. + + When the prefetch_body option is used, a deferred is also expected as a + parameter, and this will fire when the deferred initialization has + taken place, with this instance of IMAPMessage as a parameter. + + :param message: the abstract message + :type message: mail.Message + :param prefetch_body: Whether to prefetch the content doc for the body. + :type prefetch_body: bool + :param store: an instance of soledad, or anything that behaves like it. + :param d: an optional deferred, that will be fired with the instance of + the IMAPMessage being initialized + :type d: defer.Deferred """ + # TODO substitute the use of the deferred initialization by a factory + # function, maybe. + self.message = message self.__body_fd = None self.store = store diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 23ddefc..027fd7a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -500,13 +500,18 @@ class LEAPIMAPServer(imap4.IMAP4Server): select_DELETE = auth_DELETE # Need to override the command table after patching - # arg_astring and arg_literal + # arg_astring and arg_literal, except on the methods that we are already + # overriding. + # TODO -------------------------------------------- + # Check if we really need to override these + # methods, or we can monkeypatch. # do_DELETE = imap4.IMAP4Server.do_DELETE # do_CREATE = imap4.IMAP4Server.do_CREATE # do_RENAME = imap4.IMAP4Server.do_RENAME # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + # ------------------------------------------------- do_LOGIN = imap4.IMAP4Server.do_LOGIN do_STATUS = imap4.IMAP4Server.do_STATUS do_APPEND = imap4.IMAP4Server.do_APPEND @@ -530,8 +535,11 @@ class LEAPIMAPServer(imap4.IMAP4Server): auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') select_EXAMINE = auth_EXAMINE + # TODO ----------------------------------------------- + # re-add if we stop overriding DELETE # auth_DELETE = (do_DELETE, arg_astring) # select_DELETE = auth_DELETE + # ---------------------------------------------------- auth_RENAME = (do_RENAME, arg_astring, arg_astring) select_RENAME = auth_RENAME diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 93e4d62..cc76e3a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -17,6 +17,7 @@ """ IMAP service initialization """ +# TODO: leave only an implementor of IService in here import logging import os @@ -29,10 +30,9 @@ from twisted.python import log logger = logging.getLogger(__name__) from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type, leap_check +from leap.common.check import leap_assert_type, leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer -from leap.mail.incoming import IncomingMail from leap.soledad.client import Soledad from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED @@ -113,6 +113,10 @@ class LeapIMAPFactory(ServerFactory): """ Stops imap service (fetcher, factory and port). """ + # mark account as unusable, so any imap command will fail + # with unauth state. + self.theAccount.end_session() + # TODO should wait for all the pending deferreds, # the twisted way! if DO_PROFILE: @@ -123,23 +127,23 @@ class LeapIMAPFactory(ServerFactory): return ServerFactory.doStop(self) -def run_service(*args, **kwargs): +def run_service(store, **kwargs): """ Main entry point to run the service from the client. + :param store: a soledad instance + :returns: the port as returned by the reactor when starts listening, and the factory for the protocol. """ - leap_assert(len(args) == 2) - soledad = args - leap_assert_type(soledad, Soledad) + leap_assert_type(store, Soledad) port = kwargs.get('port', IMAP_PORT) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") - uuid = soledad.uuid - factory = LeapIMAPFactory(uuid, userid, soledad) + uuid = store.uuid + factory = LeapIMAPFactory(uuid, userid, store) try: tport = reactor.listenTCP(port, factory, diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 6be41cd..67a24cd 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -75,6 +75,7 @@ class TestRealm: # TestCases # +# TODO rename to IMAPMessageCollection class MessageCollectionTestCase(IMAP4HelperMixin): """ Tests for the MessageCollection class @@ -87,6 +88,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin): We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ + # FIXME -- return deferred super(MessageCollectionTestCase, self).setUp() # FIXME --- update initialization @@ -1090,64 +1092,6 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): # Okay, that was much fun indeed - # skipping close test: we just need expunge for now. - #def testClose(self): - #""" - #Test closing the mailbox. We expect to get deleted all messages flagged - #as such. - #""" - #acc = self.server.theAccount - #mailbox_name = 'mailbox-close' -# - #def add_mailbox(): - #return acc.addMailbox(mailbox_name) -# - #def login(): - #return self.client.login(TEST_USER, TEST_PASSWD) -# - #def select(): - #return self.client.select(mailbox_name) -# - #def get_mailbox(): - #def _save_mbox(mailbox): - #self.mailbox = mailbox - #d = self.server.theAccount.getMailbox(mailbox_name) - #d.addCallback(_save_mbox) - #return d -# - #def add_messages(): - #d1 = self.mailbox.addMessage( - #'test 1', flags=('\\Deleted', 'AnotherFlag')) - #d2 = self.mailbox.addMessage( - #'test 2', flags=('AnotherFlag',)) - #d3 = self.mailbox.addMessage( - #'test 3', flags=('\\Deleted',)) - #d = defer.gatherResults([d1, d2, d3]) - #return d -# - #def close(): - #return self.client.close() -# - #d = self.connected.addCallback(strip(add_mailbox)) - #d.addCallback(strip(login)) - #d.addCallbacks(strip(select), self._ebGeneral) - #d.addCallback(strip(get_mailbox)) - #d.addCallbacks(strip(add_messages), self._ebGeneral) - #d.addCallbacks(strip(close), self._ebGeneral) - #d.addCallbacks(self._cbStopClient, self._ebGeneral) - #d2 = self.loopback() - #d1 = defer.gatherResults([d, d2]) - #d1.addCallback(lambda _: self.mailbox.getMessageCount()) - #d1.addCallback(self._cbTestClose) - #return d1 -# - #def _cbTestClose(self, count): - # TODO is this correct? count should not take into account those - # flagged as deleted??? - #self.assertEqual(count, 1) - # TODO --- assert flags are those of the message #2 - #self.failUnless(self.mailbox.closed) - def testExpunge(self): """ Test expunge command @@ -1209,6 +1153,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): self.assertItemsEqual(self.results, [1, 3]) +# TODO -------- Fix this testcase class AccountTestCase(IMAP4HelperMixin): """ Test the Account. diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 9b7a562..59fd57c 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -58,9 +58,16 @@ def _write_and_rewind(payload): class MessagePart(object): - # TODO pass cdocs in init - def __init__(self, part_map, cdocs={}): + """ + :param part_map: a dictionary mapping the subparts for + this MessagePart (1-indexed). + :type part_map: dict + :param cdoc: optional, a dict of content documents + """ + # TODO document the expected keys in the part_map dict. + # TODO add abstraction layer between the cdocs and this class. Only + # adaptor should know about the format of the cdocs. self._pmap = part_map self._cdocs = cdocs @@ -266,6 +273,10 @@ class MessageCollection(object): # [ ] To guarantee synchronicity of the documents sent together during a # sync, we could get hold of a deferredLock that inhibits # synchronization while we are updating (think more about this!) + # [ ] review the serveral count_ methods. I think it's better to patch + # server to accept deferreds. + # [ ] Use inheritance for the mailbox-collection instead of handling the + # special cases everywhere? # Account should provide an adaptor instance when creating this collection. adaptor = None @@ -285,9 +296,6 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper - # TODO --- review this count shit. I think it's better to patch server - # to accept deferreds. - def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. @@ -297,22 +305,26 @@ class MessageCollection(object): @property def mbox_name(self): - wrapper = getattr(self, "mbox_wrapper", None) - if not wrapper: + # TODO raise instead? + if self.mbox_wrapper is None: return None - return wrapper.mbox + return self.mbox_wrapper.mbox @property def mbox_uuid(self): - wrapper = getattr(self, "mbox_wrapper", None) - if not wrapper: + # TODO raise instead? + if self.mbox_wrapper is None: return None - return wrapper.uuid + return self.mbox_wrapper.uuid def get_mbox_attr(self, attr): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") return getattr(self.mbox_wrapper, attr) def set_mbox_attr(self, attr, value): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") setattr(self.mbox_wrapper, attr, value) return self.mbox_wrapper.update(self.store) @@ -323,10 +335,10 @@ class MessageCollection(object): Retrieve a message by its content hash. :rtype: Deferred """ - if not self.is_mailbox_collection(): - # instead of getting the metamsg by chash, query by (meta) index - # or use the internal collection of pointers-to-docs. + # TODO instead of getting the metamsg by chash, in this case we + # should query by (meta) index or use the internal collection of + # pointers-to-docs. raise NotImplementedError() metamsg_id = _get_mdoc_id(self.mbox_name, chash) @@ -462,6 +474,9 @@ class MessageCollection(object): raise NotImplementedError() def insert_copied_mdoc_id(wrapper): + # TODO this needs to be implemented before the copy + # interface works. + newmailbox_uuid = get_mbox_uuid_from_msg_wrapper(wrapper) return self.mbox_indexer.insert_doc( newmailbox_uuid, wrapper.mdoc.doc_id) @@ -525,15 +540,6 @@ class MessageCollection(object): d.addCallback(del_all_uid) return d - def _update_flags_or_tags(self, old, new, mode): - if mode == Flagsmode.APPEND: - final = list((set(tuple(old) + new))) - elif mode == Flagsmode.REMOVE: - final = list(set(old).difference(set(new))) - elif mode == Flagsmode.SET: - final = new - return final - def update_flags(self, msg, flags, mode): """ Update flags for a given message. @@ -563,6 +569,15 @@ class MessageCollection(object): d.addCallback(newtags) return d + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + class Account(object): """ @@ -573,7 +588,7 @@ class Account(object): basic collection handled by traditional MUAs, but it can also handle other types of Collections (tag based, for instance). - leap.mail.imap.SoledadBackedAccount partially proxies methods in this + leap.mail.imap.IMAPAccount partially proxies methods in this class. """ @@ -582,7 +597,6 @@ class Account(object): # the Account class. adaptor_class = SoledadMailAdaptor - store = None def __init__(self, store, ready_cb=None): self.store = store @@ -612,6 +626,12 @@ class Account(object): return d def callWhenReady(self, cb, *args, **kw): + """ + Execute the callback when the initialization of the Account is ready. + Note that the callback will receive a first meaningless parameter. + """ + # TODO this should ignore the first parameter explicitely + # lambda _: cb(*args, **kw) self.deferred_initialization.addCallback(cb, *args, **kw) return self.deferred_initialization diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 43a1f60..4eb0fa8 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -38,23 +38,23 @@ class WrongMetaDocIDError(Exception): pass -def sanitize(mailbox_id): - return mailbox_id.replace("-", "_") +def sanitize(mailbox_uuid): + return mailbox_uuid.replace("-", "_") -def check_good_uuid(mailbox_id): +def check_good_uuid(mailbox_uuid): """ Check that the passed mailbox identifier is a valid UUID. - :param mailbox_id: the uuid to check - :type mailbox_id: str + :param mailbox_uuid: the uuid to check + :type mailbox_uuid: str :return: None :raises: AssertionError if a wrong uuid was passed. """ try: - uuid.UUID(str(mailbox_id)) + uuid.UUID(str(mailbox_uuid)) except (AttributeError, ValueError): raise AssertionError( - "the mbox_id is not a valid uuid: %s" % mailbox_id) + "the mbox_id is not a valid uuid: %s" % mailbox_uuid) class MailboxIndexer(object): @@ -88,51 +88,33 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) - def create_table(self, mailbox_id): + def create_table(self, mailbox_uuid): """ Create the UID table for a given mailbox. :param mailbox: the mailbox identifier. :type mailbox: str :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("CREATE TABLE if not exists {preffix}{name}( " "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) return self._query(sql) - def delete_table(self, mailbox_id): + def delete_table(self, mailbox_uuid): """ Delete the UID table for a given mailbox. :param mailbox: the mailbox name :type mailbox: str :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) return self._query(sql) - def rename_table(self, oldmailbox, newmailbox): - """ - Delete the UID table for a given mailbox. - :param oldmailbox: the old mailbox name - :type oldmailbox: str - :param newmailbox: the new mailbox name - :type newmailbox: str - :rtype: Deferred - """ - assert oldmailbox - assert newmailbox - assert oldmailbox != newmailbox - sql = ("ALTER TABLE {preffix}{old} " - "RENAME TO {preffix}{new}".format( - preffix=self.table_preffix, - old=sanitize(oldmailbox), new=sanitize(newmailbox))) - return self._query(sql) - - def insert_doc(self, mailbox_id, doc_id): + def insert_doc(self, mailbox_uuid, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -148,11 +130,11 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert doc_id - mailbox_id = mailbox_id.replace('-', '_') + mailbox_uuid = mailbox_uuid.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -160,12 +142,12 @@ class MailboxIndexer(object): sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) @@ -173,25 +155,25 @@ class MailboxIndexer(object): d.addErrback(lambda f: f.printTraceback()) return d - def delete_doc_by_uid(self, mailbox_id, uid): + def delete_doc_by_uid(self, mailbox_uuid, uid): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param uid: the UID of the message. :type uid: int :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert uid sql = ("DELETE FROM {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (uid,) return self._query(sql, values) - def delete_doc_by_hash(self, mailbox_id, doc_id): + def delete_doc_by_hash(self, mailbox_uuid, doc_id): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. @@ -199,7 +181,7 @@ class MailboxIndexer(object): M-- - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str @@ -207,82 +189,80 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert doc_id sql = ("DELETE FROM {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) return self._query(sql, values) - def get_doc_id_from_uid(self, mailbox_id, uid): + def get_doc_id_from_uid(self, mailbox_uuid, uid): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ - check_good_uuid(mailbox_id) - mailbox_id = mailbox_id.replace('-', '_') + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (uid,) d = self._query(sql, values) d.addCallback(get_hash) return d - def get_uid_from_doc_id(self, mailbox_id, doc_id): - check_good_uuid(mailbox_id) - mailbox_id = mailbox_id.replace('-', '_') + def get_uid_from_doc_id(self, mailbox_uuid, doc_id): + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') def get_uid(result): return _maybe_first_query_item(result) sql = ("SELECT uid from {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) d = self._query(sql, values) d.addCallback(get_uid) return d - - - def get_doc_ids_from_uids(self, mailbox_id, uids): + def get_doc_ids_from_uids(self, mailbox_uuid, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() - def count(self, mailbox_id): + def count(self, mailbox_uuid): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid - :type mailbox_id: str + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) def get_count(result): return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) d = self._query(sql) d.addCallback(get_count) d.addErrback(lambda _: 0) return d - def get_next_uid(self, mailbox_id): + def get_next_uid(self, mailbox_uuid): """ Get the next integer beyond the highest UID count for a given mailbox. @@ -291,13 +271,13 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) def increment(result): uid = _maybe_first_query_item(result) @@ -305,18 +285,18 @@ class MailboxIndexer(object): return 1 return uid + 1 - d = self.get_last_uid(mailbox_id) + d = self.get_last_uid(mailbox_uuid) d.addCallback(increment) return d - def get_last_uid(self, mailbox_id): + def get_last_uid(self, mailbox_uuid): """ Get the highest UID for a given mailbox. """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def getit(result): return _maybe_first_query_item(result) @@ -325,17 +305,17 @@ class MailboxIndexer(object): d.addCallback(getit) return d - def all_uid_iter(self, mailbox_id): + def all_uid_iter(self, mailbox_uuid): """ Get a sequence of all the uids in this mailbox. - :param mailbox_id: the mailbox uuid - :type mailbox_id: str + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("SELECT uid from {preffix}{name} ").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def get_results(result): return [x[0] for x in result] diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 8653a5f..891abdc 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -122,7 +122,7 @@ def walk_msg_tree(parts, body_phash=None): documents that will be stored in Soledad. It walks down the subparts in the parsed message tree, and collapses - the leaf docuents into a wrapper document until no multipart submessages + the leaf documents into a wrapper document until no multipart submessages are left. To achieve this, it iteratively calculates a wrapper vector of all documents in the sequence that have more than one part and have unitary documents to their right. To collapse a multipart, take as many @@ -171,7 +171,7 @@ def walk_msg_tree(parts, body_phash=None): HEADERS: dict(parts[wind][HEADERS]) } - # remove subparts and substitue wrapper + # remove subparts and substitute wrapper map(lambda i: parts.remove(i), slic) parts[wind] = cwra -- cgit v1.2.3 From 5d9b3bf1217ee220b41699a0374fd1db5e22987c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 00:47:42 -0400 Subject: fix typo --- src/leap/mail/smtp/gateway.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 222ef3f..1a187cf 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -219,7 +219,7 @@ class SMTPDelivery(object): d = self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound d.addCallbacks(found, not_found) - d.addCallbac(lambda _: EncryptedMessage(user, self._outgoing_mail)) + d.addCallback(lambda _: EncryptedMessage(user, self._outgoing_mail)) return d def validateFrom(self, helo, origin): @@ -306,4 +306,6 @@ class EncryptedMessage(object): log.err() signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) # unexpected loss of connection; don't save + + self._lines = [] -- cgit v1.2.3 From 98def315e5f48df6eec713dbe175df8bdfe406dd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 03:16:32 -0400 Subject: re-add support for basic multipart messages --- src/leap/mail/adaptors/soledad.py | 8 ++- src/leap/mail/imap/mailbox.py | 6 +- src/leap/mail/imap/messages.py | 144 ++++++++++++++++++++------------------ src/leap/mail/mail.py | 102 ++++++++++++++++----------- 4 files changed, 147 insertions(+), 113 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 9f0bb30..d21638c 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -463,8 +463,9 @@ class MessageWrapper(object): cdocs = {} cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) - self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in - cdocs.items()]) + self.cdocs = dict([ + (key, ContentDocWrapper(**doc.content)) + for (key, doc) in cdocs.items()]) for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): cdoc.set_future_doc_id(doc_id) @@ -560,6 +561,9 @@ class MessageWrapper(object): """ return self.hdoc.part_map[str(index)] + def get_subpart_indexes(self): + return self.hdoc.part_map.keys() + def get_body(self, store): """ :rtype: deferred diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 52f4dd5..045636e 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -540,7 +540,11 @@ class IMAPMailbox(object): d_msg = [] for msgid in msg_sequence: - d_msg.append(getmsg(msgid)) + # XXX We want cdocs because we "probably" are asked for the + # body. We should be smarted at do_FETCH and pass a parameter + # to this method in order not to prefetch cdocs if they're not + # going to be used. + d_msg.append(getmsg(msgid, get_cdocs=True)) d = defer.gatherResults(d_msg) d.addCallback(_get_imap_msg) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 8f4c953..b7bb6ee 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -115,13 +115,6 @@ class IMAPMessage(object): # # IMessagePart # - def __prefetch_body_file(self): - def assign_body_fd(fd): - self.__body_fd = fd - return fd - d = self.getBodyFile() - d.addCallback(assign_body_fd) - return d def getBodyFile(self, store=None): """ @@ -139,25 +132,6 @@ class IMAPMessage(object): store = self.store return self.message.get_body_file(store) - # TODO refactor with getBodyFile in MessagePart - - #body = bdoc_content.get(self.RAW_KEY, "") - #content_type = bdoc_content.get('content-type', "") - #charset = find_charset(content_type) - #if charset is None: - #charset = self._get_charset(body) - #try: - #if isinstance(body, unicode): - #body = body.encode(charset) - #except UnicodeError as exc: - #logger.error( - #"Unicode error, using 'replace'. {0!r}".format(exc)) - #logger.debug("Attempted to encode with: %s" % charset) - #body = body.encode(charset, 'replace') - #finally: - #return write_fd(body) - - def getSize(self): """ Return the total size, in octets, of this message. @@ -182,48 +156,8 @@ class IMAPMessage(object): :return: A mapping of header field names to header field values :rtype: dict """ - # TODO split in smaller methods -- format_headers()? - # XXX refactor together with MessagePart method - headers = self.message.get_headers() - - # XXX keep this in the imap imessage implementation, - # because the server impl. expects content-type to be present. - if not headers: - logger.warning("No headers found") - return {str('content-type'): str('')} - - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - if isinstance(headers, list): - headers = dict(headers) - - # default to most likely standard - charset = find_charset(headers, "utf-8") - headers2 = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() - - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - if value.endswith(";"): - # bastards - value = value[:-1] - - # filter original dict by negate-condition - if cond(key): - headers2[key] = value - return headers2 + return _format_headers(headers, negate, *names) def isMultipart(self): """ @@ -242,7 +176,81 @@ class IMAPMessage(object): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - return self.message.get_subpart(part) + subpart = self.message.get_subpart(part) + return IMAPMessagePart(subpart) + + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d + + +class IMAPMessagePart(object): + + def __init__(self, message_part): + self.message_part = message_part + + def getBodyFile(self, store=None): + return self.message_part.get_body_file() + + def getSize(self): + return self.message_part.get_size() + + def getHeaders(self, negate, *names): + headers = self.message_part.get_headers() + return _format_headers(headers, negate, *names) + + def isMultipart(self): + return self.message_part.is_multipart() + + def getSubPart(self, part): + subpart = self.message_part.get_subpart(part) + return IMAPMessagePart(subpart) + + +def _format_headers(headers, negate, *names): + # current server impl. expects content-type to be present, so if for + # some reason we do not have headers, we have to return at least that + # one + if not headers: + logger.warning("No headers found") + return {str('content-type'): str('')} + + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + if isinstance(headers, list): + headers = dict(headers) + + # default to most likely standard + charset = find_charset(headers, "utf-8") + + _headers = dict() + for key, value in headers.items(): + # twisted imap server expects *some* headers to be lowercase + # We could use a CaseInsensitiveDict here... + if key.lower() == "content-type": + key = key.lower() + + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + if value.endswith(";"): + # bastards + value = value[:-1] + + # filter original dict by negate-condition + if cond(key): + _headers[key] = value + return _headers class IMAPMessageCollection(object): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 59fd57c..aa499c0 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -22,6 +22,7 @@ import logging import StringIO from twisted.internet import defer +from twisted.python import log from leap.common.check import leap_assert_type from leap.common.mail import get_email_charset @@ -30,7 +31,7 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.utils import empty, find_charset +from leap.mail.utils import empty # find_charset logger = logging.getLogger(name=__name__) @@ -57,61 +58,57 @@ def _write_and_rewind(payload): class MessagePart(object): + # TODO This class should be better abstracted from the data model. + # TODO support arbitrarily nested multiparts (right now we only support + # the trivial case) - def __init__(self, part_map, cdocs={}): + def __init__(self, part_map, index=1, cdocs={}): """ :param part_map: a dictionary mapping the subparts for this MessagePart (1-indexed). :type part_map: dict - :param cdoc: optional, a dict of content documents + + The format for the part_map is as follows: + + {u'1': {u'ctype': u'text/plain', + u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], + [u'Content-Transfer-Encoding', u'8bit']], + u'multi': False, + u'parts': 1, + u'phash': u'02D82B29F6BB0C8612D1C', + u'size': 132}} + + :param index: which index in the content-doc is this subpart + representing. + :param cdocs: optional, a reference to the top-level dict of wrappers + for content-docs (1-indexed). """ - # TODO document the expected keys in the part_map dict. - # TODO add abstraction layer between the cdocs and this class. Only - # adaptor should know about the format of the cdocs. + # TODO: Pass only the cdoc wrapper for this part. self._pmap = part_map + self._index = index self._cdocs = cdocs def get_size(self): return self._pmap['size'] def get_body_file(self): + payload = "" pmap = self._pmap multi = pmap.get('multi') if not multi: - phash = pmap.get("phash") + payload = self._get_payload(self._index) else: - pmap_ = pmap.get('part_map') - first_part = pmap_.get('1', None) - if not empty(first_part): - phash = first_part['phash'] - else: - phash = "" - - payload = self._get_payload(phash) - + # XXX uh, multi also... should recurse" + raise NotImplementedError if payload: - # FIXME - # content_type = self._get_ctype_from_document(phash) - # charset = find_charset(content_type) - charset = None - if charset is None: - charset = get_email_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - + payload = self._format_payload(payload) return _write_and_rewind(payload) def get_headers(self): return self._pmap.get("headers", []) def is_multipart(self): - multi = self._pmap.get("multi", False) - return multi + return self._pmap.get("multi", False) def get_subpart(self, part): if not self.is_multipart(): @@ -123,10 +120,30 @@ class MessagePart(object): except KeyError: logger.debug("getSubpart for %s: KeyError" % (part,)) raise IndexError - return MessagePart(self._soledad, part_map) - - def _get_payload(self, phash): - return self._cdocs.get(phash, "") + return MessagePart(part_map, cdocs={1: self._cdocs.get(1, {})}) + + def _get_payload(self, index): + cdoc_wrapper = self._cdocs.get(index, None) + if cdoc_wrapper: + return cdoc_wrapper.raw + return "" + + def _format_payload(self, payload): + # FIXME ----------------------------------------------- + # Test against unicode payloads... + # content_type = self._get_ctype_from_document(phash) + # charset = find_charset(content_type) + charset = None + if charset is None: + charset = get_email_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + return payload class Message(object): @@ -224,17 +241,18 @@ class Message(object): raise TypeError part_index = part + 1 try: - subpart_dict = self._wrapper.get_subpart_dict( - part_index) + subpart_dict = self._wrapper.get_subpart_dict(part_index) except KeyError: - raise TypeError - # XXX pass cdocs - return MessagePart(subpart_dict) + raise IndexError + + return MessagePart( + subpart_dict, index=part_index, cdocs=self._wrapper.cdocs) # Custom methods. def get_tags(self): """ + Get the tags for this message. """ return tuple(self._wrapper.fdoc.tags) @@ -290,7 +308,7 @@ class MessageCollection(object): self.adaptor = adaptor self.store = store - # XXX I have to think about what to do when there is no mbox passed to + # XXX think about what to do when there is no mbox passed to # the initialization. We could still get the MetaMsg by index, instead # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer -- cgit v1.2.3