diff options
Diffstat (limited to 'src/leap/mail/imap')
27 files changed, 1631 insertions, 6825 deletions
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 70ed13b..cc56fff 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # account.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,22 +15,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Soledad Backed Account. +Soledad Backed IMAP Account. """ -import copy import logging import os import time +from functools import partial +from twisted.internet import defer from twisted.mail import imap4 from twisted.python import log from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.mail.imap.index import IndexedDB -from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.parser import MBoxParser -from leap.mail.imap.mailbox import SoledadMailbox + +from leap.mail.constants import MessageFlags +from leap.mail.mail import Account +from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -38,7 +39,6 @@ logger = logging.getLogger(__name__) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") @@ -46,107 +46,72 @@ if PROFILE_CMD: ####################################### -# Soledad Account +# Soledad IMAP Account ####################################### - -# TODO change name to LeapIMAPAccount, since we're using -# the memstore. -# IndexedDB should also not be here anymore. - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): +class IMAPAccount(object): """ - An implementation of IAccount and INamespacePresenteer + An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. """ implements(imap4.IAccount, imap4.INamespacePresenter) - _soledad = None selected = None - closed = False - def __init__(self, account_name, soledad, memstore=None): + def __init__(self, user_id, store, d=defer.Deferred()): """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. - - :param acct_name: The name of the account (user id). - :type acct_name: str - - :param soledad: a Soledad instance. - :type soledad: Soledad - :param memstore: a MemoryStore instance. - :type memstore: MemoryStore - """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) - - # XXX SHOULD assert too that the name matches the user/uuid with which - # soledad has been initialized. - - # XXX ??? why is this parsing mailbox name??? it's account... - # userid? homogenize. - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - self._memstore = memstore - - self.__mailboxes = set([]) - - self.initialize_db() + Keeps track of the mailboxes and subscriptions handled by this account. - # every user should have the right to an inbox folder - # at least, so let's make one! - self._load_mailboxes() + The account is not ready to be used, since the store needs to be + initialized and we also need to do some initialization routines. + You can either pass a deferred to this constructor, or use + `callWhenReady` method. - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) + :param user_id: The name of the account (user id, in the form + user@provider). + :type user_id: str - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. + :param store: a Soledad instance. + :type store: Soledad - :rtype: dict + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred """ - return copy.deepcopy(self.EMPTY_MBOX) + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) - def _get_mailbox_by_name(self, name): - """ - Return an mbox document by name. + # TODO assert too that the name matches the user/uuid with which + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. - :param name: the name of the mailbox - :type name: str + self.user_id = user_id + self.account = Account(store, ready_cb=lambda: d.callback(self)) - :rtype: SoledadDocument + def end_session(self): """ - # XXX use soledadstore instead ...; - doc = self._soledad.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) - return doc[0] if doc else None + Used to mark when the session has closed, and we should not allow any + more commands from the client. - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set + Right now it's called from the client backend. """ - return sorted(self.__mailboxes) - - def _load_mailboxes(self): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)]) + # TODO move its use to the service shutdown in leap.mail + self.account.end_session() @property - def subscriptions(self): + def session_ended(self): + return self.account.session_ended + + def callWhenReady(self, cb, *args, **kw): """ - A list of the current subscriptions for this account. + Execute callback when the account is ready to be used. + XXX note that this callback will be called with a first ignored + parameter. """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + # TODO ignore the first parameter and change tests accordingly. + d = self.account.callWhenReady(cb, *args, **kw) + return d def getMailbox(self, name): """ @@ -155,16 +120,27 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: name of the mailbox :type name: str - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox + :returns: an IMAPMailbox instance + :rtype: IMAPMailbox """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) - if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) + def check_it_exists(mailboxes): + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + return True - return SoledadMailbox(name, self._soledad, - memstore=self._memstore) + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) + return d + + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox # # IAccount @@ -182,61 +158,76 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): one is provided. :type creation_ts: int - :returns: True if successful - :rtype: bool + :returns: a Deferred that will contain the document if successful. + :rtype: defer.Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... leap_assert(name, "Need a mailbox name to create a mailbox") - if name in self.mailboxes: - raise imap4.MailboxCollision(repr(name)) - - if creation_ts is None: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # mailbox-uidvalidity. - creation_ts = int(time.time() * 10E2) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - doc = self._soledad.create_doc(mbox) - self._load_mailboxes() - return bool(doc) + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_does_not_exist) + d.addCallback(lambda _: self.account.add_mailbox( + name, creation_ts=creation_ts)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) + return d def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. + :param pathspec: + The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. :type pathspec: str - :return: A true value if the creation succeeds. - :rtype: bool + :return: + A deferred that will fire with a true value if the creation + succeeds. The deferred might fail with a MailboxException + if the mailbox cannot be added. + :rtype: Deferred - :raise MailboxException: Raised if this mailbox cannot be added. """ - # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) - for accum in range(1, len(paths)): - try: - self.addMailbox('/'.join(paths[:accum])) - except imap4.MailboxCollision: - pass - try: - self.addMailbox('/'.join(paths)) - except imap4.MailboxCollision: + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): - return False - self._load_mailboxes() - return True + return defer.succeed(False) + else: + return defer.succeed(True) + + def all_good(result): + return all(result) + + paths = filter(None, normalize_mailbox(pathspec).split('/')) + subs = [] + sep = '/' + + for accum in range(1, len(paths)): + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) + + d1 = defer.gatherResults(subs) + d1.addCallback(all_good) + return d1 def select(self, name, readwrite=1): """ @@ -248,65 +239,87 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param readwrite: 1 for readwrite permissions. :type readwrite: int - :rtype: SoledadMailbox + :rtype: IMAPMailbox """ - if PROFILE_CMD: - start = time.time() + name = normalize_mailbox(name) - name = self._parse_mailbox_name(name) - if name not in self.mailboxes: - logger.warning("No such mailbox!") - return None - self.selected = name + def check_it_exists(mailboxes): + if name not in mailboxes: + logger.warning("SELECT: No such mailbox!") + return None + return name + + def set_selected(_): + self.selected = name - sm = SoledadMailbox( - name, self._soledad, self._memstore, readwrite) - if PROFILE_CMD: - _debugProfiling(None, "SELECT", start) - return sm + def get_collection(name): + if name is None: + return None + return self.account.get_collection_by_mailbox(name) + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_collection) + d.addCallback(partial( + self._return_mailbox_from_collection, readwrite=readwrite)) + return d def delete(self, name, force=False): """ Deletes a mailbox. - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - :param name: the mailbox to be deleted :type name: str - :param force: if True, it will not check for noselect flag or inferior - names. use with care. + :param force: + if True, it will not check for noselect flag or inferior + names. use with care. :type force: bool + :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) + _mboxes = None + + def check_it_exists(mailboxes): + global _mboxes + _mboxes = mailboxes + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) - if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) - mbox = self.getMailbox(name) + def get_mailbox(_): + return self.getMailbox(name) - if force is False: + def destroy_mailbox(mbox): + return mbox.destroy() + + def check_can_be_deleted(mbox): + global _mboxes # See if this box is flagged \Noselect - # XXX use mbox.flags instead? mbox_flags = mbox.getFlags() - if self.NOSELECT_FLAG in mbox_flags: + if MessageFlags.NOSELECT_FLAG in mbox_flags: # Check for hierarchically inferior mailboxes with this one # as part of their root. - for others in self.mailboxes: + for others in _mboxes: if others != name and others.startswith(name): - raise imap4.MailboxException, ( + raise imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") - self.__mailboxes.discard(name) - mbox.destroy() + return mbox - # XXX FIXME --- not honoring the inferior names... + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_mailbox) + if not force: + d.addCallback(check_can_be_deleted) + d.addCallback(destroy_mailbox) + return d + # FIXME --- not honoring the inferior names... # if there are no hierarchically inferior names, we will # delete it from our ken. + # XXX is this right? # if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - # self._index.removeMailbox(name) + # self._index.removeMailbox(name) def rename(self, oldname, newname): """ @@ -318,27 +331,31 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param newname: new name of the mailbox :type newname: str """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) + + def rename_inferiors((inferiors, mailboxes)): + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox(repr(oldname)) + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) - for (old, new) in inferiors: - if new in self.mailboxes: - raise imap4.MailboxCollision(repr(new)) + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 - for (old, new) in inferiors: - self._memstore.rename_fdocs_mailbox(old, new) - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new - self.__mailboxes.discard(old) - self._soledad.put_doc(mbox) + d1 = self._inferiorNames(oldname) + d2 = self.account.list_all_mailbox_names() - self._load_mailboxes() + d = defer.gatherResults([d1, d2]) + d.addCallback(rename_inferiors) + return d def _inferiorNames(self, name): """ @@ -348,54 +365,87 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: list """ # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors - def isSubscribed(self, name): + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d + + def listMailboxes(self, ref, wildcard): """ - Returns True if user is subscribed to this mailbox. + List the mailboxes. - :param name: the mailbox to be checked. - :type name: str + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. - :rtype: bool + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) - def _set_subscription(self, name, value): + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + + def isSubscribed(self, name): """ - Sets the subscription value for a given mailbox + Returns True if user is subscribed to this mailbox. - :param name: the mailbox + :param name: the mailbox to be checked. :type name: str - :param value: the boolean value - :type value: bool + :rtype: Deferred (will fire with bool) """ - # maybe we should store subscriptions in another - # document... - if name not in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) + name = normalize_mailbox(name) - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) + def get_subscribed(mbox): + return mbox.collection.get_mbox_attr("subscribed") + + d = self.getMailbox(name) + d.addCallback(get_subscribed) + return d def subscribe(self, name): """ - Subscribe to this mailbox + Subscribe to this mailbox if not already subscribed. :param name: name of the mailbox :type name: str + :rtype: Deferred """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) + name = normalize_mailbox(name) + + def set_subscribed(mbox): + return mbox.collection.set_mbox_attr("subscribed", True) + + d = self.getMailbox(name) + d.addCallback(set_subscribed) + return d def unsubscribe(self, name): """ @@ -403,34 +453,27 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: name of the mailbox :type name: str + :rtype: Deferred """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException( - "Not currently subscribed to %r" % name) - self._set_subscription(name, False) + # TODO should raise MailboxException if attempted to unsubscribe + # from a mailbox that is not currently subscribed. + # TODO factor out with subscribe method. + name = normalize_mailbox(name) - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. + def set_unsubscribed(mbox): + return mbox.collection.set_mbox_attr("subscribed", False) - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. + d = self.getMailbox(name) + d.addCallback(set_unsubscribed) + return d - :param ref: reference name - :type ref: str + def getSubscriptions(self): + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d # # INamespacePresenter @@ -445,22 +488,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def getOtherNamespaces(self): return None - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - def __repr__(self): """ Representation string for this object. """ - return "<SoledadBackedAccount (%s)>" % self._account_name + return "<IMAPAccount (%s)>" % self.user_id diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py deleted file mode 100644 index 0a97752..0000000 --- a/src/leap/mail/imap/fetch.py +++ /dev/null @@ -1,655 +0,0 @@ -# -*- coding: utf-8 -*- -# fetch.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Incoming mail fetcher. -""" -import copy -import logging -import threading -import time -import sys -import traceback -import warnings - -from email.parser import Parser -from email.generator import Generator -from email.utils import parseaddr -from StringIO import StringIO - -from twisted.python import log -from twisted.internet import defer -from twisted.internet.task import LoopingCall -from twisted.internet.task import deferLater -from u1db import errors as u1db_errors -from zope.proxy import sameProxiedObjects - -from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type -from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING -from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING -from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED -from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY -from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN -from leap.common.mail import get_email_charset -from leap.keymanager import errors as keymanager_errors -from leap.keymanager.openpgp import OpenPGPKey -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.fields import fields -from leap.mail.utils import json_loads, empty, first -from leap.soledad.client import Soledad -from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY -from leap.soledad.common.errors import InvalidAuthTokenError - - -logger = logging.getLogger(__name__) - -MULTIPART_ENCRYPTED = "multipart/encrypted" -MULTIPART_SIGNED = "multipart/signed" -PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" -PGP_END = "-----END PGP MESSAGE-----" - - -class MalformedMessage(Exception): - """ - Raised when a given message is not well formed. - """ - pass - - -class LeapIncomingMail(object): - """ - Fetches and process mail from the incoming pool. - - This object has public methods start_loop and stop that will - actually initiate a LoopingCall with check_period recurrency. - The LoopingCall itself will invoke the fetch method each time - that the check_period expires. - - This loop will sync the soledad db with the remote server and - process all the documents found tagged as incoming mail. - """ - - RECENT_FLAG = "\\Recent" - CONTENT_KEY = "content" - - LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' - """ - Header added to messages when they are decrypted by the IMAP fetcher, - which states the validity of an eventual signature that might be included - in the encrypted blob. - """ - LEAP_SIGNATURE_VALID = 'valid' - LEAP_SIGNATURE_INVALID = 'invalid' - LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' - - fetching_lock = threading.Lock() - - def __init__(self, keymanager, soledad, imap_account, - check_period, userid): - - """ - Initialize LeapIncomingMail.. - - :param keymanager: a keymanager instance - :type keymanager: keymanager.KeyManager - - :param soledad: a soledad instance - :type soledad: Soledad - - :param imap_account: the account to fetch periodically - :type imap_account: SoledadBackedAccount - - :param check_period: the period to fetch new mail, in seconds. - :type check_period: int - """ - - leap_assert(keymanager, "need a keymanager to initialize") - leap_assert_type(soledad, Soledad) - leap_assert(check_period, "need a period to check incoming mail") - leap_assert_type(check_period, int) - leap_assert(userid, "need a userid to initialize") - - self._keymanager = keymanager - self._soledad = soledad - self.imapAccount = imap_account - self._inbox = self.imapAccount.getMailbox('inbox') - self._userid = userid - - self._loop = None - self._check_period = check_period - - # initialize a mail parser only once - self._parser = Parser() - - @property - def _pkey(self): - if sameProxiedObjects(self._keymanager, None): - logger.warning('tried to get key, but null keymanager found') - return None - return self._keymanager.get_key(self._userid, OpenPGPKey, private=True) - - # - # Public API: fetch, start_loop, stop. - # - - def fetch(self): - """ - Fetch incoming mail, to be called periodically. - - Calls a deferred that will execute the fetch callback - in a separate thread - """ - def syncSoledadCallback(result): - # FIXME this needs a matching change in mx!!! - # --> need to add ERROR_DECRYPTING_KEY = False - # as default. - try: - doclist = self._soledad.get_from_index( - fields.JUST_MAIL_IDX, "*", "0") - except u1db_errors.InvalidGlobbing: - # It looks like we are a dealing with an outdated - # mx. Fallback to the version of the index - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - doclist = self._soledad.get_from_index( - fields.JUST_MAIL_COMPAT_IDX, "*") - self._process_doclist(doclist) - - logger.debug("fetching mail for: %s %s" % ( - self._soledad.uuid, self._userid)) - if not self.fetching_lock.locked(): - d1 = self._sync_soledad() - d = defer.gatherResults([d1], consumeErrors=True) - d.addCallbacks(syncSoledadCallback, self._errback) - d.addCallbacks(self._signal_fetch_to_ui, self._errback) - return d - else: - logger.debug("Already fetching mail.") - - def start_loop(self): - """ - Starts a loop to fetch mail. - """ - if self._loop is None: - self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) - else: - logger.warning("Tried to start an already running fetching loop.") - - def stop(self): - # XXX change the name to stop_loop, for consistency. - """ - Stops the loop that fetches mail. - """ - if self._loop and self._loop.running is True: - self._loop.stop() - self._loop = None - - # - # Private methods. - # - - # synchronize incoming mail - - def _errback(self, failure): - logger.exception(failure.value) - traceback.print_tb(*sys.exc_info()) - - @deferred_to_thread - def _sync_soledad(self): - """ - Synchronize with remote soledad. - - :returns: a list of LeapDocuments, or None. - :rtype: iterable or None - """ - with self.fetching_lock: - try: - log.msg('FETCH: syncing soledad...') - self._soledad.sync() - log.msg('FETCH soledad SYNCED.') - except InvalidAuthTokenError: - # if the token is invalid, send an event so the GUI can - # disable mail and show an error message. - leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) - - def _signal_fetch_to_ui(self, doclist): - """ - Send leap events to ui. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: doclist - :rtype: iterable - """ - doclist = first(doclist) # gatherResults pass us a list - if doclist: - fetched_ts = time.mktime(time.gmtime()) - num_mails = len(doclist) if doclist is not None else 0 - if num_mails != 0: - log.msg("there are %s mails" % (num_mails,)) - leap_events.signal( - IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) - return doclist - - def _signal_unread_to_ui(self, *args): - """ - Sends unread event to ui. - """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) - - # process incoming mail. - - def _process_doclist(self, doclist): - """ - Iterates through the doclist, checks if each doc - looks like a message, and yields a deferred that will decrypt and - process the message. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: a list of deferreds for individual messages. - """ - log.msg('processing doclist') - if not doclist: - logger.debug("no docs found") - return - num_mails = len(doclist) - - for index, doc in enumerate(doclist): - logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - leap_events.signal( - IMAP_MSG_PROCESSING, str(index), str(num_mails)) - - keys = doc.content.keys() - - # TODO Compatibility check with the index in pre-0.6 mx - # that does not write the ERROR_DECRYPTING_KEY - # This should be removed in 0.7 - - has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) - if has_errors is None: - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - if has_errors: - logger.debug("skipping msg with decrypting errors...") - - if self._is_msg(keys) and not has_errors: - # Evaluating to bool of has_errors is intentional here. - # We don't mind at this point if it's None or False. - - # Ok, this looks like a legit msg, and with no errors. - # Let's process it! - - d1 = self._decrypt_doc(doc) - d = defer.gatherResults([d1], consumeErrors=True) - d.addCallbacks(self._add_message_locally, self._errback) - - # - # operations on individual messages - # - - @deferred_to_thread - def _decrypt_doc(self, doc): - """ - Decrypt the contents of a document. - - :param doc: A document containing an encrypted message. - :type doc: SoledadDocument - - :return: A tuple containing the document and the decrypted message. - :rtype: (SoledadDocument, str) - """ - log.msg('decrypting msg') - success = False - - try: - decrdata = self._keymanager.decrypt( - doc.content[ENC_JSON_KEY], - self._pkey) - success = True - except Exception as exc: - # XXX move this to errback !!! - logger.error("Error while decrypting msg: %r" % (exc,)) - decrdata = "" - leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - - data = self._process_decrypted_doc((doc, decrdata)) - return (doc, data) - - def _process_decrypted_doc(self, msgtuple): - """ - Process a document containing a succesfully decrypted message. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - :return: a SoledadDocument and the processed data. - :rtype: (doc, data) - """ - log.msg('processing decrypted doc') - doc, data = msgtuple - - from twisted.internet import reactor - - # XXX turn this into an errBack for each one of - # the deferreds that would process an individual document - try: - msg = json_loads(data) - except UnicodeError as exc: - logger.error("Error while decrypting %s" % (doc.doc_id,)) - logger.exception(exc) - - # we flag the message as "with decrypting errors", - # to avoid further decryption attempts during sync - # cycles until we're prepared to deal with that. - # What is the same, when Ivan deals with it... - # A new decrypting attempt event could be triggered by a - # future a library upgrade, or a cli flag to the client, - # we just `defer` that for now... :) - doc.content[fields.ERROR_DECRYPTING_KEY] = True - deferLater(reactor, 0, self._update_incoming_message, doc) - - # FIXME this is just a dirty hack to delay the proper - # deferred organization here... - # and remember, boys, do not do this at home. - return [] - - if not isinstance(msg, dict): - defer.returnValue(False) - if not msg.get(fields.INCOMING_KEY, False): - defer.returnValue(False) - - # ok, this is an incoming message - rawmsg = msg.get(self.CONTENT_KEY, None) - if not rawmsg: - return False - return self._maybe_decrypt_msg(rawmsg) - - @deferred_to_thread - def _update_incoming_message(self, doc): - """ - Do a put for a soledad document. This probably has been called only - in the case that we've needed to update the ERROR_DECRYPTING_KEY - flag in an incoming message, to get it out of the decrypting queue. - - :param doc: the SoledadDocument to update - :type doc: SoledadDocument - """ - log.msg("Updating SoledadDoc %s" % (doc.doc_id)) - self._soledad.put_doc(doc) - - @deferred_to_thread - def _delete_incoming_message(self, doc): - """ - Delete document. - - :param doc: the SoledadDocument to delete - :type doc: SoledadDocument - """ - log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) - self._soledad.delete_doc(doc) - - def _maybe_decrypt_msg(self, data): - """ - Tries to decrypt a gpg message if data looks like one. - - :param data: the text to be decrypted. - :type data: str - :return: data, possibly descrypted. - :rtype: str - """ - leap_assert_type(data, str) - log.msg('maybe decrypting doc') - - # parse the original message - encoding = get_email_charset(data) - msg = self._parser.parsestr(data) - - # try to obtain sender public key - senderPubkey = None - fromHeader = msg.get('from', None) - if (fromHeader is not None - and (msg.get_content_type() == MULTIPART_ENCRYPTED - or msg.get_content_type() == MULTIPART_SIGNED)): - _, senderAddress = parseaddr(fromHeader) - try: - senderPubkey = self._keymanager.get_key_from_cache( - senderAddress, OpenPGPKey) - except keymanager_errors.KeyNotFound: - pass - - valid_sig = False # we will add a header saying if sig is valid - decrypt_multi = self._decrypt_multipart_encrypted_msg - decrypt_inline = self._maybe_decrypt_inline_encrypted_msg - - if msg.get_content_type() == MULTIPART_ENCRYPTED: - decrmsg, valid_sig = decrypt_multi( - msg, encoding, senderPubkey) - else: - decrmsg, valid_sig = decrypt_inline( - msg, encoding, senderPubkey) - - # add x-leap-signature header - if senderPubkey is None: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_COULD_NOT_VERIFY) - else: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_VALID if valid_sig else - self.LEAP_SIGNATURE_INVALID, - pubkey=senderPubkey.key_id) - - return decrmsg.as_string() - - def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey): - """ - Decrypt a message with content-type 'multipart/encrypted'. - - :param msg: The original encrypted message. - :type msg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderPubkey: The key of the sender of the message. - :type senderPubkey: OpenPGPKey - - :return: A unitary tuple containing a decrypted message. - :rtype: (Message) - """ - log.msg('decrypting multipart encrypted msg') - msg = copy.deepcopy(msg) - self._msg_multipart_sanity_check(msg) - - # parse message and get encrypted content - pgpencmsg = msg.get_payload()[1] - encdata = pgpencmsg.get_payload() - - # decrypt or fail gracefully - try: - decrdata, valid_sig = self._decrypt_and_verify_data( - encdata, senderPubkey) - except keymanager_errors.DecryptError as e: - logger.warning('Failed to decrypt encrypted message (%s). ' - 'Storing message without modifications.' % str(e)) - # Bailing out! - return (msg, False) - - decrmsg = self._parser.parsestr(decrdata) - # remove original message's multipart/encrypted content-type - del(msg['content-type']) - - # replace headers back in original message - for hkey, hval in decrmsg.items(): - try: - # this will raise KeyError if header is not present - msg.replace_header(hkey, hval) - except KeyError: - msg[hkey] = hval - - # all ok, replace payload by unencrypted payload - msg.set_payload(decrmsg.get_payload()) - return (msg, valid_sig) - - def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, - senderPubkey): - """ - Possibly decrypt an inline OpenPGP encrypted message. - - :param origmsg: The original, possibly encrypted message. - :type origmsg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderPubkey: The key of the sender of the message. - :type senderPubkey: OpenPGPKey - - :return: A tuple containing a decrypted message and - a bool indicating whether the signature is valid. - :rtype: (Message, bool) - """ - log.msg('maybe decrypting inline encrypted msg') - # serialize the original message - buf = StringIO() - g = Generator(buf) - g.flatten(origmsg) - data = buf.getvalue() - # handle exactly one inline PGP message - valid_sig = False - if PGP_BEGIN in data: - begin = data.find(PGP_BEGIN) - end = data.find(PGP_END) - pgp_message = data[begin:end + len(PGP_END)] - try: - decrdata, valid_sig = self._decrypt_and_verify_data( - pgp_message, senderPubkey) - # replace encrypted by decrypted content - data = data.replace(pgp_message, decrdata) - except keymanager_errors.DecryptError: - logger.warning('Failed to decrypt potential inline encrypted ' - 'message. Storing message as is...') - - # if message is not encrypted, return raw data - if isinstance(data, unicode): - data = data.encode(encoding, 'replace') - return (self._parser.parsestr(data), valid_sig) - - def _decrypt_and_verify_data(self, data, senderPubkey): - """ - Decrypt C{data} using our private key and attempt to verify a - signature using C{senderPubkey}. - - :param data: The text to be decrypted. - :type data: unicode - :param senderPubkey: The public key of the sender of the message. - :type senderPubkey: OpenPGPKey - - :return: The decrypted data and a boolean stating whether the - signature could be verified. - :rtype: (str, bool) - - :raise DecryptError: Raised if failed to decrypt. - """ - log.msg('decrypting and verifying data') - valid_sig = False - try: - decrdata = self._keymanager.decrypt( - data, self._pkey, - verify=senderPubkey) - if senderPubkey is not None: - valid_sig = True - except keymanager_errors.InvalidSignature: - decrdata = self._keymanager.decrypt( - data, self._pkey) - return (decrdata, valid_sig) - - def _add_message_locally(self, result): - """ - Adds a message to local inbox and delete it from the incoming db - in soledad. - - # XXX this comes from a gatherresult... - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - """ - from twisted.internet import reactor - msgtuple = first(result) - - doc, data = msgtuple - log.msg('adding message %s to local db' % (doc.doc_id,)) - - if isinstance(data, list): - if empty(data): - return False - data = data[0] - - def msgSavedCallback(result): - if not empty(result): - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - deferLater(reactor, 0, self._delete_incoming_message, doc) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) - - d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), - notify_on_disk=True) - d.addCallbacks(msgSavedCallback, self._errback) - - # - # helpers - # - - def _msg_multipart_sanity_check(self, msg): - """ - Performs a sanity check against a multipart encrypted msg - - :param msg: The original encrypted message. - :type msg: Message - """ - # sanity check - payload = msg.get_payload() - if len(payload) != 2: - raise MalformedMessage( - 'Multipart/encrypted messages should have exactly 2 body ' - 'parts (instead of %d).' % len(payload)) - if payload[0].get_content_type() != 'application/pgp-encrypted': - raise MalformedMessage( - "Multipart/encrypted messages' first body part should " - "have content type equal to 'application/pgp-encrypted' " - "(instead of %s)." % payload[0].get_content_type()) - if payload[1].get_content_type() != 'application/octet-stream': - raise MalformedMessage( - "Multipart/encrypted messages' second body part should " - "have content type equal to 'octet-stream' (instead of " - "%s)." % payload[1].get_content_type()) - - def _is_msg(self, keys): - """ - Checks if the keys of a dictionary match the signature - of the document type we use for messages. - - :param keys: iterable containing the strings to match. - :type keys: iterable of strings. - :rtype: bool - """ - return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py deleted file mode 100644 index 4576939..0000000 --- a/src/leap/mail/imap/fields.py +++ /dev/null @@ -1,173 +0,0 @@ -# -*- coding: utf-8 -*- -# fields.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Fields for Mailbox and Message. -""" -from leap.mail.imap.parser import MBoxParser - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # indexing - CONTENT_HASH_KEY = "chash" - PAYLOAD_HASH_KEY = "phash" - - # Internal representation of Message - - # flags doc - UID_KEY = "uid" - MBOX_KEY = "mbox" - SEEN_KEY = "seen" - DEL_KEY = "deleted" - RECENT_KEY = "recent" - FLAGS_KEY = "flags" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # headers - HEADERS_KEY = "headers" - DATE_KEY = "date" - SUBJECT_KEY = "subject" - PARTS_MAP_KEY = "part_map" - BODY_KEY = "body" # link to phash of body - MSGID_KEY = "msgid" - - # content - LINKED_FROM_KEY = "lkf" # XXX not implemented yet! - RAW_KEY = "raw" - CTYPE_KEY = "ctype" - - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" - RECENTFLAGS_KEY = "rct" - HDOCS_SET_KEY = "hdocset" - - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - TYPE_HEADERS_VAL = "head" - TYPE_CONTENT_VAL = "cnt" - TYPE_RECENT_VAL = "rct" - TYPE_HDOCS_SET_VAL = "hdocset" - - INBOX_VAL = "inbox" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - - # Index types - # -------------- - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MSGID_IDX = 'by-type-and-message-id' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' - TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' - TYPE_C_HASH_IDX = 'by-type-and-contenthash' - TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' - TYPE_P_HASH_IDX = 'by-type-and-payloadhash' - - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - # Soledad index for incoming mail, without decrypting errors. - JUST_MAIL_IDX = "just-mail" - # XXX the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX = "just-mail-compat" - - INCOMING_KEY = "incoming" - ERROR_DECRYPTING_KEY = "errdecr" - - KTYPE = TYPE_KEY - MBOX_VAL = TYPE_MBOX_VAL - CHASH_VAL = CONTENT_HASH_KEY - PHASH_VAL = PAYLOAD_HASH_KEY - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # fdocs uniqueness - TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], - - # headers doc - search by msgid. - TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], - - # content, headers doc - TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - - # attachment payload dedup - TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - - # incoming queue - JUST_MAIL_IDX: [INCOMING_KEY, - "bool(%s)" % (ERROR_DECRYPTING_KEY,)], - - # the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - TYPE_KEY: MBOX_KEY, - TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - SUBJECT_KEY: "", - FLAGS_KEY: [], - CLOSED_KEY: False, - SUBSCRIBED_KEY: False, - RW_KEY: 1, - LAST_UID_KEY: 0 - } - -fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py deleted file mode 100644 index 5f0919a..0000000 --- a/src/leap/mail/imap/index.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# index.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Index for SoledadBackedAccount, Mailbox and Messages. -""" -import logging - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail.imap.fields import fields - - -logger = logging.getLogger(__name__) - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - db_indexes = dict() - if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py deleted file mode 100644 index c906278..0000000 --- a/src/leap/mail/imap/interfaces.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- -# interfaces.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Interfaces for the IMAP module. -""" -from zope.interface import Interface, Attribute - - -class IMessageContainer(Interface): - """ - I am a container around the different documents that a message - is split into. - """ - fdoc = Attribute('The flags document for this message, if any.') - hdoc = Attribute('The headers document for this message, if any.') - cdocs = Attribute('The dict of content documents for this message, ' - 'if any.') - - def walk(self): - """ - Return an iterator to the docs for all the parts. - - :rtype: iterator - """ - - -class IMessageStore(Interface): - """ - I represent a generic storage for LEAP Messages. - """ - - def create_message(self, mbox, uid, message): - """ - Put the passed message into this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :param message: a IMessageContainer implementor. - """ - - def put_message(self, mbox, uid, message): - """ - Put the passed message into this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :param message: a IMessageContainer implementor. - """ - - def remove_message(self, mbox, uid): - """ - Remove the given message from this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - """ - - def get_message(self, mbox, uid): - """ - Get a IMessageContainer for the given mbox and uid combination. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :return: IMessageContainer - """ - - -class IMessageStoreWriter(Interface): - """ - I represent a storage that is able to write its contents to another - different IMessageStore. - """ - - def write_messages(self, store): - """ - Write the documents in this IMessageStore to a different - storage. Usually this will be done from a MemoryStorage to a DbStorage. - - :param store: another IMessageStore implementor. - """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 34cf535..c52a2e3 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,36 +15,38 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Soledad Mailbox. +IMAP Mailbox. """ -import copy -import threading +import re import logging -import StringIO -import cStringIO import os +import cStringIO +import StringIO +import time from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer -from twisted.internet.task import deferLater +from twisted.internet import reactor from twisted.python import log from twisted.mail import imap4 from zope.interface import implements -from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.check import leap_assert, leap_assert_type -from leap.mail.decorators import deferred_to_thread -from leap.mail.utils import empty -from leap.mail.imap.fields import WithMsgFields, fields -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.parser import MBoxParser +from leap.common.check import leap_assert +from leap.common.check import leap_assert_type +from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage logger = logging.getLogger(__name__) +# TODO LIST +# [ ] Restore profile_cmd instrumentation +# [ ] finish the implementation of IMailboxListener +# [ ] implement the rest of ISearchableMailbox + + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. @@ -53,7 +55,6 @@ NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - import time def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 @@ -70,33 +71,32 @@ if PROFILE_CMD: d.addCallback(_debugProfiling, name, time.time()) d.addErrback(lambda f: log.msg(f.getTraceback())) +INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, + MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, + MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, + MessageFlags.LIST_FLAG) -class SoledadMailbox(WithMsgFields, MBoxParser): + +class IMAPMailbox(object): """ A Soledad-backed IMAP mailbox. Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. + The low-level database methods are contained in the generic + MessageCollection class. We receive an instance of it and it is made + accessible in the `collection` attribute. """ implements( imap4.IMailbox, imap4.IMailboxInfo, - imap4.ICloseableMailbox, imap4.ISearchableMailbox, + # XXX I think we do not need to implement CloseableMailbox, do we? + # We could remove ourselves from the collectionListener, although I + # think it simply will be garbage collected. + # imap4.ICloseableMailbox imap4.IMessageCopier) - # XXX should finish the implementation of IMailboxListener - # XXX should completely implement ISearchableMailbox too - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None + init_flags = INIT_FLAGS CMD_MSG = "MESSAGES" CMD_RECENT = "RECENT" @@ -104,65 +104,25 @@ class SoledadMailbox(WithMsgFields, MBoxParser): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" - # FIXME we should turn this into a datastructure with limited capacity + # TODO we should turn this into a datastructure with limited capacity _listeners = defaultdict(set) - next_uid_lock = threading.Lock() - last_uid_lock = threading.Lock() - - # TODO unify all the `primed` dicts - _fdoc_primed = {} - _last_uid_primed = {} - _known_uids_primed = {} - - def __init__(self, mbox, soledad, memstore, rw=1): + def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. - - :param mbox: the mailbox name - :type mbox: str - - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + :param collection: instance of MessageCollection + :type collection: MessageCollection :param rw: read-and-write flag for this mailbox :type rw: int """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") - - from twisted.internet import reactor - self.reactor = reactor - - self.mbox = self._parse_mailbox_name(mbox) self.rw = rw - - self._soledad = soledad - self._memstore = memstore - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad, memstore=self._memstore) - self._uidvalidity = None + self.collection = collection + self.collection.addListener(self) - # XXX careful with this get/set (it would be - # hitting db unconditionally, move to memstore too) - # Now it's returning a fixed amount of flags from mem - # as a workaround. - if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - if self._memstore: - self.prime_known_uids_to_memstore() - self.prime_last_uid_to_memstore() - self.prime_flag_docs_to_memstore() - - # purge memstore from empty fdocs. - self._memstore.purge_fdoc_store(mbox) + @property + def mbox_name(self): + return self.collection.mbox_name @property def listeners(self): @@ -175,11 +135,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: set """ - return self._listeners[self.mbox] + return self._listeners[self.mbox_name] + + def get_imap_message(self, message): + d = defer.Deferred() + IMAPMessage(message, store=self.collection.store, d=d) + return d - # TODO this grows too crazily when many instances are fired, like + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. + def addListener(self, listener): """ Add a listener to the listeners queue. @@ -192,8 +158,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not NOTIFY_NEW: return - logger.debug('adding mailbox listener: %s' % listener) - self.listeners.add(listener) + listeners = self.listeners + logger.debug('adding mailbox listener: %s. Total: %s' % ( + listener, len(listeners))) + listeners.add(listener) def removeListener(self, listener): """ @@ -204,17 +172,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ self.listeners.remove(listener) - def _get_mbox_doc(self): - """ - Return mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - return self._memstore.get_mbox_doc(self.mbox) - - # XXX the memstore->soledadstore method in memstore is not complete def getFlags(self): """ Returns the flags defined for this mailbox. @@ -222,12 +179,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - flags = self._memstore.get_mbox_flags(self.mbox) + flags = self.collection.mbox_wrapper.flags if not flags: - flags = self.INIT_FLAGS - return map(str, flags) + flags = self.init_flags + flags_str = map(str, flags) + return flags_str - # XXX the memstore->soledadstore method in memstore is not complete def setFlags(self, flags): """ Sets flags for this mailbox. @@ -236,87 +193,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type flags: tuple of str """ # XXX this is setting (overriding) old flags. + # Better pass a mode flag leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") - self._memstore.set_mbox_flags(self.mbox, flags) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. - - def _get_closed(self): - """ - Return the closed attribute for this mailbox. - - :return: True if the mailbox is closed - :rtype: bool - """ - return self._memstore.get_mbox_closed(self.mbox) - - def _set_closed(self, closed): - """ - Set the closed attribute for this mailbox. - - :param closed: the state to be set - :type closed: bool - """ - self._memstore.set_mbox_closed(self.mbox, closed) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - If we have a memory store, the last UID will be the highest - recorded UID in the message store, or a counter cached from - the mailbox document in soledad if this is higher. - - :return: the last uid for messages in this mailbox - :rtype: int - """ - last = self._memstore.get_last_uid(self.mbox) - logger.debug("last uid for %s: %s (from memstore)" % ( - repr(self.mbox), last)) - return last - - last_uid = property( - _get_last_uid, doc="Last_UID attribute.") - - def prime_last_uid_to_memstore(self): - """ - Prime memstore with last_uid value - """ - primed = self._last_uid_primed.get(self.mbox, False) - if not primed: - mbox = self._get_mbox_doc() - if mbox is None: - # memory-only store - return - last = mbox.content.get('lastuid', 0) - logger.info("Priming Soledad last_uid to %s" % (last,)) - self._memstore.set_last_soledad_uid(self.mbox, last) - self._last_uid_primed[self.mbox] = True - - def prime_known_uids_to_memstore(self): - """ - Prime memstore with the set of all known uids. - - We do this to be able to filter the requests efficiently. - """ - primed = self._known_uids_primed.get(self.mbox, False) - if not primed: - known_uids = self.messages.all_soledad_uid_iter() - self._memstore.set_known_uids(self.mbox, known_uids) - self._known_uids_primed[self.mbox] = True - - def prime_flag_docs_to_memstore(self): - """ - Prime memstore with all the flags documents. - """ - primed = self._fdoc_primed.get(self.mbox, False) - if not primed: - all_flag_docs = self.messages.get_all_soledad_flag_docs() - self._memstore.load_flag_docs(self.mbox, all_flag_docs) - self._fdoc_primed[self.mbox] = True + return self.collection.set_mbox_attr("flags", flags) def getUIDValidity(self): """ @@ -325,14 +205,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: unique validity identifier :rtype: int """ - if self._uidvalidity is None: - mbox = self._get_mbox_doc() - if mbox is None: - return 0 - self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) - return self._uidvalidity + return self.collection.get_mbox_attr("created") - def getUID(self, message): + def getUID(self, message_number): """ Return the UID of a message in the mailbox @@ -340,14 +215,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): but in the future will be useful to get absolute UIDs from message sequence numbers. - :param message: the message uid + :param message: the message sequence number. :type message: int :rtype: int + :return: the UID of the message. """ - msg = self.messages.get_msg_by_uid(message) - if msg is not None: - return msg.getUID() + # TODO support relative sequences. The (imap) message should + # receive a sequence number attribute: a deferred is not expected + return message_number def getUIDNext(self): """ @@ -355,23 +231,20 @@ class SoledadMailbox(WithMsgFields, MBoxParser): mailbox. Currently it returns the higher UID incremented by one. - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int + :return: deferred with int + :rtype: Deferred """ - with self.next_uid_lock: - return self.last_uid + 1 + d = self.collection.get_uid_next() + return d def getMessageCount(self): """ Returns the total count of messages in this mailbox. - :rtype: int + :return: deferred with int + :rtype: Deferred """ - return self.messages.count() + return self.collection.count() def getUnseenCount(self): """ @@ -380,7 +253,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: count of messages flagged `unseen` :rtype: int """ - return self.messages.count_unseen() + return self.collection.count_unseen() def getRecentCount(self): """ @@ -389,7 +262,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: count of messages flagged `recent` :rtype: int """ - return self.messages.count_recent() + return self.collection.count_recent() def isWriteable(self): """ @@ -398,6 +271,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: 1 if mailbox is read-writeable, 0 otherwise. :rtype: int """ + # XXX We don't need to store it in the mbox doc, do we? + # return int(self.collection.get_mbox_attr('rw')) return self.rw def getHierarchicalDelimiter(self): @@ -417,19 +292,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type names: iter """ r = {} + maybe = defer.maybeDeferred if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() + r[self.CMD_MSG] = maybe(self.getMessageCount) if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() + r[self.CMD_RECENT] = maybe(self.getRecentCount) if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) - def addMessage(self, message, flags, date=None, notify_on_disk=False): + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d + + def addMessage(self, message, flags, date=None, notify_just_mdoc=True): """ Adds a message to this mailbox. @@ -440,51 +322,69 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type flags: list of str :param date: timestamp - :type date: str - - :return: a deferred that evals to None - """ + :type date: str, or None + + :param notify_just_mdoc: + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe). + Setting it to True improves a *lot* the responsiveness of the + APPENDS: we just need to be notified when the mdoc is saved, and + let's just expect that the other parts are doing just fine. This + will not catch any errors when the inserts of the other parts + fail, but on the other hand allows us to return very quickly, + which seems a good compromise given that we have to serialize the + appends. + However, some operations like the saving of drafts need to wait for + all the parts to be saved, so if some heuristics are met down in + the call chain a Draft message will unconditionally set this flag + to False, and therefore ignoring the setting of this flag here. + :type notify_just_mdoc: bool + + :return: a deferred that will be triggered with the UID of the added + message. + """ + # TODO should raise ReadOnlyMailbox if not rw. # TODO have a look at the cases for internal date in the rfc + # XXX we could treat the message as an IMessage from here + + # TODO change notify_just_mdoc to something more meaningful, like + # fast_insert_notify? + + # TODO notify_just_mdoc *sometimes* make the append tests fail. + # have to find a better solution for this. A workaround could probably + # be to have a list of the ongoing deferreds related to append, so that + # we queue for later all the requests having to do with these. + + # A better solution will probably involve implementing MULTIAPPEND + # extension or patching imap server to support pipelining. + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): message = message.getvalue() - # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) + if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - if PROFILE_CMD: - do_profile_cmd(d, "APPEND") - - # XXX should review now that we're not using qtreactor. - # A better place for this would be the COPY/APPEND dispatcher - # in server.py, but qtreactor hangs when I do that, so this seems - # to work fine for now. - - def notifyCallback(x): - self.reactor.callLater(0, self.notify_new) - return x + if date is None: + date = formatdate(time.time()) - d.addCallback(notifyCallback) - d.addErrback(lambda f: log.msg(f.getTraceback())) - return d - - def _do_add_message(self, message, flags, date, notify_on_disk=False): - """ - Calls to the messageCollection add_msg method. - Invoked from addMessage. - """ - d = self.messages.add_msg(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) + d = self.collection.add_msg(message, flags, date=date, + notify_just_mdoc=notify_just_mdoc) + d.addErrback(lambda failure: log.err(failure)) return d def notify_new(self, *args): """ Notify of new messages to all the listeners. + This will be called indirectly by the underlying collection, that will + notify this IMAPMailbox whenever there are changes in the number of + messages in the collection, since we have added ourselves to the + collection listeners. :param args: ignored. """ @@ -493,26 +393,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def cbNotifyNew(result): exists, recent = result - for l in self.listeners: - l.newMessages(exists, recent) + for listener in self.listeners: + listener.newMessages(exists, recent) + d = self._get_notify_count() d.addCallback(cbNotifyNew) - d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda failure: log.err(failure)) - @deferred_to_thread def _get_notify_count(self): """ Get message count and recent count for this mailbox Executed in a separate thread. Called from notify_new. - :return: number of messages and number of recent messages. - :rtype: tuple + :return: a deferred that will fire with a tuple, with number of + messages and number of recent messages. + :rtype: Deferred """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( - self.mbox, exists, recent)) - return exists, recent + d_exists = defer.maybeDeferred(self.getMessageCount) + d_recent = defer.maybeDeferred(self.getRecentCount) + d_list = [d_exists, d_recent] + + def log_num_msg(result): + exists, recent = tuple(result) + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox_name, exists, recent)) + return result + + d = defer.gatherResults(d_list) + d.addCallback(log_num_msg) + return d # commands, do not rename methods @@ -522,31 +432,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Should cleanup resources, and set the \\Noselect flag on the mailbox. + """ - # XXX this will overwrite all the existing flags! + # XXX this will overwrite all the existing flags # should better simply addFlag - self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() + self.setFlags((MessageFlags.NOSELECT_FLAG,)) - # XXX removing the mailbox in situ for now, - # we should postpone the removal + def remove_mbox(_): + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d - # XXX move to memory store?? - mbox_doc = self._get_mbox_doc() - if mbox_doc is None: - # memory-only store! - return - self._soledad.delete_doc(self._get_mbox_doc()) - - def _close_cb(self, result): - self.closed = True - - def close(self): - """ - Expunge and mark as closed - """ - d = self.expunge() - d.addCallback(self._close_cb) + d = self.deleteAllDocs() + d.addCallback(remove_mbox) return d def expunge(self): @@ -555,11 +455,35 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - d = defer.Deferred() - self._memstore.expunge(self.mbox, d) + return self.collection.delete_all_flagged() + + def _get_message_fun(self, uid): + """ + Return the proper method to get a message for this mailbox, depending + on the passed uid flag. + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + :rtype: callable + """ + get_message_fun = [ + self.collection.get_message_by_sequence_number, + self.collection.get_message_by_uid][uid] + return get_message_fun + + def _get_messages_range(self, messages_asked, uid=True): + + def get_range(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = defer.maybeDeferred(self._bound_seq, messages_asked, uid) + if uid: + d.addCallback(get_range) + d.addErrback(lambda f: log.err(f)) return d - def _bound_seq(self, messages_asked): + def _bound_seq(self, messages_asked, uid): """ Put an upper bound to a messages sequence if this is open. @@ -567,15 +491,27 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type messages_asked: MessageSet :rtype: MessageSet """ + + def set_last_uid(last_uid): + messages_asked.last = last_uid + return messages_asked + + def set_last_seq(all_uid): + messages_asked.last = len(all_uid) + return messages_asked + if not messages_asked.last: try: iter(messages_asked) except TypeError: # looks like we cannot iterate - try: - messages_asked.last = self.last_uid - except ValueError: - pass + if uid: + d = self.collection.get_last_uid() + d.addCallback(set_last_uid) + else: + d = self.collection.all_uid_iter() + d.addCallback(set_last_seq) + return d return messages_asked def _filter_msg_seq(self, messages_asked): @@ -587,10 +523,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :type messages_asked: MessageSet :rtype: set """ - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - return seq_messg + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(all_msg_uid): + set_asked = set(messages_asked) + set_exist = set(all_msg_uid) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d def fetch(self, messages_asked, uid): """ @@ -607,54 +549,48 @@ class SoledadMailbox(WithMsgFields, MBoxParser): otherwise. :type uid: bool - :rtype: deferred - """ - d = defer.Deferred() - self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH") - d.addCallback(self.cb_signal_unread_to_ui) + :rtype: deferred with a generator that yields... + """ + get_msg_fun = self._get_message_fun(uid) + getimapmsg = self.get_imap_message + + def get_imap_messages_for_range(msg_range): + + def _get_imap_msg(messages): + d_imapmsg = [] + for msg in messages: + d_imapmsg.append(getimapmsg(msg)) + return defer.gatherResults(d_imapmsg, consumeErrors=True) + + def _zip_msgid(imap_messages): + zipped = zip( + list(msg_range), imap_messages) + return (item for item in zipped) + + # XXX not called?? + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in msg_range: + # XXX We want cdocs because we "probably" are asked for the + # body. We should be smarter at do_FETCH and pass a parameter + # to this method in order not to prefetch cdocs if they're not + # going to be used. + d_msg.append(get_msg_fun(msgid, get_cdocs=True)) + + d = defer.gatherResults(d_msg, consumeErrors=True) + d.addCallback(_get_imap_msg) + d.addCallback(_zip_msgid) + d.addErrback(lambda failure: log.err(failure)) + return d + + d = self._get_messages_range(messages_asked, uid) + d.addCallback(get_imap_messages_for_range) + d.addErrback(lambda failure: log.err(failure)) return d - # called in thread - def _do_fetch(self, messages_asked, uid, d): - """ - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - :param d: deferred whose callback will be called with result. - :type d: Deferred - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ - # For the moment our UID is sequential, so we - # can treat them all the same. - # Change this to the flag that twisted expects when we - # switch to content-hash based index + local UID table. - - sequence = False - # sequence = True if uid == 0 else False - - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - getmsg = lambda uid: self.messages.get_msg_by_uid(uid) - - # for sequence numbers (uid = 0) - if sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") - raise NotImplementedError - else: - got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) - result = ((msgid, msg) for msgid, msg in got_msg - if msg is not None) - self.reactor.callLater(0, self.unset_recent_flags, seq_messg) - self.reactor.callFromThread(d.callback, result) - def fetch_flags(self, messages_asked, uid): """ A fast method to fetch all flags, tricking just the @@ -679,13 +615,23 @@ class SoledadMailbox(WithMsgFields, MBoxParser): MessagePart. :rtype: tuple """ + # is_sequence = True if uid == 0 else False + # XXX FIXME ----------------------------------------------------- + # imap/tests, or muas like mutt, it will choke until we implement + # sequence numbers. This is an easy hack meanwhile. + is_sequence = False + # --------------------------------------------------------------- + + if is_sequence: + raise NotImplementedError( + "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET") + d = defer.Deferred() - self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d) + reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: do_profile_cmd(d, "FETCH-ALL-FLAGS") return d - # called in thread def _do_fetch_flags(self, messages_asked, uid, d): """ :param messages_asked: IDs of the messages to retrieve information @@ -698,8 +644,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param d: deferred whose callback will be called with result. :type d: Deferred - :rtype: A tuple of two-tuples of message sequence numbers and - flagsPart + :rtype: A generator that yields two-tuples of message sequence numbers + and flagsPart """ class flagsPart(object): def __init__(self, uid, flags): @@ -712,13 +658,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def getFlags(self): return map(str, self.flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - - all_flags = self._memstore.all_flags(self.mbox) - result = ((msgid, flagsPart( - msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - self.reactor.callFromThread(d.callback, result) + def pack_flags(result): + _uid, _flags = result + return _uid, flagsPart(_uid, _flags) + + def get_flags_for_seq(sequence): + d_all_flags = [] + for msgid in sequence: + # TODO implement sequence numbers here too + d_flags_per_uid = self.collection.get_flags_by_uid(msgid) + d_flags_per_uid.addCallback(pack_flags) + d_all_flags.append(d_flags_per_uid) + gotflags = defer.gatherResults(d_all_flags) + gotflags.addCallback(get_uid_flag_generator) + return gotflags + + def get_uid_flag_generator(result): + generator = (item for item in result) + d.callback(generator) + + d_seq = self._get_messages_range(messages_asked, uid) + d_seq.addCallback(get_flags_for_seq) + return d_seq def fetch_headers(self, messages_asked, uid): """ @@ -744,7 +705,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): MessagePart. :rtype: tuple """ - # TODO how often is thunderbird doing this? + # TODO implement sequences + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError( + "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET") class headersPart(object): def __init__(self, uid, headers): @@ -769,29 +734,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): for msgid in seq_messg) return result - def cb_signal_unread_to_ui(self, result): - """ - Sends unread event to ui. - Used as a callback in several commands. - - :param result: ignored - """ - d = self._get_unseen_deferred() - d.addCallback(self.__cb_signal_unread_to_ui) - return result - - @deferred_to_thread - def _get_unseen_deferred(self): - return self.getUnseenCount() - - def __cb_signal_unread_to_ui(self, unseen): - """ - Send the unread signal to UI. - :param unseen: number of unseen messages. - :type unseen: int - """ - leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) - def store(self, messages_asked, flags, mode, uid): """ Sets the flags of one or more messages. @@ -826,17 +768,18 @@ class SoledadMailbox(WithMsgFields, MBoxParser): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self.reactor.callLater(0, self._do_store, messages_asked, flags, - mode, uid, d) + reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") - d.addCallback(self.cb_signal_unread_to_ui) - d.addErrback(lambda f: log.msg(f.getTraceback())) + + d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda f: log.err(f)) return d def _do_store(self, messages_asked, flags, mode, uid, observer): """ - Helper method, invoke set_flags method in the MessageCollection. + Helper method, invoke set_flags method in the IMAPMessageCollection. See the documentation for the `store` method for the parameters. @@ -845,14 +788,31 @@ class SoledadMailbox(WithMsgFields, MBoxParser): done. :type observer: deferred """ - # XXX implement also sequence (uid = 0) - # XXX we should prevent client from setting Recent flag? + # TODO we should prevent client from setting Recent flag + get_msg_fun = self._get_message_fun(uid) leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer) + + def set_flags_for_seq(sequence): + def return_result_dict(list_of_flags): + result = dict(zip(list(sequence), list_of_flags)) + observer.callback(result) + return result + + d_all_set = [] + for msgid in sequence: + d = get_msg_fun(msgid) + d.addCallback(lambda msg: self.collection.update_flags( + msg, flags, mode)) + d_all_set.append(d) + got_flags_setted = defer.gatherResults(d_all_set) + got_flags_setted.addCallback(return_result_dict) + return got_flags_setted + + d_seq = self._get_messages_range(messages_asked, uid) + d_seq.addCallback(set_flags_for_seq) + return d_seq # ISearchableMailbox @@ -877,23 +837,24 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: C{list} or C{Deferred} """ # TODO see if we can raise w/o interrupting flow - #:raise IllegalQueryError: Raised when query is not valid. + # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on - # but doing a quickfix for avoiding duplicat saves in the draft folder. - # See issue #4209 + # but doing a quickfix for avoiding duplicate saves in the draft + # folder. # See issue #4209 if len(query) > 2: if query[1] == 'HEADER' and query[2].lower() == "message-id": msgid = str(query[3]).strip() logger.debug("Searching for %s" % (msgid,)) - d = self.messages._get_uid_from_msgid(str(msgid)) - d1 = defer.gatherResults([d]) - # we want a list, so return it all the same - return d1 + + d = self.collection.get_uid_from_msgid(str(msgid)) + d.addCallback(lambda result: [result]) + return d # nothing implemented for any other query logger.warning("Cannot process query: %s" % (query,)) @@ -911,94 +872,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser): uid when the copy succeed. :rtype: Deferred """ - d = defer.Deferred() if PROFILE_CMD: do_profile_cmd(d, "COPY") # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. - d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - deferLater(self.reactor, 0, self._do_copy, message, d) - return d - - def _do_copy(self, message, observer): - """ - Call invoked from the deferLater in `copy`. This will - copy the flags and header documents, and pass them to the - `create_message` method in the MemoryStore, together with - the observer deferred that we've been passed along. - - :param message: an IMessage implementor - :type message: LeapMessage - :param observer: the deferred that will fire with the - UID of the message - :type observer: Deferred - """ - memstore = self._memstore - - def createCopy(result): - exist, new_fdoc = result - if exist: - # Should we signal error on the callback? - logger.warning("Destination message already exists!") - - # XXX I'm not sure if we should raise the - # errback. This actually rases an ugly warning - # in some muas like thunderbird. - # UID 0 seems a good convention for no uid. - observer.callback(0) - else: - mbox = self.mbox - uid_next = memstore.increment_last_soledad_uid(mbox) - - new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = mbox - - flags = list(new_fdoc[self.FLAGS_KEY]) - flags.append(fields.RECENT_FLAG) - new_fdoc[self.FLAGS_KEY] = tuple(set(flags)) - - # FIXME set recent! - - self._memstore.create_message( - self.mbox, uid_next, - MessageWrapper(new_fdoc), - observer=observer, - notify_on_disk=False) - - d = self._get_msg_copy(message) - d.addCallback(createCopy) - d.addErrback(lambda f: log.msg(f.getTraceback())) - - @deferred_to_thread - def _get_msg_copy(self, message): - """ - Get a copy of the fdoc for this message, and check whether - it already exists. - - :param message: an IMessage implementor - :type message: LeapMessage - :return: exist, new_fdoc - :rtype: tuple - """ - # XXX for clarity, this could be delegated to a - # MessageCollection mixin that implements copy too, and - # moved out of here. - msg = message - memstore = self._memstore - - if empty(msg.fdoc): - logger.warning("Tried to copy a MSG with no fdoc") - return - new_fdoc = copy.deepcopy(msg.fdoc.content) - fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] + # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + # deferLater(self.reactor, 0, self._do_copy, message, d) + # return d - dest_fdoc = memstore.get_fdoc_from_chash( - fdoc_chash, self.mbox) - - exist = not empty(dest_fdoc) - return exist, new_fdoc + d = self.collection.copy_msg(message.message, + self.collection.mbox_uuid) + return d # convenience fun @@ -1006,19 +892,42 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Delete all docs in this mailbox """ - docs = self.messages.get_all_docs() - for doc in docs: - self.messages._soledad.delete_doc(doc) + # FIXME not implemented + return self.collection.delete_all_docs() def unset_recent_flags(self, uid_seq): """ Unset Recent flag for a sequence of UIDs. """ - self.messages.unset_recent_flags(uid_seq) + # FIXME not implemented + return self.collection.unset_recent_flags(uid_seq) def __repr__(self): """ Representation string for this mailbox. """ - return u"<SoledadMailbox: mbox '%s' (%s)>" % ( - self.mbox, self.messages.count()) + return u"<IMAPMailbox: mbox '%s' (%s)>" % ( + self.mbox_name, self.collection.count()) + + +_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + # XXX maybe it would make sense to normalize common folders too: + # trash, sent, drafts, etc... + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py deleted file mode 100644 index e075394..0000000 --- a/src/leap/mail/imap/memorystore.py +++ /dev/null @@ -1,1333 +0,0 @@ -# -*- coding: utf-8 -*- -# memorystore.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -In-memory transient store for a LEAPIMAPServer. -""" -import contextlib -import logging -import threading -import weakref - -from collections import defaultdict -from copy import copy - -from enum import Enum -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet.task import LoopingCall -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert_type -from leap.mail import size -from leap.mail.utils import empty, phash_iter -from leap.mail.messageflow import MessageProducer -from leap.mail.imap import interfaces -from leap.mail.imap.fields import fields -from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc -from leap.mail.imap.messageparts import RecentFlagsDoc -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.messageparts import ReferenciableDict - -from leap.mail.decorators import deferred_to_thread - -logger = logging.getLogger(__name__) - - -# The default period to do writebacks to the permanent -# soledad storage, in seconds. -SOLEDAD_WRITE_PERIOD = 15 - -FDOC = MessagePartType.fdoc.name -HDOC = MessagePartType.hdoc.name -CDOCS = MessagePartType.cdocs.name -DOCS_ID = MessagePartType.docs_id.name - - -@contextlib.contextmanager -def set_bool_flag(obj, att): - """ - Set a boolean flag to True while we're doing our thing. - Just to let the world know. - """ - setattr(obj, att, True) - try: - yield True - except RuntimeError as exc: - logger.exception(exc) - finally: - setattr(obj, att, False) - - -DirtyState = Enum("DirtyState", "none dirty new") - - -class MemoryStore(object): - """ - An in-memory store to where we can write the different parts that - we split the messages into and buffer them until we write them to the - permanent storage. - - It uses MessageWrapper instances to represent the message-parts, which are - indexed by mailbox name and UID. - - It also can be passed a permanent storage as a paremeter (any implementor - of IMessageStore, in this case a SoledadStore). In this case, a periodic - dump of the messages stored in memory will be done. The period of the - writes to the permanent storage is controled by the write_period parameter - in the constructor. - """ - implements(interfaces.IMessageStore, - interfaces.IMessageStoreWriter) - - # TODO We will want to index by chash when we transition to local-only - # UIDs. - - WRITING_FLAG = "_writing" - _last_uid_lock = threading.Lock() - _fdoc_docid_lock = threading.Lock() - - def __init__(self, permanent_store=None, - write_period=SOLEDAD_WRITE_PERIOD): - """ - Initialize a MemoryStore. - - :param permanent_store: a IMessageStore implementor to dump - messages to. - :type permanent_store: IMessageStore - :param write_period: the interval to dump messages to disk, in seconds. - :type write_period: int - """ - self.reactor = reactor - - self._permanent_store = permanent_store - self._write_period = write_period - - if permanent_store is None: - self._mbox_closed = defaultdict(lambda: False) - - # Internal Storage: messages - """ - flags document store. - _fdoc_store[mbox][uid] = { 'content': 'aaa' } - """ - self._fdoc_store = defaultdict(lambda: defaultdict( - lambda: ReferenciableDict({}))) - - # Sizes - """ - {'mbox, uid': <int>} - """ - self._sizes = {} - - # Internal Storage: payload-hash - """ - fdocs:doc-id store, stores document IDs for putting - the dirty flags-docs. - """ - self._fdoc_id_store = defaultdict(lambda: defaultdict( - lambda: '')) - - # Internal Storage: content-hash:hdoc - """ - hdoc-store keeps references to - the header-documents indexed by content-hash. - - {'chash': { dict-stuff } - } - """ - self._hdoc_store = defaultdict(lambda: ReferenciableDict({})) - - # Internal Storage: payload-hash:cdoc - """ - content-docs stored by payload-hash - {'phash': { dict-stuff } } - """ - self._cdoc_store = defaultdict(lambda: ReferenciableDict({})) - - # Internal Storage: content-hash:fdoc - """ - chash-fdoc-store keeps references to - the flag-documents indexed by content-hash. - - {'chash': {'mbox-a': weakref.proxy(dict), - 'mbox-b': weakref.proxy(dict)} - } - """ - self._chash_fdoc_store = defaultdict(lambda: defaultdict(lambda: None)) - - # Internal Storage: recent-flags store - """ - recent-flags store keeps one dict per mailbox, - with the document-id of the u1db document - and the set of the UIDs that have the recent flag. - - {'mbox-a': {'doc_id': 'deadbeef', - 'set': {1,2,3,4} - } - } - """ - # TODO this will have to transition to content-hash - # indexes after we move to local-only UIDs. - - self._rflags_store = defaultdict( - lambda: {'doc_id': None, 'set': set([])}) - - """ - last-uid store keeps the count of the highest UID - per mailbox. - - {'mbox-a': 42, - 'mbox-b': 23} - """ - self._last_uid = defaultdict(lambda: 0) - - """ - known-uids keeps a count of the uids that soledad knows for a given - mailbox - - {'mbox-a': set([1,2,3])} - """ - self._known_uids = defaultdict(set) - - """ - mbox-flags is a dict containing flags for each mailbox. this is - modified from mailbox.getFlags / mailbox.setFlags - """ - self._mbox_flags = defaultdict(set) - - # New and dirty flags, to set MessageWrapper State. - self._new = set([]) - self._new_queue = set([]) - self._new_deferreds = {} - - self._dirty = set([]) - self._dirty_queue = set([]) - self._dirty_deferreds = {} - - self._rflags_dirty = set([]) - - # Flag for signaling we're busy writing to the disk storage. - setattr(self, self.WRITING_FLAG, False) - - if self._permanent_store is not None: - # this producer spits its messages to the permanent store - # consumer using a queue. We will use that to put - # our messages to be written. - self.producer = MessageProducer(permanent_store, - period=0.1) - # looping call for dumping to SoledadStore - self._write_loop = LoopingCall(self.write_messages, - permanent_store) - - # We can start the write loop right now, why wait? - self._start_write_loop() - else: - # We have a memory-only store. - self.producer = None - self._write_loop = None - - def _start_write_loop(self): - """ - Start loop for writing to disk database. - """ - if self._write_loop is None: - return - if not self._write_loop.running: - self._write_loop.start(self._write_period, now=True) - - def _stop_write_loop(self): - """ - Stop loop for writing to disk database. - """ - if self._write_loop is None: - return - if self._write_loop.running: - self._write_loop.stop() - - # IMessageStore - - # XXX this would work well for whole message operations. - # We would have to add a put_flags operation to modify only - # the flags doc (and set the dirty flag accordingly) - - def create_message(self, mbox, uid, message, observer, - notify_on_disk=True): - """ - Create the passed message into this MemoryStore. - - By default we consider that any message is a new message. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :param message: a message to be added - :type message: MessageWrapper - :param observer: the deferred that will fire with the - UID of the message. If notify_on_disk is True, - this will happen when the message is written to - Soledad. Otherwise it will fire as soon as we've - added the message to the memory store. - :type observer: Deferred - :param notify_on_disk: whether the `observer` deferred should - wait until the message is written to disk to - be fired. - :type notify_on_disk: bool - """ - log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) - key = mbox, uid - - self._add_message(mbox, uid, message, notify_on_disk) - self._new.add(key) - - if observer is not None: - if notify_on_disk: - # We store this deferred so we can keep track of the pending - # operations internally. - # TODO this should fire with the UID !!! -- change that in - # the soledad store code. - self._new_deferreds[key] = observer - - else: - # Caller does not care, just fired and forgot, so we pass - # a defer that will inmediately have its callback triggered. - self.reactor.callFromThread(observer.callback, uid) - - def put_message(self, mbox, uid, message, notify_on_disk=True): - """ - Put an existing message. - - This will also set the dirty flag on the MemoryStore. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :param message: a message to be added - :type message: MessageWrapper - :param notify_on_disk: whether the deferred that is returned should - wait until the message is written to disk to - be fired. - :type notify_on_disk: bool - - :return: a Deferred. if notify_on_disk is True, will be fired - when written to the db on disk. - Otherwise will fire inmediately - :rtype: Deferred - """ - key = mbox, uid - d = defer.Deferred() - d.addCallback(lambda result: log.msg("message PUT save: %s" % result)) - - self._dirty.add(key) - self._dirty_deferreds[key] = d - self._add_message(mbox, uid, message, notify_on_disk) - return d - - def _add_message(self, mbox, uid, message, notify_on_disk=True): - """ - Helper method, called by both create_message and put_message. - See those for parameter documentation. - """ - msg_dict = message.as_dict() - - fdoc = msg_dict.get(FDOC, None) - if fdoc is not None: - fdoc_store = self._fdoc_store[mbox][uid] - fdoc_store.update(fdoc) - chash_fdoc_store = self._chash_fdoc_store - - # content-hash indexing - chash = fdoc.get(fields.CONTENT_HASH_KEY) - chash_fdoc_store[chash][mbox] = weakref.proxy( - self._fdoc_store[mbox][uid]) - - hdoc = msg_dict.get(HDOC, None) - if hdoc is not None: - chash = hdoc.get(fields.CONTENT_HASH_KEY) - hdoc_store = self._hdoc_store[chash] - hdoc_store.update(hdoc) - - cdocs = message.cdocs - for cdoc in cdocs.values(): - phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None) - if not phash: - continue - cdoc_store = self._cdoc_store[phash] - cdoc_store.update(cdoc) - - # Update memory store size - # XXX this should use [mbox][uid] - # TODO --- this has to be deferred to thread, - # TODO add hdoc and cdocs sizes too - # it's slowing things down here. - # key = mbox, uid - # self._sizes[key] = size.get_size(self._fdoc_store[key]) - - def purge_fdoc_store(self, mbox): - """ - Purge the empty documents from a fdoc store. - Called during initialization of the SoledadMailbox - - :param mbox: the mailbox - :type mbox: str or unicode - """ - # XXX This is really a workaround until I find the conditions - # that are making the empty items remain there. - # This happens, for instance, after running several times - # the regression test, that issues a store deleted + expunge + select - # The items are being correclty deleted, but in succesive appends - # the empty items with previously deleted uids reappear as empty - # documents. I suspect it's a timing condition with a previously - # evaluated sequence being used after the items has been removed. - - for uid, value in self._fdoc_store[mbox].items(): - if empty(value): - del self._fdoc_store[mbox][uid] - - def get_docid_for_fdoc(self, mbox, uid): - """ - Return Soledad document id for the flags-doc for a given mbox and uid, - or None of no flags document could be found. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - :rtype: unicode or None - """ - with self._fdoc_docid_lock: - doc_id = self._fdoc_id_store[mbox][uid] - - if empty(doc_id): - fdoc = self._permanent_store.get_flags_doc(mbox, uid) - if empty(fdoc) or empty(fdoc.content): - return None - doc_id = fdoc.doc_id - self._fdoc_id_store[mbox][uid] = doc_id - - return doc_id - - def get_message(self, mbox, uid, dirtystate=DirtyState.none, - flags_only=False): - """ - Get a MessageWrapper for the given mbox and uid combination. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - :param dirtystate: DirtyState enum: one of `dirty`, `new` - or `none` (default) - :type dirtystate: enum - :param flags_only: whether the message should carry only a reference - to the flags document. - :type flags_only: bool - : - - :return: MessageWrapper or None - """ - if dirtystate == DirtyState.dirty: - flags_only = True - - key = mbox, uid - - fdoc = self._fdoc_store[mbox][uid] - if empty(fdoc): - return None - - new, dirty = False, False - if dirtystate == DirtyState.none: - new, dirty = self._get_new_dirty_state(key) - if dirtystate == DirtyState.dirty: - new, dirty = False, True - if dirtystate == DirtyState.new: - new, dirty = True, False - - if flags_only: - return MessageWrapper(fdoc=fdoc, - new=new, dirty=dirty, - memstore=weakref.proxy(self)) - else: - chash = fdoc.get(fields.CONTENT_HASH_KEY) - hdoc = self._hdoc_store[chash] - if empty(hdoc): - hdoc = self._permanent_store.get_headers_doc(chash) - if empty(hdoc): - return None - if not empty(hdoc.content): - self._hdoc_store[chash] = hdoc.content - hdoc = hdoc.content - cdocs = None - - pmap = hdoc.get(fields.PARTS_MAP_KEY, None) - if new and pmap is not None: - # take the different cdocs for write... - cdoc_store = self._cdoc_store - cdocs_list = phash_iter(hdoc) - cdocs = dict(enumerate( - [cdoc_store[phash] for phash in cdocs_list], 1)) - - return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs, - new=new, dirty=dirty, - memstore=weakref.proxy(self)) - - def remove_message(self, mbox, uid): - """ - Remove a Message from this MemoryStore. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - # XXX For the moment we are only removing the flags and headers - # docs. The rest we leave there polluting your hard disk, - # until we think about a good way of deorphaning. - - # XXX implement elijah's idea of using a PUT document as a - # token to ensure consistency in the removal. - - try: - del self._fdoc_store[mbox][uid] - except KeyError: - pass - - try: - key = mbox, uid - self._new.discard(key) - self._dirty.discard(key) - if key in self._sizes: - del self._sizes[key] - self._known_uids[mbox].discard(uid) - except KeyError: - pass - except Exception as exc: - logger.error("error while removing message!") - logger.exception(exc) - try: - with self._fdoc_docid_lock: - del self._fdoc_id_store[mbox][uid] - except KeyError: - pass - except Exception as exc: - logger.error("error while removing message!") - logger.exception(exc) - - # IMessageStoreWriter - - @deferred_to_thread - def write_messages(self, store): - """ - Write the message documents in this MemoryStore to a different store. - - :param store: the IMessageStore to write to - :rtype: False if queue is not empty, None otherwise. - """ - # For now, we pass if the queue is not empty, to avoid duplicate - # queuing. - # We would better use a flag to know when we've already enqueued an - # item. - - # XXX this could return the deferred for all the enqueued operations - - if not self.producer.is_queue_empty(): - return False - - if any(map(lambda i: not empty(i), (self._new, self._dirty))): - logger.info("Writing messages to Soledad...") - - # TODO change for lock, and make the property access - # is accquired - with set_bool_flag(self, self.WRITING_FLAG): - for rflags_doc_wrapper in self.all_rdocs_iter(): - self.producer.push(rflags_doc_wrapper, - state=self.producer.STATE_DIRTY) - for msg_wrapper in self.all_new_msg_iter(): - self.producer.push(msg_wrapper, - state=self.producer.STATE_NEW) - for msg_wrapper in self.all_dirty_msg_iter(): - self.producer.push(msg_wrapper, - state=self.producer.STATE_DIRTY) - - # MemoryStore specific methods. - - def get_uids(self, mbox): - """ - Get all uids for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: list - """ - return self._fdoc_store[mbox].keys() - - def get_soledad_known_uids(self, mbox): - """ - Get all uids that soledad knows about, from the memory cache. - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: list - """ - return self._known_uids.get(mbox, []) - - # last_uid - - def get_last_uid(self, mbox): - """ - Return the highest UID for a given mbox. - It will be the highest between the highest uid in the message store for - the mailbox, and the soledad integer cache. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: int - """ - uids = self.get_uids(mbox) - last_mem_uid = uids and max(uids) or 0 - last_soledad_uid = self.get_last_soledad_uid(mbox) - return max(last_mem_uid, last_soledad_uid) - - def get_last_soledad_uid(self, mbox): - """ - Get last uid for a given mbox from the soledad integer cache. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - return self._last_uid.get(mbox, 0) - - def set_last_soledad_uid(self, mbox, value): - """ - Set last uid for a given mbox in the soledad integer cache. - SoledadMailbox should prime this value during initialization. - Other methods (during message adding) SHOULD call - `increment_last_soledad_uid` instead. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - # can be long??? - # leap_assert_type(value, int) - logger.info("setting last soledad uid for %s to %s" % - (mbox, value)) - # if we already have a value here, don't do anything - with self._last_uid_lock: - if not self._last_uid.get(mbox, None): - self._last_uid[mbox] = value - - def set_known_uids(self, mbox, value): - """ - Set the value fo the known-uids set for this mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: a sequence of integers to be added to the set. - :type value: tuple - """ - current = self._known_uids[mbox] - self._known_uids[mbox] = current.union(set(value)) - - def increment_last_soledad_uid(self, mbox): - """ - Increment by one the soledad integer cache for the last_uid for - this mbox, and fire a defer-to-thread to update the soledad value. - The caller should lock the call tho this method. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - with self._last_uid_lock: - self._last_uid[mbox] += 1 - value = self._last_uid[mbox] - self.reactor.callInThread(self.write_last_uid, mbox, value) - return value - - def write_last_uid(self, mbox, value): - """ - Increment the soledad integer cache for the highest uid value. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - leap_assert_type(value, int) - if self._permanent_store: - self._permanent_store.write_last_uid(mbox, value) - - def load_flag_docs(self, mbox, flag_docs): - """ - Load the flag documents for the given mbox. - Used during initial flag docs prefetch. - - :param mbox: the mailbox - :type mbox: str or unicode - :param flag_docs: a dict with the content for the flag docs, indexed - by uid. - :type flag_docs: dict - """ - # We can do direct assignments cause we know this will only - # be called during initialization of the mailbox. - # TODO could hook here a sanity-check - # for duplicates - - fdoc_store = self._fdoc_store[mbox] - chash_fdoc_store = self._chash_fdoc_store - for uid in flag_docs: - rdict = ReferenciableDict(flag_docs[uid]) - fdoc_store[uid] = rdict - # populate chash dict too, to avoid fdoc duplication - chash = flag_docs[uid]["chash"] - chash_fdoc_store[chash][mbox] = weakref.proxy( - self._fdoc_store[mbox][uid]) - - def update_flags(self, mbox, uid, fdoc): - """ - Update the flag document for a given mbox and uid combination, - and set the dirty flag. - We could use put_message, but this is faster. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the uid of the message - :type uid: int - - :param fdoc: a dict with the content for the flag docs - :type fdoc: dict - """ - key = mbox, uid - self._fdoc_store[mbox][uid].update(fdoc) - self._dirty.add(key) - - def load_header_docs(self, header_docs): - """ - Load the flag documents for the given mbox. - Used during header docs prefetch, and during cache after - a read from soledad if the hdoc property in message did not - find its value in here. - - :param flag_docs: a dict with the content for the flag docs. - :type flag_docs: dict - """ - hdoc_store = self._hdoc_store - for chash in header_docs: - hdoc_store[chash] = ReferenciableDict(header_docs[chash]) - - def all_flags(self, mbox): - """ - Return a dictionary with all the flags for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - fdict = {} - uids = self.get_uids(mbox) - fstore = self._fdoc_store[mbox] - - for uid in uids: - try: - fdict[uid] = fstore[uid][fields.FLAGS_KEY] - except KeyError: - continue - return fdict - - def all_headers(self, mbox): - """ - Return a dictionary with all the header docs for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - headers_dict = {} - uids = self.get_uids(mbox) - fdoc_store = self._fdoc_store[mbox] - hdoc_store = self._hdoc_store - - for uid in uids: - try: - chash = fdoc_store[uid][fields.CONTENT_HASH_KEY] - hdoc = hdoc_store[chash] - if not empty(hdoc): - headers_dict[uid] = hdoc - except KeyError: - continue - return headers_dict - - # Counting sheeps... - - def count_new_mbox(self, mbox): - """ - Count the new messages by mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: number of new messages - :rtype: int - """ - return len([(m, uid) for m, uid in self._new if mbox == mbox]) - - # XXX used at all? - def count_new(self): - """ - Count all the new messages in the MemoryStore. - - :rtype: int - """ - return len(self._new) - - def count(self, mbox): - """ - Return the count of messages for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: number of messages - :rtype: int - """ - return len(self._fdoc_store[mbox]) - - def unseen_iter(self, mbox): - """ - Get an iterator for the message UIDs with no `seen` flag - for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - fdocs = self._fdoc_store[mbox] - - return [uid for uid, value - in fdocs.items() - if fields.SEEN_FLAG not in value.get(fields.FLAGS_KEY, [])] - - def get_cdoc_from_phash(self, phash): - """ - Return a content-document by its payload-hash. - - :param phash: the payload hash to check against - :type phash: str or unicode - :rtype: MessagePartDoc - """ - doc = self._cdoc_store.get(phash, None) - - # XXX return None for consistency? - - # XXX have to keep a mapping between phash and its linkage - # info, to know if this payload is been already saved or not. - # We will be able to get this from the linkage-docs, - # not yet implemented. - new = True - dirty = False - return MessagePartDoc( - new=new, dirty=dirty, store="mem", - part=MessagePartType.cdoc, - content=doc, - doc_id=None) - - def get_fdoc_from_chash(self, chash, mbox): - """ - Return a flags-document by its content-hash and a given mailbox. - Used during content-duplication detection while copying or adding a - message. - - :param chash: the content hash to check against - :type chash: str or unicode - :param mbox: the mailbox - :type mbox: str or unicode - - :return: MessagePartDoc. It will return None if the flags document - has empty content or it is flagged as \\Deleted. - """ - fdoc = self._chash_fdoc_store[chash][mbox] - - # a couple of special cases. - # 1. We might have a doc with empty content... - if empty(fdoc): - return None - - # 2. ...Or the message could exist, but being flagged for deletion. - # We want to create a new one in this case. - # Hmmm what if the deletion is un-done?? We would end with a - # duplicate... - if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []): - return None - - uid = fdoc[fields.UID_KEY] - key = mbox, uid - new = key in self._new - dirty = key in self._dirty - - return MessagePartDoc( - new=new, dirty=dirty, store="mem", - part=MessagePartType.fdoc, - content=fdoc, - doc_id=None) - - def iter_fdoc_keys(self): - """ - Return a generator through all the mbox, uid keys in the flags-doc - store. - """ - fdoc_store = self._fdoc_store - for mbox in fdoc_store: - for uid in fdoc_store[mbox]: - yield mbox, uid - - def all_new_msg_iter(self): - """ - Return generator that iterates through all new messages. - - :return: generator of MessageWrappers - :rtype: generator - """ - gm = self.get_message - # need to freeze, set can change during iteration - new = [gm(*key, dirtystate=DirtyState.new) for key in tuple(self._new)] - # move content from new set to the queue - self._new_queue.update(self._new) - self._new.difference_update(self._new) - return new - - def all_dirty_msg_iter(self): - """ - Return generator that iterates through all dirty messages. - - :return: generator of MessageWrappers - :rtype: generator - """ - gm = self.get_message - # need to freeze, set can change during iteration - dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty) - for key in tuple(self._dirty)] - # move content from new and dirty sets to the queue - - self._dirty_queue.update(self._dirty) - self._dirty.difference_update(self._dirty) - return dirty - - def all_deleted_uid_iter(self, mbox): - """ - Return a list with the UIDs for all messags - with deleted flag in a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: list of integers - :rtype: list - """ - # This *needs* to return a fixed sequence. Otherwise the dictionary len - # will change during iteration, when we modify it - fdocs = self._fdoc_store[mbox] - return [uid for uid, value - in fdocs.items() - if fields.DELETED_FLAG in value.get(fields.FLAGS_KEY, [])] - - # new, dirty flags - - def _get_new_dirty_state(self, key): - """ - Return `new` and `dirty` flags for a given message. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - :return: tuple of bools - :rtype: tuple - """ - # TODO change indexing of sets to [mbox][key] too. - # XXX should return *first* the news, and *then* the dirty... - - # TODO should query in queues too , true? - # - return map(lambda _set: key in _set, (self._new, self._dirty)) - - def set_new_queued(self, key): - """ - Add the key value to the `new-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._new_queue.add(key) - - def unset_new_queued(self, key): - """ - Remove the key value from the `new-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._new_queue.discard(key) - deferreds = self._new_deferreds - d = deferreds.get(key, None) - if d: - # XXX use a namedtuple for passing the result - # when we check it in the other side. - d.callback('%s, ok' % str(key)) - deferreds.pop(key) - - def set_dirty_queued(self, key): - """ - Add the key value to the `dirty-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._dirty_queue.add(key) - - def unset_dirty_queued(self, key): - """ - Remove the key value from the `dirty-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._dirty_queue.discard(key) - deferreds = self._dirty_deferreds - d = deferreds.get(key, None) - if d: - # XXX use a namedtuple for passing the result - # when we check it in the other side. - d.callback('%s, ok' % str(key)) - deferreds.pop(key) - - # Recent Flags - - def set_recent_flag(self, mbox, uid): - """ - Set the `Recent` flag for a given mailbox and UID. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - self._rflags_dirty.add(mbox) - self._rflags_store[mbox]['set'].add(uid) - - # TODO --- nice but unused - def unset_recent_flag(self, mbox, uid): - """ - Unset the `Recent` flag for a given mailbox and UID. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - self._rflags_store[mbox]['set'].discard(uid) - - def set_recent_flags(self, mbox, value): - """ - Set the value for the set of the recent flags. - Used from the property in the MessageCollection. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: a sequence of flags to set - :type value: sequence - """ - self._rflags_dirty.add(mbox) - self._rflags_store[mbox]['set'] = set(value) - - def load_recent_flags(self, mbox, flags_doc): - """ - Load the passed flags document in the recent flags store, for a given - mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param flags_doc: A dictionary containing the `doc_id` of the Soledad - flags-document for this mailbox, and the `set` - of uids marked with that flag. - """ - self._rflags_store[mbox] = flags_doc - - def get_recent_flags(self, mbox): - """ - Return the set of UIDs with the `Recent` flag for this mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: set, or None - """ - rflag_for_mbox = self._rflags_store.get(mbox, None) - if not rflag_for_mbox: - return None - return self._rflags_store[mbox]['set'] - - def all_rdocs_iter(self): - """ - Return an iterator through all in-memory recent flag dicts, wrapped - under a RecentFlagsDoc namedtuple. - Used for saving to disk. - - :return: a generator of RecentFlagDoc - :rtype: generator - """ - # XXX use enums - DOC_ID = "doc_id" - SET = "set" - - rflags_store = self._rflags_store - - def get_rdoc(mbox, rdict): - mbox_rflag_set = rdict[SET] - recent_set = copy(mbox_rflag_set) - # zero it! - mbox_rflag_set.difference_update(mbox_rflag_set) - return RecentFlagsDoc( - doc_id=rflags_store[mbox][DOC_ID], - content={ - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: mbox, - fields.RECENTFLAGS_KEY: list(recent_set) - }) - - return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items() - if not empty(rdict[SET])) - - # Methods that mirror the IMailbox interface - - def remove_all_deleted(self, mbox): - """ - Remove all messages flagged \\Deleted from this Memory Store only. - Called from `expunge` - - :param mbox: the mailbox - :type mbox: str or unicode - :return: a list of UIDs - :rtype: list - """ - mem_deleted = self.all_deleted_uid_iter(mbox) - for uid in mem_deleted: - self.remove_message(mbox, uid) - return mem_deleted - - def stop_and_flush(self): - """ - Stop the write loop and trigger a write to the producer. - """ - self._stop_write_loop() - if self._permanent_store is not None: - # XXX we should check if we did get a True value on this - # operation. If we got False we should retry! (queue was not empty) - self.write_messages(self._permanent_store) - self.producer.flush() - - def expunge(self, mbox, observer): - """ - Remove all messages flagged \\Deleted, from the Memory Store - and from the permanent store also. - - It first queues up a last write, and wait for the deferreds to be done - before continuing. - - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - soledad_store = self._permanent_store - if soledad_store is None: - # just-in memory store, easy then. - self._delete_from_memory(mbox, observer) - return - - # We have a soledad storage. - try: - # Stop and trigger last write - self.stop_and_flush() - # Wait on the writebacks to finish - - # XXX what if pending deferreds is empty? - pending_deferreds = (self._new_deferreds.get(mbox, []) + - self._dirty_deferreds.get(mbox, [])) - d1 = defer.gatherResults(pending_deferreds, consumeErrors=True) - d1.addCallback( - self._delete_from_soledad_and_memory, mbox, observer) - except Exception as exc: - logger.exception(exc) - - def _delete_from_memory(self, mbox, observer): - """ - Remove all messages marked as deleted from soledad and memory. - - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - mem_deleted = self.remove_all_deleted(mbox) - observer.callback(mem_deleted) - - def _delete_from_soledad_and_memory(self, result, mbox, observer): - """ - Remove all messages marked as deleted from soledad and memory. - - :param result: ignored. the result of the deferredList that triggers - this as a callback from `expunge`. - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - all_deleted = [] - soledad_store = self._permanent_store - - try: - # 1. Delete all messages marked as deleted in soledad. - logger.debug("DELETING FROM SOLEDAD ALL FOR %r" % (mbox,)) - sol_deleted = soledad_store.remove_all_deleted(mbox) - - try: - self._known_uids[mbox].difference_update(set(sol_deleted)) - except Exception as exc: - logger.exception(exc) - - # 2. Delete all messages marked as deleted in memory. - logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,)) - mem_deleted = self.remove_all_deleted(mbox) - - all_deleted = set(mem_deleted).union(set(sol_deleted)) - logger.debug("deleted %r" % all_deleted) - except Exception as exc: - logger.exception(exc) - finally: - self._start_write_loop() - - observer.callback(all_deleted) - - # Mailbox documents and attributes - - # This could be also be cached in memstore, but proxying directly - # to soledad since it's not too performance-critical. - - def get_mbox_doc(self, mbox): - """ - Return the soledad document for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: SoledadDocument or None. - """ - if self.permanent_store is not None: - return self.permanent_store.get_mbox_document(mbox) - else: - return None - - def get_mbox_closed(self, mbox): - """ - Return the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: bool - """ - if self.permanent_store is not None: - return self.permanent_store.get_mbox_closed(mbox) - else: - return self._mbox_closed[mbox] - - def set_mbox_closed(self, mbox, closed): - """ - Set the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - if self.permanent_store is not None: - self.permanent_store.set_mbox_closed(mbox, closed) - else: - self._mbox_closed[mbox] = closed - - def get_mbox_flags(self, mbox): - """ - Get the flags for a given mbox. - :rtype: list - """ - return sorted(self._mbox_flags[mbox]) - - def set_mbox_flags(self, mbox, flags): - """ - Set the mbox flags - """ - self._mbox_flags[mbox] = set(flags) - # TODO - # This should write to the permanent store!!! - - # Rename flag-documents - - def rename_fdocs_mailbox(self, old_mbox, new_mbox): - """ - Change the mailbox name for all flag documents in a given mailbox. - Used from account.rename - - :param old_mbox: name for the old mbox - :type old_mbox: str or unicode - :param new_mbox: name for the new mbox - :type new_mbox: str or unicode - """ - fs = self._fdoc_store - keys = fs[old_mbox].keys() - for k in keys: - fdoc = fs[old_mbox][k] - fdoc['mbox'] = new_mbox - fs[new_mbox][k] = fdoc - fs[old_mbox].pop(k) - self._dirty.add((new_mbox, k)) - - # Dump-to-disk controls. - - @property - def is_writing(self): - """ - Property that returns whether the store is currently writing its - internal state to a permanent storage. - - Used to evaluate whether the CHECK command can inform that the field - is clear to proceed, or waiting for the write operations to complete - is needed instead. - - :rtype: bool - """ - # FIXME this should return a deferred !!! - # XXX ----- can fire when all new + dirty deferreds - # are done (gatherResults) - return getattr(self, self.WRITING_FLAG) - - @property - def permanent_store(self): - return self._permanent_store - - # Memory management. - - def get_size(self): - """ - Return the size of the internal storage. - Use for calculating the limit beyond which we should flush the store. - - :rtype: int - """ - return reduce(lambda x, y: x + y, self._sizes, 0) diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py deleted file mode 100644 index fb1d75a..0000000 --- a/src/leap/mail/imap/messageparts.py +++ /dev/null @@ -1,586 +0,0 @@ -# messageparts.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -MessagePart implementation. Used from LeapMessage. -""" -import logging -import StringIO -import weakref - -from collections import namedtuple - -from enum import Enum -from zope.interface import implements -from twisted.mail import imap4 - -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset -from leap.mail.imap import interfaces -from leap.mail.imap.fields import fields -from leap.mail.utils import empty, first, find_charset - -MessagePartType = Enum("MessagePartType", "hdoc fdoc cdoc cdocs docs_id") - - -logger = logging.getLogger(__name__) - - -""" -A MessagePartDoc is a light wrapper around the dictionary-like -data that we pass along for message parts. It can be used almost everywhere -that you would expect a SoledadDocument, since it has a dict under the -`content` attribute. - -We also keep some metadata on it, relative in part to the message as a whole, -and sometimes to a part in particular only. - -* `new` indicates that the document has just been created. SoledadStore - should just create a new doc for all the related message parts. -* `store` indicates the type of store a given MessagePartDoc lives in. - We currently use this to indicate that the document comes from memeory, - but we should probably get rid of it as soon as we extend the use of the - SoledadStore interface along LeapMessage, MessageCollection and Mailbox. -* `part` is one of the MessagePartType enums. - -* `dirty` indicates that, while we already have the document in Soledad, - we have modified its state in memory, so we need to put_doc instead while - dumping the MemoryStore contents. - `dirty` attribute would only apply to flags-docs and linkage-docs. -* `doc_id` is the identifier for the document in the u1db database, if any. - -""" - -MessagePartDoc = namedtuple( - 'MessagePartDoc', - ['new', 'dirty', 'part', 'store', 'content', 'doc_id']) - -""" -A RecentFlagsDoc is used to send the recent-flags document payload to the -SoledadWriter during dumps. -""" -RecentFlagsDoc = namedtuple( - 'RecentFlagsDoc', - ['content', 'doc_id']) - - -class ReferenciableDict(dict): - """ - A dict that can be weak-referenced. - - Some builtin objects are not weak-referenciable unless - subclassed. So we do. - - Used to return pointers to the items in the MemoryStore. - """ - - -class MessageWrapper(object): - """ - A simple nested dictionary container around the different message subparts. - """ - implements(interfaces.IMessageContainer) - - FDOC = "fdoc" - HDOC = "hdoc" - CDOCS = "cdocs" - DOCS_ID = "docs_id" - - # Using slots to limit some the memory use, - # Add your attribute here. - - __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"] - - def __init__(self, fdoc=None, hdoc=None, cdocs=None, - from_dict=None, memstore=None, - new=True, dirty=False, docs_id={}): - """ - Initialize a MessageWrapper. - """ - # TODO add optional reference to original message in the incoming - self._dict = {} - self.memstore = memstore - - self._new = new - self._dirty = dirty - - self._storetype = "mem" - - if from_dict is not None: - self.from_dict(from_dict) - else: - if fdoc is not None: - self._dict[self.FDOC] = ReferenciableDict(fdoc) - if hdoc is not None: - self._dict[self.HDOC] = ReferenciableDict(hdoc) - if cdocs is not None: - self._dict[self.CDOCS] = ReferenciableDict(cdocs) - - # This will keep references to the doc_ids to be able to put - # messages to soledad. It will be populated during the walk() to avoid - # the overhead of reading from the db. - - # XXX it really *only* make sense for the FDOC, the other parts - # should not be "dirty", just new...!!! - self._dict[self.DOCS_ID] = docs_id - - # properties - - # TODO Could refactor new and dirty properties together. - - def _get_new(self): - """ - Get the value for the `new` flag. - - :rtype: bool - """ - return self._new - - def _set_new(self, value=False): - """ - Set the value for the `new` flag, and propagate it - to the memory store if any. - - :param value: the value to set - :type value: bool - """ - self._new = value - if self.memstore: - mbox = self.fdoc.content.get('mbox', None) - uid = self.fdoc.content.get('uid', None) - if not mbox or not uid: - logger.warning("Malformed fdoc") - return - key = mbox, uid - fun = [self.memstore.unset_new_queued, - self.memstore.set_new_queued][int(value)] - fun(key) - else: - logger.warning("Could not find a memstore referenced from this " - "MessageWrapper. The value for new will not be " - "propagated") - - new = property(_get_new, _set_new, - doc="The `new` flag for this MessageWrapper") - - def _get_dirty(self): - """ - Get the value for the `dirty` flag. - - :rtype: bool - """ - return self._dirty - - def _set_dirty(self, value=True): - """ - Set the value for the `dirty` flag, and propagate it - to the memory store if any. - - :param value: the value to set - :type value: bool - """ - self._dirty = value - if self.memstore: - mbox = self.fdoc.content.get('mbox', None) - uid = self.fdoc.content.get('uid', None) - if not mbox or not uid: - logger.warning("Malformed fdoc") - return - key = mbox, uid - fun = [self.memstore.unset_dirty_queued, - self.memstore.set_dirty_queued][int(value)] - fun(key) - else: - logger.warning("Could not find a memstore referenced from this " - "MessageWrapper. The value for new will not be " - "propagated") - - dirty = property(_get_dirty, _set_dirty) - - # IMessageContainer - - @property - def fdoc(self): - """ - Return a MessagePartDoc wrapping around a weak reference to - the flags-document in this MemoryStore, if any. - - :rtype: MessagePartDoc - """ - _fdoc = self._dict.get(self.FDOC, None) - if _fdoc: - content_ref = weakref.proxy(_fdoc) - else: - logger.warning("NO FDOC!!!") - content_ref = {} - - return MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.fdoc, - content=content_ref, - doc_id=self._dict[self.DOCS_ID].get( - self.FDOC, None)) - - @property - def hdoc(self): - """ - Return a MessagePartDoc wrapping around a weak reference to - the headers-document in this MemoryStore, if any. - - :rtype: MessagePartDoc - """ - _hdoc = self._dict.get(self.HDOC, None) - if _hdoc: - content_ref = weakref.proxy(_hdoc) - else: - content_ref = {} - return MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.hdoc, - content=content_ref, - doc_id=self._dict[self.DOCS_ID].get( - self.HDOC, None)) - - @property - def cdocs(self): - """ - Return a weak reference to a zero-indexed dict containing - the content-documents, or an empty dict if none found. - If you want access to the MessagePartDoc for the individual - parts, use the generator returned by `walk` instead. - - :rtype: dict - """ - _cdocs = self._dict.get(self.CDOCS, None) - if _cdocs: - return weakref.proxy(_cdocs) - else: - return {} - - def walk(self): - """ - Generator that iterates through all the parts, returning - MessagePartDoc. Used for writing to SoledadStore. - - :rtype: generator - """ - if self._dirty: - try: - mbox = self.fdoc.content[fields.MBOX_KEY] - uid = self.fdoc.content[fields.UID_KEY] - docid_dict = self._dict[self.DOCS_ID] - docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc( - mbox, uid) - except Exception as exc: - logger.debug("Error while walking message...") - logger.exception(exc) - - if not empty(self.fdoc.content) and 'uid' in self.fdoc.content: - yield self.fdoc - if not empty(self.hdoc.content): - yield self.hdoc - for cdoc in self.cdocs.values(): - if not empty(cdoc): - content_ref = weakref.proxy(cdoc) - yield MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.cdoc, - content=content_ref, - doc_id=None) - - # i/o - - def as_dict(self): - """ - Return a dict representation of the parts contained. - - :rtype: dict - """ - return self._dict - - def from_dict(self, msg_dict): - """ - Populate MessageWrapper parts from a dictionary. - It expects the same format that we use in a - MessageWrapper. - - - :param msg_dict: a dictionary containing the parts to populate - the MessageWrapper from - :type msg_dict: dict - """ - fdoc, hdoc, cdocs = map( - lambda part: msg_dict.get(part, None), - [self.FDOC, self.HDOC, self.CDOCS]) - - for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc), - (self.CDOCS, cdocs)): - self._dict[t] = ReferenciableDict(doc) if doc else None - - -class MessagePart(object): - """ - IMessagePart implementor, to be passed to several methods - of the IMAP4Server. - It takes a subpart message and is able to find - the inner parts. - - See the interface documentation. - """ - - implements(imap4.IMessagePart) - - def __init__(self, soledad, part_map): - """ - Initializes the MessagePart. - - :param soledad: Soledad instance. - :type soledad: Soledad - :param part_map: a dictionary containing the parts map for this - message - :type part_map: dict - """ - # TODO - # It would be good to pass the uid/mailbox also - # for references while debugging. - - # We have a problem on bulk moves, and is - # that when the fetch on the new mailbox is done - # the parts maybe are not complete. - # So we should be able to fail with empty - # docs until we solve that. The ideal would be - # to gather the results of the deferred operations - # to signal the operation is complete. - #leap_assert(part_map, "part map dict cannot be null") - - self._soledad = soledad - self._pmap = part_map - - def getSize(self): - """ - Return the total size, in octets, of this message part. - - :return: size of the message, in octets - :rtype: int - """ - if empty(self._pmap): - return 0 - size = self._pmap.get('size', None) - if size is None: - logger.error("Message part cannot find size in the partmap") - size = 0 - return size - - def getBodyFile(self): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO - """ - fd = StringIO.StringIO() - if not empty(self._pmap): - multi = self._pmap.get('multi') - if not multi: - phash = self._pmap.get("phash", None) - else: - pmap = self._pmap.get('part_map') - first_part = pmap.get('1', None) - if not empty(first_part): - phash = first_part['phash'] - else: - phash = None - - if phash is None: - logger.warning("Could not find phash for this subpart!") - payload = "" - else: - payload = self._get_payload_from_document_memoized(phash) - if empty(payload): - payload = self._get_payload_from_document(phash) - - else: - logger.warning("Message with no part_map!") - payload = "" - - if payload: - content_type = self._get_ctype_from_document(phash) - charset = find_charset(content_type) - if charset is None: - charset = self._get_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - - fd.write(payload) - fd.seek(0) - return fd - - # TODO should memory-bound this memoize!!! - @memoized_method - def _get_payload_from_document_memoized(self, phash): - """ - Memoized method call around the regular method, to be able - to call the non-memoized method in case we got a None. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode or None - """ - return self._get_payload_from_document(phash) - - def _get_payload_from_document(self, phash): - """ - Return the message payload from the content document. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode or None - """ - cdocs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - - cdoc = first(cdocs) - if cdoc is None: - logger.warning( - "Could not find the content doc " - "for phash %s" % (phash,)) - payload = "" - else: - payload = cdoc.content.get(fields.RAW_KEY, "") - return payload - - # TODO should memory-bound this memoize!!! - @memoized_method - def _get_ctype_from_document(self, phash): - """ - Reeturn the content-type from the content document. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode - """ - cdocs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - - cdoc = first(cdocs) - if not cdoc: - logger.warning( - "Could not find the content doc " - "for phash %s" % (phash,)) - ctype = cdoc.content.get('ctype', "") - return ctype - - @memoized_method - def _get_charset(self, stuff): - # TODO put in a common class with LeapMessage - """ - Gets (guesses?) the charset of a payload. - - :param stuff: the stuff to guess about. - :type stuff: str or unicode - :return: charset - :rtype: unicode - """ - # XXX existential doubt 2. shouldn't we make the scope - # of the decorator somewhat more persistent? - # ah! yes! and put memory bounds. - return get_email_charset(stuff) - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - # XXX refactor together with MessagePart method - if not self._pmap: - logger.warning("No pmap in Subpart!") - return {} - headers = dict(self._pmap.get("headers", [])) - - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # default to most likely standard - charset = find_charset(headers, "utf-8") - headers2 = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() - - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - # filter original dict by negate-condition - if cond(key): - headers2[key] = value - return headers2 - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - if empty(self._pmap): - logger.warning("Could not get part map!") - return False - multi = self._pmap.get("multi", False) - return multi - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - if not self.isMultipart(): - raise TypeError - - sub_pmap = self._pmap.get("part_map", {}) - try: - part_map = sub_pmap[str(part + 1)] - except KeyError: - logger.debug("getSubpart for %s: KeyError" % (part,)) - raise IndexError - - # XXX check for validity - return MessagePart(self._soledad, part_map) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 0356600..d1c7b93 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# messages.py -# Copyright (C) 2013 LEAP +# imap/messages.py +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,189 +15,62 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -LeapMessage and MessageCollection. +IMAPMessage implementation. """ -import copy import logging -import re -import threading -import StringIO - -from collections import defaultdict -from email import message_from_string -from functools import partial - -from pycryptopp.hash import sha256 from twisted.mail import imap4 from twisted.internet import defer from zope.interface import implements -from zope.proxy import sameProxiedObjects - -from leap.common.check import leap_assert, leap_assert_type -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset -from leap.mail import walk -from leap.mail.utils import first, find_charset, lowerdict, empty -from leap.mail.utils import stringify_parts_map -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.index import IndexedDB -from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.memorystore import MessageWrapper -from leap.mail.imap.messageparts import MessagePart, MessagePartDoc -from leap.mail.imap.parser import MBoxParser - -logger = logging.getLogger(__name__) - -# TODO ------------------------------------------------------------ - -# [ ] Add ref to incoming message during add_msg -# [ ] Add linked-from info. -# * Need a new type of documents: linkage info. -# * HDOCS are linked from FDOCs (ref to chash) -# * CDOCS are linked from HDOCS (ref to chash) - -# [ ] Delete incoming mail only after successful write! -# [ ] Remove UID from syncable db. Store only those indexes locally. - -MSGID_PATTERN = r"""<([\w@.]+)>""" -MSGID_RE = re.compile(MSGID_PATTERN) - -def try_unique_query(curried): - """ - Try to execute a query that is expected to have a - single outcome, and log a warning if more than one document found. +from leap.mail.utils import find_charset, CaseInsensitiveDict - :param curried: a curried function - :type curried: callable - """ - leap_assert(callable(curried), "A callable is expected") - try: - query = curried() - if query: - if len(query) > 1: - # TODO we could take action, like trigger a background - # process to kill dupes. - name = getattr(curried, 'expected', 'doc') - logger.warning( - "More than one %s found for this mbox, " - "we got a duplicate!!" % (name,)) - return query.pop() - else: - return None - except Exception as exc: - logger.exception("Unhandled error %r" % exc) +logger = logging.getLogger(__name__) -""" -A dictionary that keeps one lock per mbox and uid. -""" -# XXX too much overhead? -fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# TODO +# [ ] Add ref to incoming message during add_msg. -class LeapMessage(fields, MBoxParser): +class IMAPMessage(object): """ - The main representation of a message. - - It indexes the messages in one mailbox by a combination - of uid+mailbox name. + The main representation of a message as seen by the IMAP Server. + This class implements the semantics specific to IMAP specification. """ - - # TODO this has to change. - # Should index primarily by chash, and keep a local-only - # UID table. - implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox, collection=None, container=None): - """ - Initializes a LeapMessage. - - :param soledad: a Soledad instance - :type soledad: Soledad - :param uid: the UID for the message. - :type uid: int or basestring - :param mbox: the mbox this message belongs to - :type mbox: str or unicode - :param collection: a reference to the parent collection object - :type collection: MessageCollection - :param container: a IMessageContainer implementor instance - :type container: IMessageContainer + def __init__(self, message, prefetch_body=True, + store=None, d=defer.Deferred()): """ - self._soledad = soledad - self._uid = int(uid) if uid is not None else None - self._mbox = self._parse_mailbox_name(mbox) - self._collection = collection - self._container = container + Get an IMAPMessage. A mail.Message is needed, since many of the methods + are proxied to that object. - self.__chash = None - self.__bdoc = None - from twisted.internet import reactor - self.reactor = reactor + If you do not need to prefetch the body of the message, you can set + `prefetch_body` to False, but the current imap server implementation + expect the getBodyFile method to return inmediately. - # XXX make these properties public + When the prefetch_body option is used, a deferred is also expected as a + parameter, and this will fire when the deferred initialization has + taken place, with this instance of IMAPMessage as a parameter. - @property - def fdoc(self): + :param message: the abstract message + :type message: mail.Message + :param prefetch_body: Whether to prefetch the content doc for the body. + :type prefetch_body: bool + :param store: an instance of soledad, or anything that behaves like it. + :param d: an optional deferred, that will be fired with the instance of + the IMAPMessage being initialized + :type d: defer.Deferred """ - An accessor to the flags document. - """ - if all(map(bool, (self._uid, self._mbox))): - fdoc = None - if self._container is not None: - fdoc = self._container.fdoc - if not fdoc: - fdoc = self._get_flags_doc() - if fdoc: - fdoc_content = fdoc.content - self.__chash = fdoc_content.get( - fields.CONTENT_HASH_KEY, None) - return fdoc - - @property - def hdoc(self): - """ - An accessor to the headers document. - """ - container = self._container - if container is not None: - hdoc = self._container.hdoc - if hdoc and not empty(hdoc.content): - return hdoc - hdoc = self._get_headers_doc() - - if container and not empty(hdoc.content): - # mem-cache it - hdoc_content = hdoc.content - chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - hdocs = {chash: hdoc_content} - container.memstore.load_header_docs(hdocs) - return hdoc + # TODO substitute the use of the deferred initialization by a factory + # function, maybe. - @property - def chash(self): - """ - An accessor to the content hash for this message. - """ - if not self.fdoc: - return None - if not self.__chash and self.fdoc: - self.__chash = self.fdoc.content.get( - fields.CONTENT_HASH_KEY, None) - return self.__chash - - @property - def bdoc(self): - """ - An accessor to the body document. - """ - if not self.hdoc: - return None - if not self.__bdoc: - self.__bdoc = self._get_body_doc() - return self.__bdoc + self.message = message + self.__body_fd = None + self.store = store + if prefetch_body: + gotbody = self.__prefetch_body_file() + gotbody.addCallback(lambda _: d.callback(self)) # IMessage implementation @@ -208,7 +81,7 @@ class LeapMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ - return self._uid + return self.message.get_uid() def getFlags(self): """ @@ -217,62 +90,7 @@ class LeapMessage(fields, MBoxParser): :return: The flags, represented as strings :rtype: tuple """ - uid = self._uid - - flags = set([]) - fdoc = self.fdoc - if fdoc: - flags = set(fdoc.content.get(self.FLAGS_KEY, None)) - - msgcol = self._collection - - # We treat the recent flag specially: gotten from - # a mailbox-level document. - if msgcol and uid in msgcol.recent_flags: - flags.add(fields.RECENT_FLAG) - if flags: - flags = map(str, flags) - return tuple(flags) - - # setFlags not in the interface spec but we use it with store command. - - def setFlags(self, flags, mode): - """ - Sets the flags for this message - - :param flags: the flags to update in the message. - :type flags: tuple of str - :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - :type mode: int - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - mbox, uid = self._mbox, self._uid - - APPEND = 1 - REMOVE = -1 - SET = 0 - - with fdoc_locks[mbox][uid]: - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) - - return map(str, newflags) + return self.message.get_flags() def getInternalDate(self): """ @@ -289,72 +107,27 @@ class LeapMessage(fields, MBoxParser): :return: An RFC822-formatted date string. :rtype: str """ - date = self.hdoc.content.get(fields.DATE_KEY, '') - return date + return self.message.get_internal_date() # # IMessagePart # - # XXX we should implement this interface too for the subparts - # so we allow nested parts... - - def getBodyFile(self): + def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading - :rtype: StringIO + :rtype: a deferred that will fire with a StringIO object. """ - def write_fd(body): - fd.write(body) + if self.__body_fd is not None: + fd = self.__body_fd fd.seek(0) return fd - # TODO refactor with getBodyFile in MessagePart - - fd = StringIO.StringIO() - - if self.bdoc is not None: - bdoc_content = self.bdoc.content - if empty(bdoc_content): - logger.warning("No BDOC content found for message!!!") - return write_fd("") - - body = bdoc_content.get(self.RAW_KEY, "") - content_type = bdoc_content.get('content-type', "") - charset = find_charset(content_type) - if charset is None: - charset = self._get_charset(body) - try: - if isinstance(body, unicode): - body = body.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - logger.debug("Attempted to encode with: %s" % charset) - body = body.encode(charset, 'replace') - finally: - return write_fd(body) - - # We are still returning funky characters from here. - else: - logger.warning("No BDOC found for message.") - return write_fd("") - - @memoized_method - def _get_charset(self, stuff): - """ - Gets (guesses?) the charset of a payload. - - :param stuff: the stuff to guess about. - :type stuff: basestring - :returns: charset - """ - # XXX shouldn't we make the scope - # of the decorator somewhat more persistent? - # ah! yes! and put memory bounds. - return get_email_charset(stuff) + if store is None: + store = self.store + return self.message.get_body_file(store) def getSize(self): """ @@ -363,17 +136,7 @@ class LeapMessage(fields, MBoxParser): :return: size of the message, in octets :rtype: int """ - size = None - if self.fdoc is not None: - fdoc_content = self.fdoc.content - size = fdoc_content.get(self.SIZE_KEY, False) - else: - logger.warning("No FLAGS doc for %s:%s" % (self._mbox, - self._uid)) - if not size: - # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len - return size + return self.message.get_size() def getHeaders(self, negate, *names): """ @@ -390,74 +153,14 @@ class LeapMessage(fields, MBoxParser): :return: A mapping of header field names to header field values :rtype: dict """ - # TODO split in smaller methods - # XXX refactor together with MessagePart method - - headers = self._get_headers() - if not headers: - logger.warning("No headers found") - return {str('content-type'): str('')} - - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - if isinstance(headers, list): - headers = dict(headers) - - # default to most likely standard - charset = find_charset(headers, "utf-8") - headers2 = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() - - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - if value.endswith(";"): - # bastards - value = value[:-1] - - # filter original dict by negate-condition - if cond(key): - headers2[key] = value - return headers2 - - def _get_headers(self): - """ - Return the headers dict for this message. - """ - if self.hdoc is not None: - hdoc_content = self.hdoc.content - headers = hdoc_content.get(self.HEADERS_KEY, {}) - return headers - - else: - logger.warning( - "No HEADERS doc for msg %s:%s" % ( - self._mbox, - self._uid)) + headers = self.message.get_headers() + return _format_headers(headers, negate, *names) def isMultipart(self): """ Return True if this message is multipart. """ - if self.fdoc: - fdoc_content = self.fdoc.content - is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) - return is_multipart - else: - logger.warning( - "No FLAGS doc for msg %s:%s" % ( - self._mbox, - self._uid)) + return self.message.is_multipart() def getSubPart(self, part): """ @@ -470,913 +173,82 @@ class LeapMessage(fields, MBoxParser): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - if not self.isMultipart(): - raise TypeError - try: - pmap_dict = self._get_part_from_parts_map(part + 1) - except KeyError: - raise IndexError - return MessagePart(self._soledad, pmap_dict) - - # - # accessors - # - - def _get_part_from_parts_map(self, part): - """ - Get a part map from the headers doc - - :raises: KeyError if key does not exist - :rtype: dict - """ - if not self.hdoc: - logger.warning("Tried to get part but no HDOC found!") - return None - - hdoc_content = self.hdoc.content - pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) - - # remember, lads, soledad is using strings in its keys, - # not integers! - return pmap[str(part)] - - # XXX moved to memory store - # move the rest too. ------------------------------------------ - def _get_flags_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - result = {} - try: - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % self._uid) - logger.exception(exc) - finally: - return result - - # TODO move to soledadstore instead of accessing soledad directly - def _get_headers_doc(self): - """ - Return the document that keeps the headers for this - message. - """ - head_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(self.chash)) - return first(head_docs) - - # TODO move to soledadstore instead of accessing soledad directly - def _get_body_doc(self): - """ - Return the document that keeps the body for this - message. - """ - hdoc_content = self.hdoc.content - body_phash = hdoc_content.get( - fields.BODY_KEY, None) - if not body_phash: - logger.warning("No body phash for this document!") - return None - - # XXX get from memstore too... - # if memstore: memstore.get_phrash - # memstore should keep a dict with weakrefs to the - # phash doc... - - if self._container is not None: - bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) - if not empty(bdoc) and not empty(bdoc.content): - return bdoc - - # no memstore, or no body doc found there - if self._soledad: - body_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(body_phash)) - return first(body_docs) - else: - logger.error("No phash in container, and no soledad found!") - - def __getitem__(self, key): - """ - Return an item from the content of the flags document, - for convenience. - - :param key: The key - :type key: str - - :return: The content value indexed by C{key} or None - :rtype: str - """ - return self.fdoc.content.get(key, None) - - def does_exist(self): - """ - Return True if there is actually a flags document for this - UID and mbox. - """ - return not empty(self.fdoc) - - -class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): - """ - A collection of messages, surprisingly. - - It is tied to a selected mailbox name that is passed to its constructor. - Implements a filter query over the messages contained in a soledad - database. - """ - - # XXX this should be able to produce a MessageSet methinks - # could validate these kinds of objects turning them - # into a template for the class. - FLAGS_DOC = "FLAGS" - HEADERS_DOC = "HEADERS" - CONTENT_DOC = "CONTENT" - """ - RECENT_DOC is a document that stores a list of the UIDs - with the recent flag for this mailbox. It deserves a special treatment - because: - (1) it cannot be set by the user - (2) it's a flag that we set inmediately after a fetch, which is quite - often. - (3) we need to be able to set/unset it in batches without doing a single - write for each element in the sequence. - """ - RECENT_DOC = "RECENT" - """ - HDOCS_SET_DOC is a document that stores a set of the Document-IDs - (the u1db index) for all the headers documents for a given mailbox. - We use it to prefetch massively all the headers for a mailbox. - This is the second massive query, after fetching all the FLAGS, that - a MUA will do in a case where we do not have local disk cache. - """ - HDOCS_SET_DOC = "HDOCS_SET" - - templates = { - - # Message Level - - FLAGS_DOC: { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, # XXX moe to a local table - fields.MBOX_KEY: fields.INBOX_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.SEEN_KEY: False, - fields.DEL_KEY: False, - fields.FLAGS_KEY: [], - fields.MULTIPART_KEY: False, - fields.SIZE_KEY: 0 - }, - - HEADERS_DOC: { - fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.DATE_KEY: "", - fields.SUBJECT_KEY: "", - - fields.HEADERS_KEY: {}, - fields.PARTS_MAP_KEY: {}, - }, - - CONTENT_DOC: { - fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, - fields.PAYLOAD_HASH_KEY: "", - fields.LINKED_FROM_KEY: [], - fields.CTYPE_KEY: "", # should index by this too - - # should only get inmutable headers parts - # (for indexing) - fields.HEADERS_KEY: {}, - fields.RAW_KEY: "", - fields.PARTS_MAP_KEY: {}, - fields.HEADERS_KEY: {}, - fields.MULTIPART_KEY: False, - }, - - # Mailbox Level - - RECENT_DOC: { - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, - fields.RECENTFLAGS_KEY: [], - }, - - HDOCS_SET_DOC: { - fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, - fields.HDOCS_SET_KEY: [], - } - - - } - - # Different locks for wrapping both the u1db document getting/setting - # and the property getting/settting in an atomic operation. - - # TODO we would abstract this to a SoledadProperty class - - _rdoc_lock = defaultdict(lambda: threading.Lock()) - _rdoc_write_lock = defaultdict(lambda: threading.Lock()) - _rdoc_read_lock = defaultdict(lambda: threading.Lock()) - _rdoc_property_lock = defaultdict(lambda: threading.Lock()) - - _initialized = {} - - def __init__(self, mbox=None, soledad=None, memstore=None): - """ - Constructor for MessageCollection. - - On initialization, we ensure that we have a document for - storing the recent flags. The nature of this flag make us wanting - to store the set of the UIDs with this flag at the level of the - MessageCollection for each mailbox, instead of treating them - as a property of each message. - - We are passed an instance of MemoryStore, the same for the - SoledadBackedAccount, that we use as a read cache and a buffer - for writes. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database. - :type mbox: str - :param soledad: Soledad database - :type soledad: Soledad instance - :param memstore: a MemoryStore instance - :type memstore: MemoryStore - """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") - - # okay, all in order, keep going... - - self.mbox = self._parse_mailbox_name(mbox) - - # XXX get a SoledadStore passed instead - self._soledad = soledad - self.memstore = memstore - - self.__rflags = None - - if not self._initialized.get(mbox, False): - try: - self.initialize_db() - # ensure that we have a recent-flags doc - self._get_or_create_rdoc() - except Exception: - logger.debug("Error initializing %r" % (mbox,)) - else: - self._initialized[mbox] = True - - from twisted.internet import reactor - self.reactor = reactor - - def _get_empty_doc(self, _type=FLAGS_DOC): - """ - Returns an empty doc for storing different message parts. - Defaults to returning a template for a flags document. - :return: a dict with the template - :rtype: dict - """ - if _type not in self.templates.keys(): - raise TypeError("Improper type passed to _get_empty_doc") - return copy.deepcopy(self.templates[_type]) - - def _get_or_create_rdoc(self): - """ - Try to retrieve the recent-flags doc for this MessageCollection, - and create one if not found. - """ - # XXX should move this to memstore too - with self._rdoc_write_lock[self.mbox]: - rdoc = self._get_recent_doc_from_soledad() - if rdoc is None: - rdoc = self._get_empty_doc(self.RECENT_DOC) - if self.mbox != fields.INBOX_VAL: - rdoc[fields.MBOX_KEY] = self.mbox - self._soledad.create_doc(rdoc) - - @deferred_to_thread - def _do_parse(self, raw): - """ - Parse raw message and return it along with - relevant information about its outer level. - - This is done in a separate thread, and the callback is passed - to `_do_add_msg` method. - - :param raw: the raw message - :type raw: StringIO or basestring - :return: msg, parts, chash, size, multi - :rtype: tuple - """ - msg = message_from_string(raw) - parts = walk.get_parts(msg) - size = len(raw) - chash = sha256.SHA256(raw).hexdigest() - multi = msg.is_multipart() - return msg, parts, chash, size, multi - - def _populate_flags(self, flags, uid, chash, size, multi): - """ - Return a flags doc. - - XXX Missing DOC ----------- - """ - fd = self._get_empty_doc(self.FLAGS_DOC) - - fd[self.MBOX_KEY] = self.mbox - fd[self.UID_KEY] = uid - fd[self.CONTENT_HASH_KEY] = chash - fd[self.SIZE_KEY] = size - fd[self.MULTIPART_KEY] = multi - if flags: - fd[self.FLAGS_KEY] = flags - fd[self.SEEN_KEY] = self.SEEN_FLAG in flags - fd[self.DEL_KEY] = self.DELETED_FLAG in flags - fd[self.RECENT_KEY] = True # set always by default - return fd - - def _populate_headr(self, msg, chash, subject, date): - """ - Return a headers doc. - - XXX Missing DOC ----------- - """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) + subpart = self.message.get_subpart(part + 1) + return IMAPMessagePart(subpart) - lower_headers = lowerdict(headers) - msgid = first(MSGID_RE.findall( - lower_headers.get('message-id', ''))) - - hd = self._get_empty_doc(self.HEADERS_DOC) - hd[self.CONTENT_HASH_KEY] = chash - hd[self.HEADERS_KEY] = headers - hd[self.MSGID_KEY] = msgid - - if not subject and self.SUBJECT_FIELD in headers: - hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - else: - hd[self.SUBJECT_KEY] = subject - - if not date and self.DATE_FIELD in headers: - hd[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - hd[self.DATE_KEY] = date - return hd - - def _fdoc_already_exists(self, chash): - """ - Check whether we can find a flags doc for this mailbox with the - given content-hash. It enforces that we can only have the same maessage - listed once for a a given mailbox. - - :param chash: the content-hash to check about. - :type chash: basestring - :return: False, if it does not exist, or UID. - """ - exist = False - exist = self.memstore.get_fdoc_from_chash(chash, self.mbox) - - if not exist: - exist = self._get_fdoc_from_chash(chash) - if exist and exist.content is not None: - return exist.content.get(fields.UID_KEY, "unknown-uid") - else: - return False - - def add_msg(self, raw, subject=None, flags=None, date=None, - notify_on_disk=False): - """ - Creates a new message document. - - :param raw: the raw message - :type raw: str - - :param subject: subject of the message. - :type subject: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :return: a deferred that will be fired with the message - uid when the adding succeed. - :rtype: deferred - """ - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - - observer = defer.Deferred() - d = self._do_parse(raw) - d.addCallback(lambda result: self.reactor.callInThread( - self._do_add_msg, result, flags, subject, date, - notify_on_disk, observer)) - return observer - - # Called in thread - def _do_add_msg(self, parse_result, flags, subject, - date, notify_on_disk, observer): - """ - Helper that creates a new message document. - Here lives the magic of the leap mail. Well, in soledad, really. - - See `add_msg` docstring for parameter info. - - :param parse_result: a tuple with the results of `self._do_parse` - :type parse_result: tuple - :param observer: a deferred that will be fired with the message - uid when the adding succeed. - :type observer: deferred - """ - # TODO signal that we can delete the original message!----- - # when all the processing is done. - - # TODO add the linked-from info ! - # TODO add reference to the original message - - msg, parts, chash, size, multi = parse_result - - # check for uniqueness -------------------------------- - # Watch out! We're reserving a UID right after this! - existing_uid = self._fdoc_already_exists(chash) - if existing_uid: - msg = self.get_msg_by_uid(existing_uid) - - # We can say the observer that we're done - self.reactor.callFromThread(observer.callback, existing_uid) - msg.setFlags((fields.DELETED_FLAG,), -1) - return - - # XXX get FUCKING UID from autoincremental table - uid = self.memstore.increment_last_soledad_uid(self.mbox) - - # We can say the observer that we're done at this point, but - # before that we should make sure it has no serious consequences - # if we're issued, for instance, a fetch command right after... - # self.reactor.callFromThread(observer.callback, uid) - # if we did the notify, we need to invalidate the deferred - # so not to try to fire it twice. - # observer = None - - fd = self._populate_flags(flags, uid, chash, size, multi) - hd = self._populate_headr(msg, chash, subject, date) - - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) - parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) - - # add parts map to header doc - # (body, multi, part_map) - for key in parts_map: - hd[key] = parts_map[key] - del parts_map - - hd = stringify_parts_map(hd) - - # The MessageContainer expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) - - self.set_recent_flag(uid) - msg_container = MessageWrapper(fd, hd, cdocs) - self.memstore.create_message( - self.mbox, uid, msg_container, - observer=observer, notify_on_disk=notify_on_disk) - - # - # getters: specific queries - # - - # recent flags - - def _get_recent_flags(self): - """ - An accessor for the recent-flags set for this mailbox. - """ - # XXX check if we should remove this - if self.__rflags is not None: - return self.__rflags - - if self.memstore is not None: - with self._rdoc_lock[self.mbox]: - rflags = self.memstore.get_recent_flags(self.mbox) - if not rflags: - # not loaded in the memory store yet. - # let's fetch them from soledad... - rdoc = self._get_recent_doc_from_soledad() - if rdoc is None: - return set([]) - rflags = set(rdoc.content.get( - fields.RECENTFLAGS_KEY, [])) - # ...and cache them now. - self.memstore.load_recent_flags( - self.mbox, - {'doc_id': rdoc.doc_id, 'set': rflags}) - return rflags - - def _set_recent_flags(self, value): - """ - Setter for the recent-flags set for this mailbox. - """ - if self.memstore is not None: - self.memstore.set_recent_flags(self.mbox, value) - - recent_flags = property( - _get_recent_flags, _set_recent_flags, - doc="Set of UIDs with the recent flag for this mailbox.") - - def _get_recent_doc_from_soledad(self): - """ - Get recent-flags document from Soledad for this mailbox. - :rtype: SoledadDocument or None - """ - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MBOX_IDX, - fields.TYPE_RECENT_VAL, self.mbox) - curried.expected = "rdoc" - with self._rdoc_read_lock[self.mbox]: - return try_unique_query(curried) - - # Property-set modification (protected by a different - # lock to give atomicity to the read/write operation) - - def unset_recent_flags(self, uids): - """ - Unset Recent flag for a sequence of uids. - - :param uids: the uids to unset - :type uid: sequence - """ - with self._rdoc_property_lock[self.mbox]: - self.recent_flags.difference_update( - set(uids)) - - # Individual flags operations - - def unset_recent_flag(self, uid): - """ - Unset Recent flag for a given uid. - - :param uid: the uid to unset - :type uid: int - """ - with self._rdoc_property_lock[self.mbox]: - self.recent_flags.difference_update( - set([uid])) - - @deferred_to_thread - def set_recent_flag(self, uid): - """ - Set Recent flag for a given uid. - - :param uid: the uid to set - :type uid: int - """ - with self._rdoc_property_lock[self.mbox]: - self.recent_flags = self.recent_flags.union( - set([uid])) - - # individual doc getters, message layer. - - def _get_fdoc_from_chash(self, chash): - """ - Return a flags document for this mailbox with a given chash. - - :return: A SoledadDocument containing the Flags Document, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MBOX_C_HASH_IDX, - fields.TYPE_FLAGS_VAL, self.mbox, chash) - curried.expected = "fdoc" - fdoc = try_unique_query(curried) - if fdoc is not None: - return fdoc - else: - # probably this should be the other way round, - # ie, try fist on memstore... - cf = self.memstore._chash_fdoc_store - fdoc = cf[chash][self.mbox] - # hey, I just needed to wrap fdoc thing into - # a "content" attribute, look a better way... - if not empty(fdoc): - return MessagePartDoc( - new=None, dirty=None, part=None, - store=None, doc_id=None, - content=fdoc) - - def _get_uid_from_msgidCb(self, msgid): - hdoc = None - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MSGID_IDX, - fields.TYPE_HEADERS_VAL, msgid) - curried.expected = "hdoc" - hdoc = try_unique_query(curried) - - # XXX this is only a quick hack to avoid regression - # on the "multiple copies of the draft" issue, but - # this is currently broken since it's not efficient to - # look for this. Should lookup better. - # FIXME! - - if hdoc is not None: - hdoc_dict = hdoc.content - - else: - hdocstore = self.memstore._hdoc_store - match = [x for _, x in hdocstore.items() if x['msgid'] == msgid] - hdoc_dict = first(match) - - if hdoc_dict is None: - logger.warning("Could not find hdoc for msgid %s" - % (msgid,)) - return None - msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY) - - fdoc = self._get_fdoc_from_chash(msg_chash) - if not fdoc: - logger.warning("Could not find fdoc for msgid %s" - % (msgid,)) - return None - return fdoc.content.get(fields.UID_KEY, None) - - @deferred_to_thread - def _get_uid_from_msgid(self, msgid): - """ - Return a UID for a given message-id. - - It first gets the headers-doc for that msg-id, and - it found it queries the flags doc for the current mailbox - for the matching content-hash. - - :return: A UID, or None - """ - # We need to wait a little bit, cause in some of the cases - # the query is received right after we've saved the document, - # and we cannot find it otherwise. This seems to be enough. - - # XXX do a deferLater instead ?? - # XXX is this working? - return self._get_uid_from_msgidCb(msgid) - - @deferred_to_thread - def set_flags(self, mbox, messages, flags, mode, observer): - """ - Set flags for a sequence of messages. - - :param mbox: the mbox this message belongs to - :type mbox: str or unicode - :param messages: the messages to iterate through - :type messages: sequence - :flags: the flags to be set - :type flags: tuple - :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - :type mode: int - :param observer: a deferred that will be called with the dictionary - mapping UIDs to flags after the operation has been - done. - :type observer: deferred - """ - reactor = self.reactor - getmsg = self.get_msg_by_uid - - def set_flags(uid, flags, mode): - msg = getmsg(uid, mem_only=True, flags_only=True) - if msg is not None: - return uid, msg.setFlags(flags, mode) - - setted_flags = [set_flags(uid, flags, mode) for uid in messages] - result = dict(filter(None, setted_flags)) - - reactor.callFromThread(observer.callback, result) - - # getters: generic for a mailbox - - def get_msg_by_uid(self, uid, mem_only=False, flags_only=False): - """ - Retrieves a LeapMessage by UID. - This is used primarity in the Mailbox fetch and store methods. - - :param uid: the message uid to query by - :type uid: int - :param mem_only: a flag that indicates whether this Message should - pass a reference to soledad to retrieve missing pieces - or not. - :type mem_only: bool - :param flags_only: whether the message should carry only a reference - to the flags document. - :type flags_only: bool - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - msg_container = self.memstore.get_message( - self.mbox, uid, flags_only=flags_only) - - if msg_container is not None: - if mem_only: - msg = LeapMessage(None, uid, self.mbox, collection=self, - container=msg_container) - else: - # We pass a reference to soledad just to be able to retrieve - # missing parts that cannot be found in the container, like - # the content docs after a copy. - msg = LeapMessage(self._soledad, uid, self.mbox, - collection=self, container=msg_container) - else: - msg = LeapMessage(self._soledad, uid, self.mbox, collection=self) - - if not msg.does_exist(): - return None - return msg - - def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - """ - Get all documents for the selected mailbox of the - passed type. By default, it returns the flag docs. - - If you want acess to the content, use __iter__ instead - - :return: a list of u1db documents - :rtype: list of SoledadDocument - """ - if _type not in fields.__dict__.values(): - raise TypeError("Wrong type passed to get_all_docs") - - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - all_docs = [doc for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - _type, self.mbox)] - - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) - - def all_soledad_uid_iter(self): - """ - Return an iterator through the UIDs of all messages, sorted in - ascending order. - """ - db_uids = set([doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc)]) - return db_uids - - def all_uid_iter(self): - """ - Return an iterator through the UIDs of all messages, from memory. - """ - mem_uids = self.memstore.get_uids(self.mbox) - soledad_known_uids = self.memstore.get_soledad_known_uids( - self.mbox) - combined = tuple(set(mem_uids).union(soledad_known_uids)) - return combined - - def get_all_soledad_flag_docs(self): - """ - Return a dict with the content of all the flag documents - in soledad store for the given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - # XXX we really could return a reduced version with - # just {'uid': (flags-tuple,) since the prefetch is - # only oriented to get the flag tuples. - all_docs = [( - doc.content[self.UID_KEY], - dict(doc.content)) - for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc.content)] - all_flags = dict(all_docs) - return all_flags - - def all_headers(self): - """ - Return a dict with all the header documents for this - mailbox. - - :rtype: dict - """ - return self.memstore.all_headers(self.mbox) - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - return self.memstore.count(self.mbox) - - # unseen messages - - def unseen_iter(self): - """ - Get an iterator for the message UIDs with no `seen` flag - for this mailbox. - - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - return self.memstore.unseen_iter(self.mbox) - - def count_unseen(self): - """ - Count all messages with the `Unseen` flag. - - :returns: count - :rtype: int - """ - return len(list(self.unseen_iter())) - - def get_unseen(self): - """ - Get all messages with the `Unseen` flag - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox, collection=self) - for docid in self.unseen_iter()] + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d - # recent messages - # XXX take it from memstore - def count_recent(self): - """ - Count all messages with the `Recent` flag. - It just retrieves the length of the recent_flags set, - which is stored in a specific type of document for - this collection. +class IMAPMessagePart(object): - :returns: count - :rtype: int - """ - return len(self.recent_flags) + def __init__(self, message_part): + self.message_part = message_part - def __len__(self): - """ - Returns the number of messages on this mailbox. + def getBodyFile(self, store=None): + return self.message_part.get_body_file() - :rtype: int - """ - return self.count() - - def __iter__(self): - """ - Returns an iterator over all messages. + def getSize(self): + return self.message_part.get_size() - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - return (LeapMessage(self._soledad, docuid, self.mbox, collection=self) - for docuid in self.all_uid_iter()) + def getHeaders(self, negate, *names): + headers = self.message_part.get_headers() + return _format_headers(headers, negate, *names) - def __repr__(self): - """ - Representation string for this object. - """ - return u"<MessageCollection: mbox '%s' (%s)>" % ( - self.mbox, self.count()) + def isMultipart(self): + return self.message_part.is_multipart() - # XXX should implement __eq__ also !!! - # use chash... + def getSubPart(self, part): + subpart = self.message_part.get_subpart(part + 1) + return IMAPMessagePart(subpart) + + +def _format_headers(headers, negate, *names): + # current server impl. expects content-type to be present, so if for + # some reason we do not have headers, we have to return at least that + # one + if not headers: + logger.warning("No headers found") + return {str('content-type'): str('')} + + names = map(lambda s: s.upper(), names) + + if negate: + def cond(key): + return key.upper() not in names + else: + def cond(key): + return key.upper() in names + + if isinstance(headers, list): + headers = dict(headers) + + # default to most likely standard + charset = find_charset(headers, "utf-8") + + # We will return a copy of the headers dictionary that + # will allow case-insensitive lookups. In some parts of the twisted imap + # server code the keys are expected to be in lower case, and in this way + # we avoid having to convert them. + + _headers = CaseInsensitiveDict() + for key, value in headers.items(): + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + if value.endswith(";"): + # bastards + value = value[:-1] + + # filter original dict by negate-condition + if cond(key): + _headers[key] = value + + return _headers diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py deleted file mode 100644 index 4a801b0..0000000 --- a/src/leap/mail/imap/parser.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# parser.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Mail parser mixin. -""" -import re - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - Return a normalized representation of the mailbox C{name}. - - This method ensures that an eventual initial 'inbox' part of a - mailbox name is made uppercase. - - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index fe56ea6..39f483f 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -15,18 +15,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Leap IMAP4 Server Implementation. +LEAP IMAP4 Server Implementation. """ +import StringIO from copy import copy from twisted import cred +from twisted.internet import reactor from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log -from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type -from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN +from leap.common.events import emit, catalog from leap.soledad.client import Soledad # imports for LITERAL+ patch @@ -35,9 +36,41 @@ from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile -class LeapIMAPServer(imap4.IMAP4Server): +def _getContentType(msg): """ - An IMAP4 Server with mailboxes backed by soledad + Return a two-tuple of the main and subtype of the given message. + """ + attrs = None + mm = msg.getHeaders(False, 'content-type').get('content-type', None) + if mm: + mm = ''.join(mm.splitlines()) + mimetype = mm.split(';') + if mimetype: + type = mimetype[0].split('/', 1) + if len(type) == 1: + major = type[0] + minor = None + elif len(type) == 2: + major, minor = type + else: + major = minor = None + # XXX patched --------------------------------------------- + attrs = dict(x.strip().split('=', 1) for x in mimetype[1:]) + # XXX patched --------------------------------------------- + else: + major = minor = None + else: + major = minor = None + return major, minor, attrs + +# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in +# BODYSTRUCTURE response. +imap4._getContentType = _getContentType + + +class LEAPIMAPServer(imap4.IMAP4Server): + """ + An IMAP4 Server with a LEAP Storage Backend. """ def __init__(self, *args, **kwargs): # pop extraneous arguments @@ -59,8 +92,87 @@ class LeapIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) - from twisted.internet import reactor - self.reactor = reactor + ############################################################# + # + # Twisted imap4 patch to workaround bad mime rendering in TB. + # See https://leap.se/code/issues/6773 + # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771 + # Still unclear if this is a thunderbird bug. + # TODO send this patch upstream + # + ############################################################# + + def spew_body(self, part, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + for p in part.part: + if msg.isMultipart(): + msg = msg.getSubPart(p) + elif p > 0: + # Non-multipart messages have an implicit first part but no + # other parts - reject any request for any other part. + raise TypeError("Requested subpart of non-multipart message") + + if part.header: + hdrs = msg.getHeaders(part.header.negate, *part.header.fields) + hdrs = imap4._formatHeaders(hdrs) + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # PATCHED ########################################## + elif part.text: + _w(str(part) + ' ') + _f() + return imap4.FileProducer( + msg.getBodyFile() + ).beginProducing(self.transport) + elif part.mime: + hdrs = imap4._formatHeaders(msg.getHeaders(True)) + + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # END PATCHED ###################################### + + elif part.empty: + _w(str(part) + ' ') + _f() + if part.part: + # PATCHED ############################################# + # implement partial FETCH + # TODO implement boundary checks + # TODO see if there's a more efficient way, without + # copying the original content into a new buffer. + fd = msg.getBodyFile() + begin = getattr(part, "partialBegin", None) + _len = getattr(part, "partialLength", None) + if begin is not None and _len is not None: + _fd = StringIO.StringIO() + fd.seek(part.partialBegin) + _fd.write(fd.read(part.partialLength)) + _fd.seek(0) + else: + _fd = fd + return imap4.FileProducer( + _fd + # END PATCHED #########################3 + ).beginProducing(self.transport) + else: + mf = imap4.IMessageFile(msg, None) + if mf is not None: + return imap4.FileProducer( + mf.open()).beginProducing(self.transport) + return imap4.MessageProducer( + msg, None, self._scheduler).beginProducing(self.transport) + + else: + _w('BODY ' + + imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) + + ################################################################## + # + # END Twisted imap4 patch to workaround bad mime rendering in TB. + # #6773 + # + ################################################################## def lineReceived(self, line): """ @@ -69,7 +181,7 @@ class LeapIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ - if self.theAccount.closed is True and self.state != "unauth": + if self.theAccount.session_ended is True and self.state != "unauth": log.msg("Closing the session. State: unauth") self.state = "unauth" @@ -82,6 +194,20 @@ class LeapIMAPServer(imap4.IMAP4Server): log.msg('rcv (%s): %s' % (self.state, msg)) imap4.IMAP4Server.lineReceived(self, line) + def close_server_connection(self): + """ + Send a BYE command so that the MUA at least knows that we're closing + the connection. + """ + self.sendLine( + '* BYE LEAP IMAP Proxy is shutting down; ' + 'so long and thanks for all the fish') + self.transport.loseConnection() + if self.mbox: + self.mbox.removeListener(self) + self.mbox = None + self.state = 'unauth' + def authenticateLogin(self, username, password): """ Lookup the account with the given parameters, and deny @@ -98,7 +224,7 @@ class LeapIMAPServer(imap4.IMAP4Server): # bad username, reject. raise cred.error.UnauthorizedLogin() # any dummy password is allowed so far. use realm instead! - leap_events.signal(IMAP_CLIENT_LOGIN, "1") + emit(catalog.IMAP_CLIENT_LOGIN, "1") return imap4.IAccount, self.theAccount, lambda: None def do_FETCH(self, tag, messages, query, uid=0): @@ -147,12 +273,15 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ - self.reactor.callFromThread(self.mbox.notify_new) + reactor.callFromThread(self.mbox.notify_new) def _cbSelectWork(self, mbox, cmdName, tag): """ - Callback for selectWork, patched to avoid conformance errors due to - incomplete UIDVALIDITY line. + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count """ if mbox is None: self.sendNegativeResponse(tag, 'No such mailbox') @@ -161,12 +290,22 @@ class LeapIMAPServer(imap4.IMAP4Server): self.sendNegativeResponse(tag, 'Mailbox cannot be selected') return + d1 = defer.maybeDeferred(mbox.getMessageCount) + d2 = defer.maybeDeferred(mbox.getRecentCount) + return defer.gatherResults([d1, d2]).addCallback( + self.__cbSelectWork, mbox, cmdName, tag) + + def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag): flags = mbox.getFlags() - self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') - self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) # Patched ------------------------------------------------------- + # accept deferreds for the count + self.sendUntaggedResponse(str(msg_count) + ' EXISTS') + self.sendUntaggedResponse(str(recent_count) + ' RECENT') + # ---------------------------------------------------------------- + + # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding # "UIDs valid" here. self.sendPositiveResponse( @@ -188,8 +327,7 @@ class LeapIMAPServer(imap4.IMAP4Server): a deferred, the client will only be informed of success (or failure) when the deferred's callback (or errback) is invoked. """ - # TODO return the output of _memstore.is_writing - # XXX and that should return a deferred! + # TODO implement a collection of ongoing deferreds? return None ############################################################# @@ -311,21 +449,247 @@ class LeapIMAPServer(imap4.IMAP4Server): return self._fileLiteral(size, literalPlus) ############################# - # Need to override the command table after patching - # arg_astring and arg_literal + # --------------------------------- isSubscribed patch + # TODO -- send patch upstream. + # There is a bug in twisted implementation: + # in cbListWork, it's assumed that account.isSubscribed IS a callable, + # although in the interface documentation it's stated that it can be + # a deferred. + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = self._parseMbox(mbox) + mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) + mailboxes.addCallback(self._cbSubscribed) + mailboxes.addCallback( + self._cbListWork, tag, sub, cmdName, + ).addErrback(self._ebListWork, tag) + + def _cbSubscribed(self, mailboxes): + subscribed = [ + maybeDeferred(self.account.isSubscribed, name) + for (name, box) in mailboxes] + + def get_mailboxes_and_subs(result): + subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] + return mailboxes, subscribed + + d = defer.gatherResults(subscribed) + d.addCallback(get_mailboxes_and_subs) + return d + + def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): + mailboxes, subscribed = mailboxes_subscribed + + for (name, box) in mailboxes: + if not sub or name in subscribed: + flags = box.getFlags() + delim = box.getHierarchicalDelimiter() + resp = (imap4.DontQuoteMe(cmdName), + map(imap4.DontQuoteMe, flags), + delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse( + imap4.collapseNestedLists(resp)) + self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) + # -------------------- end isSubscribed patch ----------- + + # TODO subscribe method had also to be changed to accomodate deferred + def do_SUBSCRIBE(self, tag, name): + name = self._parseMbox(name) + + def _subscribeCb(_): + self.sendPositiveResponse(tag, 'Subscribed') + + def _subscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while subscribing to mailbox") + + d = self.account.subscribe(name) + d.addCallbacks(_subscribeCb, _subscribeEb) + return d + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + def do_UNSUBSCRIBE(self, tag, name): + # unsubscribe method had also to be changed to accomodate + # deferred + name = self._parseMbox(name) + + def _unsubscribeCb(_): + self.sendPositiveResponse(tag, 'Unsubscribed') + + def _unsubscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while unsubscribing " + "from mailbox") + + d = self.account.unsubscribe(name) + d.addCallbacks(_unsubscribeCb, _unsubscribeEb) + return d + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [self._parseMbox(n) for n in oldname, newname] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse( + tag, + 'You cannot rename the inbox, or ' + 'rename another mailbox to inbox.') + return + def _renameCb(_): + self.sendPositiveResponse(tag, 'Mailbox renamed') + + def _renameEb(failure): + m = failure.value + if failure.check(TypeError): + self.sendBadResponse(tag, 'Invalid command syntax') + elif failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + log.err() + self.sendBadResponse( + tag, + "Server error encountered while " + "renaming mailbox") + + d = self.account.rename(oldname, newname) + d.addCallbacks(_renameCb, _renameEb) + return d + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + def do_CREATE(self, tag, name): + name = self._parseMbox(name) + + def _createCb(result): + if result: + self.sendPositiveResponse(tag, 'Mailbox created') + else: + self.sendNegativeResponse(tag, 'Mailbox not created') + + def _createEb(failure): + c = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(c)) + else: + log.err() + self.sendBadResponse( + tag, "Server error encountered while creating mailbox") + + d = self.account.create(name) + d.addCallbacks(_createCb, _createEb) + return d + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + def do_DELETE(self, tag, name): + name = self._parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, 'You cannot delete the inbox') + return + + def _deleteCb(result): + self.sendPositiveResponse(tag, 'Mailbox deleted') + + def _deleteEb(failure): + m = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + print "SERVER: other error" + log.err() + self.sendBadResponse( + tag, + "Server error encountered while deleting mailbox") + + d = self.account.delete(name) + d.addCallbacks(_deleteCb, _deleteEb) + return d + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + + # ----------------------------------------------------------------------- + # Patched just to allow __cbAppend to receive a deferred from messageCount + # TODO format and send upstream. + def do_APPEND(self, tag, mailbox, flags, date, message): + mailbox = self._parseMbox(mailbox) + maybeDeferred(self.account.select, mailbox).addCallback( + self._cbAppendGotMailbox, tag, flags, date, message).addErrback( + self._ebAppendGotMailbox, tag) + + def __ebAppend(self, failure, tag): + self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value)) + + def _cbAppendGotMailbox(self, mbox, tag, flags, date, message): + if not mbox: + self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox') + return + + d = mbox.addMessage(message, flags, date) + d.addCallback(self.__cbAppend, tag, mbox) + d.addErrback(self.__ebAppend, tag) + + def _ebAppendGotMailbox(self, failure, tag): + self.sendBadResponse( + tag, "Server error encountered while opening mailbox.") + log.err(failure) + + def __cbAppend(self, result, tag, mbox): + + # XXX patched --------------------------------- + def send_response(count): + self.sendUntaggedResponse('%d EXISTS' % count) + self.sendPositiveResponse(tag, 'APPEND complete') + + d = mbox.getMessageCount() + d.addCallback(send_response) + return d + # XXX patched --------------------------------- + + # ----------------------------------------------------------------------- + + auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, + imap4.IMAP4Server.opt_datetime, arg_literal) + select_APPEND = auth_APPEND + + # Need to override the command table after patching + # arg_astring and arg_literal, except on the methods that we are already + # overriding. + + # TODO -------------------------------------------- + # Check if we really need to override these + # methods, or we can monkeypatch. + # do_DELETE = imap4.IMAP4Server.do_DELETE + # do_CREATE = imap4.IMAP4Server.do_CREATE + # do_RENAME = imap4.IMAP4Server.do_RENAME + # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + # do_APPEND = imap4.IMAP4Server.do_APPEND + # ------------------------------------------------- do_LOGIN = imap4.IMAP4Server.do_LOGIN - do_CREATE = imap4.IMAP4Server.do_CREATE - do_DELETE = imap4.IMAP4Server.do_DELETE - do_RENAME = imap4.IMAP4Server.do_RENAME - do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE - do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_STATUS = imap4.IMAP4Server.do_STATUS - do_APPEND = imap4.IMAP4Server.do_APPEND do_COPY = imap4.IMAP4Server.do_COPY _selectWork = imap4.IMAP4Server._selectWork - _listWork = imap4.IMAP4Server._listWork + arg_plist = imap4.IMAP4Server.arg_plist arg_seqset = imap4.IMAP4Server.arg_seqset opt_plist = imap4.IMAP4Server.opt_plist @@ -342,8 +706,15 @@ class LeapIMAPServer(imap4.IMAP4Server): auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') select_EXAMINE = auth_EXAMINE - auth_DELETE = (do_DELETE, arg_astring) - select_DELETE = auth_DELETE + # TODO ----------------------------------------------- + # re-add if we stop overriding DELETE + # auth_DELETE = (do_DELETE, arg_astring) + # select_DELETE = auth_DELETE + # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + # arg_literal) + # select_APPEND = auth_APPEND + + # ---------------------------------------------------- auth_RENAME = (do_RENAME, arg_astring, arg_astring) select_RENAME = auth_RENAME @@ -363,13 +734,8 @@ class LeapIMAPServer(imap4.IMAP4Server): auth_STATUS = (do_STATUS, arg_astring, arg_plist) select_STATUS = auth_STATUS - auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, - arg_literal) - select_APPEND = auth_APPEND - select_COPY = (do_COPY, arg_seqset, arg_astring) - ############################################################# # END of Twisted imap4 patch to support LITERAL+ extension ############################################################# diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index 651f71b..2045757 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -23,9 +23,9 @@ For now, and for debugging/testing purposes, you need to pass a config file with the following structure: [leap_mail] -userid = "user@provider" -uuid = "deadbeefdeadabad" -passwd = "supersecret" # optional, will get prompted if not found. +userid = 'user@provider' +uuid = 'deadbeefdeadabad' +passwd = 'supersecret' # optional, will get prompted if not found. """ import ConfigParser import getpass @@ -53,38 +53,17 @@ def initialize_soledad(uuid, email, passwd, :param tempdir: path to temporal dir :rtype: Soledad instance """ - # XXX TODO unify with an authoritative source of mocks - # for soledad (or partial initializations). - # This is copied from the imap tests. - server_url = "http://provider" cert_file = "" - class Mock(object): - def __init__(self, return_value=None): - self._return = return_value - - def __call__(self, *args, **kwargs): - return self._return - - class MockSharedDB(object): - - get_doc = Mock() - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() soledad = Soledad( uuid, passwd, secrets, localdb, server_url, - cert_file) + cert_file, + syncable=False) return soledad @@ -95,9 +74,9 @@ def initialize_soledad(uuid, email, passwd, print "[+] Running LEAP IMAP Service" -bmconf = os.environ.get("LEAP_MAIL_CONF", "") +bmconf = os.environ.get("LEAP_MAIL_CONFIG", "") if not bmconf: - print ("[-] Please set LEAP_MAIL_CONF environment variable " + print ("[-] Please set LEAP_MAIL_CONFIG environment variable " "pointing to your config.") sys.exit(1) @@ -131,6 +110,7 @@ tempdir = "/tmp/" # Ad-hoc soledad/keymanager initialization. +print "[~] user:", userid soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir) km_args = (userid, "https://localhost", soledad) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 10ba32a..c3ae59a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # imap.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,58 +15,27 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Imap service initialization +IMAP service initialization """ import logging import os -import time -from twisted.internet import defer, threads -from twisted.internet.protocol import ServerFactory +from collections import defaultdict + +from twisted.internet import reactor from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory from twisted.mail import imap4 from twisted.python import log -logger = logging.getLogger(__name__) +from leap.common.events import emit, catalog +from leap.common.check import leap_check +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer -from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type, leap_check -from leap.keymanager import KeyManager -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.mail.imap.soledadstore import SoledadStore -from leap.soledad.client import Soledad - -# The default port in which imap service will run -IMAP_PORT = 1984 +# TODO: leave only an implementor of IService in here -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - -from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED -from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START - -###################################################### -# Temporary workaround for RecursionLimit when using -# qt4reactor. Do remove when we move to poll or select -# reactor, which do not show those problems. See #4974 -import resource -import sys - -try: - sys.setrecursionlimit(10**7) -except Exception: - print "Error setting recursion limit" -try: - # Increase max stack size from 8MB to 256MB - resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) -except Exception: - print "Error setting stack size" - -###################################################### +logger = logging.getLogger(__name__) DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: @@ -81,6 +50,9 @@ if DO_PROFILE: pr = cProfile.Profile() pr.enable() +# The default port in which imap service will run +IMAP_PORT = 1984 + class IMAPAuthRealm(object): """ @@ -97,6 +69,7 @@ class LeapIMAPFactory(ServerFactory): Factory for a IMAP4 server with soledad remote sync and gpg-decryption capabilities. """ + protocol = LEAPIMAPServer def __init__(self, uuid, userid, soledad): """ @@ -114,14 +87,10 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad - self._memstore = MemoryStore( - permanent_store=SoledadStore(soledad)) - theAccount = SoledadBackedAccount( - uuid, soledad=soledad, - memstore=self._memstore) + theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount - + self._connections = defaultdict() # XXX how to pass the store along? def buildProtocol(self, addr): @@ -131,91 +100,66 @@ class LeapIMAPFactory(ServerFactory): :param addr: remote ip address :type addr: str """ - imapProtocol = LeapIMAPServer( + # TODO should reject anything from addr != localhost, + # just in case. + log.msg("Building protocol for connection %s" % addr) + imapProtocol = self.protocol( uuid=self._uuid, userid=self._userid, soledad=self._soledad) imapProtocol.theAccount = self.theAccount imapProtocol.factory = self + + self._connections[addr] = imapProtocol return imapProtocol - def doStop(self, cv=None): + def stopFactory(self): + # say bye! + for conn, proto in self._connections.items(): + log.msg("Closing connections for %s" % conn) + proto.close_server_connection() + + def doStop(self): """ Stops imap service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition - :return: a Deferred that stops and flushes the in memory store data to - disk in another thread. - :rtype: Deferred """ + # mark account as unusable, so any imap command will fail + # with unauth state. + self.theAccount.end_session() + + # TODO should wait for all the pending deferreds, + # the twisted way! if DO_PROFILE: log.msg("Stopping PROFILING") pr.disable() pr.dump_stats(PROFILE_DAT) - ServerFactory.doStop(self) - - if cv is not None: - def _stop_imap_cb(): - logger.debug('Stopping in memory store.') - self._memstore.stop_and_flush() - while not self._memstore.producer.is_queue_empty(): - logger.debug('Waiting for queue to be empty.') - # TODO use a gatherResults over the new/dirty - # deferred list, - # as in memorystore's expunge() method. - time.sleep(1) - # notify that service has stopped - logger.debug('Notifying that service has stopped.') - cv.acquire() - cv.notify() - cv.release() - - return threads.deferToThread(_stop_imap_cb) + return ServerFactory.doStop(self) -def run_service(*args, **kwargs): +def run_service(store, **kwargs): """ Main entry point to run the service from the client. - :returns: the LoopingCall instance that will have to be stoppped - before shutting down the client, the port as returned by - the reactor when starts listening, and the factory for - the protocol. - """ - from twisted.internet import reactor - # it looks like qtreactor does not honor this, - # but other reactors should. - reactor.suggestThreadPoolSize(20) + :param store: a soledad instance - leap_assert(len(args) == 2) - soledad, keymanager = args - leap_assert_type(soledad, Soledad) - leap_assert_type(keymanager, KeyManager) + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + """ + leap_check(store, "store cannot be None") + # XXX this can also be a ProxiedObject, FIXME + # leap_assert_type(store, Soledad) port = kwargs.get('port', IMAP_PORT) - check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") - offline = kwargs.get('offline', False) - uuid = soledad._get_uuid() - factory = LeapIMAPFactory(uuid, userid, soledad) + uuid = store.uuid + factory = LeapIMAPFactory(uuid, userid, store) try: tport = reactor.listenTCP(port, factory, interface="localhost") - if not offline: - fetcher = LeapIncomingMail( - keymanager, - soledad, - factory.theAccount, - check_period, - userid) - else: - fetcher = None except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) @@ -223,7 +167,6 @@ def run_service(*args, **kwargs): logger.error("Error launching IMAP service: %r" % (exc,)) else: # all good. - # (the caller has still to call fetcher.start_loop) if DO_MANHOLE: # TODO get pass from env var.too. @@ -235,8 +178,10 @@ def run_service(*args, **kwargs): reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) - leap_events.signal(IMAP_SERVICE_STARTED, str(port)) - return fetcher, tport, factory + emit(catalog.IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature + return tport, factory # not ok, signal error. - leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) + emit(catalog.IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py deleted file mode 100644 index f3de8eb..0000000 --- a/src/leap/mail/imap/soledadstore.py +++ /dev/null @@ -1,620 +0,0 @@ -# -*- coding: utf-8 -*- -# soledadstore.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -A MessageStore that writes to Soledad. -""" -import logging -import threading - -from collections import defaultdict -from itertools import chain - -from u1db import errors as u1db_errors -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert_type, leap_assert -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.messageparts import MessagePartType -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.messageparts import RecentFlagsDoc -from leap.mail.imap.fields import fields -from leap.mail.imap.interfaces import IMessageStore -from leap.mail.messageflow import IMessageConsumer -from leap.mail.utils import first, empty, accumulator_queue - -logger = logging.getLogger(__name__) - - -# TODO -# [ ] Implement a retry queue? -# [ ] Consider journaling of operations. - - -class ContentDedup(object): - """ - Message deduplication. - - We do a query for the content hashes before writing to our beloved - sqlcipher backend of Soledad. This means, by now, that: - - 1. We will not store the same body/attachment twice, only the hash of it. - 2. We will not store the same message header twice, only the hash of it. - - The first case is useful if you are always receiving the same old memes - from unwary friends that still have not discovered that 4chan is the - generator of the internet. The second will save your day if you have - initiated session with the same account in two different machines. I also - wonder why would you do that, but let's respect each other choices, like - with the religious celebrations, and assume that one day we'll be able - to run Bitmask in completely free phones. Yes, I mean that, the whole GSM - Stack. - """ - # TODO refactor using unique_query - - def _header_does_exist(self, doc): - """ - Check whether we already have a header document for this - content hash in our database. - - :param doc: tentative header for document - :type doc: dict - :returns: True if it exists, False otherwise. - """ - if not doc: - return False - chash = doc[fields.CONTENT_HASH_KEY] - header_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(chash)) - if not header_docs: - return False - - # FIXME enable only to debug this problem. - #if len(header_docs) != 1: - #logger.warning("Found more than one copy of chash %s!" - #% (chash,)) - - #logger.debug("Found header doc with that hash! Skipping save!") - return True - - def _content_does_exist(self, doc): - """ - Check whether we already have a content document for a payload - with this hash in our database. - - :param doc: tentative content for document - :type doc: dict - :returns: True if it exists, False otherwise. - """ - if not doc: - return False - phash = doc[fields.PAYLOAD_HASH_KEY] - attach_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - if not attach_docs: - return False - - # FIXME enable only to debug this problem - #if len(attach_docs) != 1: - #logger.warning("Found more than one copy of phash %s!" - #% (phash,)) - #logger.debug("Found attachment doc with that hash! Skipping save!") - return True - - -class MsgWriteError(Exception): - """ - Raised if any exception is found while saving message parts. - """ - pass - - -""" -A lock per document. -""" -# TODO should bound the space of this!!! -# http://stackoverflow.com/a/2437645/1157664 -# Setting this to twice the number of threads in the threadpool -# should be safe. -put_locks = defaultdict(lambda: threading.Lock()) -mbox_doc_locks = defaultdict(lambda: threading.Lock()) - - -class SoledadStore(ContentDedup): - """ - This will create docs in the local Soledad database. - """ - _remove_lock = threading.Lock() - - implements(IMessageConsumer, IMessageStore) - - def __init__(self, soledad): - """ - Initialize the permanent store that writes to Soledad database. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - from twisted.internet import reactor - self.reactor = reactor - - self._soledad = soledad - - self._CREATE_DOC_FUN = self._soledad.create_doc - self._PUT_DOC_FUN = self._soledad.put_doc - self._GET_DOC_FUN = self._soledad.get_doc - - # we instantiate an accumulator to batch the notifications - self.docs_notify_queue = accumulator_queue( - lambda item: reactor.callFromThread(self._unset_new_dirty, item), - 20) - - # IMessageStore - - # ------------------------------------------------------------------- - # We are not yet using this interface, but it would make sense - # to implement it. - - def create_message(self, mbox, uid, message): - """ - Create the passed message into this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - :param message: a IMessageContainer implementor. - """ - raise NotImplementedError() - - def put_message(self, mbox, uid, message): - """ - Put the passed existing message into this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - :param message: a IMessageContainer implementor. - """ - raise NotImplementedError() - - def remove_message(self, mbox, uid): - """ - Remove the given message from this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - """ - raise NotImplementedError() - - def get_message(self, mbox, uid): - """ - Get a IMessageContainer for the given mbox and uid combination. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - """ - raise NotImplementedError() - - # IMessageConsumer - - # TODO should handle the delete case - # TODO should handle errors better - # TODO could generalize this method into a generic consumer - # and only implement `process` here - - def consume(self, queue): - """ - Creates a new document in soledad db. - - :param queue: a tuple of queues to get item from, with content of the - document to be inserted. - :type queue: tuple of Queues - """ - new, dirty = queue - while not new.empty(): - doc_wrapper = new.get() - self.reactor.callInThread(self._consume_doc, doc_wrapper, - self.docs_notify_queue) - while not dirty.empty(): - doc_wrapper = dirty.get() - self.reactor.callInThread(self._consume_doc, doc_wrapper, - self.docs_notify_queue) - - # Queue empty, flush the notifications queue. - self.docs_notify_queue(None, flush=True) - - def _unset_new_dirty(self, doc_wrapper): - """ - Unset the `new` and `dirty` flags for this document wrapper in the - memory store. - - :param doc_wrapper: a MessageWrapper instance - :type doc_wrapper: MessageWrapper - """ - if isinstance(doc_wrapper, MessageWrapper): - # XXX still needed for debug quite often - #logger.info("unsetting new flag!") - doc_wrapper.new = False - doc_wrapper.dirty = False - - @deferred_to_thread - def _consume_doc(self, doc_wrapper, notify_queue): - """ - Consume each document wrapper in a separate thread. - We pass an instance of an accumulator that handles the notifications - to the memorystore when the write has been done. - - :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance - :type doc_wrapper: MessageWrapper or RecentFlagsDoc - :param notify_queue: a callable that handles the writeback - notifications to the memstore. - :type notify_queue: callable - """ - def queueNotifyBack(failed, doc_wrapper): - if failed: - log.msg("There was an error writing the mesage...") - else: - notify_queue(doc_wrapper) - - def doSoledadCalls(items): - # we prime the generator, that should return the - # message or flags wrapper item in the first place. - try: - doc_wrapper = items.next() - except StopIteration: - pass - else: - failed = self._soledad_write_document_parts(items) - queueNotifyBack(failed, doc_wrapper) - - doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper)) - - # - # SoledadStore specific methods. - # - - def _soledad_write_document_parts(self, items): - """ - Write the document parts to soledad in a separate thread. - - :param items: the iterator through the different document wrappers - payloads. - :type items: iterator - :return: whether the write was successful or not - :rtype: bool - """ - failed = False - for item, call in items: - if empty(item): - continue - try: - self._try_call(call, item) - except Exception as exc: - logger.debug("ITEM WAS: %s" % repr(item)) - if hasattr(item, 'content'): - logger.debug("ITEM CONTENT WAS: %s" % - repr(item.content)) - logger.exception(exc) - failed = True - continue - return failed - - def _iter_wrapper_subparts(self, doc_wrapper): - """ - Return an iterator that will yield the doc_wrapper in the first place, - followed by the subparts item and the proper call type for every - item in the queue, if any. - - :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance - :type doc_wrapper: MessageWrapper or RecentFlagsDoc - """ - if isinstance(doc_wrapper, MessageWrapper): - return chain((doc_wrapper,), - self._get_calls_for_msg_parts(doc_wrapper)) - elif isinstance(doc_wrapper, RecentFlagsDoc): - return chain((doc_wrapper,), - self._get_calls_for_rflags_doc(doc_wrapper)) - else: - logger.warning("CANNOT PROCESS ITEM!") - return (i for i in []) - - def _try_call(self, call, item): - """ - Try to invoke a given call with item as a parameter. - - :param call: the function to call - :type call: callable - :param item: the payload to pass to the call as argument - :type item: object - """ - if call is None: - return - - if call == self._PUT_DOC_FUN: - doc_id = item.doc_id - if doc_id is None: - logger.warning("BUG! Dirty doc but has no doc_id!") - return - with put_locks[doc_id]: - doc = self._GET_DOC_FUN(doc_id) - - if doc is None: - logger.warning("BUG! Dirty doc but could not " - "find document %s" % (doc_id,)) - return - - doc.content = dict(item.content) - - item = doc - try: - call(item) - except u1db_errors.RevisionConflict as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - except Exception as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - - else: - try: - call(item) - except u1db_errors.RevisionConflict as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - except Exception as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - - def _get_calls_for_msg_parts(self, msg_wrapper): - """ - Generator that return the proper call type for a given item. - - :param msg_wrapper: A MessageWrapper - :type msg_wrapper: IMessageContainer - :return: a generator of tuples with recent-flags doc payload - and callable - :rtype: generator - """ - call = None - - if msg_wrapper.new: - call = self._CREATE_DOC_FUN - - # item is expected to be a MessagePartDoc - for item in msg_wrapper.walk(): - if item.part == MessagePartType.fdoc: - yield dict(item.content), call - - elif item.part == MessagePartType.hdoc: - if not self._header_does_exist(item.content): - yield dict(item.content), call - - elif item.part == MessagePartType.cdoc: - if not self._content_does_exist(item.content): - yield dict(item.content), call - - # For now, the only thing that will be dirty is - # the flags doc. - - elif msg_wrapper.dirty: - call = self._PUT_DOC_FUN - # item is expected to be a MessagePartDoc - for item in msg_wrapper.walk(): - # XXX FIXME Give error if dirty and not doc_id !!! - doc_id = item.doc_id # defend! - if not doc_id: - logger.warning("Dirty item but no doc_id!") - continue - - if item.part == MessagePartType.fdoc: - #logger.debug("PUT dirty fdoc") - yield item, call - - # XXX also for linkage-doc !!! - else: - logger.error("Cannot delete documents yet from the queue...!") - - def _get_calls_for_rflags_doc(self, rflags_wrapper): - """ - We always put these documents. - - :param rflags_wrapper: A wrapper around recent flags doc. - :type rflags_wrapper: RecentFlagsWrapper - :return: a tuple with recent-flags doc payload and callable - :rtype: tuple - """ - call = self._PUT_DOC_FUN - - payload = rflags_wrapper.content - if payload: - logger.debug("Saving RFLAGS to Soledad...") - yield rflags_wrapper, call - - # Mbox documents and attributes - - def get_mbox_document(self, mbox): - """ - Return mailbox document. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - with mbox_doc_locks[mbox]: - return self._get_mbox_document(mbox) - - def _get_mbox_document(self, mbox): - """ - Helper for returning the mailbox document. - """ - try: - query = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_MBOX_VAL, mbox) - if query: - return query.pop() - else: - logger.error("Could not find mbox document for %r" % - (mbox,)) - except Exception as exc: - logger.exception("Unhandled error %r" % exc) - - def get_mbox_closed(self, mbox): - """ - Return the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: bool - """ - mbox_doc = self.get_mbox_document() - return mbox_doc.content.get(fields.CLOSED_KEY, False) - - def set_mbox_closed(self, mbox, closed): - """ - Set the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param closed: the value to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - with mbox_doc_locks[mbox]: - mbox_doc = self._get_mbox_document(mbox) - if mbox_doc is None: - logger.error( - "Could not find mbox document for %r" % (mbox,)) - return - mbox_doc.content[fields.CLOSED_KEY] = closed - self._soledad.put_doc(mbox_doc) - - def write_last_uid(self, mbox, value): - """ - Write the `last_uid` integer to the proper mailbox document - in Soledad. - This is called from the deferred triggered by - memorystore.increment_last_soledad_uid, which is expected to - run in a separate thread. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - leap_assert_type(value, int) - key = fields.LAST_UID_KEY - - # XXX use accumulator to reduce number of hits - with mbox_doc_locks[mbox]: - mbox_doc = self._get_mbox_document(mbox) - old_val = mbox_doc.content[key] - if value > old_val: - mbox_doc.content[key] = value - try: - self._soledad.put_doc(mbox_doc) - except Exception as exc: - logger.error("Error while setting last_uid for %r" - % (mbox,)) - logger.exception(exc) - - def get_flags_doc(self, mbox, uid): - """ - Return the SoledadDocument for the given mbox and uid. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :rtype: SoledadDocument or None - """ - result = None - try: - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, mbox, str(uid)) - if len(flag_docs) != 1: - logger.warning("More than one flag doc for %r:%s" % - (mbox, uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % uid) - logger.exception(exc) - finally: - return result - - def get_headers_doc(self, chash): - """ - Return the document that keeps the headers for a message - indexed by its content-hash. - - :param chash: the content-hash to retrieve the document from. - :type chash: str or unicode - :rtype: SoledadDocument or None - """ - head_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(chash)) - return first(head_docs) - - # deleted messages - - def deleted_iter(self, mbox): - """ - Get an iterator for the the doc_id for SoledadDocuments for messages - with \\Deleted flag for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: iterator through deleted message docs - :rtype: iterable - """ - return [doc.doc_id for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_DEL_IDX, - fields.TYPE_FLAGS_VAL, mbox, '1')] - - def remove_all_deleted(self, mbox): - """ - Remove from Soledad all messages flagged as deleted for a given - mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - deleted = [] - for doc_id in self.deleted_iter(mbox): - with self._remove_lock: - doc = self._soledad.get_doc(doc_id) - if doc is not None: - self._soledad.delete_doc(doc) - try: - deleted.append(doc.content[fields.UID_KEY]) - except TypeError: - # empty content - pass - return deleted diff --git a/src/leap/mail/imap/tests/__init__.py b/src/leap/mail/imap/tests/__init__.py index f3d5ca6..5cf60ed 100644 --- a/src/leap/mail/imap/tests/__init__.py +++ b/src/leap/mail/imap/tests/__init__.py @@ -1,4 +1,4 @@ -#-*- encoding: utf-8 -*- +# -*- encoding: utf-8 -*- """ leap/email/imap/tests/__init__.py ---------------------------------- @@ -10,13 +10,6 @@ code, using twisted.trial, for testing leap_mx. @copyright: © 2013 Kali Kaneko, see COPYLEFT file """ -__all__ = ['test_imap'] - - -def run(): - """xxx fill me in""" - pass - import os import u1db @@ -25,11 +18,18 @@ from leap.common.testing.basetest import BaseLeapTest from leap.soledad.client import Soledad from leap.soledad.common.document import SoledadDocument +__all__ = ['test_imap'] + -#----------------------------------------------------------------------------- +def run(): + """xxx fill me in""" + pass + +# ----------------------------------------------------------------------------- # Some tests inherit from BaseSoledadTest in order to have a working Soledad # instance in each test. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + class BaseSoledadIMAPTest(BaseLeapTest): """ diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions_mime_struct index efe3f46..0332664 100755 --- a/src/leap/mail/imap/tests/regressions +++ b/src/leap/mail/imap/tests/regressions_mime_struct @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# regressions +# regression_mime_struct # Copyright (C) 2014 LEAP # Copyright (c) Twisted Matrix Laboratories. # @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Simple Regression Tests using IMAP4 client. +Simple Regression Tests for checking MIME struct handling using IMAP4 client. Iterates trough all mails under a given folder and tries to APPEND them to the server being tested. After FETCHING the pushed message, it compares @@ -40,7 +40,9 @@ from twisted.protocols import basic from twisted.python import log -REGRESSIONS_FOLDER = "regressions_test" +REGRESSIONS_FOLDER = os.environ.get( + "REGRESSIONS_FOLDER", "regressions_test") +print "[+] Using regressions folder:", REGRESSIONS_FOLDER parser = Parser() @@ -263,7 +265,7 @@ def cbSelectMbox(result, proto): if result["EXISTS"] != 0: # Flag as deleted, expunge, and do an examine again. - #print "There is mail here, will delete..." + print "There is mail here, will delete..." return cbDeleteAndExpungeTestFolder(proto) else: @@ -276,11 +278,15 @@ def ebSelectMbox(failure, proto, folder): Creates the folder. """ - print failure.getTraceback() + log.err(failure) log.msg("Folder %r does not exist. Creating..." % (folder,)) return proto.create(folder).addCallback(cbAuthentication, proto) +def ebExpunge(failure): + log.err(failure) + + def cbDeleteAndExpungeTestFolder(proto): """ Callback invoked fom cbExamineMbox when the number of messages in the @@ -292,7 +298,9 @@ def cbDeleteAndExpungeTestFolder(proto): ).addCallback( lambda r: proto.expunge() ).addCallback( - cbExpunge, proto) + cbExpunge, proto + ).addErrback( + ebExpunge) def cbExpunge(result, proto): diff --git a/src/leap/mail/imap/tests/rfc822.message b/src/leap/mail/imap/tests/rfc822.message index ee97ab9..b19cc28 100644..120000 --- a/src/leap/mail/imap/tests/rfc822.message +++ b/src/leap/mail/imap/tests/rfc822.message @@ -1,86 +1 @@ -Return-Path: <twisted-commits-admin@twistedmatrix.com> -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS <etrepum@twistedmatrix.com> -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com> -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help> -List-Post: <mailto:twisted-commits@twistedmatrix.com> -List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, - <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe> -List-Id: <twisted-commits.twistedmatrix.com> -List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, - <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe> -List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/> -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits +../../tests/rfc822.message
\ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message index 582297c..e0aa678 100644..120000 --- a/src/leap/mail/imap/tests/rfc822.multi-minimal.message +++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -1,16 +1 @@ -Content-Type: multipart/mixed; boundary="===============6203542367371144092==" -MIME-Version: 1.0 -Subject: [TEST] 010 - Inceptos cum lorem risus congue -From: testmailbitmaskspam@gmail.com -To: test_c5@dev.bitmask.net - ---===============6203542367371144092== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Howdy from python! -The subject: [TEST] 010 - Inceptos cum lorem risus congue -Current date & time: Wed Jan 8 16:36:21 2014 -Trying to attach: [] ---===============6203542367371144092==-- +../../tests/rfc822.multi-minimal.message
\ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-nested.message b/src/leap/mail/imap/tests/rfc822.multi-nested.message new file mode 120000 index 0000000..306d0de --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-nested.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-nested.message
\ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message index 9907c2d..4172244 100644..120000 --- a/src/leap/mail/imap/tests/rfc822.multi-signed.message +++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -1,238 +1 @@ -Date: Mon, 6 Jan 2014 04:40:47 -0400 -From: Kali Kaneko <kali@leap.se> -To: penguin@example.com -Subject: signed message -Message-ID: <20140106084047.GA21317@samsara.lan> -MIME-Version: 1.0 -Content-Type: multipart/signed; micalg=pgp-sha1; - protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" -Content-Disposition: inline -User-Agent: Mutt/1.5.21 (2012-12-30) - - ---z9ECzHErBrwFF8sy -Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" -Content-Disposition: inline - - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=utf-8 -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -This is an example of a signed message, -with attachments. - - ---=20 -Nihil sine chao! =E2=88=B4 - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=us-ascii -Content-Disposition: attachment; filename="attach.txt" - -this is attachment in plain text. - ---z0eOaCaDLjvTGF2l -Content-Type: application/octet-stream -Content-Disposition: attachment; filename="hack.ico" -Content-Transfer-Encoding: base64 - -AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA -KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG -RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA -PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl -5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA -/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ -yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A -Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK -ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK -LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP -QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy -AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs -AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA -AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA -gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d -HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA -x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 -+wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA -AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 -+QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA -OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK -igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA -JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra -2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA -xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj -owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB -AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA -AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d -XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d -XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA -AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB -AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm -X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC -AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B -bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ -S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu -J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y -AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N -KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB -XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A -AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA -AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d -XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA -AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr -RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA -AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A -Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI -yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA -CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys -rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA -vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d -HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA -urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx -cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA -CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo -6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA -2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 -OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA -UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp -qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA -lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa -WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB -AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB -AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA -ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA -AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB -AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB -AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA -tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA -AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB -AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB -AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA -AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd -AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB -AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB -AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 -ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 -NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF -RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB -lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA -AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa -WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA -AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX -AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB -AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB -AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA -AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA -AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA -AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA -AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB -AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB -AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA -ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA -AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 -LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA -AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA -ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 -RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi -JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 -NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK -T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN -UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA -AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA -W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA -AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB -l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB -AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ -WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA -AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv -RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA -AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj -AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB -AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA -AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA -AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA -dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A -AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB -AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW -pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= - ---z0eOaCaDLjvTGF2l-- - ---z9ECzHErBrwFF8sy -Content-Type: application/pgp-signature - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.15 (GNU/Linux) - -iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv -kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl -vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK -PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC -w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw -sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr -BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN -QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt -mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ -jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 -gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X -sSdfcAhT7Tno7PB/Acoh -=+okv ------END PGP SIGNATURE----- - ---z9ECzHErBrwFF8sy-- +../../tests/rfc822.multi-signed.message
\ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message index 30f74e5..62057d2 100644..120000 --- a/src/leap/mail/imap/tests/rfc822.multi.message +++ b/src/leap/mail/imap/tests/rfc822.multi.message @@ -1,96 +1 @@ -Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
-From: Doug Sauder <doug@penguin.example.com>
-To: Joe Blow <blow@example.com>
-Subject: Test message from PINE
-Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
-MIME-Version: 1.0
-Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
-
- This message is in MIME format. The first part should be readable text,
- while the remaining parts are likely unreadable without MIME-aware tools.
- Send mail to mime@docserver.cac.washington.edu for more info.
-
----1463757054-952513540-958744548=:8452
-Content-Type: TEXT/PLAIN; charset=US-ASCII
-
-This is a test message from PINE MUA.
-
-
----1463757054-952513540-958744548=:8452
-Content-Type: APPLICATION/octet-stream; name="redball.png"
-Content-Transfer-Encoding: BASE64
-Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
-Content-Description: A PNG graphic file
-Content-Disposition: attachment; filename="redball.png"
-
-iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
-AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
-AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
-AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
-GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
-hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
-rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
-ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
-AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
-AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
-AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
-AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
-AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
-AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
-AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
-AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
-LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
-MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
-BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
-6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
-8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
-ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
-OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
-a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
-VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
-Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
-lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
-joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
-1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
-JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
-vAAAAABJRU5ErkJggg==
----1463757054-952513540-958744548=:8452
-Content-Type: APPLICATION/octet-stream; name="blueball.png"
-Content-Transfer-Encoding: BASE64
-Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
-Content-Description: A PNG graphic file
-Content-Disposition: attachment; filename="blueball.png"
-
-iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
-AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
-IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
-Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
-hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
-vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
-Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
-MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
-fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
-Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
-LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
-P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
-jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
-+VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
-1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
-YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
-mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
-Z3VldDZzO7wAAAAASUVORK5CYII=
----1463757054-952513540-958744548=:8452--
+../../tests/rfc822.multi.message
\ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message index fc627c3..5bab0e8 100644..120000 --- a/src/leap/mail/imap/tests/rfc822.plain.message +++ b/src/leap/mail/imap/tests/rfc822.plain.message @@ -1,66 +1 @@ -From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 -Return-Path: <pyar-bounces@python.org.ar> -X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net -X-Spam-Level: ** -X-Spam-Pyzor: Reported 0 times. -X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, - CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, - NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled - version=3.3.2 -Delivered-To: kali@leap.se -Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) - (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) - (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) - by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F - for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) -Received: from pyar.usla.org.ar (unknown [190.228.30.157]) - by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 - for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST) -Received: from [127.0.0.1] (localhost [127.0.0.1]) - by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F - for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART) -MIME-Version: 1.0 -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: quoted-printable -From: pyar-request@python.org.ar -To: kali@leap.se -Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 -Reply-To: pyar-request@python.org.ar -Auto-Submitted: auto-replied -Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar> -Date: Wed, 08 Jan 2014 15:45:59 -0300 -Precedence: bulk -X-BeenThere: pyar@python.org.ar -X-Mailman-Version: 2.1.15 -List-Id: Python Argentina <pyar.python.org.ar> -X-List-Administrivia: yes -Errors-To: pyar-bounces@python.org.ar -Sender: "pyar" <pyar-bounces@python.org.ar> -X-Virus-Scanned: clamav-milter 0.97.8 at mx1 -X-Virus-Status: Clean - -Mailing list subscription confirmation notice for mailing list pyar - -We have received a request de kaliyuga@riseup.net for subscription of -your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar -mailing list. To confirm that you want to be added to this mailing -list, simply reply to this message, keeping the Subject: header -intact. Or visit this web page: - - http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= -3377148ac2 - - -Or include the following line -- and only the following line -- in a -message to pyar-request@python.org.ar: - - confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 - -Note that simply sending a `reply' to this message should work from -most mail readers, since that usually leaves the Subject: line in the -right form (additional "Re:" text in the Subject: is okay). - -If you do not wish to be subscribed to this list, please simply -disregard this message. If you think you are being maliciously -subscribed to the list, or have any other questions, send them to -pyar-owner@python.org.ar. +../../tests/rfc822.plain.message
\ No newline at end of file diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/stress_tests_imap.zsh index 544faca..544faca 100755 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ b/src/leap/mail/imap/tests/stress_tests_imap.zsh diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 631a2c1..62c3c41 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -17,7 +17,7 @@ """ Test case for leap.email.imap.server TestCases taken from twisted tests and modified to make them work -against SoledadBackedAccount. +against our implementation of the IMAPAccount. @authors: Kali Kaneko, <kali@leap.se> XXX add authors from the original twisted tests. @@ -25,31 +25,20 @@ XXX add authors from the original twisted tests. @license: GPLv3, see included LICENSE file """ # XXX review license of the original tests!!! - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - import os +import string import types from twisted.mail import imap4 from twisted.internet import defer -from twisted.trial import unittest from twisted.python import util from twisted.python import failure from twisted import cred - -# import u1db - -from leap.mail.imap.mailbox import SoledadMailbox -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import CaseInsensitiveDict from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -73,7 +62,6 @@ def sortNest(l): class TestRealm: - """ A minimal auth realm for testing purposes only """ @@ -82,153 +70,21 @@ class TestRealm: def requestAvatar(self, avatarId, mind, *interfaces): return imap4.IAccount, self.theAccount, lambda: None - # # TestCases # -class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): - - """ - Tests for the MessageCollection class - """ - count = 0 - - def setUp(self): - """ - setUp method for each test - We override mixin method since we are only testing - MessageCollection interface in this particular TestCase - """ - super(MessageCollectionTestCase, self).setUp() - memstore = MemoryStore() - self.messages = MessageCollection("testmbox%s" % (self.count,), - self._soledad, memstore=memstore) - MessageCollectionTestCase.count += 1 - - def tearDown(self): - """ - tearDown method for each test - """ - del self.messages - - def testEmptyMessage(self): - """ - Test empty message and collection - """ - em = self.messages._get_empty_doc() - self.assertEqual( - em, - { - "chash": '', - "deleted": False, - "flags": [], - "mbox": "inbox", - "seen": False, - "multi": False, - "size": 0, - "type": "flags", - "uid": 1, - }) - self.assertEqual(self.messages.count(), 0) - - def testMultipleAdd(self): - """ - Add multiple messages - """ - mc = self.messages - self.assertEqual(self.messages.count(), 0) - - def add_first(): - d = defer.gatherResults([ - mc.add_msg('Stuff 1', subject="test1"), - mc.add_msg('Stuff 2', subject="test2"), - mc.add_msg('Stuff 3', subject="test3"), - mc.add_msg('Stuff 4', subject="test4")]) - return d - - def add_second(result): - d = defer.gatherResults([ - mc.add_msg('Stuff 5', subject="test5"), - mc.add_msg('Stuff 6', subject="test6"), - mc.add_msg('Stuff 7', subject="test7")]) - return d - - def check_second(result): - return self.assertEqual(mc.count(), 7) - - d1 = add_first() - d1.addCallback(add_second) - d1.addCallback(check_second) - - def testRecentCount(self): - """ - Test the recent count - """ - mc = self.messages - countrecent = mc.count_recent - eq = self.assertEqual - - self.assertEqual(countrecent(), 0) - - d = mc.add_msg('Stuff', subject="test1") - # For the semantics defined in the RFC, we auto-add the - # recent flag by default. - - def add2(_): - return mc.add_msg('Stuff', subject="test2", - flags=('\\Deleted',)) - - def add3(_): - return mc.add_msg('Stuff', subject="test3", - flags=('\\Recent',)) - - def add4(_): - return mc.add_msg('Stuff', subject="test4", - flags=('\\Deleted', '\\Recent')) - - d.addCallback(lambda r: eq(countrecent(), 1)) - d.addCallback(add2) - d.addCallback(lambda r: eq(countrecent(), 2)) - d.addCallback(add3) - d.addCallback(lambda r: eq(countrecent(), 3)) - d.addCallback(add4) - d.addCallback(lambda r: eq(countrecent(), 4)) - - def testFilterByMailbox(self): - """ - Test that queries filter by selected mailbox - """ - mc = self.messages - self.assertEqual(self.messages.count(), 0) - - def add_1(): - d1 = mc.add_msg('msg 1', subject="test1") - d2 = mc.add_msg('msg 2', subject="test2") - d3 = mc.add_msg('msg 3', subject="test3") - d = defer.gatherResults([d1, d2, d3]) - return d - - add_1().addCallback(lambda ignored: self.assertEqual( - mc.count(), 3)) - - # XXX this has to be redone to fit memstore ------------# - #newmsg = mc._get_empty_doc() - #newmsg['mailbox'] = "mailbox/foo" - #mc._soledad.create_doc(newmsg) - #self.assertEqual(mc.count(), 3) - #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) +# DEBUG --- +# from twisted.internet.base import DelayedCall +# DelayedCall.debug = True -class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): - # TODO this currently will use a memory-only store. - # create a different one for testing soledad sync. +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ - Tests for the generic behavior of the LeapIMAP4Server + Tests for the generic behavior of the LEAPIMAP4Server which, right now, it's just implemented in this test file as - LeapIMAPServer. We will move the implementation, together with + LEAPIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -248,6 +104,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') + acc = self.server.theAccount def cb(): self.result.append(1) @@ -259,46 +116,54 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.login(TEST_USER, TEST_PASSWD) def create(): + create_deferreds = [] for name in succeed + fail: d = self.client.create(name) d.addCallback(strip(cb)).addErrback(eb) - d.addCallbacks(self._cbStopClient, self._ebGeneral) + create_deferreds.append(d) + dd = defer.gatherResults(create_deferreds) + dd.addCallbacks(self._cbStopClient, self._ebGeneral) + return dd self.result = [] - d1 = self.connected.addCallback(strip(login)).addCallback( - strip(create)) + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create)) d2 = self.loopback() - d = defer.gatherResults([d1, d2]) + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestCreate, succeed, fail) - def _cbTestCreate(self, ignored, succeed, fail): + def _cbTestCreate(self, mailboxes, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = list(LeapIMAPServer.theAccount.mailboxes) - answers = ([u'INBOX', u'foobox', 'test', u'test/box', - u'test/box/box', 'testbox']) - self.assertEqual(mboxes, [a for a in answers]) + answers = ([u'INBOX', u'testbox', u'test/box', u'test', + u'test/box/box', 'foobox']) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - LeapIMAPServer.theAccount.addMailbox('delete/me') + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) def delete(): - return self.client.delete('delete/me') + return self.client.delete('test-delete/me') - d1 = self.connected.addCallback(strip(login)) + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback( - lambda _: self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, ['INBOX'])) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) return d def testIllegalInboxDelete(self): @@ -357,24 +222,34 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - LeapIMAPServer.theAccount.addMailbox('delete') - to_delete = LeapIMAPServer.theAccount.getMailbox('delete') - to_delete.setFlags((r'\Noselect',)) - to_delete.getFlags() - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = self.server.theAccount def login(): return self.client.login(TEST_USER, TEST_PASSWD) - def delete(): + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): return self.client.delete('delete') def deleteFailed(failure): self.failure = failure self.failure = None + d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -386,11 +261,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly def testRename(self): """ Test whether we can rename a mailbox """ - LeapIMAPServer.theAccount.addMailbox('oldmbox') + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -398,15 +277,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, - ['INBOX', 'newname'])) + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) return d def testIllegalInboxRename(self): @@ -440,8 +320,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - LeapIMAPServer.theAccount.create('oldmbox/m1') - LeapIMAPServer.theAccount.create('oldmbox/m2') + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('oldmbox/m1'), + acc.addMailbox('oldmbox/m2')]) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -449,45 +333,65 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestHierarchicalRename) - def _cbTestHierarchicalRename(self, ignored): - mboxes = LeapIMAPServer.theAccount.mailboxes - expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] - self.assertEqual(mboxes, [s for s in expected]) + def _cbTestHierarchicalRename(self, mailboxes): + expected = ['INBOX', 'newname/m1', 'newname/m2'] + self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) def testSubscribe(self): """ Test whether we can mark a mailbox as subscribed to """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + def login(): return self.client.login(TEST_USER, TEST_PASSWD) def subscribe(): return self.client.subscribe('this/mbox') - d1 = self.connected.addCallback(strip(login)) + def get_subscriptions(ignored): + return self.server.theAccount.getSubscriptions() + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['this/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['this/mbox'])) return d def testUnsubscribe(self): """ Test whether we can unsubscribe from a set of mailboxes """ - LeapIMAPServer.theAccount.subscribe('this/mbox') - LeapIMAPServer.theAccount.subscribe('that/mbox') + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) + + def dc1(): + return acc.subscribe('this/mbox') + + def dc2(): + return acc.subscribe('that/mbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -495,24 +399,35 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def unsubscribe(): return self.client.unsubscribe('this/mbox') - d1 = self.connected.addCallback(strip(login)) + def get_subscriptions(ignored): + return acc.getSubscriptions() + + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['that/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['that/mbox'])) return d def testSelect(self): """ Try to select a mailbox """ - self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + mbox_name = "TESTMAILBOXSELECT" self.selectedArgs = None + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -520,29 +435,26 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def selected(args): self.selectedArgs = args self._cbStopClient(None) - d = self.client.select('TESTMAILBOX-SELECT') + d = self.client.select(mbox_name) d.addCallback(selected) return d - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(select)) - d1.addErrback(self._ebGeneral) + # d1.addErrback(self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d def _cbTestSelect(self, ignored): - mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. + self.assertTrue(self.selectedArgs is not None) self.assertEqual(self.selectedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': True @@ -560,13 +472,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): caps.update(c) self.server.transport.loseConnection() return self.client.getCapabilities().addCallback(gotCaps) - d1 = self.connected.addCallback( + + d1 = self.connected + d1.addCallback( strip(getCaps)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, 'IDLE': None} - - return d.addCallback(lambda _: self.assertEqual(expected, caps)) + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d def testCapabilityWithAuth(self): caps = {} @@ -587,7 +502,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'IDLE': None, 'LITERAL+': None, 'AUTH': ['CRAM-MD5']} - return d.addCallback(lambda _: self.assertEqual(expCap, caps)) + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d # # authentication @@ -635,7 +551,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') def testFailedLogin(self): @@ -673,7 +588,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # @@ -720,11 +634,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): for details. """ # TODO implement the IMAP4ClientExamineTests testcase. - - self.server.theAccount.addMailbox('test-mailbox-e', - creation_ts=42) + mbox_name = "test_mailbox_e" + acc = self.server.theAccount self.examinedArgs = None + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -732,11 +648,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def examined(args): self.examinedArgs = args self._cbStopClient(None) - d = self.client.examine('test-mailbox-e') + d = self.client.examine(mbox_name) d.addCallback(examined) return d - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(examine)) d1.addErrback(self._ebGeneral) d2 = self.loopback() @@ -744,28 +661,24 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('test-mailbox-e') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. self.assertEqual(self.examinedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) - def _listSetup(self, f): - LeapIMAPServer.theAccount.addMailbox('root/subthingl', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('root/another-thing', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('non-root/subthing', - creation_ts=42) + def _listSetup(self, f, f2=None): + + acc = self.server.theAccount + + def dc1(): + return acc.addMailbox('root_subthing', creation_ts=42) + + def dc2(): + return acc.addMailbox('root_another_thing', creation_ts=42) + + def dc3(): + return acc.addMailbox('non_root_subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -775,6 +688,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.listed = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallback(strip(dc3)) + + if f2 is not None: + d1.addCallback(f2) + d1.addCallbacks(strip(f), self._ebGeneral) d1.addCallbacks(listed, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -787,12 +707,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ def list(): return self.client.list('root', '%') + d = self._listSetup(list) d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"), - (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") ]) )) return d @@ -801,20 +722,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - LeapIMAPServer.theAccount.subscribe('root/subthingl2') + acc = self.server.theAccount + + def subs_mailbox(): + # why not client.subscribe instead? + return acc.subscribe('root_subthing') def lsub(): return self.client.lsub('root', '%') - d = self._listSetup(lsub) + + d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")]) + [(IMAPMailbox.init_flags, "/", "root_subthing")]) return d def testStatus(self): """ Test Status command """ - LeapIMAPServer.theAccount.addMailbox('root/subthings') + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('root_subthings') + # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -823,13 +753,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def status(): return self.client.status( - 'root/subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') def statused(result): self.statused = result self.statused = None - d1 = self.connected.addCallback(strip(login)) + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(status), self._ebGeneral) d1.addCallbacks(statused, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -886,56 +818,86 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) - LeapIMAPServer.theAccount.addMailbox('root/subthing') + acc = self.server.theAccount + mailbox_name = "appendmbox/subthing" + + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def append(): return self.client.append( - 'root/subthing', - message, + mailbox_name, message, ('\\SEEN', '\\DELETED'), 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', ) - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback(self._cbTestFullAppend, infile) - def _cbTestFullAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('root/subthing') - self.assertEqual(1, len(mb.messages)) + def _cbTestFullAppend(self, fetched, infile): + fetched = list(fetched) + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = CaseInsensitiveDict(parsed.items()) - msg = mb.messages.get_msg_by_uid(1) - self.assertEqual( - set(('\\Recent', '\\SEEN', '\\DELETED')), - set(msg.getFlags())) + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) - self.assertEqual( - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - msg.getInternalDate()) + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) - parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - headers = dict(parsed.items()) - self.assertEqual( - body, - msg.getBodyFile().read()) - gotheaders = msg.getHeaders(True) + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(map(string.lower, expected_headers), headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) - self.assertItemsEqual( - headers, gotheaders) + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) + + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d def testPartialAppend(self): """ Test partially appending a message to the mailbox """ + # TODO this test sometimes will fail because of the notify_just_mdoc infile = util.sibpath(__file__, 'rfc822.message') - LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -950,33 +912,47 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): (), self.client._IMAP4Client__cbContinueAppend, message ) ) - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback( self._cbTestPartialAppend, infile) - def _cbTestPartialAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - self.assertEqual(1, len(mb.messages)) - msg = mb.messages.get_msg_by_uid(1) - self.assertEqual( - set(('\\SEEN', '\\Recent')), - set(msg.getFlags()) - ) + def _cbTestPartialAppend(self, fetched, infile): + fetched = list(fetched) + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - self.assertEqual( - body, - msg.getBodyFile().read()) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d def testCheck(self): """ Test check command """ - LeapIMAPServer.theAccount.addMailbox('root/subthing') + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -987,89 +963,51 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def check(): return self.client.check() - d = self.connected.addCallback(strip(login)) + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(check), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) - return self.loopback() - - # Okay, that was fun - - def testClose(self): - """ - Test closing the mailbox. We expect to get deleted all messages flagged - as such. - """ - name = 'mailbox-close' - self.server.theAccount.addMailbox(name) - - m = LeapIMAPServer.theAccount.getMailbox(name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select(name) - - def add_messages(): - d1 = m.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = m.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) - return d - - def close(): - return self.client.close() - - d = self.connected.addCallback(strip(login)) - d.addCallbacks(strip(select), self._ebGeneral) - d.addCallbacks(strip(add_messages), self._ebGeneral) - d.addCallbacks(strip(close), self._ebGeneral) - d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m) - - def _cbTestClose(self, ignored, m): - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) - self.assertTrue(msg is not None) + return defer.gatherResults([d, d2]) - self.assertEqual( - dict(msg.hdoc.content)['subject'], - 'Message 2') - self.failUnless(m.closed) + # Okay, that was much fun indeed def testExpunge(self): """ Test expunge command """ - name = 'mailbox-expunge' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + acc = self.server.theAccount + mailbox_name = 'mailboxexpunge' + + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def select(): - return self.client.select('mailbox-expunge') + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox + + def get_mailbox(): + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d def add_messages(): - d1 = m.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = m.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag'), + notify_just_mdoc=False) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',), + notify_just_mdoc=False)) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',), + notify_just_mdoc=False)) return d def expunge(): @@ -1080,47 +1018,41 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = results self.results = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallbacks(strip(select), self._ebGeneral) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestExpunge, m) + d.addCallback(lambda _: self.mailbox.getMessageCount()) + return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored, m): + def _cbTestExpunge(self, count): # we only left 1 mssage with no deleted flag - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) - - msg = list(m.messages)[0] - self.assertTrue(msg is not None) - - self.assertEqual( - msg.hdoc.content['subject'], - 'Message 2') - + self.assertEqual(count, 1) # the uids of the deleted messages self.assertItemsEqual(self.results, [1, 3]) -class AccountTestCase(IMAP4HelperMixin, unittest.TestCase): +class AccountTestCase(IMAP4HelperMixin): """ Test the Account. """ def _create_empty_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('') + return self.server.theAccount.addMailbox('') def _create_one_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('one') + return self.server.theAccount.addMailbox('one') def test_illegalMailboxCreate(self): self.assertRaises(AssertionError, self._create_empty_mailbox) -class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): """ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ diff --git a/src/leap/mail/imap/tests/test_imap_store_fetch.py b/src/leap/mail/imap/tests/test_imap_store_fetch.py deleted file mode 100644 index 6da8581..0000000 --- a/src/leap/mail/imap/tests/test_imap_store_fetch.py +++ /dev/null @@ -1,71 +0,0 @@ -from twisted.protocols import loopback -from twisted.python import util - -from leap.mail.imap.tests.utils import IMAP4HelperMixin - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" - - -class StoreAndFetchTestCase(IMAP4HelperMixin): - """ - Several tests to check that the internal storage representation - is able to render the message structures as we expect them. - """ - - def setUp(self): - IMAP4HelperMixin.setUp(self) - self.received_messages = self.received_uid = None - self.result = None - - def addListener(self, x): - pass - - def removeListener(self, x): - pass - - def _addSignedMessage(self, _): - self.server.state = 'select' - infile = util.sibpath(__file__, 'rfc822.multi-signed.message') - raw = open(infile).read() - MBOX_NAME = "multipart/SIGNED" - - self.server.theAccount.addMailbox(MBOX_NAME) - mbox = self.server.theAccount.getMailbox(MBOX_NAME) - self.server.mbox = mbox - # return a deferred that will fire with UID - return self.server.mbox.messages.add_msg(raw) - - def _fetchWork(self, uids): - - def result(R): - self.result = R - - self.connected.addCallback( - self._addSignedMessage).addCallback( - lambda uid: self.function( - uids, uid=uid) # do NOT use seq numbers! - ).addCallback(result).addCallback( - self._cbStopClient).addErrback(self._ebGeneral) - - d = loopback.loopbackTCP(self.server, self.client, noisy=False) - d.addCallback(lambda x: self.assertEqual(self.result, self.expected)) - return d - - def testMultiBody(self): - """ - Test that a multipart signed message is retrieved the same - as we stored it. - """ - self.function = self.client.fetchBody - messages = '1' - - # XXX review. This probably should give everything? - - self.expected = {1: { - 'RFC822.TEXT': 'This is an example of a signed message,\n' - 'with attachments.\n\n\n--=20\n' - 'Nihil sine chao! =E2=88=B4\n', - 'UID': '1'}} - # print "test multi: fetch uid", messages - return self._fetchWork(messages) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 0932bd4..a34538b 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -1,30 +1,44 @@ -import os -import tempfile -import shutil - +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Common utilities for testing Soledad IMAP Server. +""" from email import parser from mock import Mock from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback +from twisted.python import log -from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.soledad.client import Soledad +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.tests.common import SoledadTestMixin TEST_USER = "testuser@leap.se" TEST_PASSWD = "1234" + # # Simple IMAP4 Client for testing # - class SimpleClient(imap4.IMAP4Client): - """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -51,161 +65,59 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "http://provider" - cert_file = "" - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - - _soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file) - - return _soledad - - -# XXX this is not properly a mixin, since helper already inherits from -# uniittest.Testcase -class IMAP4HelperMixin(BaseLeapTest): +class IMAP4HelperMixin(SoledadTestMixin): """ MixIn containing several utilities to be shared across different TestCases """ - serverCTX = None clientCTX = None - # setUpClass cannot be a classmethod in trial, see: - # https://twistedmatrix.com/trac/ticket/1870 - def setUp(self): - """ - Setup method for each test. - - Initializes and run a LEAP IMAP4 Server, - but passing the same Soledad instance (it's costly to initialize), - so we have to be sure to restore state across tests. - """ - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - UUID = 'deadbeef', - USERID = TEST_USER - memstore = MemoryStore() - - ########### - d = defer.Deferred() - self.server = LeapIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - # XXX do we really need this?? - soledad=self._soledad) + soledad_adaptor.cleanup_deferred_locks() - self.client = SimpleClient(d, contextFactory=self.clientCTX) - self.connected = d - - # XXX REVIEW-ME. - # We're adding theAccount here to server - # but it was also passed to initialization - # as it was passed to realm. - # I THINK we ONLY need to do it at one place now. - - theAccount = SoledadBackedAccount( - USERID, - soledad=self._soledad, - memstore=memstore) - LeapIMAPServer.theAccount = theAccount - - # in case we get something from previous tests... - for mb in self.server.theAccount.mailboxes: - self.server.theAccount.delete(mb) + UUID = 'deadbeef', + USERID = TEST_USER - # email parser - self.parser = parser.Parser() + def setup_server(account): + self.server = LEAPIMAPServer( + uuid=UUID, userid=USERID, + contextFactory=self.serverCTX, + soledad=self._soledad) + self.server.theAccount = account + + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready + + def setup_account(_): + self.parser = parser.Parser() + + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + # TODO see here, possibly related? + # -- http://www.pythoneye.com/83_20424875/ + self._soledad.sync = Mock() + + d = defer.Deferred() + self.acc = IMAPAccount(USERID, self._soledad, d=d) + return d + + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d def tearDown(self): - """ - tearDown method called after each test. - - Deletes all documents in the Index, and deletes - instances of server and client. - """ - try: - self._soledad.close() - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" - - def populateMessages(self): - """ - Populates soledad instance with several simple messages - """ - # XXX we should encapsulate this thru SoledadBackedAccount - # instead. - - # XXX we also should put this in a mailbox! - - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") - # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") - - def delete_all_docs(self): - """ - Deletes all the docs in the testing instance of the - SoledadBackedAccount. - """ - self.server.theAccount.deleteAllMessages( - iknowhatiamdoing=True) + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected def _cbStopClient(self, ignore): self.client.transport.loseConnection() @@ -213,13 +125,8 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - # can we do something similar? - # I guess this was ok with trial, but not in noseland... - # log.err(failure, "Problem with %r" % (self.function,)) - raise failure.value - # failure.trap(Exception) + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) def loopback(self): return loopback.loopbackAsync(self.server, self.client) - - diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index 695f487..f259a55 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -1,4 +1,4 @@ -#t -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # walktree.py # Copyright (C) 2013 LEAP # @@ -19,6 +19,7 @@ Tests for the walktree module. """ import os import sys +import pprint from email import parser from leap.mail import walk as W @@ -118,7 +119,6 @@ if DEBUG and DO_CHECK: print "Structure: OK" -import pprint print print "RAW DOCS" pprint.pprint(raw_docs) |