diff options
| author | Tomás Touceda <chiiph@leap.se> | 2014-04-04 16:43:40 -0300 |
|---|---|---|
| committer | Tomás Touceda <chiiph@leap.se> | 2014-04-04 16:43:40 -0300 |
| commit | 8b1d8e88955fe5c1af8cee973abfdec4043a650c (patch) | |
| tree | 0cad3ee91e02335d0cbfc30bdc42d46cc83fbe8a | |
| parent | dc45a6098b798a2866c2ab6af097307913cf542b (diff) | |
| parent | 3258025ea15e3f20c8054fd67ec617b7a87eb309 (diff) | |
Merge branch 'release-0.3.9'0.3.9
43 files changed, 9487 insertions, 2159 deletions
@@ -19,3 +19,4 @@ lib/ local/ share/ MANIFEST +twistd.pid @@ -1,3 +1,57 @@ +0.3.9 Apr 4: + o Footer url shouldn't end in period. Closes #4791. + o Handle non-ascii headers. Closes #5021. + o Soledad writer consumes messages eagerly. Fixes failing + tests. Closes #4715. + o Convert unicode to str when raising exceptions in IMAP server. + Fixes #4830. + o Remove conversion of IMAP folder names to string. This makes the + IMAP server use twisted's transparent 7bit conversion. Fixes + #4830. + o Add a flag to be able to reset the session. Closes #4925. + o Check for none in payload detection. Closes #4933. + o Check for flags doc uniqueness before adding a message. Avoids + duplicates of a single message in the same mailbox while copying + or moving. Closes #4949. + o Correctly process attachments when signing. Fixes #5014. + o Fix bug in which destination folder sometimes was not showing + messages after copy/append. Closes #5167. + o Fix unread notifications to client UI. Only INBOX is + notified. Closes #5177. + o Fix bug in which deleted folder wouldn't show its messages + inside. Closes #5179. + o Keep processing after a decryption error. Closes #5307. + o Enqueue unsetting of recent flag. this was holding the new mails + from being displayed soonish. + o Properly parse emails crafted by Mail.app. Fixes #5013. + o Restrict adding outgoing footer to text/plain messages. + o Sanity check on last_uid setter. Avoids incomplete fetches. + o Stop providing hostname for helo in smtp gateway. Fixes #4335. + o Only try to fetch keys for multipart signed or encrypted emails. + Fixes #4671. + o Add a flag for offline mode in imap. Related to #4943. + o Flush IMAP data to disk when stopping. Closes #5095. + o Signal the client when auth token is invalid for syncing Soledad. + Fixes #5191. + o Ability to support SEARCH Commands, limited to HEADER Message-ID. + This is a quick workaround for avoiding duplicate saves in Drafts + Folder. Closes #4209. + o Use a memory store as write-buffer and read-cache. + o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs + can be made in a single round-trip. Closes #5190. + o Defer costly operations to a pool of threads. + o Split the internal representation of messages into three distinct + documents: 1) Flags 2) Headers 3) Content. + o Make use of the Twisted MIME interface. + o Add deduplication ability to the save operation, for body and + attachments. + o Add IMessageCopier interface to mailbox implementation, so bulk + moves are costless. Closes #4654. + o Makes efficient use of indexes and count method. Closes #4616. + o Handle correctly unicode characters in emails. Closes #4838. + +-- 2014 -- + 0.3.8 Dec 6: o Fail gracefully when failing to decrypt incoming messages. Closes #4589. diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index 032b26a..cc00ecf 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -8,4 +8,3 @@ # # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z - diff --git a/pkg/requirements-testing.pip b/pkg/requirements-testing.pip index 7233634..41222f7 100644 --- a/pkg/requirements-testing.pip +++ b/pkg/requirements-testing.pip @@ -1,2 +1,7 @@ setuptools-trial mock +nose +rednose +nose-progressive +coverage +pep8>=1.1 diff --git a/pkg/requirements.pip b/pkg/requirements.pip index dc0635c..17ceba6 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -1,6 +1,7 @@ zope.interface -leap.soledad.client>=0.3.0 -leap.common>=0.3.5 -leap.keymanager>=0.3.7 +leap.soledad.client>=0.4.5 +leap.common>=0.3.7 +leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy +enum diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py index 8a66c1f..d80ec47 100644 --- a/src/leap/mail/_version.py +++ b/src/leap/mail/_version.py @@ -17,6 +17,7 @@ git_full = "$Format:%H$" import subprocess import sys + def run_command(args, cwd=None, verbose=False): try: # remember shell=False, so use git.cmd on windows, not just git @@ -41,6 +42,7 @@ import sys import re import os.path + def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import @@ -48,7 +50,7 @@ def get_expanded_variables(versionfile_source): # used from _version.py. variables = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_source, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -63,12 +65,13 @@ def get_expanded_variables(versionfile_source): pass return variables + def versions_from_expanded_variables(variables, tag_prefix, verbose=False): refnames = variables["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + return {} # unexpanded, so not in an unpacked git-archive tarball refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -84,7 +87,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) + print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -93,13 +96,14 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } + return {"version": r, + "full": variables["full"].strip()} # no suitable tags, so we use the full revision id if verbose: print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } + return {"version": variables["full"].strip(), + "full": variables["full"].strip()} + def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): # this runs 'git' from the root of the source tree. That either means @@ -116,7 +120,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): here = os.path.abspath(__file__) except NameError: # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct + return {} # not always correct # versionfile_source is the relative path from the top of the source tree # (where the .git directory might live) to this file. Invert this to find @@ -141,7 +145,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): return {} if not stdout.startswith(tag_prefix): if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) + print("tag '%s' doesn't start with prefix '%s'" % + (stdout, tag_prefix)) return {} tag = stdout[len(tag_prefix):] stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) @@ -153,7 +158,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): return {"version": tag, "full": full} -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): +def versions_from_parentdir(parentdir_prefix, versionfile_source, + verbose=False): if IN_LONG_VERSION_PY: # We're running from _version.py. If it's from a source tree # (execute-in-place), we can work upwards to find the root of the @@ -163,7 +169,7 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) here = os.path.abspath(__file__) except NameError: # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + return {} # without __file__, we have no hope # versionfile_source is the relative path from the top of the source # tree to _version.py. Invert this to find the root from __file__. root = here @@ -180,7 +186,8 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) dirname = os.path.basename(root) if not dirname.startswith(parentdir_prefix): if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % + print("guessing rootdir is '%s', but '%s' doesn't " + "start with prefix '%s'" % (root, dirname, parentdir_prefix)) return None return {"version": dirname[len(parentdir_prefix):], "full": ""} @@ -189,8 +196,9 @@ tag_prefix = "" parentdir_prefix = "leap-mail" versionfile_source = "src/leap/mail/_version.py" + def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } + variables = {"refnames": git_refnames, "full": git_full} ver = versions_from_expanded_variables(variables, tag_prefix, verbose) if not ver: ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) @@ -200,4 +208,3 @@ def get_versions(default={"version": "unknown", "full": ""}, verbose=False): if not ver: ver = default return ver - diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py new file mode 100644 index 0000000..5105de9 --- /dev/null +++ b/src/leap/mail/decorators.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# decorators.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Useful decorators for mail package. +""" +import logging +import os + +from functools import wraps + +from twisted.internet.threads import deferToThread + + +logger = logging.getLogger(__name__) + + +# TODO +# Should write a helper to be able to pass a timeout argument. +# See this answer: http://stackoverflow.com/a/19019648/1157664 +# And the notes by glyph and jpcalderone + +def deferred_to_thread(f): + """ + Decorator, for deferring methods to Threads. + + It will do a deferToThread of the decorated method + unless the environment variable LEAPMAIL_DEBUG is set. + + It uses a descriptor to delay the definition of the + method wrapper. + """ + class descript(object): + """ + The class to be used as decorator. + + It takes any method as the passed object. + """ + + def __init__(self, f): + """ + Initializes the decorator object. + + :param f: the decorated function + :type f: callable + """ + self.f = f + + def __get__(self, instance, klass): + """ + Descriptor implementation. + + At creation time, the decorated `method` is unbound. + + It will dispatch the make_unbound method if we still do not + have an instance available, and the make_bound method when the + method has already been bound to the instance. + + :param instance: the instance of the class, or None if not exist. + :type instance: instantiated class or None. + """ + if instance is None: + # Class method was requested + return self.make_unbound(klass) + return self.make_bound(instance) + + def _errback(self, failure): + """ + Errorback that logs the exception catched. + + :param failure: a twisted failure + :type failure: Failure + """ + logger.warning('Error in method: %s' % (self.f.__name__)) + logger.exception(failure.getTraceback()) + + def make_unbound(self, klass): + """ + Return a wrapped function with the unbound call, during the + early access to the decortad method. This gets passed + only the class (not the instance since it does not yet exist). + + :param klass: the class to which the still unbound method belongs + :type klass: type + """ + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + We're temporarily wrapping the decorated method, but this + should not be called, since our application should use + the bound-wrapped method after this decorator class has been + used. + + This documentation will vanish at runtime. + """ + raise TypeError( + 'unbound method {}() must be called with {} instance ' + 'as first argument (got nothing instead)'.format( + self.f.__name__, + klass.__name__) + ) + return wrapper + + def make_bound(self, instance): + """ + Return a function that wraps the bound method call, + after we are able to access the instance object. + + :param instance: an instance of the class the decorated method, + now bound, belongs to. + :type instance: object + """ + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + Do a proper function wrapper that defers the decorated method + call to a separated thread if the LEAPMAIL_DEBUG + environment variable is set. + + This documentation will vanish at runtime. + """ + if not os.environ.get('LEAPMAIL_DEBUG'): + d = deferToThread(self.f, instance, *args, **kwargs) + d.addErrback(self._errback) + return d + else: + return self.f(instance, *args, **kwargs) + + # This instance does not need the descriptor anymore, + # let it find the wrapper directly next time: + setattr(instance, self.f.__name__, wrapper) + return wrapper + + return descript(f) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py new file mode 100644 index 0000000..199a2a4 --- /dev/null +++ b/src/leap/mail/imap/account.py @@ -0,0 +1,462 @@ +# -*- coding: utf-8 -*- +# account.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad Backed Account. +""" +import copy +import logging +import os +import time + +from twisted.mail import imap4 +from twisted.python import log +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.imap.index import IndexedDB +from leap.mail.imap.fields import WithMsgFields +from leap.mail.imap.parser import MBoxParser +from leap.mail.imap.mailbox import SoledadMailbox +from leap.soledad.client import Soledad + +logger = logging.getLogger(__name__) + +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + +####################################### +# Soledad Account +####################################### + + +# TODO change name to LeapIMAPAccount, since we're using +# the memstore. +# IndexedDB should also not be here anymore. + +class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): + """ + An implementation of IAccount and INamespacePresenteer + that is backed by Soledad Encrypted Documents. + """ + + implements(imap4.IAccount, imap4.INamespacePresenter) + + _soledad = None + selected = None + closed = False + + def __init__(self, account_name, soledad, memstore=None): + """ + Creates a SoledadAccountIndex that keeps track of the mailboxes + and subscriptions handled by this account. + + :param acct_name: The name of the account (user id). + :type acct_name: str + + :param soledad: a Soledad instance. + :type soledad: Soledad + :param memstore: a MemoryStore instance. + :type memstore: MemoryStore + """ + leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert_type(soledad, Soledad) + + # XXX SHOULD assert too that the name matches the user/uuid with which + # soledad has been initialized. + + # XXX ??? why is this parsing mailbox name??? it's account... + # userid? homogenize. + self._account_name = self._parse_mailbox_name(account_name) + self._soledad = soledad + self._memstore = memstore + + self.__mailboxes = set([]) + + self.initialize_db() + + # every user should have the right to an inbox folder + # at least, so let's make one! + self._load_mailboxes() + + if not self.mailboxes: + self.addMailbox(self.INBOX_NAME) + + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + return copy.deepcopy(self.EMPTY_MBOX) + + def _get_mailbox_by_name(self, name): + """ + Return an mbox document by name. + + :param name: the name of the mailbox + :type name: str + + :rtype: SoledadDocument + """ + # XXX use soledadstore instead ...; + doc = self._soledad.get_from_index( + self.TYPE_MBOX_IDX, self.MBOX_KEY, + self._parse_mailbox_name(name)) + return doc[0] if doc else None + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + """ + return self.__mailboxes + + def _load_mailboxes(self): + self.__mailboxes.update( + [doc.content[self.MBOX_KEY] + for doc in self._soledad.get_from_index( + self.TYPE_IDX, self.MBOX_KEY)]) + + @property + def subscriptions(self): + """ + A list of the current subscriptions for this account. + """ + return [doc.content[self.MBOX_KEY] + for doc in self._soledad.get_from_index( + self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + + def getMailbox(self, name): + """ + Returns a Mailbox with that name, without selecting it. + + :param name: name of the mailbox + :type name: str + + :returns: a a SoledadMailbox instance + :rtype: SoledadMailbox + """ + name = self._parse_mailbox_name(name) + + if name not in self.mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + + return SoledadMailbox(name, self._soledad, + memstore=self._memstore) + + ## + ## IAccount + ## + + def addMailbox(self, name, creation_ts=None): + """ + Add a mailbox to the account. + + :param name: the name of the mailbox + :type name: str + + :param creation_ts: an optional creation timestamp to be used as + mailbox id. A timestamp will be used if no + one is provided. + :type creation_ts: int + + :returns: True if successful + :rtype: bool + """ + name = self._parse_mailbox_name(name) + + if name in self.mailboxes: + raise imap4.MailboxCollision(repr(name)) + + if not creation_ts: + # by default, we pass an int value + # taken from the current time + # we make sure to take enough decimals to get a unique + # mailbox-uidvalidity. + creation_ts = int(time.time() * 10E2) + + mbox = self._get_empty_mailbox() + mbox[self.MBOX_KEY] = name + mbox[self.CREATED_KEY] = creation_ts + + doc = self._soledad.create_doc(mbox) + self._load_mailboxes() + return bool(doc) + + def create(self, pathspec): + """ + Create a new mailbox from the given hierarchical name. + + :param pathspec: The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. + :type pathspec: str + + :return: A true value if the creation succeeds. + :rtype: bool + + :raise MailboxException: Raised if this mailbox cannot be added. + """ + # TODO raise MailboxException + paths = filter( + None, + self._parse_mailbox_name(pathspec).split('/')) + for accum in range(1, len(paths)): + try: + self.addMailbox('/'.join(paths[:accum])) + except imap4.MailboxCollision: + pass + try: + self.addMailbox('/'.join(paths)) + except imap4.MailboxCollision: + if not pathspec.endswith('/'): + return False + self._load_mailboxes() + return True + + def select(self, name, readwrite=1): + """ + Selects a mailbox. + + :param name: the mailbox to select + :type name: str + + :param readwrite: 1 for readwrite permissions. + :type readwrite: int + + :rtype: SoledadMailbox + """ + if PROFILE_CMD: + start = time.time() + + name = self._parse_mailbox_name(name) + if name not in self.mailboxes: + logger.warning("No such mailbox!") + return None + self.selected = name + + sm = SoledadMailbox( + name, self._soledad, self._memstore, readwrite) + if PROFILE_CMD: + _debugProfiling(None, "SELECT", start) + return sm + + def delete(self, name, force=False): + """ + Deletes a mailbox. + + Right now it does not purge the messages, but just removes the mailbox + name from the mailboxes list!!! + + :param name: the mailbox to be deleted + :type name: str + + :param force: if True, it will not check for noselect flag or inferior + names. use with care. + :type force: bool + """ + name = self._parse_mailbox_name(name) + + if not name in self.mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + + mbox = self.getMailbox(name) + + if force is False: + # See if this box is flagged \Noselect + # XXX use mbox.flags instead? + if self.NOSELECT_FLAG in mbox.getFlags(): + # Check for hierarchically inferior mailboxes with this one + # as part of their root. + for others in self.mailboxes: + if others != name and others.startswith(name): + raise imap4.MailboxException, ( + "Hierarchically inferior mailboxes " + "exist and \\Noselect is set") + mbox.destroy() + self._load_mailboxes() + + # XXX FIXME --- not honoring the inferior names... + + # if there are no hierarchically inferior names, we will + # delete it from our ken. + #if self._inferiorNames(name) > 1: + # ??! -- can this be rite? + #self._index.removeMailbox(name) + + def rename(self, oldname, newname): + """ + Renames a mailbox. + + :param oldname: old name of the mailbox + :type oldname: str + + :param newname: new name of the mailbox + :type newname: str + """ + oldname = self._parse_mailbox_name(oldname) + newname = self._parse_mailbox_name(newname) + + if oldname not in self.mailboxes: + raise imap4.NoSuchMailbox(repr(oldname)) + + inferiors = self._inferiorNames(oldname) + inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + + for (old, new) in inferiors: + if new in self.mailboxes: + raise imap4.MailboxCollision(repr(new)) + + for (old, new) in inferiors: + self._memstore.rename_fdocs_mailbox(old, new) + mbox = self._get_mailbox_by_name(old) + mbox.content[self.MBOX_KEY] = new + self._soledad.put_doc(mbox) + + self._load_mailboxes() + + def _inferiorNames(self, name): + """ + Return hierarchically inferior mailboxes. + + :param name: name of the mailbox + :rtype: list + """ + # XXX use wildcard query instead + inferiors = [] + for infname in self.mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + def isSubscribed(self, name): + """ + Returns True if user is subscribed to this mailbox. + + :param name: the mailbox to be checked. + :type name: str + + :rtype: bool + """ + mbox = self._get_mailbox_by_name(name) + return mbox.content.get('subscribed', False) + + def _set_subscription(self, name, value): + """ + Sets the subscription value for a given mailbox + + :param name: the mailbox + :type name: str + + :param value: the boolean value + :type value: bool + """ + # maybe we should store subscriptions in another + # document... + if not name in self.mailboxes: + self.addMailbox(name) + mbox = self._get_mailbox_by_name(name) + + if mbox: + mbox.content[self.SUBSCRIBED_KEY] = value + self._soledad.put_doc(mbox) + + def subscribe(self, name): + """ + Subscribe to this mailbox + + :param name: name of the mailbox + :type name: str + """ + name = self._parse_mailbox_name(name) + if name not in self.subscriptions: + self._set_subscription(name, True) + + def unsubscribe(self, name): + """ + Unsubscribe from this mailbox + + :param name: name of the mailbox + :type name: str + """ + name = self._parse_mailbox_name(name) + if name not in self.subscriptions: + raise imap4.MailboxException( + "Not currently subscribed to %r" % name) + self._set_subscription(name, False) + + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames( + self._parse_mailbox_name(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + ## + ## INamespacePresenter + ## + + def getPersonalNamespaces(self): + return [["", "/"]] + + def getSharedNamespaces(self): + return None + + def getOtherNamespaces(self): + return None + + # extra, for convenience + + def deleteAllMessages(self, iknowhatiamdoing=False): + """ + Deletes all messages from all mailboxes. + Danger! high voltage! + + :param iknowhatiamdoing: confirmation parameter, needs to be True + to proceed. + """ + if iknowhatiamdoing is True: + for mbox in self.mailboxes: + self.delete(mbox, force=True) + + def __repr__(self): + """ + Representation string for this object. + """ + return "<SoledadBackedAccount (%s)>" % self._account_name diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 14f7a9b..0a97752 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -17,21 +17,24 @@ """ Incoming mail fetcher. """ +import copy import logging -import json -import ssl import threading import time -import copy -from StringIO import StringIO +import sys +import traceback +import warnings from email.parser import Parser from email.generator import Generator from email.utils import parseaddr +from StringIO import StringIO from twisted.python import log +from twisted.internet import defer from twisted.internet.task import LoopingCall -from twisted.internet.threads import deferToThread +from twisted.internet.task import deferLater +from u1db import errors as u1db_errors from zope.proxy import sameProxiedObjects from leap.common import events as leap_events @@ -42,15 +45,25 @@ from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.decorators import deferred_to_thread +from leap.mail.imap.fields import fields +from leap.mail.utils import json_loads, empty, first from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY +from leap.soledad.common.errors import InvalidAuthTokenError logger = logging.getLogger(__name__) +MULTIPART_ENCRYPTED = "multipart/encrypted" +MULTIPART_SIGNED = "multipart/signed" +PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" +PGP_END = "-----END PGP MESSAGE-----" + class MalformedMessage(Exception): """ @@ -73,8 +86,6 @@ class LeapIncomingMail(object): """ RECENT_FLAG = "\\Recent" - - INCOMING_KEY = "incoming" CONTENT_KEY = "content" LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' @@ -123,13 +134,8 @@ class LeapIncomingMail(object): self._loop = None self._check_period = check_period - self._create_soledad_indexes() - - def _create_soledad_indexes(self): - """ - Create needed indexes on soledad. - """ - self._soledad.create_index("just-mail", "incoming") + # initialize a mail parser only once + self._parser = Parser() @property def _pkey(self): @@ -149,12 +155,29 @@ class LeapIncomingMail(object): Calls a deferred that will execute the fetch callback in a separate thread """ + def syncSoledadCallback(result): + # FIXME this needs a matching change in mx!!! + # --> need to add ERROR_DECRYPTING_KEY = False + # as default. + try: + doclist = self._soledad.get_from_index( + fields.JUST_MAIL_IDX, "*", "0") + except u1db_errors.InvalidGlobbing: + # It looks like we are a dealing with an outdated + # mx. Fallback to the version of the index + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + doclist = self._soledad.get_from_index( + fields.JUST_MAIL_COMPAT_IDX, "*") + self._process_doclist(doclist) + logger.debug("fetching mail for: %s %s" % ( self._soledad.uuid, self._userid)) if not self.fetching_lock.locked(): - d = deferToThread(self._sync_soledad) - d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error) - d.addCallbacks(self._process_doclist, self._sync_soledad_error) + d1 = self._sync_soledad() + d = defer.gatherResults([d1], consumeErrors=True) + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) return d else: logger.debug("Already fetching mail.") @@ -184,81 +207,53 @@ class LeapIncomingMail(object): # synchronize incoming mail + def _errback(self, failure): + logger.exception(failure.value) + traceback.print_tb(*sys.exc_info()) + + @deferred_to_thread def _sync_soledad(self): """ - Synchronizes with remote soledad. + Synchronize with remote soledad. :returns: a list of LeapDocuments, or None. :rtype: iterable or None """ with self.fetching_lock: - log.msg('syncing soledad...') - self._soledad.sync() - log.msg('soledad synced.') - doclist = self._soledad.get_from_index("just-mail", "*") - - return doclist - - def _signal_unread_to_ui(self): - """ - Sends unread event to ui. - """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) + try: + log.msg('FETCH: syncing soledad...') + self._soledad.sync() + log.msg('FETCH soledad SYNCED.') + except InvalidAuthTokenError: + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) def _signal_fetch_to_ui(self, doclist): """ - Sends leap events to ui. + Send leap events to ui. :param doclist: iterable with msg documents. :type doclist: iterable. :returns: doclist :rtype: iterable """ - fetched_ts = time.mktime(time.gmtime()) - num_mails = len(doclist) - log.msg("there are %s mails" % (num_mails,)) - leap_events.signal( - IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) - self._signal_unread_to_ui() - return doclist - - def _sync_soledad_error(self, failure): - """ - Errback for sync errors. - """ - # XXX should signal unrecoverable maybe. - err = failure.value - logger.error("error syncing soledad: %s" % (err,)) - if failure.check(ssl.SSLError): - logger.warning('SSL Error while ' - 'syncing soledad: %r' % (err,)) - elif failure.check(Exception): - logger.warning('Unknown error while ' - 'syncing soledad: %r' % (err,)) - - def _log_err(self, failure): - """ - Generic errback - """ - err = failure.value - logger.exception("error!: %r" % (err,)) - - def _decryption_error(self, failure): - """ - Errback for decryption errors. - """ - # XXX should signal unrecoverable maybe. - err = failure.value - logger.error("error decrypting msg: %s" % (err,)) + doclist = first(doclist) # gatherResults pass us a list + if doclist: + fetched_ts = time.mktime(time.gmtime()) + num_mails = len(doclist) if doclist is not None else 0 + if num_mails != 0: + log.msg("there are %s mails" % (num_mails,)) + leap_events.signal( + IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) + return doclist - def _saving_error(self, failure): + def _signal_unread_to_ui(self, *args): """ - Errback for local save errors. + Sends unread event to ui. """ - # XXX should signal unrecoverable maybe. - err = failure.value - logger.error("error saving msg locally: %s" % (err,)) + leap_events.signal( + IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) # process incoming mail. @@ -278,44 +273,40 @@ class LeapIncomingMail(object): return num_mails = len(doclist) - docs_cb = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) leap_events.signal( IMAP_MSG_PROCESSING, str(index), str(num_mails)) + keys = doc.content.keys() - if self._is_msg(keys): - # Ok, this looks like a legit msg. + + # TODO Compatibility check with the index in pre-0.6 mx + # that does not write the ERROR_DECRYPTING_KEY + # This should be removed in 0.7 + + has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) + if has_errors is None: + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + if has_errors: + logger.debug("skipping msg with decrypting errors...") + + if self._is_msg(keys) and not has_errors: + # Evaluating to bool of has_errors is intentional here. + # We don't mind at this point if it's None or False. + + # Ok, this looks like a legit msg, and with no errors. # Let's process it! - # Deferred chain for individual messages - - # XXX use an IConsumer instead... ? - d = deferToThread(self._decrypt_doc, doc) - d.addCallback(self._process_decrypted_doc) - d.addErrback(self._log_err) - d.addCallback(self._add_message_locally) - d.addErrback(self._log_err) - docs_cb.append(d) - else: - # Ooops, this does not. - logger.debug('This does not look like a proper msg.') - return docs_cb + + d1 = self._decrypt_doc(doc) + d = defer.gatherResults([d1], consumeErrors=True) + d.addCallbacks(self._add_message_locally, self._errback) # # operations on individual messages # - def _is_msg(self, keys): - """ - Checks if the keys of a dictionary match the signature - of the document type we use for messages. - - :param keys: iterable containing the strings to match. - :type keys: iterable of strings. - :rtype: bool - """ - return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys - + @deferred_to_thread def _decrypt_doc(self, doc): """ Decrypt the contents of a document. @@ -339,7 +330,9 @@ class LeapIncomingMail(object): logger.error("Error while decrypting msg: %r" % (exc,)) decrdata = "" leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - return doc, decrdata + + data = self._process_decrypted_doc((doc, decrdata)) + return (doc, data) def _process_decrypted_doc(self, msgtuple): """ @@ -355,54 +348,105 @@ class LeapIncomingMail(object): """ log.msg('processing decrypted doc') doc, data = msgtuple - msg = json.loads(data) + + from twisted.internet import reactor + + # XXX turn this into an errBack for each one of + # the deferreds that would process an individual document + try: + msg = json_loads(data) + except UnicodeError as exc: + logger.error("Error while decrypting %s" % (doc.doc_id,)) + logger.exception(exc) + + # we flag the message as "with decrypting errors", + # to avoid further decryption attempts during sync + # cycles until we're prepared to deal with that. + # What is the same, when Ivan deals with it... + # A new decrypting attempt event could be triggered by a + # future a library upgrade, or a cli flag to the client, + # we just `defer` that for now... :) + doc.content[fields.ERROR_DECRYPTING_KEY] = True + deferLater(reactor, 0, self._update_incoming_message, doc) + + # FIXME this is just a dirty hack to delay the proper + # deferred organization here... + # and remember, boys, do not do this at home. + return [] + if not isinstance(msg, dict): - return False - if not msg.get(self.INCOMING_KEY, False): - return False + defer.returnValue(False) + if not msg.get(fields.INCOMING_KEY, False): + defer.returnValue(False) # ok, this is an incoming message rawmsg = msg.get(self.CONTENT_KEY, None) if not rawmsg: return False - data = self._maybe_decrypt_msg(rawmsg) - return doc, data + return self._maybe_decrypt_msg(rawmsg) + + @deferred_to_thread + def _update_incoming_message(self, doc): + """ + Do a put for a soledad document. This probably has been called only + in the case that we've needed to update the ERROR_DECRYPTING_KEY + flag in an incoming message, to get it out of the decrypting queue. + + :param doc: the SoledadDocument to update + :type doc: SoledadDocument + """ + log.msg("Updating SoledadDoc %s" % (doc.doc_id)) + self._soledad.put_doc(doc) + + @deferred_to_thread + def _delete_incoming_message(self, doc): + """ + Delete document. + + :param doc: the SoledadDocument to delete + :type doc: SoledadDocument + """ + log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) + self._soledad.delete_doc(doc) def _maybe_decrypt_msg(self, data): """ Tries to decrypt a gpg message if data looks like one. :param data: the text to be decrypted. - :type data: unicode + :type data: str :return: data, possibly descrypted. :rtype: str """ + leap_assert_type(data, str) log.msg('maybe decrypting doc') - leap_assert_type(data, unicode) # parse the original message - parser = Parser() encoding = get_email_charset(data) - data = data.encode(encoding) - msg = parser.parsestr(data) + msg = self._parser.parsestr(data) # try to obtain sender public key senderPubkey = None fromHeader = msg.get('from', None) - if fromHeader is not None: + if (fromHeader is not None + and (msg.get_content_type() == MULTIPART_ENCRYPTED + or msg.get_content_type() == MULTIPART_SIGNED)): _, senderAddress = parseaddr(fromHeader) try: - senderPubkey = self._keymanager.get_key( + senderPubkey = self._keymanager.get_key_from_cache( senderAddress, OpenPGPKey) except keymanager_errors.KeyNotFound: pass valid_sig = False # we will add a header saying if sig is valid - if msg.get_content_type() == 'multipart/encrypted': - decrmsg, valid_sig = self._decrypt_multipart_encrypted_msg( + decrypt_multi = self._decrypt_multipart_encrypted_msg + decrypt_inline = self._maybe_decrypt_inline_encrypted_msg + + if msg.get_content_type() == MULTIPART_ENCRYPTED: + decrmsg, valid_sig = decrypt_multi( msg, encoding, senderPubkey) else: - decrmsg, valid_sig = self._maybe_decrypt_inline_encrypted_msg( + decrmsg, valid_sig = decrypt_inline( msg, encoding, senderPubkey) # add x-leap-signature header @@ -435,25 +479,12 @@ class LeapIncomingMail(object): """ log.msg('decrypting multipart encrypted msg') msg = copy.deepcopy(msg) - # sanity check - payload = msg.get_payload() - if len(payload) != 2: - raise MalformedMessage( - 'Multipart/encrypted messages should have exactly 2 body ' - 'parts (instead of %d).' % len(payload)) - if payload[0].get_content_type() != 'application/pgp-encrypted': - raise MalformedMessage( - "Multipart/encrypted messages' first body part should " - "have content type equal to 'application/pgp-encrypted' " - "(instead of %s)." % payload[0].get_content_type()) - if payload[1].get_content_type() != 'application/octet-stream': - raise MalformedMessage( - "Multipart/encrypted messages' second body part should " - "have content type equal to 'octet-stream' (instead of " - "%s)." % payload[1].get_content_type()) + self._msg_multipart_sanity_check(msg) + # parse message and get encrypted content pgpencmsg = msg.get_payload()[1] encdata = pgpencmsg.get_payload() + # decrypt or fail gracefully try: decrdata, valid_sig = self._decrypt_and_verify_data( @@ -461,17 +492,13 @@ class LeapIncomingMail(object): except keymanager_errors.DecryptError as e: logger.warning('Failed to decrypt encrypted message (%s). ' 'Storing message without modifications.' % str(e)) - return msg, False # return original message - # decrypted successully, now fix encoding and parse - try: - decrdata = decrdata.encode(encoding) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - decrdata = decrdata.encode(encoding, 'replace') - parser = Parser() - decrmsg = parser.parsestr(decrdata) + # Bailing out! + return (msg, False) + + decrmsg = self._parser.parsestr(decrdata) # remove original message's multipart/encrypted content-type del(msg['content-type']) + # replace headers back in original message for hkey, hval in decrmsg.items(): try: @@ -479,9 +506,10 @@ class LeapIncomingMail(object): msg.replace_header(hkey, hval) except KeyError: msg[hkey] = hval - # replace payload by unencrypted payload + + # all ok, replace payload by unencrypted payload msg.set_payload(decrmsg.get_payload()) - return msg, valid_sig + return (msg, valid_sig) def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, senderPubkey): @@ -495,8 +523,9 @@ class LeapIncomingMail(object): :param senderPubkey: The key of the sender of the message. :type senderPubkey: OpenPGPKey - :return: A unitary tuple containing a decrypted message. - :rtype: (Message) + :return: A tuple containing a decrypted message and + a bool indicating whether the signature is valid. + :rtype: (Message, bool) """ log.msg('maybe decrypting inline encrypted msg') # serialize the original message @@ -505,13 +534,11 @@ class LeapIncomingMail(object): g.flatten(origmsg) data = buf.getvalue() # handle exactly one inline PGP message - PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" - PGP_END = "-----END PGP MESSAGE-----" valid_sig = False if PGP_BEGIN in data: begin = data.find(PGP_BEGIN) end = data.find(PGP_END) - pgp_message = data[begin:end+len(PGP_END)] + pgp_message = data[begin:end + len(PGP_END)] try: decrdata, valid_sig = self._decrypt_and_verify_data( pgp_message, senderPubkey) @@ -520,11 +547,11 @@ class LeapIncomingMail(object): except keymanager_errors.DecryptError: logger.warning('Failed to decrypt potential inline encrypted ' 'message. Storing message as is...') + # if message is not encrypted, return raw data if isinstance(data, unicode): data = data.encode(encoding, 'replace') - parser = Parser() - return parser.parsestr(data), valid_sig + return (self._parser.parsestr(data), valid_sig) def _decrypt_and_verify_data(self, data, senderPubkey): """ @@ -553,25 +580,76 @@ class LeapIncomingMail(object): except keymanager_errors.InvalidSignature: decrdata = self._keymanager.decrypt( data, self._pkey) - return decrdata, valid_sig + return (decrdata, valid_sig) - def _add_message_locally(self, msgtuple): + def _add_message_locally(self, result): """ Adds a message to local inbox and delete it from the incoming db in soledad. + # XXX this comes from a gatherresult... :param msgtuple: a tuple consisting of a SoledadDocument instance containing the incoming message and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) """ - log.msg('adding message to local db') + from twisted.internet import reactor + msgtuple = first(result) + doc, data = msgtuple - self._inbox.addMessage(data, (self.RECENT_FLAG,)) - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - doc_id = doc.doc_id - self._soledad.delete_doc(doc) - log.msg("deleted doc %s from incoming" % doc_id) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) - self._signal_unread_to_ui() + log.msg('adding message %s to local db' % (doc.doc_id,)) + + if isinstance(data, list): + if empty(data): + return False + data = data[0] + + def msgSavedCallback(result): + if not empty(result): + leap_events.signal(IMAP_MSG_SAVED_LOCALLY) + deferLater(reactor, 0, self._delete_incoming_message, doc) + leap_events.signal(IMAP_MSG_DELETED_INCOMING) + + d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), + notify_on_disk=True) + d.addCallbacks(msgSavedCallback, self._errback) + + # + # helpers + # + + def _msg_multipart_sanity_check(self, msg): + """ + Performs a sanity check against a multipart encrypted msg + + :param msg: The original encrypted message. + :type msg: Message + """ + # sanity check + payload = msg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py new file mode 100644 index 0000000..4576939 --- /dev/null +++ b/src/leap/mail/imap/fields.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# fields.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Fields for Mailbox and Message. +""" +from leap.mail.imap.parser import MBoxParser + + +class WithMsgFields(object): + """ + Container class for class-attributes to be shared by + several message-related classes. + """ + # indexing + CONTENT_HASH_KEY = "chash" + PAYLOAD_HASH_KEY = "phash" + + # Internal representation of Message + + # flags doc + UID_KEY = "uid" + MBOX_KEY = "mbox" + SEEN_KEY = "seen" + DEL_KEY = "deleted" + RECENT_KEY = "recent" + FLAGS_KEY = "flags" + MULTIPART_KEY = "multi" + SIZE_KEY = "size" + + # headers + HEADERS_KEY = "headers" + DATE_KEY = "date" + SUBJECT_KEY = "subject" + PARTS_MAP_KEY = "part_map" + BODY_KEY = "body" # link to phash of body + MSGID_KEY = "msgid" + + # content + LINKED_FROM_KEY = "lkf" # XXX not implemented yet! + RAW_KEY = "raw" + CTYPE_KEY = "ctype" + + # Mailbox specific keys + CLOSED_KEY = "closed" + CREATED_KEY = "created" + SUBSCRIBED_KEY = "subscribed" + RW_KEY = "rw" + LAST_UID_KEY = "lastuid" + RECENTFLAGS_KEY = "rct" + HDOCS_SET_KEY = "hdocset" + + # Document Type, for indexing + TYPE_KEY = "type" + TYPE_MBOX_VAL = "mbox" + TYPE_FLAGS_VAL = "flags" + TYPE_HEADERS_VAL = "head" + TYPE_CONTENT_VAL = "cnt" + TYPE_RECENT_VAL = "rct" + TYPE_HDOCS_SET_VAL = "hdocset" + + INBOX_VAL = "inbox" + + # Flags in Mailbox and Message + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) + + # Fields in mail object + SUBJECT_FIELD = "Subject" + DATE_FIELD = "Date" + + # Index types + # -------------- + + TYPE_IDX = 'by-type' + TYPE_MBOX_IDX = 'by-type-and-mbox' + TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' + TYPE_SUBS_IDX = 'by-type-and-subscribed' + TYPE_MSGID_IDX = 'by-type-and-message-id' + TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' + TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' + TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' + TYPE_C_HASH_IDX = 'by-type-and-contenthash' + TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' + TYPE_P_HASH_IDX = 'by-type-and-payloadhash' + + # Tomas created the `recent and seen index`, but the semantic is not too + # correct since the recent flag is volatile. + TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + + # Soledad index for incoming mail, without decrypting errors. + JUST_MAIL_IDX = "just-mail" + # XXX the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX = "just-mail-compat" + + INCOMING_KEY = "incoming" + ERROR_DECRYPTING_KEY = "errdecr" + + KTYPE = TYPE_KEY + MBOX_VAL = TYPE_MBOX_VAL + CHASH_VAL = CONTENT_HASH_KEY + PHASH_VAL = PAYLOAD_HASH_KEY + + INDEXES = { + # generic + TYPE_IDX: [KTYPE], + TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], + TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], + + # mailboxes + TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], + + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], + + # headers doc - search by msgid. + TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], + + # content, headers doc + TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], + + # attachment payload dedup + TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], + + # messages + TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], + TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, + 'bool(recent)', 'bool(seen)'], + + # incoming queue + JUST_MAIL_IDX: [INCOMING_KEY, + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], + + # the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], + } + + MBOX_KEY = MBOX_VAL + + EMPTY_MBOX = { + TYPE_KEY: MBOX_KEY, + TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, + SUBJECT_KEY: "", + FLAGS_KEY: [], + CLOSED_KEY: False, + SUBSCRIBED_KEY: False, + RW_KEY: 1, + LAST_UID_KEY: 0 + } + +fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py new file mode 100644 index 0000000..5f0919a --- /dev/null +++ b/src/leap/mail/imap/index.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# index.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Index for SoledadBackedAccount, Mailbox and Messages. +""" +import logging + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail.imap.fields import fields + + +logger = logging.getLogger(__name__) + + +class IndexedDB(object): + """ + Methods dealing with the index. + + This is a MixIn that needs access to the soledad instance, + and also assumes that a INDEXES attribute is accessible to the instance. + + INDEXES must be a dictionary of type: + {'index-name': ['field1', 'field2']} + """ + # TODO we might want to move this to soledad itself, check + + def initialize_db(self): + """ + Initialize the database. + """ + leap_assert(self._soledad, + "Need a soledad attribute accesible in the instance") + leap_assert_type(self.INDEXES, dict) + + # Ask the database for currently existing indexes. + if not self._soledad: + logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") + return + db_indexes = dict() + if self._soledad is not None: + db_indexes = dict(self._soledad.list_indexes()) + for name, expression in fields.INDEXES.items(): + if name not in db_indexes: + # The index does not yet exist. + self._soledad.create_index(name, *expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so we + # delete it and add the proper index expression. + self._soledad.delete_index(name) + self._soledad.create_index(name, *expression) diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py new file mode 100644 index 0000000..c906278 --- /dev/null +++ b/src/leap/mail/imap/interfaces.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Interfaces for the IMAP module. +""" +from zope.interface import Interface, Attribute + + +class IMessageContainer(Interface): + """ + I am a container around the different documents that a message + is split into. + """ + fdoc = Attribute('The flags document for this message, if any.') + hdoc = Attribute('The headers document for this message, if any.') + cdocs = Attribute('The dict of content documents for this message, ' + 'if any.') + + def walk(self): + """ + Return an iterator to the docs for all the parts. + + :rtype: iterator + """ + + +class IMessageStore(Interface): + """ + I represent a generic storage for LEAP Messages. + """ + + def create_message(self, mbox, uid, message): + """ + Put the passed message into this IMessageStore. + + :param mbox: the mbox this message belongs. + :param uid: the UID that identifies this message in this mailbox. + :param message: a IMessageContainer implementor. + """ + + def put_message(self, mbox, uid, message): + """ + Put the passed message into this IMessageStore. + + :param mbox: the mbox this message belongs. + :param uid: the UID that identifies this message in this mailbox. + :param message: a IMessageContainer implementor. + """ + + def remove_message(self, mbox, uid): + """ + Remove the given message from this IMessageStore. + + :param mbox: the mbox this message belongs. + :param uid: the UID that identifies this message in this mailbox. + """ + + def get_message(self, mbox, uid): + """ + Get a IMessageContainer for the given mbox and uid combination. + + :param mbox: the mbox this message belongs. + :param uid: the UID that identifies this message in this mailbox. + :return: IMessageContainer + """ + + +class IMessageStoreWriter(Interface): + """ + I represent a storage that is able to write its contents to another + different IMessageStore. + """ + + def write_messages(self, store): + """ + Write the documents in this IMessageStore to a different + storage. Usually this will be done from a MemoryStorage to a DbStorage. + + :param store: another IMessageStore implementor. + """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py new file mode 100644 index 0000000..47c7ff1 --- /dev/null +++ b/src/leap/mail/imap/mailbox.py @@ -0,0 +1,1026 @@ +# *- coding: utf-8 -*- +# mailbox.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad Mailbox. +""" +import copy +import threading +import logging +import StringIO +import cStringIO +import os + +from collections import defaultdict + +from twisted.internet import defer +from twisted.internet.task import deferLater +from twisted.python import log + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common import events as leap_events +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.decorators import deferred_to_thread +from leap.mail.utils import empty +from leap.mail.imap.fields import WithMsgFields, fields +from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.messageparts import MessageWrapper +from leap.mail.imap.parser import MBoxParser + +logger = logging.getLogger(__name__) + +""" +If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid +notifying clients of new messages. Use during stress tests. +""" +NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + import time + + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + def do_profile_cmd(d, name): + """ + Add the profiling debug to the passed callback. + :param d: deferred + :param name: name of the command + :type name: str + """ + d.addCallback(_debugProfiling, name, time.time()) + d.addErrback(lambda f: log.msg(f.getTraceback())) + + +class SoledadMailbox(WithMsgFields, MBoxParser): + """ + A Soledad-backed IMAP mailbox. + + Implements the high-level method needed for the Mailbox interfaces. + The low-level database methods are contained in MessageCollection class, + which we instantiate and make accessible in the `messages` attribute. + """ + implements( + imap4.IMailbox, + imap4.IMailboxInfo, + imap4.ICloseableMailbox, + imap4.ISearchableMailbox, + imap4.IMessageCopier) + + # XXX should finish the implementation of IMailboxListener + # XXX should completely implement ISearchableMailbox too + + messages = None + _closed = False + + INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, + WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, + WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, + WithMsgFields.LIST_FLAG) + flags = None + + CMD_MSG = "MESSAGES" + CMD_RECENT = "RECENT" + CMD_UIDNEXT = "UIDNEXT" + CMD_UIDVALIDITY = "UIDVALIDITY" + CMD_UNSEEN = "UNSEEN" + + # FIXME we should turn this into a datastructure with limited capacity + _listeners = defaultdict(set) + + next_uid_lock = threading.Lock() + last_uid_lock = threading.Lock() + + # TODO unify all the `primed` dicts + _fdoc_primed = {} + _last_uid_primed = {} + _known_uids_primed = {} + + def __init__(self, mbox, soledad, memstore, rw=1): + """ + SoledadMailbox constructor. Needs to get passed a name, plus a + Soledad instance. + + :param mbox: the mailbox name + :type mbox: str + + :param soledad: a Soledad instance. + :type soledad: Soledad + + :param memstore: a MemoryStore instance + :type memstore: MemoryStore + + :param rw: read-and-write flag for this mailbox + :type rw: int + """ + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(soledad, "Need a soledad instance to initialize") + + from twisted.internet import reactor + self.reactor = reactor + + self.mbox = self._parse_mailbox_name(mbox) + self.rw = rw + + self._soledad = soledad + self._memstore = memstore + + self.messages = MessageCollection( + mbox=mbox, soledad=self._soledad, memstore=self._memstore) + + self._uidvalidity = None + + # XXX careful with this get/set (it would be + # hitting db unconditionally, move to memstore too) + # Now it's returning a fixed amount of flags from mem + # as a workaround. + if not self.getFlags(): + self.setFlags(self.INIT_FLAGS) + + if self._memstore: + self.prime_known_uids_to_memstore() + self.prime_last_uid_to_memstore() + self.prime_flag_docs_to_memstore() + + # purge memstore from empty fdocs. + self._memstore.purge_fdoc_store(mbox) + + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox] + + # TODO this grows too crazily when many instances are fired, like + # during imaptest stress testing. Should have a queue of limited size + # instead. + def addListener(self, listener): + """ + Add a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + if not NOTIFY_NEW: + return + + logger.debug('adding mailbox listener: %s' % listener) + self.listeners.add(listener) + + def removeListener(self, listener): + """ + Remove a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + self.listeners.remove(listener) + + def _get_mbox_doc(self): + """ + Return mailbox document. + + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. + """ + return self._memstore.get_mbox_doc(self.mbox) + + def getFlags(self): + """ + Returns the flags defined for this mailbox. + + :returns: tuple of flags for this mailbox + :rtype: tuple of str + """ + flags = self.INIT_FLAGS + + # XXX returning fixed flags always + # Since I have not found a case where the client + # wants to modify this, as a way of speeding up + # selects. To do it right, we probably should keep + # track of the set of all flags used by msgs + # in this mailbox. Does it matter? + #mbox = self._get_mbox_doc() + #if not mbox: + #return None + #flags = mbox.content.get(self.FLAGS_KEY, []) + return map(str, flags) + + # XXX move to memstore->soledadstore + def setFlags(self, flags): + """ + Sets flags for this mailbox. + + :param flags: a tuple with the flags + :type flags: tuple of str + """ + leap_assert(isinstance(flags, tuple), + "flags expected to be a tuple") + mbox = self._get_mbox_doc() + if not mbox: + return None + mbox.content[self.FLAGS_KEY] = map(str, flags) + logger.debug("Writing mbox document for %r to Soledad" + % (self.mbox,)) + self._soledad.put_doc(mbox) + + # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + + def _get_closed(self): + """ + Return the closed attribute for this mailbox. + + :return: True if the mailbox is closed + :rtype: bool + """ + return self._memstore.get_mbox_closed(self.mbox) + + def _set_closed(self, closed): + """ + Set the closed attribute for this mailbox. + + :param closed: the state to be set + :type closed: bool + """ + self._memstore.set_mbox_closed(self.mbox, closed) + + closed = property( + _get_closed, _set_closed, doc="Closed attribute.") + + def _get_last_uid(self): + """ + Return the last uid for this mailbox. + If we have a memory store, the last UID will be the highest + recorded UID in the message store, or a counter cached from + the mailbox document in soledad if this is higher. + + :return: the last uid for messages in this mailbox + :rtype: int + """ + last = self._memstore.get_last_uid(self.mbox) + logger.debug("last uid for %s: %s (from memstore)" % ( + repr(self.mbox), last)) + return last + + last_uid = property( + _get_last_uid, doc="Last_UID attribute.") + + def prime_last_uid_to_memstore(self): + """ + Prime memstore with last_uid value + """ + primed = self._last_uid_primed.get(self.mbox, False) + if not primed: + mbox = self._get_mbox_doc() + last = mbox.content.get('lastuid', 0) + logger.info("Priming Soledad last_uid to %s" % (last,)) + self._memstore.set_last_soledad_uid(self.mbox, last) + self._last_uid_primed[self.mbox] = True + + def prime_known_uids_to_memstore(self): + """ + Prime memstore with the set of all known uids. + + We do this to be able to filter the requests efficiently. + """ + primed = self._known_uids_primed.get(self.mbox, False) + if not primed: + known_uids = self.messages.all_soledad_uid_iter() + self._memstore.set_known_uids(self.mbox, known_uids) + self._known_uids_primed[self.mbox] = True + + def prime_flag_docs_to_memstore(self): + """ + Prime memstore with all the flags documents. + """ + primed = self._fdoc_primed.get(self.mbox, False) + if not primed: + all_flag_docs = self.messages.get_all_soledad_flag_docs() + self._memstore.load_flag_docs(self.mbox, all_flag_docs) + self._fdoc_primed[self.mbox] = True + + def getUIDValidity(self): + """ + Return the unique validity identifier for this mailbox. + + :return: unique validity identifier + :rtype: int + """ + if self._uidvalidity is None: + mbox = self._get_mbox_doc() + if mbox is None: + return 0 + self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) + return self._uidvalidity + + def getUID(self, message): + """ + Return the UID of a message in the mailbox + + .. note:: this implementation does not make much sense RIGHT NOW, + but in the future will be useful to get absolute UIDs from + message sequence numbers. + + :param message: the message uid + :type message: int + + :rtype: int + """ + msg = self.messages.get_msg_by_uid(message) + if msg is not None: + return msg.getUID() + + def getUIDNext(self): + """ + Return the likely UID for the next message added to this + mailbox. Currently it returns the higher UID incremented by + one. + + We increment the next uid *each* time this function gets called. + In this way, there will be gaps if the message with the allocated + uid cannot be saved. But that is preferable to having race conditions + if we get to parallel message adding. + + :rtype: int + """ + with self.next_uid_lock: + return self.last_uid + 1 + + def getMessageCount(self): + """ + Returns the total count of messages in this mailbox. + + :rtype: int + """ + return self.messages.count() + + def getUnseenCount(self): + """ + Returns the number of messages with the 'Unseen' flag. + + :return: count of messages flagged `unseen` + :rtype: int + """ + return self.messages.count_unseen() + + def getRecentCount(self): + """ + Returns the number of messages with the 'Recent' flag. + + :return: count of messages flagged `recent` + :rtype: int + """ + return self.messages.count_recent() + + def isWriteable(self): + """ + Get the read/write status of the mailbox. + + :return: 1 if mailbox is read-writeable, 0 otherwise. + :rtype: int + """ + return self.rw + + def getHierarchicalDelimiter(self): + """ + Returns the character used to delimite hierarchies in mailboxes. + + :rtype: str + """ + return '/' + + def requestStatus(self, names): + """ + Handles a status request by gathering the output of the different + status commands. + + :param names: a list of strings containing the status commands + :type names: iter + """ + r = {} + if self.CMD_MSG in names: + r[self.CMD_MSG] = self.getMessageCount() + if self.CMD_RECENT in names: + r[self.CMD_RECENT] = self.getRecentCount() + if self.CMD_UIDNEXT in names: + r[self.CMD_UIDNEXT] = self.last_uid + 1 + if self.CMD_UIDVALIDITY in names: + r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + if self.CMD_UNSEEN in names: + r[self.CMD_UNSEEN] = self.getUnseenCount() + return defer.succeed(r) + + def addMessage(self, message, flags, date=None, notify_on_disk=False): + """ + Adds a message to this mailbox. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str + + :return: a deferred that evals to None + """ + # TODO have a look at the cases for internal date in the rfc + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() + + # XXX we could treat the message as an IMessage from here + leap_assert_type(message, basestring) + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + d = self._do_add_message(message, flags=flags, date=date, + notify_on_disk=notify_on_disk) + if PROFILE_CMD: + do_profile_cmd(d, "APPEND") + # A better place for this would be the COPY/APPEND dispatcher + # in server.py, but qtreactor hangs when I do that, so this seems + # to work fine for now. + + def notifyCallback(x): + self.reactor.callLater(0, self.notify_new) + return x + + d.addCallback(notifyCallback) + d.addErrback(lambda f: log.msg(f.getTraceback())) + return d + + def _do_add_message(self, message, flags, date, notify_on_disk=False): + """ + Calls to the messageCollection add_msg method. + Invoked from addMessage. + """ + d = self.messages.add_msg(message, flags=flags, date=date, + notify_on_disk=notify_on_disk) + return d + + def notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + :param args: ignored. + """ + if not NOTIFY_NEW: + return + + def cbNotifyNew(result): + exists, recent = result + for l in self.listeners: + l.newMessages(exists, recent) + d = self._get_notify_count() + d.addCallback(cbNotifyNew) + d.addCallback(self.cb_signal_unread_to_ui) + + @deferred_to_thread + def _get_notify_count(self): + """ + Get message count and recent count for this mailbox + Executed in a separate thread. Called from notify_new. + + :return: number of messages and number of recent messages. + :rtype: tuple + """ + exists = self.getMessageCount() + recent = self.getRecentCount() + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox, exists, recent)) + return exists, recent + + # commands, do not rename methods + + def destroy(self): + """ + Called before this mailbox is permanently deleted. + + Should cleanup resources, and set the \\Noselect flag + on the mailbox. + """ + self.setFlags((self.NOSELECT_FLAG,)) + self.deleteAllDocs() + + # XXX removing the mailbox in situ for now, + # we should postpone the removal + + # XXX move to memory store?? + self._soledad.delete_doc(self._get_mbox_doc()) + + def _close_cb(self, result): + self.closed = True + + def close(self): + """ + Expunge and mark as closed + """ + d = self.expunge() + d.addCallback(self._close_cb) + return d + + def expunge(self): + """ + Remove all messages flagged \\Deleted + """ + if not self.isWriteable(): + raise imap4.ReadOnlyMailbox + d = defer.Deferred() + self._memstore.expunge(self.mbox, d) + return d + + def _bound_seq(self, messages_asked): + """ + Put an upper bound to a messages sequence if this is open. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :rtype: MessageSet + """ + if not messages_asked.last: + try: + iter(messages_asked) + except TypeError: + # looks like we cannot iterate + try: + messages_asked.last = self.last_uid + except ValueError: + pass + return messages_asked + + def _filter_msg_seq(self, messages_asked): + """ + Filter a message sequence returning only the ones that do exist in the + collection. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :rtype: set + """ + set_asked = set(messages_asked) + set_exist = set(self.messages.all_uid_iter()) + seq_messg = set_asked.intersection(set_exist) + return seq_messg + + def fetch(self, messages_asked, uid): + """ + Retrieve one or more messages in this mailbox. + + from rfc 3501: The data items to be fetched can be either a single atom + or a parenthesized list. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :rtype: deferred + """ + d = defer.Deferred() + self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "FETCH") + d.addCallback(self.cb_signal_unread_to_ui) + return d + + # called in thread + def _do_fetch(self, messages_asked, uid, d): + """ + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + :param d: deferred whose callback will be called with result. + :type d: Deferred + + :rtype: A tuple of two-tuples of message sequence numbers and + LeapMessage + """ + # For the moment our UID is sequential, so we + # can treat them all the same. + # Change this to the flag that twisted expects when we + # switch to content-hash based index + local UID table. + + sequence = False + #sequence = True if uid == 0 else False + + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) + getmsg = lambda uid: self.messages.get_msg_by_uid(uid) + + # for sequence numbers (uid = 0) + if sequence: + logger.debug("Getting msg by index: INEFFICIENT call!") + raise NotImplementedError + else: + got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) + result = ((msgid, msg) for msgid, msg in got_msg + if msg is not None) + self.reactor.callLater(0, self.unset_recent_flags, seq_messg) + self.reactor.callFromThread(d.callback, result) + + def fetch_flags(self, messages_asked, uid): + """ + A fast method to fetch all flags, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic FLAGS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + it's not bad to fetch all the FLAGS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + + :return: A tuple of two-tuples of message sequence numbers and + flagsPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + d = defer.Deferred() + self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "FETCH-ALL-FLAGS") + return d + + # called in thread + def _do_fetch_flags(self, messages_asked, uid, d): + """ + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + :param d: deferred whose callback will be called with result. + :type d: Deferred + + :rtype: A tuple of two-tuples of message sequence numbers and + flagsPart + """ + class flagsPart(object): + def __init__(self, uid, flags): + self.uid = uid + self.flags = flags + + def getUID(self): + return self.uid + + def getFlags(self): + return map(str, self.flags) + + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) + + all_flags = self._memstore.all_flags(self.mbox) + result = ((msgid, flagsPart( + msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) + self.reactor.callFromThread(d.callback, result) + + def fetch_headers(self, messages_asked, uid): + """ + A fast method to fetch all headers, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic HEADERS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + **MAYBE** it's not too bad to fetch all the HEADERS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :return: A tuple of two-tuples of message sequence numbers and + headersPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + # TODO how often is thunderbird doing this? + + class headersPart(object): + def __init__(self, uid, headers): + self.uid = uid + self.headers = headers + + def getUID(self): + return self.uid + + def getHeaders(self, _): + return dict( + (str(key), str(value)) + for key, value in + self.headers.items()) + + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) + + all_headers = self.messages.all_headers() + result = ((msgid, headersPart( + msgid, all_headers.get(msgid, {}))) + for msgid in seq_messg) + return result + + def cb_signal_unread_to_ui(self, result): + """ + Sends unread event to ui. + Used as a callback in several commands. + + :param result: ignored + """ + d = self._get_unseen_deferred() + d.addCallback(self.__cb_signal_unread_to_ui) + return result + + @deferred_to_thread + def _get_unseen_deferred(self): + return self.getUnseenCount() + + def __cb_signal_unread_to_ui(self, unseen): + """ + Send the unread signal to UI. + :param unseen: number of unseen messages. + :type unseen: int + """ + leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + + def store(self, messages_asked, flags, mode, uid): + """ + Sets the flags of one or more messages. + + :param messages: The identifiers of the messages to set the flags + :type messages: A MessageSet object with the list of messages requested + + :param flags: The flags to set, unset, or add. + :type flags: sequence of str + + :param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be + added to the specified messages. If mode is 0, all + existing flags should be cleared and these flags should be + added. + :type mode: -1, 0, or 1 + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A deferred, that will be called with a dict mapping message + sequence numbers to sequences of str representing the flags + set on the message after this operation has been performed. + :rtype: deferred + + :raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + d = defer.Deferred() + self.reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "STORE") + d.addCallback(self.cb_signal_unread_to_ui) + d.addErrback(lambda f: log.msg(f.getTraceback())) + return d + + def _do_store(self, messages_asked, flags, mode, uid, observer): + """ + Helper method, invoke set_flags method in the MessageCollection. + + See the documentation for the `store` method for the parameters. + + :param observer: a deferred that will be called with the dictionary + mapping UIDs to flags after the operation has been + done. + :type observer: deferred + """ + # XXX implement also sequence (uid = 0) + # XXX we should prevent client from setting Recent flag? + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) + messages_asked = self._bound_seq(messages_asked) + seq_messg = self._filter_msg_seq(messages_asked) + self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer) + + # ISearchableMailbox + + def search(self, query, uid): + """ + Search for messages that meet the given query criteria. + + Warning: this is half-baked, and it might give problems since + it offers the SearchableInterface. + We'll be implementing it asap. + + :param query: The search criteria + :type query: list + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A list of message sequence numbers or message UIDs which + match the search criteria or a C{Deferred} whose callback + will be invoked with such a list. + :rtype: C{list} or C{Deferred} + """ + # TODO see if we can raise w/o interrupting flow + #:raise IllegalQueryError: Raised when query is not valid. + # example query: + # ['UNDELETED', 'HEADER', 'Message-ID', + # '52D44F11.9060107@dev.bitmask.net'] + + # TODO hardcoding for now! -- we'll support generic queries later on + # but doing a quickfix for avoiding duplicat saves in the draft folder. + # See issue #4209 + + if len(query) > 2: + if query[1] == 'HEADER' and query[2].lower() == "message-id": + msgid = str(query[3]).strip() + logger.debug("Searching for %s" % (msgid,)) + d = self.messages._get_uid_from_msgid(str(msgid)) + d1 = defer.gatherResults([d]) + # we want a list, so return it all the same + return d1 + + # nothing implemented for any other query + logger.warning("Cannot process query: %s" % (query,)) + return [] + + # IMessageCopier + + def copy(self, message): + """ + Copy the given message object into this mailbox. + + :param message: an IMessage implementor + :type message: LeapMessage + :return: a deferred that will be fired with the message + uid when the copy succeed. + :rtype: Deferred + """ + d = defer.Deferred() + if PROFILE_CMD: + do_profile_cmd(d, "COPY") + + # A better place for this would be the COPY/APPEND dispatcher + # in server.py, but qtreactor hangs when I do that, so this seems + # to work fine for now. + d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + deferLater(self.reactor, 0, self._do_copy, message, d) + return d + + def _do_copy(self, message, observer): + """ + Call invoked from the deferLater in `copy`. This will + copy the flags and header documents, and pass them to the + `create_message` method in the MemoryStore, together with + the observer deferred that we've been passed along. + + :param message: an IMessage implementor + :type message: LeapMessage + :param observer: the deferred that will fire with the + UID of the message + :type observer: Deferred + """ + memstore = self._memstore + + def createCopy(result): + exist, new_fdoc = result + if exist: + # Should we signal error on the callback? + logger.warning("Destination message already exists!") + + # XXX I'm not sure if we should raise the + # errback. This actually rases an ugly warning + # in some muas like thunderbird. + # UID 0 seems a good convention for no uid. + observer.callback(0) + else: + mbox = self.mbox + uid_next = memstore.increment_last_soledad_uid(mbox) + + new_fdoc[self.UID_KEY] = uid_next + new_fdoc[self.MBOX_KEY] = mbox + + flags = list(new_fdoc[self.FLAGS_KEY]) + flags.append(fields.RECENT_FLAG) + new_fdoc[self.FLAGS_KEY] = tuple(set(flags)) + + # FIXME set recent! + + self._memstore.create_message( + self.mbox, uid_next, + MessageWrapper(new_fdoc), + observer=observer, + notify_on_disk=False) + + d = self._get_msg_copy(message) + d.addCallback(createCopy) + d.addErrback(lambda f: log.msg(f.getTraceback())) + + @deferred_to_thread + def _get_msg_copy(self, message): + """ + Get a copy of the fdoc for this message, and check whether + it already exists. + + :param message: an IMessage implementor + :type message: LeapMessage + :return: exist, new_fdoc + :rtype: tuple + """ + # XXX for clarity, this could be delegated to a + # MessageCollection mixin that implements copy too, and + # moved out of here. + msg = message + memstore = self._memstore + + if empty(msg.fdoc): + logger.warning("Tried to copy a MSG with no fdoc") + return + new_fdoc = copy.deepcopy(msg.fdoc.content) + fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] + + dest_fdoc = memstore.get_fdoc_from_chash( + fdoc_chash, self.mbox) + + exist = not empty(dest_fdoc) + return exist, new_fdoc + + # convenience fun + + def deleteAllDocs(self): + """ + Delete all docs in this mailbox + """ + docs = self.messages.get_all_docs() + for doc in docs: + self.messages._soledad.delete_doc(doc) + + def unset_recent_flags(self, uid_seq): + """ + Unset Recent flag for a sequence of UIDs. + """ + self.messages.unset_recent_flags(uid_seq) + + def __repr__(self): + """ + Representation string for this mailbox. + """ + return u"<SoledadMailbox: mbox '%s' (%s)>" % ( + self.mbox, self.messages.count()) diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py new file mode 100644 index 0000000..d383b79 --- /dev/null +++ b/src/leap/mail/imap/memorystore.py @@ -0,0 +1,1300 @@ +# -*- coding: utf-8 -*- +# memorystore.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +In-memory transient store for a LEAPIMAPServer. +""" +import contextlib +import logging +import threading +import weakref + +from collections import defaultdict +from copy import copy + +from enum import Enum +from twisted.internet import defer +from twisted.internet.task import LoopingCall +from twisted.python import log +from zope.interface import implements + +from leap.common.check import leap_assert_type +from leap.mail import size +from leap.mail.utils import empty, phash_iter +from leap.mail.messageflow import MessageProducer +from leap.mail.imap import interfaces +from leap.mail.imap.fields import fields +from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc +from leap.mail.imap.messageparts import RecentFlagsDoc +from leap.mail.imap.messageparts import MessageWrapper +from leap.mail.imap.messageparts import ReferenciableDict + +from leap.mail.decorators import deferred_to_thread + +logger = logging.getLogger(__name__) + + +# The default period to do writebacks to the permanent +# soledad storage, in seconds. +SOLEDAD_WRITE_PERIOD = 15 + +FDOC = MessagePartType.fdoc.key +HDOC = MessagePartType.hdoc.key +CDOCS = MessagePartType.cdocs.key +DOCS_ID = MessagePartType.docs_id.key + + +@contextlib.contextmanager +def set_bool_flag(obj, att): + """ + Set a boolean flag to True while we're doing our thing. + Just to let the world know. + """ + setattr(obj, att, True) + try: + yield True + except RuntimeError as exc: + logger.exception(exc) + finally: + setattr(obj, att, False) + + +DirtyState = Enum("none", "dirty", "new") + + +class MemoryStore(object): + """ + An in-memory store to where we can write the different parts that + we split the messages into and buffer them until we write them to the + permanent storage. + + It uses MessageWrapper instances to represent the message-parts, which are + indexed by mailbox name and UID. + + It also can be passed a permanent storage as a paremeter (any implementor + of IMessageStore, in this case a SoledadStore). In this case, a periodic + dump of the messages stored in memory will be done. The period of the + writes to the permanent storage is controled by the write_period parameter + in the constructor. + """ + implements(interfaces.IMessageStore, + interfaces.IMessageStoreWriter) + + # TODO We will want to index by chash when we transition to local-only + # UIDs. + + WRITING_FLAG = "_writing" + _last_uid_lock = threading.Lock() + _fdoc_docid_lock = threading.Lock() + + def __init__(self, permanent_store=None, + write_period=SOLEDAD_WRITE_PERIOD): + """ + Initialize a MemoryStore. + + :param permanent_store: a IMessageStore implementor to dump + messages to. + :type permanent_store: IMessageStore + :param write_period: the interval to dump messages to disk, in seconds. + :type write_period: int + """ + from twisted.internet import reactor + self.reactor = reactor + + self._permanent_store = permanent_store + self._write_period = write_period + + # Internal Storage: messages + """ + flags document store. + _fdoc_store[mbox][uid] = { 'content': 'aaa' } + """ + self._fdoc_store = defaultdict(lambda: defaultdict( + lambda: ReferenciableDict({}))) + + # Sizes + """ + {'mbox, uid': <int>} + """ + self._sizes = {} + + # Internal Storage: payload-hash + """ + fdocs:doc-id store, stores document IDs for putting + the dirty flags-docs. + """ + self._fdoc_id_store = defaultdict(lambda: defaultdict( + lambda: '')) + + # Internal Storage: content-hash:hdoc + """ + hdoc-store keeps references to + the header-documents indexed by content-hash. + + {'chash': { dict-stuff } + } + """ + self._hdoc_store = defaultdict(lambda: ReferenciableDict({})) + + # Internal Storage: payload-hash:cdoc + """ + content-docs stored by payload-hash + {'phash': { dict-stuff } } + """ + self._cdoc_store = defaultdict(lambda: ReferenciableDict({})) + + # Internal Storage: content-hash:fdoc + """ + chash-fdoc-store keeps references to + the flag-documents indexed by content-hash. + + {'chash': {'mbox-a': weakref.proxy(dict), + 'mbox-b': weakref.proxy(dict)} + } + """ + self._chash_fdoc_store = defaultdict(lambda: defaultdict(lambda: None)) + + # Internal Storage: recent-flags store + """ + recent-flags store keeps one dict per mailbox, + with the document-id of the u1db document + and the set of the UIDs that have the recent flag. + + {'mbox-a': {'doc_id': 'deadbeef', + 'set': {1,2,3,4} + } + } + """ + # TODO this will have to transition to content-hash + # indexes after we move to local-only UIDs. + + self._rflags_store = defaultdict( + lambda: {'doc_id': None, 'set': set([])}) + + """ + last-uid store keeps the count of the highest UID + per mailbox. + + {'mbox-a': 42, + 'mbox-b': 23} + """ + self._last_uid = defaultdict(lambda: 0) + + """ + known-uids keeps a count of the uids that soledad knows for a given + mailbox + + {'mbox-a': set([1,2,3])} + """ + self._known_uids = defaultdict(set) + + # New and dirty flags, to set MessageWrapper State. + self._new = set([]) + self._new_queue = set([]) + self._new_deferreds = {} + + self._dirty = set([]) + self._dirty_queue = set([]) + self._dirty_deferreds = {} + + self._rflags_dirty = set([]) + + # Flag for signaling we're busy writing to the disk storage. + setattr(self, self.WRITING_FLAG, False) + + if self._permanent_store is not None: + # this producer spits its messages to the permanent store + # consumer using a queue. We will use that to put + # our messages to be written. + self.producer = MessageProducer(permanent_store, + period=0.1) + # looping call for dumping to SoledadStore + self._write_loop = LoopingCall(self.write_messages, + permanent_store) + + # We can start the write loop right now, why wait? + self._start_write_loop() + else: + # We have a memory-only store. + self.producer = None + self._write_loop = None + + def _start_write_loop(self): + """ + Start loop for writing to disk database. + """ + if self._write_loop is None: + return + if not self._write_loop.running: + self._write_loop.start(self._write_period, now=True) + + def _stop_write_loop(self): + """ + Stop loop for writing to disk database. + """ + if self._write_loop is None: + return + if self._write_loop.running: + self._write_loop.stop() + + # IMessageStore + + # XXX this would work well for whole message operations. + # We would have to add a put_flags operation to modify only + # the flags doc (and set the dirty flag accordingly) + + def create_message(self, mbox, uid, message, observer, + notify_on_disk=True): + """ + Create the passed message into this MemoryStore. + + By default we consider that any message is a new message. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the UID for the message + :type uid: int + :param message: a message to be added + :type message: MessageWrapper + :param observer: the deferred that will fire with the + UID of the message. If notify_on_disk is True, + this will happen when the message is written to + Soledad. Otherwise it will fire as soon as we've + added the message to the memory store. + :type observer: Deferred + :param notify_on_disk: whether the `observer` deferred should + wait until the message is written to disk to + be fired. + :type notify_on_disk: bool + """ + log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) + key = mbox, uid + + self._add_message(mbox, uid, message, notify_on_disk) + self._new.add(key) + + if observer is not None: + if notify_on_disk: + # We store this deferred so we can keep track of the pending + # operations internally. + # TODO this should fire with the UID !!! -- change that in + # the soledad store code. + self._new_deferreds[key] = observer + + else: + # Caller does not care, just fired and forgot, so we pass + # a defer that will inmediately have its callback triggered. + self.reactor.callFromThread(observer.callback, uid) + + def put_message(self, mbox, uid, message, notify_on_disk=True): + """ + Put an existing message. + + This will also set the dirty flag on the MemoryStore. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the UID for the message + :type uid: int + :param message: a message to be added + :type message: MessageWrapper + :param notify_on_disk: whether the deferred that is returned should + wait until the message is written to disk to + be fired. + :type notify_on_disk: bool + + :return: a Deferred. if notify_on_disk is True, will be fired + when written to the db on disk. + Otherwise will fire inmediately + :rtype: Deferred + """ + key = mbox, uid + d = defer.Deferred() + d.addCallback(lambda result: log.msg("message PUT save: %s" % result)) + + self._dirty.add(key) + self._dirty_deferreds[key] = d + self._add_message(mbox, uid, message, notify_on_disk) + return d + + def _add_message(self, mbox, uid, message, notify_on_disk=True): + """ + Helper method, called by both create_message and put_message. + See those for parameter documentation. + """ + msg_dict = message.as_dict() + + fdoc = msg_dict.get(FDOC, None) + if fdoc is not None: + fdoc_store = self._fdoc_store[mbox][uid] + fdoc_store.update(fdoc) + chash_fdoc_store = self._chash_fdoc_store + + # content-hash indexing + chash = fdoc.get(fields.CONTENT_HASH_KEY) + chash_fdoc_store[chash][mbox] = weakref.proxy( + self._fdoc_store[mbox][uid]) + + hdoc = msg_dict.get(HDOC, None) + if hdoc is not None: + chash = hdoc.get(fields.CONTENT_HASH_KEY) + hdoc_store = self._hdoc_store[chash] + hdoc_store.update(hdoc) + + cdocs = message.cdocs + for cdoc in cdocs.values(): + phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None) + if not phash: + continue + cdoc_store = self._cdoc_store[phash] + cdoc_store.update(cdoc) + + # Update memory store size + # XXX this should use [mbox][uid] + # TODO --- this has to be deferred to thread, + # TODO add hdoc and cdocs sizes too + # it's slowing things down here. + #key = mbox, uid + #self._sizes[key] = size.get_size(self._fdoc_store[key]) + + def purge_fdoc_store(self, mbox): + """ + Purge the empty documents from a fdoc store. + Called during initialization of the SoledadMailbox + + :param mbox: the mailbox + :type mbox: str or unicode + """ + # XXX This is really a workaround until I find the conditions + # that are making the empty items remain there. + # This happens, for instance, after running several times + # the regression test, that issues a store deleted + expunge + select + # The items are being correclty deleted, but in succesive appends + # the empty items with previously deleted uids reappear as empty + # documents. I suspect it's a timing condition with a previously + # evaluated sequence being used after the items has been removed. + + for uid, value in self._fdoc_store[mbox].items(): + if empty(value): + del self._fdoc_store[mbox][uid] + + def get_docid_for_fdoc(self, mbox, uid): + """ + Return Soledad document id for the flags-doc for a given mbox and uid, + or None of no flags document could be found. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the message UID + :type uid: int + :rtype: unicode or None + """ + with self._fdoc_docid_lock: + doc_id = self._fdoc_id_store[mbox][uid] + + if empty(doc_id): + fdoc = self._permanent_store.get_flags_doc(mbox, uid) + if empty(fdoc) or empty(fdoc.content): + return None + doc_id = fdoc.doc_id + self._fdoc_id_store[mbox][uid] = doc_id + + return doc_id + + def get_message(self, mbox, uid, dirtystate=DirtyState.none, + flags_only=False): + """ + Get a MessageWrapper for the given mbox and uid combination. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the message UID + :type uid: int + :param dirtystate: DirtyState enum: one of `dirty`, `new` + or `none` (default) + :type dirtystate: enum + :param flags_only: whether the message should carry only a reference + to the flags document. + :type flags_only: bool + : + + :return: MessageWrapper or None + """ + if dirtystate == DirtyState.dirty: + flags_only = True + + key = mbox, uid + + fdoc = self._fdoc_store[mbox][uid] + if empty(fdoc): + return None + + new, dirty = False, False + if dirtystate == DirtyState.none: + new, dirty = self._get_new_dirty_state(key) + if dirtystate == DirtyState.dirty: + new, dirty = False, True + if dirtystate == DirtyState.new: + new, dirty = True, False + + if flags_only: + return MessageWrapper(fdoc=fdoc, + new=new, dirty=dirty, + memstore=weakref.proxy(self)) + else: + chash = fdoc.get(fields.CONTENT_HASH_KEY) + hdoc = self._hdoc_store[chash] + if empty(hdoc): + hdoc = self._permanent_store.get_headers_doc(chash) + if empty(hdoc): + return None + if not empty(hdoc.content): + self._hdoc_store[chash] = hdoc.content + hdoc = hdoc.content + cdocs = None + + pmap = hdoc.get(fields.PARTS_MAP_KEY, None) + if new and pmap is not None: + # take the different cdocs for write... + cdoc_store = self._cdoc_store + cdocs_list = phash_iter(hdoc) + cdocs = dict(enumerate( + [cdoc_store[phash] for phash in cdocs_list], 1)) + + return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs, + new=new, dirty=dirty, + memstore=weakref.proxy(self)) + + def remove_message(self, mbox, uid): + """ + Remove a Message from this MemoryStore. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the message UID + :type uid: int + """ + # XXX For the moment we are only removing the flags and headers + # docs. The rest we leave there polluting your hard disk, + # until we think about a good way of deorphaning. + + # XXX implement elijah's idea of using a PUT document as a + # token to ensure consistency in the removal. + + try: + del self._fdoc_store[mbox][uid] + except KeyError: + pass + + try: + key = mbox, uid + self._new.discard(key) + self._dirty.discard(key) + if key in self._sizes: + del self._sizes[key] + self._known_uids[mbox].discard(uid) + except KeyError: + pass + except Exception as exc: + logger.error("error while removing message!") + logger.exception(exc) + try: + with self._fdoc_docid_lock: + del self._fdoc_id_store[mbox][uid] + except KeyError: + pass + except Exception as exc: + logger.error("error while removing message!") + logger.exception(exc) + + # IMessageStoreWriter + + @deferred_to_thread + def write_messages(self, store): + """ + Write the message documents in this MemoryStore to a different store. + + :param store: the IMessageStore to write to + :rtype: False if queue is not empty, None otherwise. + """ + # For now, we pass if the queue is not empty, to avoid duplicate + # queuing. + # We would better use a flag to know when we've already enqueued an + # item. + + # XXX this could return the deferred for all the enqueued operations + + if not self.producer.is_queue_empty(): + return False + + if any(map(lambda i: not empty(i), (self._new, self._dirty))): + logger.info("Writing messages to Soledad...") + + # TODO change for lock, and make the property access + # is accquired + with set_bool_flag(self, self.WRITING_FLAG): + for rflags_doc_wrapper in self.all_rdocs_iter(): + self.producer.push(rflags_doc_wrapper, + state=self.producer.STATE_DIRTY) + for msg_wrapper in self.all_new_msg_iter(): + self.producer.push(msg_wrapper, + state=self.producer.STATE_NEW) + for msg_wrapper in self.all_dirty_msg_iter(): + self.producer.push(msg_wrapper, + state=self.producer.STATE_DIRTY) + + # MemoryStore specific methods. + + def get_uids(self, mbox): + """ + Get all uids for a given mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: list + """ + return self._fdoc_store[mbox].keys() + + def get_soledad_known_uids(self, mbox): + """ + Get all uids that soledad knows about, from the memory cache. + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: list + """ + return self._known_uids.get(mbox, []) + + # last_uid + + def get_last_uid(self, mbox): + """ + Return the highest UID for a given mbox. + It will be the highest between the highest uid in the message store for + the mailbox, and the soledad integer cache. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: int + """ + uids = self.get_uids(mbox) + last_mem_uid = uids and max(uids) or 0 + last_soledad_uid = self.get_last_soledad_uid(mbox) + return max(last_mem_uid, last_soledad_uid) + + def get_last_soledad_uid(self, mbox): + """ + Get last uid for a given mbox from the soledad integer cache. + + :param mbox: the mailbox + :type mbox: str or unicode + """ + return self._last_uid.get(mbox, 0) + + def set_last_soledad_uid(self, mbox, value): + """ + Set last uid for a given mbox in the soledad integer cache. + SoledadMailbox should prime this value during initialization. + Other methods (during message adding) SHOULD call + `increment_last_soledad_uid` instead. + + :param mbox: the mailbox + :type mbox: str or unicode + :param value: the value to set + :type value: int + """ + # can be long??? + #leap_assert_type(value, int) + logger.info("setting last soledad uid for %s to %s" % + (mbox, value)) + # if we already have a value here, don't do anything + with self._last_uid_lock: + if not self._last_uid.get(mbox, None): + self._last_uid[mbox] = value + + def set_known_uids(self, mbox, value): + """ + Set the value fo the known-uids set for this mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :param value: a sequence of integers to be added to the set. + :type value: tuple + """ + current = self._known_uids[mbox] + self._known_uids[mbox] = current.union(set(value)) + + def increment_last_soledad_uid(self, mbox): + """ + Increment by one the soledad integer cache for the last_uid for + this mbox, and fire a defer-to-thread to update the soledad value. + The caller should lock the call tho this method. + + :param mbox: the mailbox + :type mbox: str or unicode + """ + with self._last_uid_lock: + self._last_uid[mbox] += 1 + value = self._last_uid[mbox] + self.reactor.callInThread(self.write_last_uid, mbox, value) + return value + + def write_last_uid(self, mbox, value): + """ + Increment the soledad integer cache for the highest uid value. + + :param mbox: the mailbox + :type mbox: str or unicode + :param value: the value to set + :type value: int + """ + leap_assert_type(value, int) + if self._permanent_store: + self._permanent_store.write_last_uid(mbox, value) + + def load_flag_docs(self, mbox, flag_docs): + """ + Load the flag documents for the given mbox. + Used during initial flag docs prefetch. + + :param mbox: the mailbox + :type mbox: str or unicode + :param flag_docs: a dict with the content for the flag docs, indexed + by uid. + :type flag_docs: dict + """ + # We can do direct assignments cause we know this will only + # be called during initialization of the mailbox. + # TODO could hook here a sanity-check + # for duplicates + + fdoc_store = self._fdoc_store[mbox] + chash_fdoc_store = self._chash_fdoc_store + for uid in flag_docs: + rdict = ReferenciableDict(flag_docs[uid]) + fdoc_store[uid] = rdict + # populate chash dict too, to avoid fdoc duplication + chash = flag_docs[uid]["chash"] + chash_fdoc_store[chash][mbox] = weakref.proxy( + self._fdoc_store[mbox][uid]) + + def update_flags(self, mbox, uid, fdoc): + """ + Update the flag document for a given mbox and uid combination, + and set the dirty flag. + We could use put_message, but this is faster. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the uid of the message + :type uid: int + + :param fdoc: a dict with the content for the flag docs + :type fdoc: dict + """ + key = mbox, uid + self._fdoc_store[mbox][uid].update(fdoc) + self._dirty.add(key) + + def load_header_docs(self, header_docs): + """ + Load the flag documents for the given mbox. + Used during header docs prefetch, and during cache after + a read from soledad if the hdoc property in message did not + find its value in here. + + :param flag_docs: a dict with the content for the flag docs. + :type flag_docs: dict + """ + hdoc_store = self._hdoc_store + for chash in header_docs: + hdoc_store[chash] = ReferenciableDict(header_docs[chash]) + + def all_flags(self, mbox): + """ + Return a dictionary with all the flags for a given mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: dict + """ + fdict = {} + uids = self.get_uids(mbox) + fstore = self._fdoc_store[mbox] + + for uid in uids: + try: + fdict[uid] = fstore[uid][fields.FLAGS_KEY] + except KeyError: + continue + return fdict + + def all_headers(self, mbox): + """ + Return a dictionary with all the header docs for a given mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: dict + """ + headers_dict = {} + uids = self.get_uids(mbox) + fdoc_store = self._fdoc_store[mbox] + hdoc_store = self._hdoc_store + + for uid in uids: + try: + chash = fdoc_store[uid][fields.CONTENT_HASH_KEY] + hdoc = hdoc_store[chash] + if not empty(hdoc): + headers_dict[uid] = hdoc + except KeyError: + continue + return headers_dict + + # Counting sheeps... + + def count_new_mbox(self, mbox): + """ + Count the new messages by mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: number of new messages + :rtype: int + """ + return len([(m, uid) for m, uid in self._new if mbox == mbox]) + + # XXX used at all? + def count_new(self): + """ + Count all the new messages in the MemoryStore. + + :rtype: int + """ + return len(self._new) + + def count(self, mbox): + """ + Return the count of messages for a given mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: number of messages + :rtype: int + """ + return len(self._fdoc_store[mbox]) + + def unseen_iter(self, mbox): + """ + Get an iterator for the message UIDs with no `seen` flag + for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: iterator through unseen message doc UIDs + :rtype: iterable + """ + fdocs = self._fdoc_store[mbox] + + return [uid for uid, value + in fdocs.items() + if fields.SEEN_FLAG not in value.get(fields.FLAGS_KEY, [])] + + def get_cdoc_from_phash(self, phash): + """ + Return a content-document by its payload-hash. + + :param phash: the payload hash to check against + :type phash: str or unicode + :rtype: MessagePartDoc + """ + doc = self._cdoc_store.get(phash, None) + + # XXX return None for consistency? + + # XXX have to keep a mapping between phash and its linkage + # info, to know if this payload is been already saved or not. + # We will be able to get this from the linkage-docs, + # not yet implemented. + new = True + dirty = False + return MessagePartDoc( + new=new, dirty=dirty, store="mem", + part=MessagePartType.cdoc, + content=doc, + doc_id=None) + + def get_fdoc_from_chash(self, chash, mbox): + """ + Return a flags-document by its content-hash and a given mailbox. + Used during content-duplication detection while copying or adding a + message. + + :param chash: the content hash to check against + :type chash: str or unicode + :param mbox: the mailbox + :type mbox: str or unicode + + :return: MessagePartDoc. It will return None if the flags document + has empty content or it is flagged as \\Deleted. + """ + fdoc = self._chash_fdoc_store[chash][mbox] + + # a couple of special cases. + # 1. We might have a doc with empty content... + if empty(fdoc): + return None + + # 2. ...Or the message could exist, but being flagged for deletion. + # We want to create a new one in this case. + # Hmmm what if the deletion is un-done?? We would end with a + # duplicate... + if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []): + return None + + uid = fdoc[fields.UID_KEY] + key = mbox, uid + new = key in self._new + dirty = key in self._dirty + + return MessagePartDoc( + new=new, dirty=dirty, store="mem", + part=MessagePartType.fdoc, + content=fdoc, + doc_id=None) + + def iter_fdoc_keys(self): + """ + Return a generator through all the mbox, uid keys in the flags-doc + store. + """ + fdoc_store = self._fdoc_store + for mbox in fdoc_store: + for uid in fdoc_store[mbox]: + yield mbox, uid + + def all_new_msg_iter(self): + """ + Return generator that iterates through all new messages. + + :return: generator of MessageWrappers + :rtype: generator + """ + gm = self.get_message + # need to freeze, set can change during iteration + new = [gm(*key, dirtystate=DirtyState.new) for key in tuple(self._new)] + # move content from new set to the queue + self._new_queue.update(self._new) + self._new.difference_update(self._new) + return new + + def all_dirty_msg_iter(self): + """ + Return generator that iterates through all dirty messages. + + :return: generator of MessageWrappers + :rtype: generator + """ + gm = self.get_message + # need to freeze, set can change during iteration + dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty) + for key in tuple(self._dirty)] + # move content from new and dirty sets to the queue + + self._dirty_queue.update(self._dirty) + self._dirty.difference_update(self._dirty) + return dirty + + def all_deleted_uid_iter(self, mbox): + """ + Return a list with the UIDs for all messags + with deleted flag in a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: list of integers + :rtype: list + """ + # This *needs* to return a fixed sequence. Otherwise the dictionary len + # will change during iteration, when we modify it + fdocs = self._fdoc_store[mbox] + return [uid for uid, value + in fdocs.items() + if fields.DELETED_FLAG in value.get(fields.FLAGS_KEY, [])] + + # new, dirty flags + + def _get_new_dirty_state(self, key): + """ + Return `new` and `dirty` flags for a given message. + + :param key: the key for the message, in the form mbox, uid + :type key: tuple + :return: tuple of bools + :rtype: tuple + """ + # TODO change indexing of sets to [mbox][key] too. + # XXX should return *first* the news, and *then* the dirty... + + # TODO should query in queues too , true? + # + return map(lambda _set: key in _set, (self._new, self._dirty)) + + def set_new_queued(self, key): + """ + Add the key value to the `new-queue` set. + + :param key: the key for the message, in the form mbox, uid + :type key: tuple + """ + self._new_queue.add(key) + + def unset_new_queued(self, key): + """ + Remove the key value from the `new-queue` set. + + :param key: the key for the message, in the form mbox, uid + :type key: tuple + """ + self._new_queue.discard(key) + deferreds = self._new_deferreds + d = deferreds.get(key, None) + if d: + # XXX use a namedtuple for passing the result + # when we check it in the other side. + d.callback('%s, ok' % str(key)) + deferreds.pop(key) + + def set_dirty_queued(self, key): + """ + Add the key value to the `dirty-queue` set. + + :param key: the key for the message, in the form mbox, uid + :type key: tuple + """ + self._dirty_queue.add(key) + + def unset_dirty_queued(self, key): + """ + Remove the key value from the `dirty-queue` set. + + :param key: the key for the message, in the form mbox, uid + :type key: tuple + """ + self._dirty_queue.discard(key) + deferreds = self._dirty_deferreds + d = deferreds.get(key, None) + if d: + # XXX use a namedtuple for passing the result + # when we check it in the other side. + d.callback('%s, ok' % str(key)) + deferreds.pop(key) + + # Recent Flags + + def set_recent_flag(self, mbox, uid): + """ + Set the `Recent` flag for a given mailbox and UID. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the message UID + :type uid: int + """ + self._rflags_dirty.add(mbox) + self._rflags_store[mbox]['set'].add(uid) + + # TODO --- nice but unused + def unset_recent_flag(self, mbox, uid): + """ + Unset the `Recent` flag for a given mailbox and UID. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the message UID + :type uid: int + """ + self._rflags_store[mbox]['set'].discard(uid) + + def set_recent_flags(self, mbox, value): + """ + Set the value for the set of the recent flags. + Used from the property in the MessageCollection. + + :param mbox: the mailbox + :type mbox: str or unicode + :param value: a sequence of flags to set + :type value: sequence + """ + self._rflags_dirty.add(mbox) + self._rflags_store[mbox]['set'] = set(value) + + def load_recent_flags(self, mbox, flags_doc): + """ + Load the passed flags document in the recent flags store, for a given + mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :param flags_doc: A dictionary containing the `doc_id` of the Soledad + flags-document for this mailbox, and the `set` + of uids marked with that flag. + """ + self._rflags_store[mbox] = flags_doc + + def get_recent_flags(self, mbox): + """ + Return the set of UIDs with the `Recent` flag for this mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: set, or None + """ + rflag_for_mbox = self._rflags_store.get(mbox, None) + if not rflag_for_mbox: + return None + return self._rflags_store[mbox]['set'] + + def all_rdocs_iter(self): + """ + Return an iterator through all in-memory recent flag dicts, wrapped + under a RecentFlagsDoc namedtuple. + Used for saving to disk. + + :return: a generator of RecentFlagDoc + :rtype: generator + """ + # XXX use enums + DOC_ID = "doc_id" + SET = "set" + + rflags_store = self._rflags_store + + def get_rdoc(mbox, rdict): + mbox_rflag_set = rdict[SET] + recent_set = copy(mbox_rflag_set) + # zero it! + mbox_rflag_set.difference_update(mbox_rflag_set) + return RecentFlagsDoc( + doc_id=rflags_store[mbox][DOC_ID], + content={ + fields.TYPE_KEY: fields.TYPE_RECENT_VAL, + fields.MBOX_KEY: mbox, + fields.RECENTFLAGS_KEY: list(recent_set) + }) + + return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items() + if not empty(rdict[SET])) + + # Methods that mirror the IMailbox interface + + def remove_all_deleted(self, mbox): + """ + Remove all messages flagged \\Deleted from this Memory Store only. + Called from `expunge` + + :param mbox: the mailbox + :type mbox: str or unicode + :return: a list of UIDs + :rtype: list + """ + mem_deleted = self.all_deleted_uid_iter(mbox) + for uid in mem_deleted: + self.remove_message(mbox, uid) + return mem_deleted + + def stop_and_flush(self): + """ + Stop the write loop and trigger a write to the producer. + """ + self._stop_write_loop() + if self._permanent_store is not None: + # XXX we should check if we did get a True value on this + # operation. If we got False we should retry! (queue was not empty) + self.write_messages(self._permanent_store) + self.producer.flush() + + def expunge(self, mbox, observer): + """ + Remove all messages flagged \\Deleted, from the Memory Store + and from the permanent store also. + + It first queues up a last write, and wait for the deferreds to be done + before continuing. + + :param mbox: the mailbox + :type mbox: str or unicode + :param observer: a deferred that will be fired when expunge is done + :type observer: Deferred + """ + soledad_store = self._permanent_store + if soledad_store is None: + # just-in memory store, easy then. + self._delete_from_memory(mbox, observer) + return + + # We have a soledad storage. + try: + # Stop and trigger last write + self.stop_and_flush() + # Wait on the writebacks to finish + + # XXX what if pending deferreds is empty? + pending_deferreds = (self._new_deferreds.get(mbox, []) + + self._dirty_deferreds.get(mbox, [])) + d1 = defer.gatherResults(pending_deferreds, consumeErrors=True) + d1.addCallback( + self._delete_from_soledad_and_memory, mbox, observer) + except Exception as exc: + logger.exception(exc) + + def _delete_from_memory(self, mbox, observer): + """ + Remove all messages marked as deleted from soledad and memory. + + :param mbox: the mailbox + :type mbox: str or unicode + :param observer: a deferred that will be fired when expunge is done + :type observer: Deferred + """ + mem_deleted = self.remove_all_deleted(mbox) + observer.callback(mem_deleted) + + def _delete_from_soledad_and_memory(self, result, mbox, observer): + """ + Remove all messages marked as deleted from soledad and memory. + + :param result: ignored. the result of the deferredList that triggers + this as a callback from `expunge`. + :param mbox: the mailbox + :type mbox: str or unicode + :param observer: a deferred that will be fired when expunge is done + :type observer: Deferred + """ + all_deleted = [] + soledad_store = self._permanent_store + + try: + # 1. Delete all messages marked as deleted in soledad. + logger.debug("DELETING FROM SOLEDAD ALL FOR %r" % (mbox,)) + sol_deleted = soledad_store.remove_all_deleted(mbox) + + try: + self._known_uids[mbox].difference_update(set(sol_deleted)) + except Exception as exc: + logger.exception(exc) + + # 2. Delete all messages marked as deleted in memory. + logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,)) + mem_deleted = self.remove_all_deleted(mbox) + + all_deleted = set(mem_deleted).union(set(sol_deleted)) + logger.debug("deleted %r" % all_deleted) + except Exception as exc: + logger.exception(exc) + finally: + self._start_write_loop() + + observer.callback(all_deleted) + + # Mailbox documents and attributes + + # This could be also be cached in memstore, but proxying directly + # to soledad since it's not too performance-critical. + + def get_mbox_doc(self, mbox): + """ + Return the soledad document for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: SoledadDocument or None. + """ + return self.permanent_store.get_mbox_document(mbox) + + def get_mbox_closed(self, mbox): + """ + Return the closed attribute for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: bool + """ + return self.permanent_store.get_mbox_closed(mbox) + + def set_mbox_closed(self, mbox, closed): + """ + Set the closed attribute for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + """ + self.permanent_store.set_mbox_closed(mbox, closed) + + # Rename flag-documents + + def rename_fdocs_mailbox(self, old_mbox, new_mbox): + """ + Change the mailbox name for all flag documents in a given mailbox. + Used from account.rename + + :param old_mbox: name for the old mbox + :type old_mbox: str or unicode + :param new_mbox: name for the new mbox + :type new_mbox: str or unicode + """ + fs = self._fdoc_store + keys = fs[old_mbox].keys() + for k in keys: + fdoc = fs[old_mbox][k] + fdoc['mbox'] = new_mbox + fs[new_mbox][k] = fdoc + fs[old_mbox].pop(k) + self._dirty.add((new_mbox, k)) + + # Dump-to-disk controls. + + @property + def is_writing(self): + """ + Property that returns whether the store is currently writing its + internal state to a permanent storage. + + Used to evaluate whether the CHECK command can inform that the field + is clear to proceed, or waiting for the write operations to complete + is needed instead. + + :rtype: bool + """ + # FIXME this should return a deferred !!! + # XXX ----- can fire when all new + dirty deferreds + # are done (gatherResults) + return getattr(self, self.WRITING_FLAG) + + @property + def permanent_store(self): + return self._permanent_store + + # Memory management. + + def get_size(self): + """ + Return the size of the internal storage. + Use for calculating the limit beyond which we should flush the store. + + :rtype: int + """ + return reduce(lambda x, y: x + y, self._sizes, 0) diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py new file mode 100644 index 0000000..257721c --- /dev/null +++ b/src/leap/mail/imap/messageparts.py @@ -0,0 +1,586 @@ +# messageparts.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +MessagePart implementation. Used from LeapMessage. +""" +import logging +import StringIO +import weakref + +from collections import namedtuple + +from enum import Enum +from zope.interface import implements +from twisted.mail import imap4 + +from leap.common.decorators import memoized_method +from leap.common.mail import get_email_charset +from leap.mail.imap import interfaces +from leap.mail.imap.fields import fields +from leap.mail.utils import empty, first, find_charset + +MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id") + + +logger = logging.getLogger(__name__) + + +""" +A MessagePartDoc is a light wrapper around the dictionary-like +data that we pass along for message parts. It can be used almost everywhere +that you would expect a SoledadDocument, since it has a dict under the +`content` attribute. + +We also keep some metadata on it, relative in part to the message as a whole, +and sometimes to a part in particular only. + +* `new` indicates that the document has just been created. SoledadStore + should just create a new doc for all the related message parts. +* `store` indicates the type of store a given MessagePartDoc lives in. + We currently use this to indicate that the document comes from memeory, + but we should probably get rid of it as soon as we extend the use of the + SoledadStore interface along LeapMessage, MessageCollection and Mailbox. +* `part` is one of the MessagePartType enums. + +* `dirty` indicates that, while we already have the document in Soledad, + we have modified its state in memory, so we need to put_doc instead while + dumping the MemoryStore contents. + `dirty` attribute would only apply to flags-docs and linkage-docs. +* `doc_id` is the identifier for the document in the u1db database, if any. + +""" + +MessagePartDoc = namedtuple( + 'MessagePartDoc', + ['new', 'dirty', 'part', 'store', 'content', 'doc_id']) + +""" +A RecentFlagsDoc is used to send the recent-flags document payload to the +SoledadWriter during dumps. +""" +RecentFlagsDoc = namedtuple( + 'RecentFlagsDoc', + ['content', 'doc_id']) + + +class ReferenciableDict(dict): + """ + A dict that can be weak-referenced. + + Some builtin objects are not weak-referenciable unless + subclassed. So we do. + + Used to return pointers to the items in the MemoryStore. + """ + + +class MessageWrapper(object): + """ + A simple nested dictionary container around the different message subparts. + """ + implements(interfaces.IMessageContainer) + + FDOC = "fdoc" + HDOC = "hdoc" + CDOCS = "cdocs" + DOCS_ID = "docs_id" + + # Using slots to limit some the memory use, + # Add your attribute here. + + __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"] + + def __init__(self, fdoc=None, hdoc=None, cdocs=None, + from_dict=None, memstore=None, + new=True, dirty=False, docs_id={}): + """ + Initialize a MessageWrapper. + """ + # TODO add optional reference to original message in the incoming + self._dict = {} + self.memstore = memstore + + self._new = new + self._dirty = dirty + + self._storetype = "mem" + + if from_dict is not None: + self.from_dict(from_dict) + else: + if fdoc is not None: + self._dict[self.FDOC] = ReferenciableDict(fdoc) + if hdoc is not None: + self._dict[self.HDOC] = ReferenciableDict(hdoc) + if cdocs is not None: + self._dict[self.CDOCS] = ReferenciableDict(cdocs) + + # This will keep references to the doc_ids to be able to put + # messages to soledad. It will be populated during the walk() to avoid + # the overhead of reading from the db. + + # XXX it really *only* make sense for the FDOC, the other parts + # should not be "dirty", just new...!!! + self._dict[self.DOCS_ID] = docs_id + + # properties + + # TODO Could refactor new and dirty properties together. + + def _get_new(self): + """ + Get the value for the `new` flag. + + :rtype: bool + """ + return self._new + + def _set_new(self, value=False): + """ + Set the value for the `new` flag, and propagate it + to the memory store if any. + + :param value: the value to set + :type value: bool + """ + self._new = value + if self.memstore: + mbox = self.fdoc.content.get('mbox', None) + uid = self.fdoc.content.get('uid', None) + if not mbox or not uid: + logger.warning("Malformed fdoc") + return + key = mbox, uid + fun = [self.memstore.unset_new_queued, + self.memstore.set_new_queued][int(value)] + fun(key) + else: + logger.warning("Could not find a memstore referenced from this " + "MessageWrapper. The value for new will not be " + "propagated") + + new = property(_get_new, _set_new, + doc="The `new` flag for this MessageWrapper") + + def _get_dirty(self): + """ + Get the value for the `dirty` flag. + + :rtype: bool + """ + return self._dirty + + def _set_dirty(self, value=True): + """ + Set the value for the `dirty` flag, and propagate it + to the memory store if any. + + :param value: the value to set + :type value: bool + """ + self._dirty = value + if self.memstore: + mbox = self.fdoc.content.get('mbox', None) + uid = self.fdoc.content.get('uid', None) + if not mbox or not uid: + logger.warning("Malformed fdoc") + return + key = mbox, uid + fun = [self.memstore.unset_dirty_queued, + self.memstore.set_dirty_queued][int(value)] + fun(key) + else: + logger.warning("Could not find a memstore referenced from this " + "MessageWrapper. The value for new will not be " + "propagated") + + dirty = property(_get_dirty, _set_dirty) + + # IMessageContainer + + @property + def fdoc(self): + """ + Return a MessagePartDoc wrapping around a weak reference to + the flags-document in this MemoryStore, if any. + + :rtype: MessagePartDoc + """ + _fdoc = self._dict.get(self.FDOC, None) + if _fdoc: + content_ref = weakref.proxy(_fdoc) + else: + logger.warning("NO FDOC!!!") + content_ref = {} + + return MessagePartDoc(new=self.new, dirty=self.dirty, + store=self._storetype, + part=MessagePartType.fdoc, + content=content_ref, + doc_id=self._dict[self.DOCS_ID].get( + self.FDOC, None)) + + @property + def hdoc(self): + """ + Return a MessagePartDoc wrapping around a weak reference to + the headers-document in this MemoryStore, if any. + + :rtype: MessagePartDoc + """ + _hdoc = self._dict.get(self.HDOC, None) + if _hdoc: + content_ref = weakref.proxy(_hdoc) + else: + content_ref = {} + return MessagePartDoc(new=self.new, dirty=self.dirty, + store=self._storetype, + part=MessagePartType.hdoc, + content=content_ref, + doc_id=self._dict[self.DOCS_ID].get( + self.HDOC, None)) + + @property + def cdocs(self): + """ + Return a weak reference to a zero-indexed dict containing + the content-documents, or an empty dict if none found. + If you want access to the MessagePartDoc for the individual + parts, use the generator returned by `walk` instead. + + :rtype: dict + """ + _cdocs = self._dict.get(self.CDOCS, None) + if _cdocs: + return weakref.proxy(_cdocs) + else: + return {} + + def walk(self): + """ + Generator that iterates through all the parts, returning + MessagePartDoc. Used for writing to SoledadStore. + + :rtype: generator + """ + if self._dirty: + try: + mbox = self.fdoc.content[fields.MBOX_KEY] + uid = self.fdoc.content[fields.UID_KEY] + docid_dict = self._dict[self.DOCS_ID] + docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc( + mbox, uid) + except Exception as exc: + logger.debug("Error while walking message...") + logger.exception(exc) + + if not empty(self.fdoc.content) and 'uid' in self.fdoc.content: + yield self.fdoc + if not empty(self.hdoc.content): + yield self.hdoc + for cdoc in self.cdocs.values(): + if not empty(cdoc): + content_ref = weakref.proxy(cdoc) + yield MessagePartDoc(new=self.new, dirty=self.dirty, + store=self._storetype, + part=MessagePartType.cdoc, + content=content_ref, + doc_id=None) + + # i/o + + def as_dict(self): + """ + Return a dict representation of the parts contained. + + :rtype: dict + """ + return self._dict + + def from_dict(self, msg_dict): + """ + Populate MessageWrapper parts from a dictionary. + It expects the same format that we use in a + MessageWrapper. + + + :param msg_dict: a dictionary containing the parts to populate + the MessageWrapper from + :type msg_dict: dict + """ + fdoc, hdoc, cdocs = map( + lambda part: msg_dict.get(part, None), + [self.FDOC, self.HDOC, self.CDOCS]) + + for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc), + (self.CDOCS, cdocs)): + self._dict[t] = ReferenciableDict(doc) if doc else None + + +class MessagePart(object): + """ + IMessagePart implementor, to be passed to several methods + of the IMAP4Server. + It takes a subpart message and is able to find + the inner parts. + + See the interface documentation. + """ + + implements(imap4.IMessagePart) + + def __init__(self, soledad, part_map): + """ + Initializes the MessagePart. + + :param soledad: Soledad instance. + :type soledad: Soledad + :param part_map: a dictionary containing the parts map for this + message + :type part_map: dict + """ + # TODO + # It would be good to pass the uid/mailbox also + # for references while debugging. + + # We have a problem on bulk moves, and is + # that when the fetch on the new mailbox is done + # the parts maybe are not complete. + # So we should be able to fail with empty + # docs until we solve that. The ideal would be + # to gather the results of the deferred operations + # to signal the operation is complete. + #leap_assert(part_map, "part map dict cannot be null") + + self._soledad = soledad + self._pmap = part_map + + def getSize(self): + """ + Return the total size, in octets, of this message part. + + :return: size of the message, in octets + :rtype: int + """ + if empty(self._pmap): + return 0 + size = self._pmap.get('size', None) + if size is None: + logger.error("Message part cannot find size in the partmap") + size = 0 + return size + + def getBodyFile(self): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: StringIO + """ + fd = StringIO.StringIO() + if not empty(self._pmap): + multi = self._pmap.get('multi') + if not multi: + phash = self._pmap.get("phash", None) + else: + pmap = self._pmap.get('part_map') + first_part = pmap.get('1', None) + if not empty(first_part): + phash = first_part['phash'] + else: + phash = None + + if phash is None: + logger.warning("Could not find phash for this subpart!") + payload = "" + else: + payload = self._get_payload_from_document_memoized(phash) + if empty(payload): + payload = self._get_payload_from_document(phash) + + else: + logger.warning("Message with no part_map!") + payload = "" + + if payload: + content_type = self._get_ctype_from_document(phash) + charset = find_charset(content_type) + if charset is None: + charset = self._get_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + + fd.write(payload) + fd.seek(0) + return fd + + # TODO should memory-bound this memoize!!! + @memoized_method + def _get_payload_from_document_memoized(self, phash): + """ + Memoized method call around the regular method, to be able + to call the non-memoized method in case we got a None. + + :param phash: the payload hash to retrieve by. + :type phash: str or unicode + :rtype: str or unicode or None + """ + return self._get_payload_from_document(phash) + + def _get_payload_from_document(self, phash): + """ + Return the message payload from the content document. + + :param phash: the payload hash to retrieve by. + :type phash: str or unicode + :rtype: str or unicode or None + """ + cdocs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(phash)) + + cdoc = first(cdocs) + if cdoc is None: + logger.warning( + "Could not find the content doc " + "for phash %s" % (phash,)) + payload = "" + else: + payload = cdoc.content.get(fields.RAW_KEY, "") + return payload + + # TODO should memory-bound this memoize!!! + @memoized_method + def _get_ctype_from_document(self, phash): + """ + Reeturn the content-type from the content document. + + :param phash: the payload hash to retrieve by. + :type phash: str or unicode + :rtype: str or unicode + """ + cdocs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(phash)) + + cdoc = first(cdocs) + if not cdoc: + logger.warning( + "Could not find the content doc " + "for phash %s" % (phash,)) + ctype = cdoc.content.get('ctype', "") + return ctype + + @memoized_method + def _get_charset(self, stuff): + # TODO put in a common class with LeapMessage + """ + Gets (guesses?) the charset of a payload. + + :param stuff: the stuff to guess about. + :type stuff: str or unicode + :return: charset + :rtype: unicode + """ + # XXX existential doubt 2. shouldn't we make the scope + # of the decorator somewhat more persistent? + # ah! yes! and put memory bounds. + return get_email_charset(stuff) + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + # XXX refactor together with MessagePart method + if not self._pmap: + logger.warning("No pmap in Subpart!") + return {} + headers = dict(self._pmap.get("headers", [])) + + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + # default to most likely standard + charset = find_charset(headers, "utf-8") + headers2 = dict() + for key, value in headers.items(): + # twisted imap server expects *some* headers to be lowercase + # We could use a CaseInsensitiveDict here... + if key.lower() == "content-type": + key = key.lower() + + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + # filter original dict by negate-condition + if cond(key): + headers2[key] = value + return headers2 + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + if empty(self._pmap): + logger.warning("Could not get part map!") + return False + multi = self._pmap.get("multi", False) + return multi + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + if not self.isMultipart(): + raise TypeError + + sub_pmap = self._pmap.get("part_map", {}) + try: + part_map = sub_pmap[str(part + 1)] + except KeyError: + logger.debug("getSubpart for %s: KeyError" % (part,)) + raise IndexError + + # XXX check for validity + return MessagePart(self._soledad, part_map) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py new file mode 100644 index 0000000..b0b2f95 --- /dev/null +++ b/src/leap/mail/imap/messages.py @@ -0,0 +1,1384 @@ +# -*- coding: utf-8 -*- +# messages.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +LeapMessage and MessageCollection. +""" +import copy +import logging +import re +import threading +import StringIO + +from collections import defaultdict +from email import message_from_string +from functools import partial + +from pycryptopp.hash import sha256 +from twisted.mail import imap4 +from twisted.internet import defer +from zope.interface import implements +from zope.proxy import sameProxiedObjects + +from leap.common.check import leap_assert, leap_assert_type +from leap.common.decorators import memoized_method +from leap.common.mail import get_email_charset +from leap.mail import walk +from leap.mail.utils import first, find_charset, lowerdict, empty +from leap.mail.utils import stringify_parts_map +from leap.mail.decorators import deferred_to_thread +from leap.mail.imap.index import IndexedDB +from leap.mail.imap.fields import fields, WithMsgFields +from leap.mail.imap.memorystore import MessageWrapper +from leap.mail.imap.messageparts import MessagePart, MessagePartDoc +from leap.mail.imap.parser import MBoxParser + +logger = logging.getLogger(__name__) + +# TODO ------------------------------------------------------------ + +# [ ] Add ref to incoming message during add_msg +# [ ] Add linked-from info. +# * Need a new type of documents: linkage info. +# * HDOCS are linked from FDOCs (ref to chash) +# * CDOCS are linked from HDOCS (ref to chash) + +# [ ] Delete incoming mail only after successful write! +# [ ] Remove UID from syncable db. Store only those indexes locally. + +MSGID_PATTERN = r"""<([\w@.]+)>""" +MSGID_RE = re.compile(MSGID_PATTERN) + + +def try_unique_query(curried): + """ + Try to execute a query that is expected to have a + single outcome, and log a warning if more than one document found. + + :param curried: a curried function + :type curried: callable + """ + leap_assert(callable(curried), "A callable is expected") + try: + query = curried() + if query: + if len(query) > 1: + # TODO we could take action, like trigger a background + # process to kill dupes. + name = getattr(curried, 'expected', 'doc') + logger.warning( + "More than one %s found for this mbox, " + "we got a duplicate!!" % (name,)) + return query.pop() + else: + return None + except Exception as exc: + logger.exception("Unhandled error %r" % exc) + + +""" +A dictionary that keeps one lock per mbox and uid. +""" +# XXX too much overhead? +fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) + + +class LeapMessage(fields, MBoxParser): + """ + The main representation of a message. + + It indexes the messages in one mailbox by a combination + of uid+mailbox name. + """ + + # TODO this has to change. + # Should index primarily by chash, and keep a local-only + # UID table. + + implements(imap4.IMessage) + + def __init__(self, soledad, uid, mbox, collection=None, container=None): + """ + Initializes a LeapMessage. + + :param soledad: a Soledad instance + :type soledad: Soledad + :param uid: the UID for the message. + :type uid: int or basestring + :param mbox: the mbox this message belongs to + :type mbox: str or unicode + :param collection: a reference to the parent collection object + :type collection: MessageCollection + :param container: a IMessageContainer implementor instance + :type container: IMessageContainer + """ + self._soledad = soledad + self._uid = int(uid) if uid is not None else None + self._mbox = self._parse_mailbox_name(mbox) + self._collection = collection + self._container = container + + self.__chash = None + self.__bdoc = None + + from twisted.internet import reactor + self.reactor = reactor + + # XXX make these properties public + + @property + def fdoc(self): + """ + An accessor to the flags document. + """ + if all(map(bool, (self._uid, self._mbox))): + fdoc = None + if self._container is not None: + fdoc = self._container.fdoc + if not fdoc: + fdoc = self._get_flags_doc() + if fdoc: + fdoc_content = fdoc.content + self.__chash = fdoc_content.get( + fields.CONTENT_HASH_KEY, None) + return fdoc + + @property + def hdoc(self): + """ + An accessor to the headers document. + """ + container = self._container + if container is not None: + hdoc = self._container.hdoc + if hdoc and not empty(hdoc.content): + return hdoc + hdoc = self._get_headers_doc() + + if container and not empty(hdoc.content): + # mem-cache it + hdoc_content = hdoc.content + chash = hdoc_content.get(fields.CONTENT_HASH_KEY) + hdocs = {chash: hdoc_content} + container.memstore.load_header_docs(hdocs) + return hdoc + + @property + def chash(self): + """ + An accessor to the content hash for this message. + """ + if not self.fdoc: + return None + if not self.__chash and self.fdoc: + self.__chash = self.fdoc.content.get( + fields.CONTENT_HASH_KEY, None) + return self.__chash + + @property + def bdoc(self): + """ + An accessor to the body document. + """ + if not self.hdoc: + return None + if not self.__bdoc: + self.__bdoc = self._get_body_doc() + return self.__bdoc + + # IMessage implementation + + def getUID(self): + """ + Retrieve the unique identifier associated with this Message. + + :return: uid for this message + :rtype: int + """ + return self._uid + + def getFlags(self): + """ + Retrieve the flags associated with this Message. + + :return: The flags, represented as strings + :rtype: tuple + """ + uid = self._uid + + flags = set([]) + fdoc = self.fdoc + if fdoc: + flags = set(fdoc.content.get(self.FLAGS_KEY, None)) + + msgcol = self._collection + + # We treat the recent flag specially: gotten from + # a mailbox-level document. + if msgcol and uid in msgcol.recent_flags: + flags.add(fields.RECENT_FLAG) + if flags: + flags = map(str, flags) + return tuple(flags) + + # setFlags not in the interface spec but we use it with store command. + + def setFlags(self, flags, mode): + """ + Sets the flags for this message + + :param flags: the flags to update in the message. + :type flags: tuple of str + :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. + :type mode: int + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + mbox, uid = self._mbox, self._uid + + APPEND = 1 + REMOVE = -1 + SET = 0 + + with fdoc_locks[mbox][uid]: + doc = self.fdoc + if not doc: + logger.warning( + "Could not find FDOC for %r:%s while setting flags!" % + (mbox, uid)) + return + current = doc.content[self.FLAGS_KEY] + if mode == APPEND: + newflags = tuple(set(tuple(current) + flags)) + elif mode == REMOVE: + newflags = tuple(set(current).difference(set(flags))) + elif mode == SET: + newflags = flags + new_fdoc = { + self.FLAGS_KEY: newflags, + self.SEEN_KEY: self.SEEN_FLAG in newflags, + self.DEL_KEY: self.DELETED_FLAG in newflags} + self._collection.memstore.update_flags(mbox, uid, new_fdoc) + + return map(str, newflags) + + def getInternalDate(self): + """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str + """ + date = self.hdoc.content.get(fields.DATE_KEY, '') + return date + + # + # IMessagePart + # + + # XXX we should implement this interface too for the subparts + # so we allow nested parts... + + def getBodyFile(self): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: StringIO + """ + def write_fd(body): + fd.write(body) + fd.seek(0) + return fd + + # TODO refactor with getBodyFile in MessagePart + + fd = StringIO.StringIO() + + if self.bdoc is not None: + bdoc_content = self.bdoc.content + if empty(bdoc_content): + logger.warning("No BDOC content found for message!!!") + return write_fd("") + + body = bdoc_content.get(self.RAW_KEY, "") + content_type = bdoc_content.get('content-type', "") + charset = find_charset(content_type) + if charset is None: + charset = self._get_charset(body) + try: + if isinstance(body, unicode): + body = body.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + logger.debug("Attempted to encode with: %s" % charset) + body = body.encode(charset, 'replace') + finally: + return write_fd(body) + + # We are still returning funky characters from here. + else: + logger.warning("No BDOC found for message.") + return write_fd("") + + @memoized_method + def _get_charset(self, stuff): + """ + Gets (guesses?) the charset of a payload. + + :param stuff: the stuff to guess about. + :type stuff: basestring + :returns: charset + """ + # XXX shouldn't we make the scope + # of the decorator somewhat more persistent? + # ah! yes! and put memory bounds. + return get_email_charset(stuff) + + def getSize(self): + """ + Return the total size, in octets, of this message. + + :return: size of the message, in octets + :rtype: int + """ + size = None + if self.fdoc is not None: + fdoc_content = self.fdoc.content + size = fdoc_content.get(self.SIZE_KEY, False) + else: + logger.warning("No FLAGS doc for %s:%s" % (self._mbox, + self._uid)) + if not size: + # XXX fallback, should remove when all migrated. + size = self.getBodyFile().len + return size + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + # TODO split in smaller methods + # XXX refactor together with MessagePart method + + headers = self._get_headers() + if not headers: + logger.warning("No headers found") + return {str('content-type'): str('')} + + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + if isinstance(headers, list): + headers = dict(headers) + + # default to most likely standard + charset = find_charset(headers, "utf-8") + headers2 = dict() + for key, value in headers.items(): + # twisted imap server expects *some* headers to be lowercase + # We could use a CaseInsensitiveDict here... + if key.lower() == "content-type": + key = key.lower() + + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + if value.endswith(";"): + # bastards + value = value[:-1] + + # filter original dict by negate-condition + if cond(key): + headers2[key] = value + return headers2 + + def _get_headers(self): + """ + Return the headers dict for this message. + """ + if self.hdoc is not None: + hdoc_content = self.hdoc.content + headers = hdoc_content.get(self.HEADERS_KEY, {}) + return headers + + else: + logger.warning( + "No HEADERS doc for msg %s:%s" % ( + self._mbox, + self._uid)) + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + if self.fdoc: + fdoc_content = self.fdoc.content + is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) + return is_multipart + else: + logger.warning( + "No FLAGS doc for msg %s:%s" % ( + self._mbox, + self._uid)) + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + if not self.isMultipart(): + raise TypeError + try: + pmap_dict = self._get_part_from_parts_map(part + 1) + except KeyError: + raise IndexError + return MessagePart(self._soledad, pmap_dict) + + # + # accessors + # + + def _get_part_from_parts_map(self, part): + """ + Get a part map from the headers doc + + :raises: KeyError if key does not exist + :rtype: dict + """ + if not self.hdoc: + logger.warning("Tried to get part but no HDOC found!") + return None + + hdoc_content = self.hdoc.content + pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) + + # remember, lads, soledad is using strings in its keys, + # not integers! + return pmap[str(part)] + + # XXX moved to memory store + # move the rest too. ------------------------------------------ + def _get_flags_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + result = {} + try: + flag_docs = self._soledad.get_from_index( + fields.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + result = first(flag_docs) + except Exception as exc: + # ugh! Something's broken down there! + logger.warning("ERROR while getting flags for UID: %s" % self._uid) + logger.exception(exc) + finally: + return result + + # TODO move to soledadstore instead of accessing soledad directly + def _get_headers_doc(self): + """ + Return the document that keeps the headers for this + message. + """ + head_docs = self._soledad.get_from_index( + fields.TYPE_C_HASH_IDX, + fields.TYPE_HEADERS_VAL, str(self.chash)) + return first(head_docs) + + # TODO move to soledadstore instead of accessing soledad directly + def _get_body_doc(self): + """ + Return the document that keeps the body for this + message. + """ + hdoc_content = self.hdoc.content + body_phash = hdoc_content.get( + fields.BODY_KEY, None) + if not body_phash: + logger.warning("No body phash for this document!") + return None + + # XXX get from memstore too... + # if memstore: memstore.get_phrash + # memstore should keep a dict with weakrefs to the + # phash doc... + + if self._container is not None: + bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) + if not empty(bdoc) and not empty(bdoc.content): + return bdoc + + # no memstore, or no body doc found there + if self._soledad: + body_docs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(body_phash)) + return first(body_docs) + else: + logger.error("No phash in container, and no soledad found!") + + def __getitem__(self, key): + """ + Return an item from the content of the flags document, + for convenience. + + :param key: The key + :type key: str + + :return: The content value indexed by C{key} or None + :rtype: str + """ + return self.fdoc.content.get(key, None) + + def does_exist(self): + """ + Return True if there is actually a flags document for this + UID and mbox. + """ + return not empty(self.fdoc) + + +class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): + """ + A collection of messages, surprisingly. + + It is tied to a selected mailbox name that is passed to its constructor. + Implements a filter query over the messages contained in a soledad + database. + """ + + # XXX this should be able to produce a MessageSet methinks + # could validate these kinds of objects turning them + # into a template for the class. + FLAGS_DOC = "FLAGS" + HEADERS_DOC = "HEADERS" + CONTENT_DOC = "CONTENT" + """ + RECENT_DOC is a document that stores a list of the UIDs + with the recent flag for this mailbox. It deserves a special treatment + because: + (1) it cannot be set by the user + (2) it's a flag that we set inmediately after a fetch, which is quite + often. + (3) we need to be able to set/unset it in batches without doing a single + write for each element in the sequence. + """ + RECENT_DOC = "RECENT" + """ + HDOCS_SET_DOC is a document that stores a set of the Document-IDs + (the u1db index) for all the headers documents for a given mailbox. + We use it to prefetch massively all the headers for a mailbox. + This is the second massive query, after fetching all the FLAGS, that + a MUA will do in a case where we do not have local disk cache. + """ + HDOCS_SET_DOC = "HDOCS_SET" + + templates = { + + # Message Level + + FLAGS_DOC: { + fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, + fields.UID_KEY: 1, # XXX moe to a local table + fields.MBOX_KEY: fields.INBOX_VAL, + fields.CONTENT_HASH_KEY: "", + + fields.SEEN_KEY: False, + fields.DEL_KEY: False, + fields.FLAGS_KEY: [], + fields.MULTIPART_KEY: False, + fields.SIZE_KEY: 0 + }, + + HEADERS_DOC: { + fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, + fields.CONTENT_HASH_KEY: "", + + fields.DATE_KEY: "", + fields.SUBJECT_KEY: "", + + fields.HEADERS_KEY: {}, + fields.PARTS_MAP_KEY: {}, + }, + + CONTENT_DOC: { + fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, + fields.PAYLOAD_HASH_KEY: "", + fields.LINKED_FROM_KEY: [], + fields.CTYPE_KEY: "", # should index by this too + + # should only get inmutable headers parts + # (for indexing) + fields.HEADERS_KEY: {}, + fields.RAW_KEY: "", + fields.PARTS_MAP_KEY: {}, + fields.HEADERS_KEY: {}, + fields.MULTIPART_KEY: False, + }, + + # Mailbox Level + + RECENT_DOC: { + fields.TYPE_KEY: fields.TYPE_RECENT_VAL, + fields.MBOX_KEY: fields.INBOX_VAL, + fields.RECENTFLAGS_KEY: [], + }, + + HDOCS_SET_DOC: { + fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL, + fields.MBOX_KEY: fields.INBOX_VAL, + fields.HDOCS_SET_KEY: [], + } + + + } + + # Different locks for wrapping both the u1db document getting/setting + # and the property getting/settting in an atomic operation. + + # TODO we would abstract this to a SoledadProperty class + + _rdoc_lock = defaultdict(lambda: threading.Lock()) + _rdoc_write_lock = defaultdict(lambda: threading.Lock()) + _rdoc_read_lock = defaultdict(lambda: threading.Lock()) + _rdoc_property_lock = defaultdict(lambda: threading.Lock()) + + _initialized = {} + + def __init__(self, mbox=None, soledad=None, memstore=None): + """ + Constructor for MessageCollection. + + On initialization, we ensure that we have a document for + storing the recent flags. The nature of this flag make us wanting + to store the set of the UIDs with this flag at the level of the + MessageCollection for each mailbox, instead of treating them + as a property of each message. + + We are passed an instance of MemoryStore, the same for the + SoledadBackedAccount, that we use as a read cache and a buffer + for writes. + + :param mbox: the name of the mailbox. It is the name + with which we filter the query over the + messages database. + :type mbox: str + :param soledad: Soledad database + :type soledad: Soledad instance + :param memstore: a MemoryStore instance + :type memstore: MemoryStore + """ + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(mbox.strip() != "", "mbox cannot be blank space") + leap_assert(isinstance(mbox, (str, unicode)), + "mbox needs to be a string") + leap_assert(soledad, "Need a soledad instance to initialize") + + # okay, all in order, keep going... + + self.mbox = self._parse_mailbox_name(mbox) + + # XXX get a SoledadStore passed instead + self._soledad = soledad + self.memstore = memstore + + self.__rflags = None + + if not self._initialized.get(mbox, False): + try: + self.initialize_db() + # ensure that we have a recent-flags doc + self._get_or_create_rdoc() + except Exception: + logger.debug("Error initializing %r" % (mbox,)) + else: + self._initialized[mbox] = True + + from twisted.internet import reactor + self.reactor = reactor + + def _get_empty_doc(self, _type=FLAGS_DOC): + """ + Returns an empty doc for storing different message parts. + Defaults to returning a template for a flags document. + :return: a dict with the template + :rtype: dict + """ + if not _type in self.templates.keys(): + raise TypeError("Improper type passed to _get_empty_doc") + return copy.deepcopy(self.templates[_type]) + + def _get_or_create_rdoc(self): + """ + Try to retrieve the recent-flags doc for this MessageCollection, + and create one if not found. + """ + # XXX should move this to memstore too + with self._rdoc_write_lock[self.mbox]: + rdoc = self._get_recent_doc_from_soledad() + if rdoc is None: + rdoc = self._get_empty_doc(self.RECENT_DOC) + if self.mbox != fields.INBOX_VAL: + rdoc[fields.MBOX_KEY] = self.mbox + self._soledad.create_doc(rdoc) + + @deferred_to_thread + def _do_parse(self, raw): + """ + Parse raw message and return it along with + relevant information about its outer level. + + This is done in a separate thread, and the callback is passed + to `_do_add_msg` method. + + :param raw: the raw message + :type raw: StringIO or basestring + :return: msg, parts, chash, size, multi + :rtype: tuple + """ + msg = message_from_string(raw) + parts = walk.get_parts(msg) + size = len(raw) + chash = sha256.SHA256(raw).hexdigest() + multi = msg.is_multipart() + return msg, parts, chash, size, multi + + def _populate_flags(self, flags, uid, chash, size, multi): + """ + Return a flags doc. + + XXX Missing DOC ----------- + """ + fd = self._get_empty_doc(self.FLAGS_DOC) + + fd[self.MBOX_KEY] = self.mbox + fd[self.UID_KEY] = uid + fd[self.CONTENT_HASH_KEY] = chash + fd[self.SIZE_KEY] = size + fd[self.MULTIPART_KEY] = multi + if flags: + fd[self.FLAGS_KEY] = flags + fd[self.SEEN_KEY] = self.SEEN_FLAG in flags + fd[self.DEL_KEY] = self.DELETED_FLAG in flags + fd[self.RECENT_KEY] = True # set always by default + return fd + + def _populate_headr(self, msg, chash, subject, date): + """ + Return a headers doc. + + XXX Missing DOC ----------- + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + + # "fix" for repeated headers. + for k, v in headers.items(): + newline = "\n%s: " % (k,) + headers[k] = newline.join(v) + + lower_headers = lowerdict(headers) + msgid = first(MSGID_RE.findall( + lower_headers.get('message-id', ''))) + + hd = self._get_empty_doc(self.HEADERS_DOC) + hd[self.CONTENT_HASH_KEY] = chash + hd[self.HEADERS_KEY] = headers + hd[self.MSGID_KEY] = msgid + + if not subject and self.SUBJECT_FIELD in headers: + hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + else: + hd[self.SUBJECT_KEY] = subject + + if not date and self.DATE_FIELD in headers: + hd[self.DATE_KEY] = headers[self.DATE_FIELD] + else: + hd[self.DATE_KEY] = date + return hd + + def _fdoc_already_exists(self, chash): + """ + Check whether we can find a flags doc for this mailbox with the + given content-hash. It enforces that we can only have the same maessage + listed once for a a given mailbox. + + :param chash: the content-hash to check about. + :type chash: basestring + :return: False, if it does not exist, or UID. + """ + exist = False + exist = self.memstore.get_fdoc_from_chash(chash, self.mbox) + + if not exist: + exist = self._get_fdoc_from_chash(chash) + if exist and exist.content is not None: + return exist.content.get(fields.UID_KEY, "unknown-uid") + else: + return False + + def add_msg(self, raw, subject=None, flags=None, date=None, uid=None, + notify_on_disk=False): + """ + Creates a new message document. + + :param raw: the raw message + :type raw: str + + :param subject: subject of the message. + :type subject: str + + :param flags: flags + :type flags: list + + :param date: the received date for the message + :type date: str + + :param uid: the message uid for this mailbox + :type uid: int + + :return: a deferred that will be fired with the message + uid when the adding succeed. + :rtype: deferred + """ + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + + observer = defer.Deferred() + d = self._do_parse(raw) + d.addCallback(lambda result: self.reactor.callInThread( + self._do_add_msg, result, flags, subject, date, + notify_on_disk, observer)) + return observer + + # Called in thread + def _do_add_msg(self, parse_result, flags, subject, + date, notify_on_disk, observer): + """ + Helper that creates a new message document. + Here lives the magic of the leap mail. Well, in soledad, really. + + See `add_msg` docstring for parameter info. + + :param parse_result: a tuple with the results of `self._do_parse` + :type parse_result: tuple + :param observer: a deferred that will be fired with the message + uid when the adding succeed. + :type observer: deferred + """ + # TODO signal that we can delete the original message!----- + # when all the processing is done. + + # TODO add the linked-from info ! + # TODO add reference to the original message + + msg, parts, chash, size, multi = parse_result + + # check for uniqueness -------------------------------- + # Watch out! We're reserving a UID right after this! + existing_uid = self._fdoc_already_exists(chash) + if existing_uid: + msg = self.get_msg_by_uid(existing_uid) + + # We can say the observer that we're done + self.reactor.callFromThread(observer.callback, existing_uid) + msg.setFlags((fields.DELETED_FLAG,), -1) + return + + uid = self.memstore.increment_last_soledad_uid(self.mbox) + + # We can say the observer that we're done at this point, but + # before that we should make sure it has no serious consequences + # if we're issued, for instance, a fetch command right after... + #self.reactor.callFromThread(observer.callback, uid) + # if we did the notify, we need to invalidate the deferred + # so not to try to fire it twice. + #observer = None + + fd = self._populate_flags(flags, uid, chash, size, multi) + hd = self._populate_headr(msg, chash, subject, date) + + body_phash_fun = [walk.get_body_phash_simple, + walk.get_body_phash_multi][int(multi)] + body_phash = body_phash_fun(walk.get_payloads(msg)) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + + # add parts map to header doc + # (body, multi, part_map) + for key in parts_map: + hd[key] = parts_map[key] + del parts_map + + hd = stringify_parts_map(hd) + + # The MessageContainer expects a dict, one-indexed + cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + + self.set_recent_flag(uid) + msg_container = MessageWrapper(fd, hd, cdocs) + self.memstore.create_message( + self.mbox, uid, msg_container, + observer=observer, notify_on_disk=notify_on_disk) + + # + # getters: specific queries + # + + # recent flags + + def _get_recent_flags(self): + """ + An accessor for the recent-flags set for this mailbox. + """ + # XXX check if we should remove this + if self.__rflags is not None: + return self.__rflags + + if self.memstore is not None: + with self._rdoc_lock[self.mbox]: + rflags = self.memstore.get_recent_flags(self.mbox) + if not rflags: + # not loaded in the memory store yet. + # let's fetch them from soledad... + rdoc = self._get_recent_doc_from_soledad() + if rdoc is None: + return set([]) + rflags = set(rdoc.content.get( + fields.RECENTFLAGS_KEY, [])) + # ...and cache them now. + self.memstore.load_recent_flags( + self.mbox, + {'doc_id': rdoc.doc_id, 'set': rflags}) + return rflags + + def _set_recent_flags(self, value): + """ + Setter for the recent-flags set for this mailbox. + """ + if self.memstore is not None: + self.memstore.set_recent_flags(self.mbox, value) + + recent_flags = property( + _get_recent_flags, _set_recent_flags, + doc="Set of UIDs with the recent flag for this mailbox.") + + def _get_recent_doc_from_soledad(self): + """ + Get recent-flags document from Soledad for this mailbox. + :rtype: SoledadDocument or None + """ + curried = partial( + self._soledad.get_from_index, + fields.TYPE_MBOX_IDX, + fields.TYPE_RECENT_VAL, self.mbox) + curried.expected = "rdoc" + with self._rdoc_read_lock[self.mbox]: + return try_unique_query(curried) + + # Property-set modification (protected by a different + # lock to give atomicity to the read/write operation) + + def unset_recent_flags(self, uids): + """ + Unset Recent flag for a sequence of uids. + + :param uids: the uids to unset + :type uid: sequence + """ + with self._rdoc_property_lock[self.mbox]: + self.recent_flags.difference_update( + set(uids)) + + # Individual flags operations + + def unset_recent_flag(self, uid): + """ + Unset Recent flag for a given uid. + + :param uid: the uid to unset + :type uid: int + """ + with self._rdoc_property_lock[self.mbox]: + self.recent_flags.difference_update( + set([uid])) + + @deferred_to_thread + def set_recent_flag(self, uid): + """ + Set Recent flag for a given uid. + + :param uid: the uid to set + :type uid: int + """ + with self._rdoc_property_lock[self.mbox]: + self.recent_flags = self.recent_flags.union( + set([uid])) + + # individual doc getters, message layer. + + def _get_fdoc_from_chash(self, chash): + """ + Return a flags document for this mailbox with a given chash. + + :return: A SoledadDocument containing the Flags Document, or None if + the query failed. + :rtype: SoledadDocument or None. + """ + curried = partial( + self._soledad.get_from_index, + fields.TYPE_MBOX_C_HASH_IDX, + fields.TYPE_FLAGS_VAL, self.mbox, chash) + curried.expected = "fdoc" + fdoc = try_unique_query(curried) + if fdoc is not None: + return fdoc + else: + # probably this should be the other way round, + # ie, try fist on memstore... + cf = self.memstore._chash_fdoc_store + fdoc = cf[chash][self.mbox] + # hey, I just needed to wrap fdoc thing into + # a "content" attribute, look a better way... + if not empty(fdoc): + return MessagePartDoc( + new=None, dirty=None, part=None, + store=None, doc_id=None, + content=fdoc) + + def _get_uid_from_msgidCb(self, msgid): + hdoc = None + curried = partial( + self._soledad.get_from_index, + fields.TYPE_MSGID_IDX, + fields.TYPE_HEADERS_VAL, msgid) + curried.expected = "hdoc" + hdoc = try_unique_query(curried) + + # XXX this is only a quick hack to avoid regression + # on the "multiple copies of the draft" issue, but + # this is currently broken since it's not efficient to + # look for this. Should lookup better. + # FIXME! + + if hdoc is not None: + hdoc_dict = hdoc.content + + else: + hdocstore = self.memstore._hdoc_store + match = [x for _, x in hdocstore.items() if x['msgid'] == msgid] + hdoc_dict = first(match) + + if hdoc_dict is None: + logger.warning("Could not find hdoc for msgid %s" + % (msgid,)) + return None + msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY) + + fdoc = self._get_fdoc_from_chash(msg_chash) + if not fdoc: + logger.warning("Could not find fdoc for msgid %s" + % (msgid,)) + return None + return fdoc.content.get(fields.UID_KEY, None) + + @deferred_to_thread + def _get_uid_from_msgid(self, msgid): + """ + Return a UID for a given message-id. + + It first gets the headers-doc for that msg-id, and + it found it queries the flags doc for the current mailbox + for the matching content-hash. + + :return: A UID, or None + """ + # We need to wait a little bit, cause in some of the cases + # the query is received right after we've saved the document, + # and we cannot find it otherwise. This seems to be enough. + + # XXX do a deferLater instead ?? + # XXX is this working? + return self._get_uid_from_msgidCb(msgid) + + @deferred_to_thread + def set_flags(self, mbox, messages, flags, mode, observer): + """ + Set flags for a sequence of messages. + + :param mbox: the mbox this message belongs to + :type mbox: str or unicode + :param messages: the messages to iterate through + :type messages: sequence + :flags: the flags to be set + :type flags: tuple + :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. + :type mode: int + :param observer: a deferred that will be called with the dictionary + mapping UIDs to flags after the operation has been + done. + :type observer: deferred + """ + reactor = self.reactor + getmsg = self.get_msg_by_uid + + def set_flags(uid, flags, mode): + msg = getmsg(uid, mem_only=True, flags_only=True) + if msg is not None: + return uid, msg.setFlags(flags, mode) + + setted_flags = [set_flags(uid, flags, mode) for uid in messages] + result = dict(filter(None, setted_flags)) + + reactor.callFromThread(observer.callback, result) + + # getters: generic for a mailbox + + def get_msg_by_uid(self, uid, mem_only=False, flags_only=False): + """ + Retrieves a LeapMessage by UID. + This is used primarity in the Mailbox fetch and store methods. + + :param uid: the message uid to query by + :type uid: int + :param mem_only: a flag that indicates whether this Message should + pass a reference to soledad to retrieve missing pieces + or not. + :type mem_only: bool + :param flags_only: whether the message should carry only a reference + to the flags document. + :type flags_only: bool + + :return: A LeapMessage instance matching the query, + or None if not found. + :rtype: LeapMessage + """ + msg_container = self.memstore.get_message( + self.mbox, uid, flags_only=flags_only) + + if msg_container is not None: + if mem_only: + msg = LeapMessage(None, uid, self.mbox, collection=self, + container=msg_container) + else: + # We pass a reference to soledad just to be able to retrieve + # missing parts that cannot be found in the container, like + # the content docs after a copy. + msg = LeapMessage(self._soledad, uid, self.mbox, + collection=self, container=msg_container) + else: + msg = LeapMessage(self._soledad, uid, self.mbox, collection=self) + + if not msg.does_exist(): + return None + return msg + + def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + """ + Get all documents for the selected mailbox of the + passed type. By default, it returns the flag docs. + + If you want acess to the content, use __iter__ instead + + :return: a list of u1db documents + :rtype: list of SoledadDocument + """ + if _type not in fields.__dict__.values(): + raise TypeError("Wrong type passed to get_all_docs") + + if sameProxiedObjects(self._soledad, None): + logger.warning('Tried to get messages but soledad is None!') + return [] + + all_docs = [doc for doc in self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + _type, self.mbox)] + + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad + # FIXME ---------------------------------------------- + return sorted(all_docs, key=lambda item: item.content['uid']) + + def all_soledad_uid_iter(self): + """ + Return an iterator through the UIDs of all messages, sorted in + ascending order. + """ + db_uids = set([doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + if not empty(doc)]) + return db_uids + + def all_uid_iter(self): + """ + Return an iterator through the UIDs of all messages, from memory. + """ + mem_uids = self.memstore.get_uids(self.mbox) + soledad_known_uids = self.memstore.get_soledad_known_uids( + self.mbox) + combined = tuple(set(mem_uids).union(soledad_known_uids)) + return combined + + def get_all_soledad_flag_docs(self): + """ + Return a dict with the content of all the flag documents + in soledad store for the given mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: dict + """ + # XXX we really could return a reduced version with + # just {'uid': (flags-tuple,) since the prefetch is + # only oriented to get the flag tuples. + all_docs = [( + doc.content[self.UID_KEY], + dict(doc.content)) + for doc in + self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + if not empty(doc.content)] + all_flags = dict(all_docs) + return all_flags + + def all_headers(self): + """ + Return a dict with all the header documents for this + mailbox. + + :rtype: dict + """ + return self.memstore.all_headers(self.mbox) + + def count(self): + """ + Return the count of messages for this mailbox. + + :rtype: int + """ + return self.memstore.count(self.mbox) + + # unseen messages + + def unseen_iter(self): + """ + Get an iterator for the message UIDs with no `seen` flag + for this mailbox. + + :return: iterator through unseen message doc UIDs + :rtype: iterable + """ + return self.memstore.unseen_iter(self.mbox) + + def count_unseen(self): + """ + Count all messages with the `Unseen` flag. + + :returns: count + :rtype: int + """ + return len(list(self.unseen_iter())) + + def get_unseen(self): + """ + Get all messages with the `Unseen` flag + + :returns: a list of LeapMessages + :rtype: list + """ + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.unseen_iter()] + + # recent messages + + # XXX take it from memstore + def count_recent(self): + """ + Count all messages with the `Recent` flag. + It just retrieves the length of the recent_flags set, + which is stored in a specific type of document for + this collection. + + :returns: count + :rtype: int + """ + return len(self.recent_flags) + + def __len__(self): + """ + Returns the number of messages on this mailbox. + + :rtype: int + """ + return self.count() + + def __iter__(self): + """ + Returns an iterator over all messages. + + :returns: iterator of dicts with content for all messages. + :rtype: iterable + """ + return (LeapMessage(self._soledad, docuid, self.mbox) + for docuid in self.all_uid_iter()) + + def __repr__(self): + """ + Representation string for this object. + """ + return u"<MessageCollection: mbox '%s' (%s)>" % ( + self.mbox, self.count()) + + # XXX should implement __eq__ also !!! + # use chash... diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py new file mode 100644 index 0000000..4a801b0 --- /dev/null +++ b/src/leap/mail/imap/parser.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# parser.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Mail parser mixin. +""" +import re + + +class MBoxParser(object): + """ + Utility function to parse mailbox names. + """ + INBOX_NAME = "INBOX" + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + def _parse_mailbox_name(self, name): + """ + Return a normalized representation of the mailbox C{name}. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 6320a51..fe56ea6 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # server.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,1661 +15,361 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Soledad-backed IMAP Server. +Leap IMAP4 Server Implementation. """ -import copy -import logging -import StringIO -import cStringIO -import time - -from collections import defaultdict -from email.parser import Parser - -from zope.interface import implements -from zope.proxy import sameProxiedObjects +from copy import copy +from twisted import cred +from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 -from twisted.internet import defer from twisted.python import log from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type -from leap.common.mail import get_email_charset -from leap.mail.messageflow import IMessageConsumer, MessageProducer +from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN from leap.soledad.client import Soledad -logger = logging.getLogger(__name__) - - -class MissingIndexError(Exception): - """ - Raises when tried to access a non existent index document. - """ - - -class BadIndexError(Exception): - """ - Raises when index is malformed or has the wrong cardinality. - """ - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # Internal representation of Message - DATE_KEY = "date" - HEADERS_KEY = "headers" - FLAGS_KEY = "flags" - MBOX_KEY = "mbox" - RAW_KEY = "raw" - SUBJECT_KEY = "subject" - UID_KEY = "uid" - - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MESSAGE_VAL = "msg" - TYPE_MBOX_VAL = "mbox" - - INBOX_VAL = "inbox" - - # Flags for SoledadDocument for indexing. - SEEN_KEY = "seen" - RECENT_KEY = "recent" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - db_indexes = dict() - if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in SoledadBackedAccount.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) - +# imports for LITERAL+ patch +from twisted.internet import defer, interfaces +from twisted.mail.imap4 import IllegalClientResponse +from twisted.mail.imap4 import LiteralString, LiteralFile -####################################### -# Soledad Account -####################################### - -class SoledadBackedAccount(WithMsgFields, IndexedDB): +class LeapIMAPServer(imap4.IMAP4Server): """ - An implementation of IAccount and INamespacePresenteer - that is backed by Soledad Encrypted Documents. + An IMAP4 Server with mailboxes backed by soledad """ + def __init__(self, *args, **kwargs): + # pop extraneous arguments + soledad = kwargs.pop('soledad', None) + uuid = kwargs.pop('uuid', None) + userid = kwargs.pop('userid', None) - implements(imap4.IAccount, imap4.INamespacePresenter) - - _soledad = None - selected = None - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - KTYPE = WithMsgFields.TYPE_KEY - MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - } - - INBOX_NAME = "INBOX" - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - WithMsgFields.TYPE_KEY: MBOX_KEY, - WithMsgFields.TYPE_MBOX_VAL: INBOX_NAME, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.CLOSED_KEY: False, - WithMsgFields.SUBSCRIBED_KEY: False, - WithMsgFields.RW_KEY: 1, - } - - def __init__(self, account_name, soledad=None): - """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. - - :param acct_name: The name of the account (user id). - :type acct_name: str - - :param soledad: a Soledad instance. - :param soledad: Soledad - """ - leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert(soledad, "need a soledad instance") leap_assert_type(soledad, Soledad) + leap_assert(uuid, "need a user in the initialization") - # XXX SHOULD assert too that the name matches the user/uuid with which - # soledad has been initialized. + self._userid = userid - self._account_name = account_name.upper() - self._soledad = soledad + # initialize imap server! + imap4.IMAP4Server.__init__(self, *args, **kwargs) - self.initialize_db() + # we should initialize the account here, + # but we move it to the factory so we can + # populate the test account properly (and only once + # per session) - # every user should have the right to an inbox folder - # at least, so let's make one! + from twisted.internet import reactor + self.reactor = reactor - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - - def _get_mailbox_by_name(self, name): - """ - Returns an mbox document by name. - - :param name: the name of the mailbox - :type name: str - - :rtype: SoledadDocument + def lineReceived(self, line): """ - # XXX only upper for INBOX --- - name = name.upper() - doc = self._soledad.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, name) - return doc[0] if doc else None - - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - """ - return [str(doc.content[self.MBOX_KEY]) - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)] - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [str(doc.content[self.MBOX_KEY]) - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] - - def getMailbox(self, name): - """ - Returns a Mailbox with that name, without selecting it. - - :param name: name of the mailbox - :type name: str - - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox - """ - # XXX only upper for INBOX - name = name.upper() - if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - return SoledadMailbox(name, soledad=self._soledad) - - ## - ## IAccount - ## - - def addMailbox(self, name, creation_ts=None): - """ - Adds a mailbox to the account. - - :param name: the name of the mailbox - :type name: str - - :param creation_ts: a optional creation timestamp to be used as - mailbox id. A timestamp will be used if no - one is provided. - :type creation_ts: int - - :returns: True if successful - :rtype: bool - """ - # XXX only upper for INBOX - name = name.upper() - # XXX should check mailbox name for RFC-compliant form - - if name in self.mailboxes: - raise imap4.MailboxCollision, name - - if not creation_ts: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # maibox-uidvalidity. - creation_ts = int(time.time() * 10E2) - - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - doc = self._soledad.create_doc(mbox) - return bool(doc) - - def create(self, pathspec): - """ - Create a new mailbox from the given hierarchical name. - - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. - :type pathspec: str - - :return: A true value if the creation succeeds. - :rtype: bool - - :raise MailboxException: Raised if this mailbox cannot be added. - """ - # TODO raise MailboxException - - paths = filter(None, pathspec.split('/')) - for accum in range(1, len(paths)): - try: - self.addMailbox('/'.join(paths[:accum])) - except imap4.MailboxCollision: - pass - try: - self.addMailbox('/'.join(paths)) - except imap4.MailboxCollision: - if not pathspec.endswith('/'): - return False - return True - - def select(self, name, readwrite=1): - """ - Selects a mailbox. - - :param name: the mailbox to select - :type name: str - - :param readwrite: 1 for readwrite permissions. - :type readwrite: int - - :rtype: bool - """ - # XXX only upper for INBOX - name = name.upper() - - if name not in self.mailboxes: - return None - - self.selected = str(name) - - return SoledadMailbox( - name, rw=readwrite, - soledad=self._soledad) - - def delete(self, name, force=False): - """ - Deletes a mailbox. - - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - - :param name: the mailbox to be deleted - :type name: str - - :param force: if True, it will not check for noselect flag or inferior - names. use with care. - :type force: bool - """ - # XXX only upper for INBOX - name = name.upper() - if not name in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - mbox = self.getMailbox(name) - - if force is False: - # See if this box is flagged \Noselect - # XXX use mbox.flags instead? - if self.NOSELECT_FLAG in mbox.getFlags(): - # Check for hierarchically inferior mailboxes with this one - # as part of their root. - for others in self.mailboxes: - if others != name and others.startswith(name): - raise imap4.MailboxException, ( - "Hierarchically inferior mailboxes " - "exist and \\Noselect is set") - mbox.destroy() - - # XXX FIXME --- not honoring the inferior names... - - # if there are no hierarchically inferior names, we will - # delete it from our ken. - #if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - #self._index.removeMailbox(name) - - def rename(self, oldname, newname): - """ - Renames a mailbox. - - :param oldname: old name of the mailbox - :type oldname: str - - :param newname: new name of the mailbox - :type newname: str - """ - # XXX only upper for INBOX - oldname = oldname.upper() - newname = newname.upper() - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox, oldname - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] - - for (old, new) in inferiors: - if new in self.mailboxes: - raise imap4.MailboxCollision, new - - for (old, new) in inferiors: - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new - self._soledad.put_doc(mbox) - - # XXX ---- FIXME!!!! ------------------------------------ - # until here we just renamed the index... - # We have to rename also the occurrence of this - # mailbox on ALL the messages that are contained in it!!! - # ... we maybe could use a reference to the doc_id - # in each msg, instead of the "mbox" field in msgs - # ------------------------------------------------------- - - def _inferiorNames(self, name): - """ - Return hierarchically inferior mailboxes. - - :param name: name of the mailbox - :rtype: list - """ - # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors - - def isSubscribed(self, name): - """ - Returns True if user is subscribed to this mailbox. - - :param name: the mailbox to be checked. - :type name: str + Attempt to parse a single line from the server. - :rtype: bool + :param line: the line from the server, without the line delimiter. + :type line: str """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) + if self.theAccount.closed is True and self.state != "unauth": + log.msg("Closing the session. State: unauth") + self.state = "unauth" - def _set_subscription(self, name, value): - """ - Sets the subscription value for a given mailbox - - :param name: the mailbox - :type name: str - - :param value: the boolean value - :type value: bool - """ - # maybe we should store subscriptions in another - # document... - if not name in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) - - def subscribe(self, name): - """ - Subscribe to this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = name.upper() - if name not in self.subscriptions: - self._set_subscription(name, True) - - def unsubscribe(self, name): - """ - Unsubscribe from this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = name.upper() - if name not in self.subscriptions: - raise imap4.MailboxException, "Not currently subscribed to " + name - self._set_subscription(name, False) - - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames(ref.upper()) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - - ## - ## INamespacePresenter - ## - - def getPersonalNamespaces(self): - return [["", "/"]] - - def getSharedNamespaces(self): - return None - - def getOtherNamespaces(self): - return None - - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - - def __repr__(self): - """ - Representation string for this object. - """ - return "<SoledadBackedAccount (%s)>" % self._account_name - -####################################### -# LeapMessage, MessageCollection -# and Mailbox -####################################### - - -class LeapMessage(WithMsgFields): - - implements(imap4.IMessage, imap4.IMessageFile) + if "login" in line.lower(): + # avoid to log the pass, even though we are using a dummy auth + # by now. + msg = line[:7] + " [...]" + else: + msg = copy(line) + log.msg('rcv (%s): %s' % (self.state, msg)) + imap4.IMAP4Server.lineReceived(self, line) - def __init__(self, doc): + def authenticateLogin(self, username, password): """ - Initializes a LeapMessage. + Lookup the account with the given parameters, and deny + the improper combinations. - :param doc: A SoledadDocument containing the internal - representation of the message - :type doc: SoledadDocument + :param username: the username that is attempting authentication. + :type username: str + :param password: the password to authenticate with. + :type password: str """ - self._doc = doc + # XXX this should use portal: + # return portal.login(cred.credentials.UsernamePassword(user, pass) + if username != self._userid: + # bad username, reject. + raise cred.error.UnauthorizedLogin() + # any dummy password is allowed so far. use realm instead! + leap_events.signal(IMAP_CLIENT_LOGIN, "1") + return imap4.IAccount, self.theAccount, lambda: None - def getUID(self): + def do_FETCH(self, tag, messages, query, uid=0): """ - Retrieve the unique identifier associated with this message - - :return: uid for this message - :rtype: int + Overwritten fetch dispatcher to use the fast fetch_flags + method """ - # XXX debug, to remove after a while... - if not self._doc: - log.msg('BUG!!! ---- message has no doc!') + if not query: + self.sendPositiveResponse(tag, 'FETCH complete') return - return self._doc.content[self.UID_KEY] - - def getFlags(self): - """ - Retrieve the flags associated with this message - - :return: The flags, represented as strings - :rtype: iterable - """ - if self._doc is None: - return [] - flags = self._doc.content.get(self.FLAGS_KEY, None) - if flags: - flags = map(str, flags) - return flags - - # setFlags, addFlags, removeFlags are not in the interface spec - # but we use them with store command. - - def setFlags(self, flags): - """ - Sets the flags for this message - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to update in the message. - :type flags: sequence of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - log.msg('setting flags') - doc = self._doc - doc.content[self.FLAGS_KEY] = flags - doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags - doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags - return doc - - def addFlags(self, flags): - """ - Adds flags to this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to add to the message. - :type flags: sequence of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - oldflags = self.getFlags() - return self.setFlags(list(set(flags + oldflags))) - - def removeFlags(self, flags): - """ - Remove flags from this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to be removed from the message. - :type flags: sequence of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - oldflags = self.getFlags() - return self.setFlags(list(set(oldflags) - set(flags))) - - def getInternalDate(self): - """ - Retrieve the date internally associated with this message - - @rtype: C{str} - @retur: An RFC822-formatted date string. - """ - return str(self._doc.content.get(self.DATE_KEY, '')) - - # - # IMessageFile - # - """ - Optional message interface for representing messages as files. - - If provided by message objects, this interface will be used instead - the more complex MIME-based interface. - """ - - def open(self): - """ - Return an file-like object opened for reading. - - Reading from the returned file will return all the bytes - of which this message consists. - - :return: file-like object opened fore reading. - :rtype: StringIO - """ - fd = cStringIO.StringIO() - charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) - content = self._doc.content.get(self.RAW_KEY, '') - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - fd.write(content) - fd.seek(0) - return fd - - # - # IMessagePart - # + cbFetch = self._IMAP4Server__cbFetch + ebFetch = self._IMAP4Server__ebFetch + + if len(query) == 1 and str(query[0]) == "flags": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_flags, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + + elif len(query) == 1 and str(query[0]) == "rfc822.header": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_headers, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + else: + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback( + ebFetch, tag) - # XXX should implement the rest of IMessagePart interface: - # (and do not use the open above) + select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_fetchatt) - def getBodyFile(self): + def notifyNew(self, ignored=None): """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO + Notify new messages to listeners. """ - fd = StringIO.StringIO() - charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) - content = self._doc.content.get(self.RAW_KEY, '') - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - fd.write(content) - # SHOULD use a separate BODY FIELD ... - fd.seek(0) - return fd + self.reactor.callFromThread(self.mbox.notify_new) - def getSize(self): + def _cbSelectWork(self, mbox, cmdName, tag): """ - Return the total size, in octets, of this message. - - :return: size of the message, in octets - :rtype: int + Callback for selectWork, patched to avoid conformance errors due to + incomplete UIDVALIDITY line. """ - return self.getBodyFile().len + if mbox is None: + self.sendNegativeResponse(tag, 'No such mailbox') + return + if '\\noselect' in [s.lower() for s in mbox.getFlags()]: + self.sendNegativeResponse(tag, 'Mailbox cannot be selected') + return - def _get_headers(self): - """ - Return the headers dict stored in this message document. - """ - return self._doc.content.get(self.HEADERS_KEY, {}) + flags = mbox.getFlags() + self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') + self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) + + # Patched ------------------------------------------------------- + # imaptest was complaining about the incomplete line, we're adding + # "UIDs valid" here. + self.sendPositiveResponse( + None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) + # ---------------------------------------------------------------- + + s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' + mbox.addListener(self) + self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) + self.state = 'select' + self.mbox = mbox - def getHeaders(self, negate, *names): + def checkpoint(self): """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool + Called when the client issues a CHECK command. - :return: A mapping of header field names to header field values - :rtype: dict + This should perform any checkpoint operations required by the server. + It may be a long running operation, but may not block. If it returns + a deferred, the client will only be informed of success (or failure) + when the deferred's callback (or errback) is invoked. """ - headers = self._get_headers() - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # unpack and filter original dict by negate-condition - filter_by_cond = [ - map(str, (key, val)) for - key, val in headers.items() - if cond(key)] - return dict(filter_by_cond) - - # --- no multipart for now - # XXX Fix MULTIPART SUPPORT! - - def isMultipart(self): - return False - - def getSubPart(part): + # TODO return the output of _memstore.is_writing + # XXX and that should return a deferred! return None + ############################################################# # - # accessors + # Twisted imap4 patch to support LITERAL+ extension + # TODO send this patch upstream asap! # - - def __getitem__(self, key): - """ - Return the content of the message document. - - @param key: The key - @type key: str - - @return: The content value indexed by C{key} or None - @rtype: str - """ - return self._doc.content.get(key, None) - - -class SoledadDocWriter(object): - """ - This writer will create docs serially in the local soledad database. - """ - - implements(IMessageConsumer) - - def __init__(self, soledad): - """ - Initialize the writer. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - self._soledad = soledad - - def consume(self, item): - """ - Creates a new document in soledad db. - - :param item: object to update. content of the document to be inserted. - :type item: dict - """ - self._soledad.create_doc(item) - - -class MessageCollection(WithMsgFields, IndexedDB): - """ - A collection of messages, surprisingly. - - It is tied to a selected mailbox name that is passed to constructor. - Implements a filter query over the messages contained in a soledad - database. - """ - # XXX this should be able to produce a MessageSet methinks - - EMPTY_MSG = { - WithMsgFields.TYPE_KEY: WithMsgFields.TYPE_MESSAGE_VAL, - WithMsgFields.UID_KEY: 1, - WithMsgFields.MBOX_KEY: WithMsgFields.INBOX_VAL, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.DATE_KEY: "", - WithMsgFields.SEEN_KEY: False, - WithMsgFields.RECENT_KEY: True, - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.HEADERS_KEY: {}, - WithMsgFields.RAW_KEY: "", - } - - # get from SoledadBackedAccount the needed index-related constants - INDEXES = SoledadBackedAccount.INDEXES - TYPE_IDX = SoledadBackedAccount.TYPE_IDX - - def __init__(self, mbox=None, soledad=None): - """ - Constructor for MessageCollection. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database - :type mbox: str - - :param soledad: Soledad database - :type soledad: Soledad instance - """ - # XXX pass soledad directly - - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") - - # This is a wrapper now!... - # should move assertion there... - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - - # okay, all in order, keep going... - - self.mbox = mbox.upper() - self._soledad = soledad - self.initialize_db() - self._parser = Parser() - - # I think of someone like nietzsche when reading this - - # this will be the producer that will enqueue the content - # to be processed serially by the consumer (the writer). We just - # need to `put` the new material on its plate. - - self._soledad_writer = MessageProducer( - SoledadDocWriter(soledad), - period=0.2) - - def _get_empty_msg(self): - """ - Returns an empty message. - - :return: a dict containing a default empty message - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MSG) - - def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): - """ - Creates a new message document. - - :param raw: the raw message - :type raw: str - - :param subject: subject of the message. - :type subject: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :param uid: the message uid for this mailbox - :type uid: int - """ - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - - def stringify(o): - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() + ############################################################# + + def capabilities(self): + cap = {'AUTH': self.challengers.keys()} + if self.ctx and self.canStartTLS: + t = self.transport + ti = interfaces.ISSLTransport + if not self.startedTLS and ti(t, None) is None: + cap['LOGINDISABLED'] = None + cap['STARTTLS'] = None + cap['NAMESPACE'] = None + cap['IDLE'] = None + # patched ############ + cap['LITERAL+'] = None + ###################### + return cap + + def _stringLiteral(self, size, literal_plus=False): + if size > self._literalStringLimit: + raise IllegalClientResponse( + "Literal too long! I accept at most %d octets" % + (self._literalStringLimit,)) + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralString(size, d) + # Patched ########################################################### + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of text' % size) + ##################################################################### + self.setRawMode() + return d + + def _fileLiteral(self, size, literal_plus=False): + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralFile(size, d) + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of data' % size) + self.setRawMode() + return d + + def arg_astring(self, line): + """ + Parse an astring from the line, return (arg, rest), possibly + via a deferred (to handle literals) + """ + line = line.strip() + if not line: + raise IllegalClientResponse("Missing argument") + d = None + arg, rest = None, None + if line[0] == '"': + try: + spam, arg, rest = line.split('"', 2) + rest = rest[1:] # Strip space + except ValueError: + raise IllegalClientResponse("Unmatched quotes") + elif line[0] == '{': + # literal + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################ + if line[-2] == "+": + literalPlus = True + size_end = -2 else: - return o - - content = self._get_empty_msg() - content[self.MBOX_KEY] = self.mbox - - if flags: - content[self.FLAGS_KEY] = map(stringify, flags) - content[self.SEEN_KEY] = self.SEEN_FLAG in flags - - def _get_parser_fun(o): - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, (str, unicode)): - return self._parser.parsestr - - msg = _get_parser_fun(raw)(raw, True) - headers = dict(msg) - - # XXX get lower case for keys? - content[self.HEADERS_KEY] = headers - # set subject based on message headers and eventually replace by - # subject given as param - if self.SUBJECT_FIELD in headers: - content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - if subject is not None: - content[self.SUBJECT_KEY] = subject - content[self.RAW_KEY] = stringify(raw) - - if not date and self.DATE_FIELD in headers: - content[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - content[self.DATE_KEY] = date - - # ...should get a sanity check here. - content[self.UID_KEY] = uid - - self._soledad_writer.put(content) - # XXX have to decide what shall we do with errors with this change... - #return self._soledad.create_doc(content) - - def remove(self, msg): - """ - Removes a message. - - :param msg: a u1db doc containing the message - :type msg: SoledadDocument - """ - self._soledad.delete_doc(msg) - - # getters - - def get_by_uid(self, uid): - """ - Retrieves a message document by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A SoledadDocument instance matching the query, - or None if not found. - :rtype: SoledadDocument - """ - docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, str(uid)) - - return docs[0] if docs else None - - def get_msg_by_uid(self, uid): - """ - Retrieves a LeapMessage by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - doc = self.get_by_uid(uid) - if doc: - return LeapMessage(doc) - - def get_by_index(self, index): - """ - Retrieves a mesage document by mailbox index. - - :param index: the index of the sequence (zero-indexed) - :type index: int - """ - try: - return self.get_all()[index] - except IndexError: - return None - - def get_msg_by_index(self, index): - """ - Retrieves a LeapMessage by sequence index. - - :param index: the index of the sequence (zero-indexed) - :type index: int - """ - doc = self.get_by_index(index) - if doc: - return LeapMessage(doc) - - def is_deleted(self, doc): - """ - Returns whether a given doc is deleted or not. - - :param doc: the document to check - :rtype: bool - """ - return self.DELETED_FLAG in doc.content[self.FLAGS_KEY] - - def get_last(self): - """ - Gets the last LeapMessage - """ - _all = self.get_all() - if not _all: - return None - return LeapMessage(_all[-1]) - - def get_all(self): - """ - Get all message documents for the selected mailbox. - If you want acess to the content, use __iter__ instead - - :return: a list of u1db documents - :rtype: list of SoledadDocument - """ - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - #f XXX this should return LeapMessage instances - all_docs = [doc for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox)] - #if not self.is_deleted(doc)] - # highly inneficient, but first let's grok it and then - # let's worry about efficiency. - return sorted(all_docs, key=lambda item: item.content['uid']) - - def unseen_iter(self): - """ - Get an iterator for the message docs with no `seen` flag - - :return: iterator through unseen message docs - :rtype: iterable - """ - return (doc for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_SEEN_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1', '0')) - - def get_unseen(self): - """ - Get all messages with the `Unseen` flag - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(doc) for doc in self.unseen_iter()] - - def recent_iter(self): - """ - Get an iterator for the message docs with `recent` flag. - - :return: iterator through recent message docs - :rtype: iterable - """ - return (doc for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1')) - - def get_recent(self): - """ - Get all messages with the `Recent` flag. - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(doc) for doc in self.recent_iter()] - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - return len(self.get_all()) - - def __len__(self): - """ - Returns the number of messages on this mailbox. - - :rtype: int - """ - return self.count() - - def __iter__(self): - """ - Returns an iterator over all messages. - - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - # XXX return LeapMessage instead?! (change accordingly) - return (m.content for m in self.get_all()) - - def __getitem__(self, uid): - """ - Allows indexing as a list, with msg uid as the index. - - :param uid: an integer index - :type uid: int - - :return: LeapMessage or None if not found. - :rtype: LeapMessage - """ - #try: - #return self.get_msg_by_uid(uid) - try: - return [doc - for doc in self.get_all()][uid - 1] - except IndexError: - return None - - def __repr__(self): - """ - Representation string for this object. - """ - return u"<MessageCollection: mbox '%s' (%s)>" % ( - self.mbox, self.count()) - - # XXX should implement __eq__ also - - -class SoledadMailbox(WithMsgFields): - """ - A Soledad-backed IMAP mailbox. - - Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. - """ - implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) - # XXX should finish the implementation of IMailboxListener - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None - - CMD_MSG = "MESSAGES" - CMD_RECENT = "RECENT" - CMD_UIDNEXT = "UIDNEXT" - CMD_UIDVALIDITY = "UIDVALIDITY" - CMD_UNSEEN = "UNSEEN" - - _listeners = defaultdict(set) - - def __init__(self, mbox, soledad=None, rw=1): - """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. - - :param mbox: the mailbox name - :type mbox: str + literalPlus = False + size_end = -1 - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param rw: read-and-write flags - :type rw: int - """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") - - # XXX should move to wrapper - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - - self.mbox = mbox - self.rw = rw - - self._soledad = soledad - - self.messages = MessageCollection( - mbox=mbox, soledad=soledad) - - if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - @property - def listeners(self): - """ - Returns listeners for this mbox. - - The server itself is a listener to the mailbox. - so we can notify it (and should!) after changes in flags - and number of messages. - - :rtype: set - """ - return self._listeners[self.mbox] - - def addListener(self, listener): - """ - Rdds a listener to the listeners queue. - - :param listener: listener to add - :type listener: an object that implements IMailboxListener - """ - logger.debug('adding mailbox listener: %s' % listener) - self.listeners.add(listener) - - def removeListener(self, listener): - """ - Removes a listener from the listeners queue. - - :param listener: listener to remove - :type listener: an object that implements IMailboxListener - """ - self.listeners.remove(listener) - - def _get_mbox(self): - """ - Returns mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - try: - query = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() - except Exception as exc: - logger.error("Unhandled error %r" % exc) - - def getFlags(self): - """ - Returns the flags defined for this mailbox. - - :returns: tuple of flags for this mailbox - :rtype: tuple of str - """ - #return map(str, self.INIT_FLAGS) - - # XXX CHECK against thunderbird XXX - # XXX I think this is slightly broken.. :/ - - mbox = self._get_mbox() - if not mbox: - return None - flags = mbox.content.get(self.FLAGS_KEY, []) - return map(str, flags) - - def setFlags(self, flags): - """ - Sets flags for this mailbox. - - :param flags: a tuple with the flags - :type flags: tuple of str - """ - # TODO -- fix also getFlags - leap_assert(isinstance(flags, tuple), - "flags expected to be a tuple") - mbox = self._get_mbox() - if not mbox: - return None - mbox.content[self.FLAGS_KEY] = map(str, flags) - self._soledad.put_doc(mbox) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. - - def _get_closed(self): - """ - Return the closed attribute for this mailbox. - - :return: True if the mailbox is closed - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.CLOSED_KEY, False) - - def _set_closed(self, closed): - """ - Set the closed attribute for this mailbox. - - :param closed: the state to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - mbox = self._get_mbox() - mbox.content[self.CLOSED_KEY] = closed - self._soledad.put_doc(mbox) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def getUIDValidity(self): - """ - Return the unique validity identifier for this mailbox. - - :return: unique validity identifier - :rtype: int - """ - mbox = self._get_mbox() - return mbox.content.get(self.CREATED_KEY, 1) - - def getUID(self, message): - """ - Return the UID of a message in the mailbox - - .. note:: this implementation does not make much sense RIGHT NOW, - but in the future will be useful to get absolute UIDs from - message sequence numbers. - - :param message: the message uid - :type message: int - - :rtype: int - """ - msg = self.messages.get_msg_by_uid(message) - return msg.getUID() - - def getUIDNext(self): - """ - Return the likely UID for the next message added to this - mailbox. Currently it returns the current length incremented - by one. - - :rtype: int - """ - last = self.messages.get_last() - if last: - nextuid = last.getUID() + 1 + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + d = self._stringLiteral(size, literalPlus) + ########################## else: - nextuid = 1 - return nextuid + arg = line.split(' ', 1) + if len(arg) == 1: + arg.append('') + arg, rest = arg + return d or (arg, rest) - def getMessageCount(self): + def arg_literal(self, line): """ - Returns the total count of messages in this mailbox. - - :rtype: int - """ - return self.messages.count() - - def getUnseenCount(self): - """ - Returns the number of messages with the 'Unseen' flag. - - :return: count of messages flagged `unseen` - :rtype: int - """ - return len(self.messages.get_unseen()) - - def getRecentCount(self): - """ - Returns the number of messages with the 'Recent' flag. - - :return: count of messages flagged `recent` - :rtype: int - """ - return len(self.messages.get_recent()) - - def isWriteable(self): - """ - Get the read/write status of the mailbox. - - :return: 1 if mailbox is read-writeable, 0 otherwise. - :rtype: int + Parse a literal from the line """ - return self.rw + if not line: + raise IllegalClientResponse("Missing argument") - def getHierarchicalDelimiter(self): - """ - Returns the character used to delimite hierarchies in mailboxes. + if line[0] != '{': + raise IllegalClientResponse("Missing literal") - :rtype: str - """ - return '/' + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") - def requestStatus(self, names): - """ - Handles a status request by gathering the output of the different - status commands. - - :param names: a list of strings containing the status commands - :type names: iter - """ - r = {} - if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() - if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() - if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getMessageCount() + 1 - if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUID() - if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) - - def addMessage(self, message, flags, date=None): - """ - Adds a message to this mailbox. - - :param message: the raw message - :type message: str - - :param flags: flag list - :type flags: list of str - - :param date: timestamp - :type date: str - - :return: a deferred that evals to None - """ - # XXX we should treat the message as an IMessage from here - uid_next = self.getUIDNext() - if flags is None: - flags = tuple() + # Patched ################## + if line[-2] == "+": + literalPlus = True + size_end = -2 else: - flags = tuple(str(flag) for flag in flags) - - self.messages.add_msg(message, flags=flags, date=date, - uid=uid_next) - - exists = len(self.messages) - recent = len(self.messages.get_recent()) - for listener in self.listeners: - listener.newMessages(exists, recent) - return defer.succeed(None) - - # commands, do not rename methods + literalPlus = False + size_end = -1 - def destroy(self): - """ - Called before this mailbox is permanently deleted. - - Should cleanup resources, and set the \\Noselect flag - on the mailbox. - """ - self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - self._soledad.delete_doc(self._get_mbox()) - - def expunge(self): - """ - Remove all messages flagged \\Deleted - """ - if not self.isWriteable(): - raise imap4.ReadOnlyMailbox - - delete = [] - deleted = [] - for m in self.messages.get_all(): - if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: - delete.append(m) - for m in delete: - deleted.append(m.content) - self.messages.remove(m) - - # XXX should return the UIDs of the deleted messages - # more generically - return [x for x in range(len(deleted))] - - def fetch(self, messages, uid): - """ - Retrieve one or more messages in this mailbox. - - from rfc 3501: The data items to be fetched can be either a single atom - or a parenthesized list. - - :param messages: IDs of the messages to retrieve information about - :type messages: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ - result = [] - sequence = True if uid == 0 else False - - if not messages.last: - try: - iter(messages) - except TypeError: - # looks like we cannot iterate - last = self.messages.get_last() - if last is None: - uid_last = 1 - else: - uid_last = last.getUID() - messages.last = uid_last + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) - # for sequence numbers (uid = 0) - if sequence: - for msg_id in messages: - msg = self.messages.get_msg_by_index(msg_id - 1) - if msg: - result.append((msg.getUID(), msg)) - else: - print "fetch %s, no msg found!!!" % msg_id + return self._fileLiteral(size, literalPlus) + ############################# - else: - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) - else: - print "fetch %s, no msg found!!!" % msg_id + # Need to override the command table after patching + # arg_astring and arg_literal - return tuple(result) + do_LOGIN = imap4.IMAP4Server.do_LOGIN + do_CREATE = imap4.IMAP4Server.do_CREATE + do_DELETE = imap4.IMAP4Server.do_DELETE + do_RENAME = imap4.IMAP4Server.do_RENAME + do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + do_STATUS = imap4.IMAP4Server.do_STATUS + do_APPEND = imap4.IMAP4Server.do_APPEND + do_COPY = imap4.IMAP4Server.do_COPY - def _signal_unread_to_ui(self): - """ - Sends unread event to ui. - """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self.getUnseenCount())) + _selectWork = imap4.IMAP4Server._selectWork + _listWork = imap4.IMAP4Server._listWork + arg_plist = imap4.IMAP4Server.arg_plist + arg_seqset = imap4.IMAP4Server.arg_seqset + opt_plist = imap4.IMAP4Server.opt_plist + opt_datetime = imap4.IMAP4Server.opt_datetime - def store(self, messages, flags, mode, uid): - """ - Sets the flags of one or more messages. + unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring) - :param messages: The identifiers of the messages to set the flags - :type messages: A MessageSet object with the list of messages requested + auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT') + select_SELECT = auth_SELECT - :param flags: The flags to set, unset, or add. - :type flags: sequence of str + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE - :param mode: If mode is -1, these flags should be removed from the - specified messages. If mode is 1, these flags should be - added to the specified messages. If mode is 0, all - existing flags should be cleared and these flags should be - added. - :type mode: -1, 0, or 1 + auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') + select_EXAMINE = auth_EXAMINE - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE - :return: A dict mapping message sequence numbers to sequences of - str representing the flags set on the message after this - operation has been performed. - :rtype: dict + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME - :raise ReadOnlyMailbox: Raised if this mailbox is not open for - read-write. - """ - # XXX implement also sequence (uid = 0) + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE - if not self.isWriteable(): - log.msg('read only mailbox!') - raise imap4.ReadOnlyMailbox + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE - if not messages.last: - messages.last = self.messages.count() + auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST') + select_LIST = auth_LIST - result = {} - for msg_id in messages: - print "MSG ID = %s" % msg_id - msg = self.messages.get_msg_by_uid(msg_id) - if mode == 1: - self._update(msg.addFlags(flags)) - elif mode == -1: - self._update(msg.removeFlags(flags)) - elif mode == 0: - self._update(msg.setFlags(flags)) - result[msg_id] = msg.getFlags() + auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB') + select_LSUB = auth_LSUB - self._signal_unread_to_ui() - return result + auth_STATUS = (do_STATUS, arg_astring, arg_plist) + select_STATUS = auth_STATUS - def close(self): - """ - Expunge and mark as closed - """ - self.expunge() - self.closed = True + auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + arg_literal) + select_APPEND = auth_APPEND - # convenience fun + select_COPY = (do_COPY, arg_seqset, arg_astring) - def deleteAllDocs(self): - """ - Deletes all docs in this mailbox - """ - docs = self.messages.get_all() - for doc in docs: - self.messages._soledad.delete_doc(doc) - def _update(self, doc): - """ - Updates document in u1db database - """ - #log.msg('updating doc... %s ' % doc) - self._soledad.put_doc(doc) - - def __repr__(self): - """ - Representation string for this mailbox. - """ - return u"<SoledadMailbox: mbox '%s' (%s)>" % ( - self.mbox, self.messages.count()) + ############################################################# + # END of Twisted imap4 patch to support LITERAL+ extension + ############################################################# diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index da72cae..651f71b 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -1,69 +1,160 @@ +# -*- coding: utf-8 -*- +# imap-server.tac +# Copyright (C) 2013,2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +TAC file for initialization of the imap service using twistd. + +Use this for debugging and testing the imap server using a native reactor. + +For now, and for debugging/testing purposes, you need +to pass a config file with the following structure: + +[leap_mail] +userid = "user@provider" +uuid = "deadbeefdeadabad" +passwd = "supersecret" # optional, will get prompted if not found. +""" import ConfigParser +import getpass import os +import sys -from leap.soledad.client import Soledad +from leap.keymanager import KeyManager from leap.mail.imap.service import imap -from leap.common.config import get_path_prefix - - -config = ConfigParser.ConfigParser() -config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')]) - -userID = config.get('mail', 'address') -privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read() -nickserver_url = "" +from leap.soledad.client import Soledad -d = {} +from twisted.application import service, internet -for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'): - d[key] = config.get('mail', key) +# TODO should get this initializers from some authoritative mocked source +# We might want to put them the soledad itself. -def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url, - server_pemfile, token): +def initialize_soledad(uuid, email, passwd, + secrets, localdb, + gnupg_home, tempdir): """ Initializes soledad by hand - :param user_uuid: - :param soledad_pass: - :param server_url: - :param server_pemfile: - :param token: - + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir :rtype: Soledad instance """ + # XXX TODO unify with an authoritative source of mocks + # for soledad (or partial initializations). + # This is copied from the imap tests. + + server_url = "http://provider" + cert_file = "" - base_config = get_path_prefix() + class Mock(object): + def __init__(self, return_value=None): + self._return = return_value - secret_path = os.path.join( - base_config, "leap", "soledad", "%s.secret" % user_uuid) - soledad_path = os.path.join( - base_config, "leap", "soledad", "%s-mailbox.db" % user_uuid) + def __call__(self, *args, **kwargs): + return self._return - _soledad = Soledad( - user_uuid, - soledad_pass, - secret_path, - soledad_path, + class MockSharedDB(object): + + get_doc = Mock() + put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + + def __call__(self): + return self + + Soledad._shared_db = MockSharedDB() + soledad = Soledad( + uuid, + passwd, + secrets, + localdb, server_url, - server_pemfile, - token) + cert_file) + + return soledad + +###################################################################### +# Remember to set your config files, see module documentation above! +###################################################################### + +print "[+] Running LEAP IMAP Service" + + +bmconf = os.environ.get("LEAP_MAIL_CONF", "") +if not bmconf: + print ("[-] Please set LEAP_MAIL_CONF environment variable " + "pointing to your config.") + sys.exit(1) + +SECTION = "leap_mail" +cp = ConfigParser.ConfigParser() +cp.read(bmconf) + +userid = cp.get(SECTION, "userid") +uuid = cp.get(SECTION, "uuid") +passwd = unicode(cp.get(SECTION, "passwd")) + +# XXX get this right from the environment variable !!! +port = 1984 + +if not userid or not uuid: + print "[-] Config file missing userid or uuid field" + sys.exit(1) + +if not passwd: + passwd = unicode(getpass.getpass("Soledad passphrase: ")) + + +secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) +localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,)) + +# XXX Is this really used? Should point it to user var dirs defined in xdg? +gnupg_home = "/tmp/" +tempdir = "/tmp/" + +################################################### + +# Ad-hoc soledad/keymanager initialization. + +soledad = initialize_soledad(uuid, userid, passwd, secrets, + localdb, gnupg_home, tempdir) +km_args = (userid, "https://localhost", soledad) +km_kwargs = { + "token": "", + "ca_cert_path": "", + "api_uri": "", + "api_version": "", + "uid": uuid, + "gpgbinary": "/usr/bin/gpg" +} +keymanager = KeyManager(*km_args, **km_kwargs) + +################################################## - return _soledad +# Ok, let's expose the application object for the twistd application +# framework to pick up from here... -soledad = initialize_soledad_mailbox( - d['uid'], - d['passphrase'], - d['server'], - d['pemfile'], - d['token']) -# import the private key ---- should sync it from remote! -from leap.common.keymanager.openpgp import OpenPGPScheme -opgp = OpenPGPScheme(soledad) -opgp.put_ascii_key(privkey) +def getIMAPService(): + factory = imap.LeapIMAPFactory(uuid, userid, soledad) + return internet.TCPServer(port, factory, interface="localhost") -from leap.common.keymanager import KeyManager -keymanager = KeyManager(userID, nickserver_url, soledad, d['token']) -imap.run_service(soledad, keymanager) +application = service.Application("LEAP IMAP Application") +service = getIMAPService() +service.setServiceParent(application) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8756ddc..10ba32a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -17,23 +17,26 @@ """ Imap service initialization """ -from copy import copy - import logging +import os +import time +from twisted.internet import defer, threads from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError from twisted.mail import imap4 from twisted.python import log -from twisted import cred logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.server import SoledadBackedAccount +from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail +from leap.mail.imap.memorystore import MemoryStore +from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.soledadstore import SoledadStore from leap.soledad.client import Soledad # The default port in which imap service will run @@ -45,75 +48,38 @@ INCOMING_CHECK_PERIOD = 60 from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN - - -class LeapIMAPServer(imap4.IMAP4Server): - """ - An IMAP4 Server with mailboxes backed by soledad - """ - def __init__(self, *args, **kwargs): - # pop extraneous arguments - soledad = kwargs.pop('soledad', None) - uuid = kwargs.pop('uuid', None) - userid = kwargs.pop('userid', None) - leap_assert(soledad, "need a soledad instance") - leap_assert_type(soledad, Soledad) - leap_assert(uuid, "need a user in the initialization") - - self._userid = userid - # initialize imap server! - imap4.IMAP4Server.__init__(self, *args, **kwargs) +###################################################### +# Temporary workaround for RecursionLimit when using +# qt4reactor. Do remove when we move to poll or select +# reactor, which do not show those problems. See #4974 +import resource +import sys - # we should initialize the account here, - # but we move it to the factory so we can - # populate the test account properly (and only once - # per session) +try: + sys.setrecursionlimit(10**7) +except Exception: + print "Error setting recursion limit" +try: + # Increase max stack size from 8MB to 256MB + resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) +except Exception: + print "Error setting stack size" - # theAccount = SoledadBackedAccount( - # user, soledad=soledad) +###################################################### - # --------------------------------- - # XXX pre-populate acct for tests!! - # populate_test_account(theAccount) - # --------------------------------- - #self.theAccount = theAccount +DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) +if DO_MANHOLE: + from leap.mail.imap.service import manhole - def lineReceived(self, line): - """ - Attempt to parse a single line from the server. +DO_PROFILE = os.environ.get("LEAP_PROFILE", None) +if DO_PROFILE: + import cProfile + log.msg("Starting PROFILING...") - :param line: the line from the server, without the line delimiter. - :type line: str - """ - if "login" in line.lower(): - # avoid to log the pass, even though we are using a dummy auth - # by now. - msg = line[:7] + " [...]" - else: - msg = copy(line) - log.msg('rcv: %s' % msg) - imap4.IMAP4Server.lineReceived(self, line) - - def authenticateLogin(self, username, password): - """ - Lookup the account with the given parameters, and deny - the improper combinations. - - :param username: the username that is attempting authentication. - :type username: str - :param password: the password to authenticate with. - :type password: str - """ - # XXX this should use portal: - # return portal.login(cred.credentials.UsernamePassword(user, pass) - if username != self._userid: - # bad username, reject. - raise cred.error.UnauthorizedLogin() - # any dummy password is allowed so far. use realm instead! - leap_events.signal(IMAP_CLIENT_LOGIN, "1") - return imap4.IAccount, self.theAccount, lambda: None + PROFILE_DAT = "/tmp/leap_mail_profile.pstats" + pr = cProfile.Profile() + pr.enable() class IMAPAuthRealm(object): @@ -148,13 +114,23 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad + self._memstore = MemoryStore( + permanent_store=SoledadStore(soledad)) theAccount = SoledadBackedAccount( - uuid, soledad=soledad) + uuid, soledad=soledad, + memstore=self._memstore) self.theAccount = theAccount + # XXX how to pass the store along? + def buildProtocol(self, addr): - "Return a protocol suitable for the job." + """ + Return a protocol suitable for the job. + + :param addr: remote ip address + :type addr: str + """ imapProtocol = LeapIMAPServer( uuid=self._uuid, userid=self._userid, @@ -163,6 +139,42 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol + def doStop(self, cv=None): + """ + Stops imap service (fetcher, factory and port). + + :param cv: A condition variable to which we can signal when imap + indeed stops. + :type cv: threading.Condition + :return: a Deferred that stops and flushes the in memory store data to + disk in another thread. + :rtype: Deferred + """ + if DO_PROFILE: + log.msg("Stopping PROFILING") + pr.disable() + pr.dump_stats(PROFILE_DAT) + + ServerFactory.doStop(self) + + if cv is not None: + def _stop_imap_cb(): + logger.debug('Stopping in memory store.') + self._memstore.stop_and_flush() + while not self._memstore.producer.is_queue_empty(): + logger.debug('Waiting for queue to be empty.') + # TODO use a gatherResults over the new/dirty + # deferred list, + # as in memorystore's expunge() method. + time.sleep(1) + # notify that service has stopped + logger.debug('Notifying that service has stopped.') + cv.acquire() + cv.notify() + cv.release() + + return threads.deferToThread(_stop_imap_cb) + def run_service(*args, **kwargs): """ @@ -173,6 +185,11 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ + from twisted.internet import reactor + # it looks like qtreactor does not honor this, + # but other reactors should. + reactor.suggestThreadPoolSize(20) + leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -182,21 +199,23 @@ def run_service(*args, **kwargs): check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") + offline = kwargs.get('offline', False) uuid = soledad._get_uuid() factory = LeapIMAPFactory(uuid, userid, soledad) - from twisted.internet import reactor - try: tport = reactor.listenTCP(port, factory, interface="localhost") - fetcher = LeapIncomingMail( - keymanager, - soledad, - factory.theAccount, - check_period, - userid) + if not offline: + fetcher = LeapIncomingMail( + keymanager, + soledad, + factory.theAccount, + check_period, + userid) + else: + fetcher = None except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) @@ -204,7 +223,17 @@ def run_service(*args, **kwargs): logger.error("Error launching IMAP service: %r" % (exc,)) else: # all good. - fetcher.start_loop() + # (the caller has still to call fetcher.start_loop) + + if DO_MANHOLE: + # TODO get pass from env var.too. + manhole_factory = manhole.getManholeFactory( + {'f': factory, + 'a': factory.theAccount, + 'gm': factory.theAccount.getMailbox}, + "boss", "leap") + reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, + interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) return fetcher, tport, factory diff --git a/src/leap/mail/imap/service/manhole.py b/src/leap/mail/imap/service/manhole.py new file mode 100644 index 0000000..c83ae89 --- /dev/null +++ b/src/leap/mail/imap/service/manhole.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# manhole.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Utilities for enabling the manhole administrative interface into the +LEAP Mail application. +""" +MANHOLE_PORT = 2222 + + +def getManholeFactory(namespace, user, secret): + """ + Get an administrative manhole into the application. + + :param namespace: the namespace to show in the manhole + :type namespace: dict + :param user: the user to authenticate into the administrative shell. + :type user: str + :param secret: pass for this manhole + :type secret: str + """ + import string + + from twisted.cred.portal import Portal + from twisted.conch import manhole, manhole_ssh + from twisted.conch.insults import insults + from twisted.cred.checkers import ( + InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) + + from rlcompleter import Completer + + class EnhancedColoredManhole(manhole.ColoredManhole): + """ + A Manhole with some primitive autocomplete support. + """ + # TODO use introspection to make life easier + + def find_common(self, l): + """ + find common parts in thelist items + ex: 'ab' for ['abcd','abce','abf'] + requires an ordered list + """ + if len(l) == 1: + return l[0] + + init = l[0] + for item in l[1:]: + for i, (x, y) in enumerate(zip(init, item)): + if x != y: + init = "".join(init[:i]) + break + + if not init: + return None + return init + + def handle_TAB(self): + """ + Trap the TAB keystroke. + """ + necessarypart = "".join(self.lineBuffer).split(' ')[-1] + completer = Completer(globals()) + if completer.complete(necessarypart, 0): + matches = list(set(completer.matches)) # has multiples + + if len(matches) == 1: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(matches[0]) + self.lineBufferIndex = len(self.lineBuffer) + else: + matches.sort() + commons = self.find_common(matches) + if commons: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(commons) + self.lineBufferIndex = len(self.lineBuffer) + + self.terminal.nextLine() + while matches: + matches, part = matches[4:], matches[:4] + for item in part: + self.terminal.write('%s' % item.ljust(30)) + self.terminal.write('\n') + self.terminal.nextLine() + + self.terminal.eraseLine() + self.terminal.cursorBackward(self.lineBufferIndex + 5) + self.terminal.write("%s %s" % ( + self.ps[self.pn], "".join(self.lineBuffer))) + + def keystrokeReceived(self, keyID, modifier): + """ + Act upon any keystroke received. + """ + self.keyHandlers.update({'\b': self.handle_BACKSPACE}) + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in string.printable: + self.characterReceived(keyID, False) + + sshRealm = manhole_ssh.TerminalRealm() + + def chainedProtocolFactory(): + return insults.ServerProtocol(EnhancedColoredManhole, namespace) + + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory + + portal = Portal( + sshRealm, [MemoryDB(**{user: secret})]) + + f = manhole_ssh.ConchFactory(portal) + return f diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py new file mode 100644 index 0000000..f3de8eb --- /dev/null +++ b/src/leap/mail/imap/soledadstore.py @@ -0,0 +1,620 @@ +# -*- coding: utf-8 -*- +# soledadstore.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +A MessageStore that writes to Soledad. +""" +import logging +import threading + +from collections import defaultdict +from itertools import chain + +from u1db import errors as u1db_errors +from twisted.python import log +from zope.interface import implements + +from leap.common.check import leap_assert_type, leap_assert +from leap.mail.decorators import deferred_to_thread +from leap.mail.imap.messageparts import MessagePartType +from leap.mail.imap.messageparts import MessageWrapper +from leap.mail.imap.messageparts import RecentFlagsDoc +from leap.mail.imap.fields import fields +from leap.mail.imap.interfaces import IMessageStore +from leap.mail.messageflow import IMessageConsumer +from leap.mail.utils import first, empty, accumulator_queue + +logger = logging.getLogger(__name__) + + +# TODO +# [ ] Implement a retry queue? +# [ ] Consider journaling of operations. + + +class ContentDedup(object): + """ + Message deduplication. + + We do a query for the content hashes before writing to our beloved + sqlcipher backend of Soledad. This means, by now, that: + + 1. We will not store the same body/attachment twice, only the hash of it. + 2. We will not store the same message header twice, only the hash of it. + + The first case is useful if you are always receiving the same old memes + from unwary friends that still have not discovered that 4chan is the + generator of the internet. The second will save your day if you have + initiated session with the same account in two different machines. I also + wonder why would you do that, but let's respect each other choices, like + with the religious celebrations, and assume that one day we'll be able + to run Bitmask in completely free phones. Yes, I mean that, the whole GSM + Stack. + """ + # TODO refactor using unique_query + + def _header_does_exist(self, doc): + """ + Check whether we already have a header document for this + content hash in our database. + + :param doc: tentative header for document + :type doc: dict + :returns: True if it exists, False otherwise. + """ + if not doc: + return False + chash = doc[fields.CONTENT_HASH_KEY] + header_docs = self._soledad.get_from_index( + fields.TYPE_C_HASH_IDX, + fields.TYPE_HEADERS_VAL, str(chash)) + if not header_docs: + return False + + # FIXME enable only to debug this problem. + #if len(header_docs) != 1: + #logger.warning("Found more than one copy of chash %s!" + #% (chash,)) + + #logger.debug("Found header doc with that hash! Skipping save!") + return True + + def _content_does_exist(self, doc): + """ + Check whether we already have a content document for a payload + with this hash in our database. + + :param doc: tentative content for document + :type doc: dict + :returns: True if it exists, False otherwise. + """ + if not doc: + return False + phash = doc[fields.PAYLOAD_HASH_KEY] + attach_docs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(phash)) + if not attach_docs: + return False + + # FIXME enable only to debug this problem + #if len(attach_docs) != 1: + #logger.warning("Found more than one copy of phash %s!" + #% (phash,)) + #logger.debug("Found attachment doc with that hash! Skipping save!") + return True + + +class MsgWriteError(Exception): + """ + Raised if any exception is found while saving message parts. + """ + pass + + +""" +A lock per document. +""" +# TODO should bound the space of this!!! +# http://stackoverflow.com/a/2437645/1157664 +# Setting this to twice the number of threads in the threadpool +# should be safe. +put_locks = defaultdict(lambda: threading.Lock()) +mbox_doc_locks = defaultdict(lambda: threading.Lock()) + + +class SoledadStore(ContentDedup): + """ + This will create docs in the local Soledad database. + """ + _remove_lock = threading.Lock() + + implements(IMessageConsumer, IMessageStore) + + def __init__(self, soledad): + """ + Initialize the permanent store that writes to Soledad database. + + :param soledad: the soledad instance + :type soledad: Soledad + """ + from twisted.internet import reactor + self.reactor = reactor + + self._soledad = soledad + + self._CREATE_DOC_FUN = self._soledad.create_doc + self._PUT_DOC_FUN = self._soledad.put_doc + self._GET_DOC_FUN = self._soledad.get_doc + + # we instantiate an accumulator to batch the notifications + self.docs_notify_queue = accumulator_queue( + lambda item: reactor.callFromThread(self._unset_new_dirty, item), + 20) + + # IMessageStore + + # ------------------------------------------------------------------- + # We are not yet using this interface, but it would make sense + # to implement it. + + def create_message(self, mbox, uid, message): + """ + Create the passed message into this SoledadStore. + + :param mbox: the mbox this message belongs. + :type mbox: str or unicode + :param uid: the UID that identifies this message in this mailbox. + :type uid: int + :param message: a IMessageContainer implementor. + """ + raise NotImplementedError() + + def put_message(self, mbox, uid, message): + """ + Put the passed existing message into this SoledadStore. + + :param mbox: the mbox this message belongs. + :type mbox: str or unicode + :param uid: the UID that identifies this message in this mailbox. + :type uid: int + :param message: a IMessageContainer implementor. + """ + raise NotImplementedError() + + def remove_message(self, mbox, uid): + """ + Remove the given message from this SoledadStore. + + :param mbox: the mbox this message belongs. + :type mbox: str or unicode + :param uid: the UID that identifies this message in this mailbox. + :type uid: int + """ + raise NotImplementedError() + + def get_message(self, mbox, uid): + """ + Get a IMessageContainer for the given mbox and uid combination. + + :param mbox: the mbox this message belongs. + :type mbox: str or unicode + :param uid: the UID that identifies this message in this mailbox. + :type uid: int + """ + raise NotImplementedError() + + # IMessageConsumer + + # TODO should handle the delete case + # TODO should handle errors better + # TODO could generalize this method into a generic consumer + # and only implement `process` here + + def consume(self, queue): + """ + Creates a new document in soledad db. + + :param queue: a tuple of queues to get item from, with content of the + document to be inserted. + :type queue: tuple of Queues + """ + new, dirty = queue + while not new.empty(): + doc_wrapper = new.get() + self.reactor.callInThread(self._consume_doc, doc_wrapper, + self.docs_notify_queue) + while not dirty.empty(): + doc_wrapper = dirty.get() + self.reactor.callInThread(self._consume_doc, doc_wrapper, + self.docs_notify_queue) + + # Queue empty, flush the notifications queue. + self.docs_notify_queue(None, flush=True) + + def _unset_new_dirty(self, doc_wrapper): + """ + Unset the `new` and `dirty` flags for this document wrapper in the + memory store. + + :param doc_wrapper: a MessageWrapper instance + :type doc_wrapper: MessageWrapper + """ + if isinstance(doc_wrapper, MessageWrapper): + # XXX still needed for debug quite often + #logger.info("unsetting new flag!") + doc_wrapper.new = False + doc_wrapper.dirty = False + + @deferred_to_thread + def _consume_doc(self, doc_wrapper, notify_queue): + """ + Consume each document wrapper in a separate thread. + We pass an instance of an accumulator that handles the notifications + to the memorystore when the write has been done. + + :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance + :type doc_wrapper: MessageWrapper or RecentFlagsDoc + :param notify_queue: a callable that handles the writeback + notifications to the memstore. + :type notify_queue: callable + """ + def queueNotifyBack(failed, doc_wrapper): + if failed: + log.msg("There was an error writing the mesage...") + else: + notify_queue(doc_wrapper) + + def doSoledadCalls(items): + # we prime the generator, that should return the + # message or flags wrapper item in the first place. + try: + doc_wrapper = items.next() + except StopIteration: + pass + else: + failed = self._soledad_write_document_parts(items) + queueNotifyBack(failed, doc_wrapper) + + doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper)) + + # + # SoledadStore specific methods. + # + + def _soledad_write_document_parts(self, items): + """ + Write the document parts to soledad in a separate thread. + + :param items: the iterator through the different document wrappers + payloads. + :type items: iterator + :return: whether the write was successful or not + :rtype: bool + """ + failed = False + for item, call in items: + if empty(item): + continue + try: + self._try_call(call, item) + except Exception as exc: + logger.debug("ITEM WAS: %s" % repr(item)) + if hasattr(item, 'content'): + logger.debug("ITEM CONTENT WAS: %s" % + repr(item.content)) + logger.exception(exc) + failed = True + continue + return failed + + def _iter_wrapper_subparts(self, doc_wrapper): + """ + Return an iterator that will yield the doc_wrapper in the first place, + followed by the subparts item and the proper call type for every + item in the queue, if any. + + :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance + :type doc_wrapper: MessageWrapper or RecentFlagsDoc + """ + if isinstance(doc_wrapper, MessageWrapper): + return chain((doc_wrapper,), + self._get_calls_for_msg_parts(doc_wrapper)) + elif isinstance(doc_wrapper, RecentFlagsDoc): + return chain((doc_wrapper,), + self._get_calls_for_rflags_doc(doc_wrapper)) + else: + logger.warning("CANNOT PROCESS ITEM!") + return (i for i in []) + + def _try_call(self, call, item): + """ + Try to invoke a given call with item as a parameter. + + :param call: the function to call + :type call: callable + :param item: the payload to pass to the call as argument + :type item: object + """ + if call is None: + return + + if call == self._PUT_DOC_FUN: + doc_id = item.doc_id + if doc_id is None: + logger.warning("BUG! Dirty doc but has no doc_id!") + return + with put_locks[doc_id]: + doc = self._GET_DOC_FUN(doc_id) + + if doc is None: + logger.warning("BUG! Dirty doc but could not " + "find document %s" % (doc_id,)) + return + + doc.content = dict(item.content) + + item = doc + try: + call(item) + except u1db_errors.RevisionConflict as exc: + logger.exception("Error: %r" % (exc,)) + raise exc + except Exception as exc: + logger.exception("Error: %r" % (exc,)) + raise exc + + else: + try: + call(item) + except u1db_errors.RevisionConflict as exc: + logger.exception("Error: %r" % (exc,)) + raise exc + except Exception as exc: + logger.exception("Error: %r" % (exc,)) + raise exc + + def _get_calls_for_msg_parts(self, msg_wrapper): + """ + Generator that return the proper call type for a given item. + + :param msg_wrapper: A MessageWrapper + :type msg_wrapper: IMessageContainer + :return: a generator of tuples with recent-flags doc payload + and callable + :rtype: generator + """ + call = None + + if msg_wrapper.new: + call = self._CREATE_DOC_FUN + + # item is expected to be a MessagePartDoc + for item in msg_wrapper.walk(): + if item.part == MessagePartType.fdoc: + yield dict(item.content), call + + elif item.part == MessagePartType.hdoc: + if not self._header_does_exist(item.content): + yield dict(item.content), call + + elif item.part == MessagePartType.cdoc: + if not self._content_does_exist(item.content): + yield dict(item.content), call + + # For now, the only thing that will be dirty is + # the flags doc. + + elif msg_wrapper.dirty: + call = self._PUT_DOC_FUN + # item is expected to be a MessagePartDoc + for item in msg_wrapper.walk(): + # XXX FIXME Give error if dirty and not doc_id !!! + doc_id = item.doc_id # defend! + if not doc_id: + logger.warning("Dirty item but no doc_id!") + continue + + if item.part == MessagePartType.fdoc: + #logger.debug("PUT dirty fdoc") + yield item, call + + # XXX also for linkage-doc !!! + else: + logger.error("Cannot delete documents yet from the queue...!") + + def _get_calls_for_rflags_doc(self, rflags_wrapper): + """ + We always put these documents. + + :param rflags_wrapper: A wrapper around recent flags doc. + :type rflags_wrapper: RecentFlagsWrapper + :return: a tuple with recent-flags doc payload and callable + :rtype: tuple + """ + call = self._PUT_DOC_FUN + + payload = rflags_wrapper.content + if payload: + logger.debug("Saving RFLAGS to Soledad...") + yield rflags_wrapper, call + + # Mbox documents and attributes + + def get_mbox_document(self, mbox): + """ + Return mailbox document. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. + """ + with mbox_doc_locks[mbox]: + return self._get_mbox_document(mbox) + + def _get_mbox_document(self, mbox): + """ + Helper for returning the mailbox document. + """ + try: + query = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_MBOX_VAL, mbox) + if query: + return query.pop() + else: + logger.error("Could not find mbox document for %r" % + (mbox,)) + except Exception as exc: + logger.exception("Unhandled error %r" % exc) + + def get_mbox_closed(self, mbox): + """ + Return the closed attribute for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: bool + """ + mbox_doc = self.get_mbox_document() + return mbox_doc.content.get(fields.CLOSED_KEY, False) + + def set_mbox_closed(self, mbox, closed): + """ + Set the closed attribute for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :param closed: the value to be set + :type closed: bool + """ + leap_assert(isinstance(closed, bool), "closed needs to be boolean") + with mbox_doc_locks[mbox]: + mbox_doc = self._get_mbox_document(mbox) + if mbox_doc is None: + logger.error( + "Could not find mbox document for %r" % (mbox,)) + return + mbox_doc.content[fields.CLOSED_KEY] = closed + self._soledad.put_doc(mbox_doc) + + def write_last_uid(self, mbox, value): + """ + Write the `last_uid` integer to the proper mailbox document + in Soledad. + This is called from the deferred triggered by + memorystore.increment_last_soledad_uid, which is expected to + run in a separate thread. + + :param mbox: the mailbox + :type mbox: str or unicode + :param value: the value to set + :type value: int + """ + leap_assert_type(value, int) + key = fields.LAST_UID_KEY + + # XXX use accumulator to reduce number of hits + with mbox_doc_locks[mbox]: + mbox_doc = self._get_mbox_document(mbox) + old_val = mbox_doc.content[key] + if value > old_val: + mbox_doc.content[key] = value + try: + self._soledad.put_doc(mbox_doc) + except Exception as exc: + logger.error("Error while setting last_uid for %r" + % (mbox,)) + logger.exception(exc) + + def get_flags_doc(self, mbox, uid): + """ + Return the SoledadDocument for the given mbox and uid. + + :param mbox: the mailbox + :type mbox: str or unicode + :param uid: the UID for the message + :type uid: int + :rtype: SoledadDocument or None + """ + result = None + try: + flag_docs = self._soledad.get_from_index( + fields.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, mbox, str(uid)) + if len(flag_docs) != 1: + logger.warning("More than one flag doc for %r:%s" % + (mbox, uid)) + result = first(flag_docs) + except Exception as exc: + # ugh! Something's broken down there! + logger.warning("ERROR while getting flags for UID: %s" % uid) + logger.exception(exc) + finally: + return result + + def get_headers_doc(self, chash): + """ + Return the document that keeps the headers for a message + indexed by its content-hash. + + :param chash: the content-hash to retrieve the document from. + :type chash: str or unicode + :rtype: SoledadDocument or None + """ + head_docs = self._soledad.get_from_index( + fields.TYPE_C_HASH_IDX, + fields.TYPE_HEADERS_VAL, str(chash)) + return first(head_docs) + + # deleted messages + + def deleted_iter(self, mbox): + """ + Get an iterator for the the doc_id for SoledadDocuments for messages + with \\Deleted flag for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: iterator through deleted message docs + :rtype: iterable + """ + return [doc.doc_id for doc in self._soledad.get_from_index( + fields.TYPE_MBOX_DEL_IDX, + fields.TYPE_FLAGS_VAL, mbox, '1')] + + def remove_all_deleted(self, mbox): + """ + Remove from Soledad all messages flagged as deleted for a given + mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + """ + deleted = [] + for doc_id in self.deleted_iter(mbox): + with self._remove_lock: + doc = self._soledad.get_doc(doc_id) + if doc is not None: + self._soledad.delete_doc(doc) + try: + deleted.append(doc.content[fields.UID_KEY]) + except TypeError: + # empty content + pass + return deleted diff --git a/src/leap/mail/imap/tests/.gitignore b/src/leap/mail/imap/tests/.gitignore new file mode 100644 index 0000000..60baa9c --- /dev/null +++ b/src/leap/mail/imap/tests/.gitignore @@ -0,0 +1 @@ +data/* diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail new file mode 100755 index 0000000..0fb00d2 --- /dev/null +++ b/src/leap/mail/imap/tests/getmail @@ -0,0 +1,280 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE in twisted for details. + +# Modifications by LEAP Developers 2014 to fit +# Bitmask configuration settings. +""" +Simple IMAP4 client which displays the subjects of all messages in a +particular mailbox. +""" + +import sys + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +class TrivialPrompter(basic.LineReceiver): + from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + insecure-login. + """ + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + """ + result = [e[2] for e in result] + s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, result + ) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.examine(mbox + ).addCallback(cbExamineMbox, proto + ) + + +def cbExamineMbox(result, proto): + """ + Callback invoked when examine command completes. + + Retrieve the subject header of every message in the mailbox. + """ + return proto.fetchSpecific('1:*', + headerType='HEADER.FIELDS', + headerArgs=['SUBJECT'], + ).addCallback(cbFetch, proto, + ) + + +def cbFetch(result, proto): + """ + Display headers. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty mailbox!" + + return proto.prompt("\nWhich message? [1] (Q quits) " + ).addCallback(cbPickMessage, proto) + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + if result == "Q": + print "Bye!" + return proto.logout() + + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbShowmessage, proto) + + +def cbShowmessage(result, proto): + """ + Display message. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty message!" + + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import sys + + if len(sys.argv) != 3: + print "Usage: getmail <user> <pass>" + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + if port == '993': + reactor.connectSSL( + hostname, int(port), factory, ssl.ClientContextFactory()) + else: + if not port: + port = 143 + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/src/leap/mail/imap/tests/imapclient.py b/src/leap/mail/imap/tests/imapclient.py index 027396c..c353cee 100755 --- a/src/leap/mail/imap/tests/imapclient.py +++ b/src/leap/mail/imap/tests/imapclient.py @@ -21,7 +21,7 @@ from twisted.python import log class TrivialPrompter(basic.LineReceiver): - #from os import linesep as delimiter + # from os import linesep as delimiter promptDeferred = None @@ -42,6 +42,7 @@ class TrivialPrompter(basic.LineReceiver): class SimpleIMAP4Client(imap4.IMAP4Client): + """ Add callbacks when the client receives greeting messages from an IMAP server. @@ -98,8 +99,8 @@ def cbServerGreeting(proto, username, password): # Try to authenticate securely return proto.authenticate( password).addCallback( - cbAuthentication, proto).addErrback( - ebAuthentication, proto, username, password) + cbAuthentication, proto).addErrback( + ebAuthentication, proto, username, password) def ebConnection(reason): diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh new file mode 100755 index 0000000..544faca --- /dev/null +++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh @@ -0,0 +1,178 @@ +#!/bin/zsh +# BATCH STRESS TEST FOR IMAP ---------------------- +# http://imgs.xkcd.com/comics/science.jpg +# +# Run imaptest against a LEAP IMAP server +# for a fixed period of time, and collect output. +# +# Author: Kali Kaneko +# Date: 2014 01 26 +# +# To run, you need to have `imaptest` in your path. +# See: +# http://www.imapwiki.org/ImapTest/Installation +# +# For the tests, I'm using a 10MB file sample that +# can be downloaded from: +# http://www.dovecot.org/tmp/dovecot-crlf +# +# Want to contribute to benchmarking? +# +# 1. Create a pristine account in a bitmask provider. +# +# 2. Launch your bitmask client, with different flags +# if you desire. +# +# For example to try the nosync flag in sqlite: +# +# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log +# +# 3. Run at several points in time (ie: just after +# launching the bitmask client. one minute after, +# ten minutes after) +# +# mkdir data +# cd data +# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log +# +# 4. Submit your results to: kali at leap dot se +# together with the logs of the bitmask run. +# +# Please provide also details about your system, and +# the type of hard disk setup you are running against. +# + +# ------------------------------------------------ +# Edit these variables if you are too lazy to pass +# the user and mbox as parameters. Like me. + +USER="test_f14@dev.bitmask.net" +MBOX="~/leap/imaptest/data/dovecot-crlf" + +HOST="localhost" +PORT="1984" + +# in case you have it aliased +GREP="/bin/grep" +IMAPTEST="imaptest" + +# ----------------------------------------------- +# +# These should be kept constant across benchmarking +# runs across different machines, for comparability. + +DURATION=200 +NUM_MSG=200 + + +# TODO add another function, and a cli flag, to be able +# to take several aggretates spaced in time, along a period +# of several minutes. + +imaptest_cmd() { + stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ + port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ + no_pipelining 2>/dev/null +} + +stress_imap() { + mkfifo imap_pipe + cat imap_pipe | tee output & + imaptest_cmd >> imap_pipe +} + +wait_and_kill() { + while : + do + sleep $DURATION + pkill -2 imaptest + rm imap_pipe + break + done +} + +print_results() { + sleep 1 + echo + echo + echo "AGGREGATED RESULTS" + echo "----------------------" + echo "\tavg\tstdev" + $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ + gawk ' +function avg(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + return sum/count; +} +function std_dev(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + average = sum/count; + + sumsq=0; + for( x=0; x <= count-1; x++) { + sumsq += (data[x] - average)^2; + } + return sqrt(sumsq/count); +} +BEGIN { + cnt = 0 +} END { + +printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); +printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); +printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); +printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); +printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); +printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); +printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); +printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); +printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); +printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); +printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); + +print "" +print "TOT samples", NR; +} +{ + it = cnt++; + array[1][it] = $1; + array[2][it] = $2; + array[3][it] = $3; + array[4][it] = $4; + array[5][it] = $5; + array[6][it] = $6; + array[7][it] = $7; + array[8][it] = $8; + array[9][it] = $9; + array[10][it] = $10; + array[11][it] = $11; +}' +} + + +{ test $1 = "--help" } && { + echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" + exit 0 +} + +# If the first parameter is passed, take it as the user +{ test $1 } && { + USER=$1 +} + +# If the second parameter is passed, take it as the mbox +{ test $2 } && { + MBOX=$2 +} + +echo "[+] LEAP IMAP TESTS" +echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" +wait_and_kill & +stress_imap +print_results diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions new file mode 100755 index 0000000..efe3f46 --- /dev/null +++ b/src/leap/mail/imap/tests/regressions @@ -0,0 +1,453 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# regressions +# Copyright (C) 2014 LEAP +# Copyright (c) Twisted Matrix Laboratories. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Simple Regression Tests using IMAP4 client. + +Iterates trough all mails under a given folder and tries to APPEND them to +the server being tested. After FETCHING the pushed message, it compares +the received version with the one that was saved, and exits with an error +code if they do not match. +""" +import os +import StringIO +import sys + +from email.parser import Parser + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +REGRESSIONS_FOLDER = "regressions_test" + +parser = Parser() + + +def get_msg_parts(raw): + """ + Return a representation of the parts of a message suitable for + comparison. + + :param raw: string for the message + :type raw: str + """ + m = parser.parsestr(raw) + return [dict(part.items()) + if part.is_multipart() + else part.get_payload() + for part in m.walk()] + + +def compare_msg_parts(a, b): + """ + Compare two sequences of parts of messages. + + :param a: part sequence for message a + :param b: part sequence for message b + + :return: True if both message sequences are equivalent. + :rtype: bool + """ + # XXX This could be smarter and show the differences in the + # different parts when/where they differ. + #import pprint; pprint.pprint(a[0]) + #import pprint; pprint.pprint(b[0]) + + def lowerkey(d): + return dict((k.lower(), v.replace('\r', '')) + for k, v in d.iteritems()) + + def eq(x, y): + # For dicts, we compare a variation with their keys + # in lowercase, and \r removed from their values + if all(map(lambda i: isinstance(i, dict), (x, y))): + x, y = map(lowerkey, (x, y)) + return x == y + + compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) + all_match = all(compare_vector) + + if not all_match: + print "PARTS MISMATCH!" + print "vector: ", compare_vector + index = compare_vector.index(False) + from pprint import pprint + print "Expected:" + pprint(a[index]) + print ("***") + print "Found:" + pprint(b[index]) + print + + return all_match + + +def get_fd(string): + """ + Return a file descriptor with the passed string + as content. + """ + fd = StringIO.StringIO() + fd.write(string) + fd.seek(0) + return fd + + +class TrivialPrompter(basic.LineReceiver): + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback( + ebSelectMbox, proto, REGRESSIONS_FOLDER) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + Raise insecure-login error. + """ + return proto.login( + username, password + ).addCallback( + cbAuthentication, proto) + + +def cbSelectMbox(result, proto): + """ + Callback invoked when select command finishes successfully. + + If any message is in the test folder, it will flag them as deleted and + expunge. + If no messages found, it will start with the APPEND tests. + """ + print "SELECT: %s EXISTS " % result.get("EXISTS", "??") + + if result["EXISTS"] != 0: + # Flag as deleted, expunge, and do an examine again. + #print "There is mail here, will delete..." + return cbDeleteAndExpungeTestFolder(proto) + + else: + return cbAppendNextMessage(proto) + + +def ebSelectMbox(failure, proto, folder): + """ + Errback invoked when the examine command fails. + + Creates the folder. + """ + print failure.getTraceback() + log.msg("Folder %r does not exist. Creating..." % (folder,)) + return proto.create(folder).addCallback(cbAuthentication, proto) + + +def cbDeleteAndExpungeTestFolder(proto): + """ + Callback invoked fom cbExamineMbox when the number of messages in the + mailbox is not zero. It flags all messages as deleted and expunge the + mailbox. + """ + return proto.setFlags( + "1:*", ("\\Deleted",) + ).addCallback( + lambda r: proto.expunge() + ).addCallback( + cbExpunge, proto) + + +def cbExpunge(result, proto): + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback(ebSettingDeleted, proto) + + +def ebSettingDeleted(failure, proto): + """ + Report errors during deletion of messages in the mailbox. + """ + print failure.getTraceback() + + +def cbAppendNextMessage(proto): + """ + Appends the next message in the global queue to the test folder. + """ + # 1. Get the next test message from global tuple. + try: + next_sample = SAMPLES.pop() + except IndexError: + # we're done! + return proto.logout() + + print "\nAPPEND %s" % (next_sample,) + raw = open(next_sample).read() + msg = get_fd(raw) + return proto.append( + REGRESSIONS_FOLDER, msg + ).addCallback( + lambda r: proto.select(REGRESSIONS_FOLDER) + ).addCallback( + cbAppend, proto, raw + ).addErrback( + ebAppend, proto, raw) + + +def cbAppend(result, proto, orig_msg): + """ + Fetches the message right after an append. + """ + # XXX keep account of highest UID + uid = "1:*" + + return proto.fetchSpecific( + '%s' % uid, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback( + cbCompareMessage, proto, orig_msg + ).addErrback(ebAppend, proto, orig_msg) + + +def ebAppend(failure, proto, raw): + """ + Errorback for the append operation + """ + print "ERROR WHILE APPENDING!" + print failure.getTraceback() + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbCompareMessage, proto) + + +def cbCompareMessage(result, proto, raw): + """ + Display message and compare it with the original one. + """ + parts_orig = get_msg_parts(raw) + + if result: + keys = result.keys() + keys.sort() + else: + print "[-] GOT NO RESULT" + return proto.logout() + + latest = max(keys) + + fetched_msg = result[latest][0][2] + parts_fetched = get_msg_parts(fetched_msg) + + equal = compare_msg_parts( + parts_orig, + parts_fetched) + + if equal: + print "[+] MESSAGES MATCH" + return cbAppendNextMessage(proto) + else: + print "[-] ERROR: MESSAGES DO NOT MATCH !!!" + print " ABORTING COMPARISON..." + # FIXME logout and print the subject ... + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import glob + import sys + + if len(sys.argv) != 4: + print "Usage: regressions <user> <pass> <samples-folder>" + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + samplesdir = sys.argv[3] + + if not os.path.isdir(samplesdir): + print ("Could not find samples folder! " + "Make sure of copying mail_breaker contents there.") + sys.exit() + + samples = glob.glob(samplesdir + '/*') + + global SAMPLES + SAMPLES = [] + SAMPLES += samples + + onConn = defer.Deferred( + ).addCallback( + cbServerGreeting, username, password + ).addErrback( + ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message new file mode 100644 index 0000000..582297c --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message new file mode 100644 index 0000000..9907c2d --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko <kali@leap.se> +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message new file mode 100644 index 0000000..30f74e5 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
+From: Doug Sauder <doug@penguin.example.com>
+To: Joe Blow <blow@example.com>
+Subject: Test message from PINE
+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
+MIME-Version: 1.0
+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
+
+ This message is in MIME format. The first part should be readable text,
+ while the remaining parts are likely unreadable without MIME-aware tools.
+ Send mail to mime@docserver.cac.washington.edu for more info.
+
+---1463757054-952513540-958744548=:8452
+Content-Type: TEXT/PLAIN; charset=US-ASCII
+
+This is a test message from PINE MUA.
+
+
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="redball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="redball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
+vAAAAABJRU5ErkJggg==
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="blueball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="blueball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
+Z3VldDZzO7wAAAAASUVORK5CYII=
+---1463757054-952513540-958744548=:8452--
diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message new file mode 100644 index 0000000..fc627c3 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: <pyar-bounces@python.org.ar> +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar> +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina <pyar.python.org.ar> +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" <pyar-bounces@python.org.ar> +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index ca73a11..fd88440 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -25,6 +25,7 @@ XXX add authors from the original twisted tests. @license: GPLv3, see included LICENSE file """ # XXX review license of the original tests!!! +from email import parser try: from cStringIO import StringIO @@ -35,9 +36,14 @@ import os import types import tempfile import shutil +import time + +from itertools import chain from mock import Mock +from nose.twistedtools import deferred, stop_reactor +from unittest import skip from twisted.mail import imap4 @@ -54,16 +60,21 @@ import twisted.cred.credentials import twisted.cred.portal -#import u1db +# import u1db from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.server import SoledadMailbox -from leap.mail.imap.server import SoledadBackedAccount -from leap.mail.imap.server import MessageCollection +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.memorystore import MemoryStore +from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.server import LeapIMAPServer from leap.soledad.client import Soledad from leap.soledad.client import SoledadCrypto +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + def strip(f): return lambda result, f=f: f() @@ -84,10 +95,10 @@ def initialize_soledad(email, gnupg_home, tempdir): """ Initializes soledad by hand - @param email: ID for the user - @param gnupg_home: path to home used by gnupg - @param tempdir: path to temporal dir - @rtype: Soledad instance + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance """ uuid = "foobar-uuid" @@ -120,54 +131,8 @@ def initialize_soledad(email, gnupg_home, tempdir): return _soledad -########################################## -# Simple LEAP IMAP4 Server for testing -########################################## - -class SimpleLEAPServer(imap4.IMAP4Server): - """ - A Simple IMAP4 Server with mailboxes backed by Soledad. - - This should be pretty close to the real LeapIMAP4Server that we - will be instantiating as a service, minus the authentication bits. - """ - def __init__(self, *args, **kw): - - soledad = kw.pop('soledad', None) - - imap4.IMAP4Server.__init__(self, *args, **kw) - realm = TestRealm() - - # XXX Why I AM PASSING THE ACCOUNT TO - # REALM? I AM NOT USING THAT NOW, AM I??? - realm.theAccount = SoledadBackedAccount( - 'testuser', - soledad=soledad) - - portal = cred.portal.Portal(realm) - c = cred.checkers.InMemoryUsernamePasswordDatabaseDontUse() - self.checker = c - self.portal = portal - portal.registerChecker(c) - self.timeoutTest = False - - def lineReceived(self, line): - if self.timeoutTest: - #Do not send a respones - return - - imap4.IMAP4Server.lineReceived(self, line) - - _username = 'testuser' - _password = 'password-test' - - def authenticateLogin(self, username, password): - if username == self._username and password == self._password: - return imap4.IAccount, self.theAccount, lambda: None - raise cred.error.UnauthorizedLogin() - - class TestRealm: + """ A minimal auth realm for testing purposes only """ @@ -177,12 +142,13 @@ class TestRealm: return imap4.IAccount, self.theAccount, lambda: None -###################################### +# # Simple IMAP4 Client for testing -###################################### +# class SimpleClient(imap4.IMAP4Client): + """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -210,6 +176,7 @@ class SimpleClient(imap4.IMAP4Client): class IMAP4HelperMixin(BaseLeapTest): + """ MixIn containing several utilities to be shared across different TestCases @@ -245,13 +212,6 @@ class IMAP4HelperMixin(BaseLeapTest): # Soledad: config info cls.gnupg_home = "%s/gnupg" % cls.tempdir cls.email = 'leap@leap.se' - #cls.db1_file = "%s/db1.u1db" % cls.tempdir - #cls.db2_file = "%s/db2.u1db" % cls.tempdir - # open test dbs - #cls._db1 = u1db.open(cls.db1_file, create=True, - #document_factory=SoledadDocument) - #cls._db2 = u1db.open(cls.db2_file, create=True, - #document_factory=SoledadDocument) # initialize soledad by hand so we can control keys cls._soledad = initialize_soledad( @@ -261,7 +221,7 @@ class IMAP4HelperMixin(BaseLeapTest): # now we're passing the mailbox name, so we # should get this into a partial or something. - #cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad) + # cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad) # XXX REFACTOR --- self.server (in setUp) is initializing # a SoledadBackedAccount @@ -273,8 +233,6 @@ class IMAP4HelperMixin(BaseLeapTest): Restores the old path and home environment variables. Removes the temporal dir created for tests. """ - #cls._db1.close() - #cls._db2.close() cls._soledad.close() os.environ["PATH"] = cls.old_path @@ -291,8 +249,13 @@ class IMAP4HelperMixin(BaseLeapTest): but passing the same Soledad instance (it's costly to initialize), so we have to be sure to restore state across tests. """ + UUID = 'deadbeef', + USERID = TEST_USER + memstore = MemoryStore() + d = defer.Deferred() - self.server = SimpleLEAPServer( + self.server = LeapIMAPServer( + uuid=UUID, userid=USERID, contextFactory=self.serverCTX, # XXX do we really need this?? soledad=self._soledad) @@ -307,14 +270,18 @@ class IMAP4HelperMixin(BaseLeapTest): # I THINK we ONLY need to do it at one place now. theAccount = SoledadBackedAccount( - 'testuser', - soledad=self._soledad) - SimpleLEAPServer.theAccount = theAccount + USERID, + soledad=self._soledad, + memstore=memstore) + LeapIMAPServer.theAccount = theAccount # in case we get something from previous tests... for mb in self.server.theAccount.mailboxes: self.server.theAccount.delete(mb) + # email parser + self.parser = parser.Parser() + def tearDown(self): """ tearDown method called after each test. @@ -328,8 +295,8 @@ class IMAP4HelperMixin(BaseLeapTest): acct.delete(mb) # FIXME add again - #for subs in acct.subscriptions: - #acct.unsubscribe(subs) + # for subs in acct.subscriptions: + # acct.unsubscribe(subs) del self.server del self.client @@ -344,11 +311,11 @@ class IMAP4HelperMixin(BaseLeapTest): # XXX we also should put this in a mailbox! - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") + self._soledad.messages.add_msg('', uid=1, subject="test1") + self._soledad.messages.add_msg('', uid=2, subject="test2") + self._soledad.messages.add_msg('', uid=3, subject="test3") # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") + self._soledad.messages.add_msg('', uid=4, subject="test4") def delete_all_docs(self): """ @@ -364,7 +331,11 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - log.err(failure, "Problem with %r" % (self.function,)) + # can we do something similar? + # I guess this was ok with trial, but not in noseland... + #log.err(failure, "Problem with %r" % (self.function,)) + raise failure.value + #failure.trap(Exception) def loopback(self): return loopback.loopbackAsync(self.server, self.client) @@ -375,21 +346,26 @@ class IMAP4HelperMixin(BaseLeapTest): # class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): + """ Tests for the MessageCollection class """ + count = 0 + def setUp(self): """ setUp method for each test We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ - self.messages = MessageCollection("testmbox", self._soledad._db) + memstore = MemoryStore() + self.messages = MessageCollection("testmbox%s" % (self.count,), + self._soledad, memstore=memstore) + MessageCollectionTestCase.count += 1 def tearDown(self): """ tearDown method for each test - Delete the message collection """ del self.messages @@ -397,47 +373,120 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test empty message and collection """ - em = self.messages._get_empty_msg() + em = self.messages._get_empty_doc() self.assertEqual( em, { - "date": '', + "chash": '', + "deleted": False, "flags": [], - "headers": {}, "mbox": "inbox", - "raw": "", - "recent": True, "seen": False, - "subject": "", - "type": "msg", + "multi": False, + "size": 0, + "type": "flags", "uid": 1, }) self.assertEqual(self.messages.count(), 0) + def testMultipleAdd(self): + """ + Add multiple messages + """ + mc = self.messages + self.assertEqual(self.messages.count(), 0) + + def add_first(): + d = defer.gatherResults([ + mc.add_msg('Stuff 1', uid=1, subject="test1"), + mc.add_msg('Stuff 2', uid=2, subject="test2"), + mc.add_msg('Stuff 3', uid=3, subject="test3"), + mc.add_msg('Stuff 4', uid=4, subject="test4")]) + return d + + def add_second(result): + d = defer.gatherResults([ + mc.add_msg('Stuff 5', uid=5, subject="test5"), + mc.add_msg('Stuff 6', uid=6, subject="test6"), + mc.add_msg('Stuff 7', uid=7, subject="test7")]) + return d + + def check_second(result): + return self.assertEqual(mc.count(), 7) + + d1 = add_first() + d1.addCallback(add_second) + d1.addCallback(check_second) + + @skip("needs update!") + def testRecentCount(self): + """ + Test the recent count + """ + mc = self.messages + countrecent = mc.count_recent + eq = self.assertEqual + + self.assertEqual(countrecent(), 0) + + d = mc.add_msg('Stuff', uid=1, subject="test1") + # For the semantics defined in the RFC, we auto-add the + # recent flag by default. + + def add2(_): + return mc.add_msg('Stuff', subject="test2", uid=2, + flags=('\\Deleted',)) + + def add3(_): + return mc.add_msg('Stuff', subject="test3", uid=3, + flags=('\\Recent',)) + + def add4(_): + return mc.add_msg('Stuff', subject="test4", uid=4, + flags=('\\Deleted', '\\Recent')) + + d.addCallback(lambda r: eq(countrecent(), 1)) + d.addCallback(add2) + d.addCallback(lambda r: eq(countrecent(), 2)) + d.addCallback(add3) + d.addCallback(lambda r: eq(countrecent(), 3)) + d.addCallback(add4) + d.addCallback(lambda r: eq(countrecent(), 4)) + def testFilterByMailbox(self): """ Test that queries filter by selected mailbox """ mc = self.messages - mc.add_msg('', subject="test1") - mc.add_msg('', subject="test2") - mc.add_msg('', subject="test3") - self.assertEqual(self.messages.count(), 3) - - newmsg = mc._get_empty_msg() - newmsg['mailbox'] = "mailbox/foo" - newmsg['subject'] = "test another mailbox" - mc._soledad.create_doc(newmsg) - self.assertEqual(mc.count(), 3) - self.assertEqual( - len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4) + self.assertEqual(self.messages.count(), 0) + + def add_1(): + d1 = mc.add_msg('msg 1', uid=1, subject="test1") + d2 = mc.add_msg('msg 2', uid=2, subject="test2") + d3 = mc.add_msg('msg 3', uid=3, subject="test3") + d = defer.gatherResults([d1, d2, d3]) + return d + + add_1().addCallback(lambda ignored: self.assertEqual( + mc.count(), 3)) + + # XXX this has to be redone to fit memstore ------------# + #newmsg = mc._get_empty_doc() + #newmsg['mailbox'] = "mailbox/foo" + #mc._soledad.create_doc(newmsg) + #self.assertEqual(mc.count(), 3) + #self.assertEqual( + #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): + # TODO this currently will use a memory-only store. + # create a different one for testing soledad sync. + """ Tests for the generic behavior of the LeapIMAP4Server which, right now, it's just implemented in this test file as - SimpleLEAPServer. We will move the implementation, together with + LeapIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -451,11 +500,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # mailboxes operations # + @deferred(timeout=None) def testCreate(self): """ Test whether we can create mailboxes """ - succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'FOOBOX') + succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') def cb(): @@ -465,7 +515,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.result.append(0) def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def create(): for name in succeed + fail: @@ -483,20 +533,21 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestCreate(self, ignored, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mbox = SimpleLEAPServer.theAccount.mailboxes + mbox = LeapIMAPServer.theAccount.mailboxes answers = ['foobox', 'testbox', 'test/box', 'test', 'test/box/box'] mbox.sort() answers.sort() - self.assertEqual(mbox, [a.upper() for a in answers]) + self.assertEqual(mbox, [a for a in answers]) + @deferred(timeout=None) def testDelete(self): """ Test whether we can delete mailboxes """ - SimpleLEAPServer.theAccount.addMailbox('delete/me') + LeapIMAPServer.theAccount.addMailbox('delete/me') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('delete/me') @@ -508,7 +559,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback( lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.mailboxes, [])) + LeapIMAPServer.theAccount.mailboxes, [])) return d def testIllegalInboxDelete(self): @@ -519,7 +570,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.stashed = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('inbox') @@ -537,13 +588,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): failure.Failure))) return d + @deferred(timeout=None) def testNonExistentDelete(self): """ Test what happens if we try to delete a non-existent mailbox. - We expect an error raised stating 'No such inbox' + We expect an error raised stating 'No such mailbox' """ def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('delete/me') @@ -558,10 +610,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.assertEqual(str(self.failure.value), - 'No such mailbox')) + d.addCallback(lambda _: self.assertTrue( + str(self.failure.value).startswith('No such mailbox'))) return d + @deferred(timeout=None) def testIllegalDelete(self): """ Try deleting a mailbox with sub-folders, and \NoSelect flag set. @@ -569,14 +622,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Obs: this test will fail if SoledadMailbox returns hardcoded flags. """ - SimpleLEAPServer.theAccount.addMailbox('delete') - to_delete = SimpleLEAPServer.theAccount.getMailbox('delete') + LeapIMAPServer.theAccount.addMailbox('delete') + to_delete = LeapIMAPServer.theAccount.getMailbox('delete') to_delete.setFlags((r'\Noselect',)) to_delete.getFlags() - SimpleLEAPServer.theAccount.addMailbox('delete/me') + LeapIMAPServer.theAccount.addMailbox('delete/me') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('delete') @@ -596,14 +649,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + @deferred(timeout=None) def testRename(self): """ Test whether we can rename a mailbox """ - SimpleLEAPServer.theAccount.addMailbox('oldmbox') + LeapIMAPServer.theAccount.addMailbox('oldmbox') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def rename(): return self.client.rename('oldmbox', 'newname') @@ -615,10 +669,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.mailboxes, - ['NEWNAME'])) + LeapIMAPServer.theAccount.mailboxes, + ['newname'])) return d + @deferred(timeout=None) def testIllegalInboxRename(self): """ Try to rename inbox. We expect it to fail. Then it would be not @@ -627,7 +682,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.stashed = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def rename(): return self.client.rename('inbox', 'frotz') @@ -646,15 +701,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.stashed, failure.Failure))) return d + @deferred(timeout=None) def testHierarchicalRename(self): """ Try to rename hierarchical mailboxes """ - SimpleLEAPServer.theAccount.create('oldmbox/m1') - SimpleLEAPServer.theAccount.create('oldmbox/m2') + LeapIMAPServer.theAccount.create('oldmbox/m1') + LeapIMAPServer.theAccount.create('oldmbox/m2') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def rename(): return self.client.rename('oldmbox', 'newname') @@ -667,17 +723,18 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, ignored): - mboxes = SimpleLEAPServer.theAccount.mailboxes + mboxes = LeapIMAPServer.theAccount.mailboxes expected = ['newname', 'newname/m1', 'newname/m2'] mboxes.sort() - self.assertEqual(mboxes, [s.upper() for s in expected]) + self.assertEqual(mboxes, [s for s in expected]) + @deferred(timeout=None) def testSubscribe(self): """ Test whether we can mark a mailbox as subscribed to """ def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def subscribe(): return self.client.subscribe('this/mbox') @@ -689,19 +746,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.subscriptions, - ['THIS/MBOX'])) + LeapIMAPServer.theAccount.subscriptions, + ['this/mbox'])) return d + @deferred(timeout=None) def testUnsubscribe(self): """ Test whether we can unsubscribe from a set of mailboxes """ - SimpleLEAPServer.theAccount.subscribe('THIS/MBOX') - SimpleLEAPServer.theAccount.subscribe('THAT/MBOX') + LeapIMAPServer.theAccount.subscribe('this/mbox') + LeapIMAPServer.theAccount.subscribe('that/mbox') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def unsubscribe(): return self.client.unsubscribe('this/mbox') @@ -713,10 +771,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.subscriptions, - ['THAT/MBOX'])) + LeapIMAPServer.theAccount.subscriptions, + ['that/mbox'])) return d + @deferred(timeout=None) def testSelect(self): """ Try to select a mailbox @@ -725,7 +784,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.selectedArgs = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def select(): def selected(args): @@ -743,7 +802,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) def _cbTestSelect(self, ignored): - mbox = SimpleLEAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') + mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) self.assertEqual(self.selectedArgs, { 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, @@ -756,6 +815,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # capabilities # + @deferred(timeout=None) def testCapability(self): caps = {} @@ -771,6 +831,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(lambda _: self.assertEqual(expected, caps)) + @deferred(timeout=None) def testCapabilityWithAuth(self): caps = {} self.server.challengers[ @@ -795,6 +856,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # authentication # + @deferred(timeout=None) def testLogout(self): """ Test log out @@ -809,6 +871,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = self.loopback() return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1)) + @deferred(timeout=None) def testNoop(self): """ Test noop command @@ -824,12 +887,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = self.loopback() return d.addCallback(lambda _: self.assertEqual(self.responses, [])) + @deferred(timeout=None) def testLogin(self): """ Test login """ def login(): - d = self.client.login('testuser', 'password-test') + d = self.client.login(TEST_USER, TEST_PASSWD) d.addCallback(self._cbStopClient) d1 = self.connected.addCallback( strip(login)).addErrback(self._ebGeneral) @@ -837,15 +901,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, SimpleLEAPServer.theAccount) + self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') + @deferred(timeout=None) def testFailedLogin(self): """ Test bad login """ def login(): - d = self.client.login('testuser', 'wrong-password') + d = self.client.login("bad_user@leap.se", TEST_PASSWD) d.addBoth(self._cbStopClient) d1 = self.connected.addCallback( @@ -855,18 +920,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestFailedLogin) def _cbTestFailedLogin(self, ignored): - self.assertEqual(self.server.account, None) self.assertEqual(self.server.state, 'unauth') + self.assertEqual(self.server.account, None) + @deferred(timeout=None) def testLoginRequiringQuoting(self): """ Test login requiring quoting """ - self.server._username = '{test}user' + self.server._userid = '{test}user@leap.se' self.server._password = '{test}password' def login(): - d = self.client.login('{test}user', '{test}password') + d = self.client.login('{test}user@leap.se', '{test}password') d.addBoth(self._cbStopClient) d1 = self.connected.addCallback( @@ -875,13 +941,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, SimpleLEAPServer.theAccount) + self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # # Inspection # + @deferred(timeout=None) def testNamespace(self): """ Test retrieving namespace @@ -889,7 +956,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.namespaceArgs = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def namespace(): def gotNamespace(args): @@ -906,6 +973,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): [[['', '/']], [], []])) return d + @deferred(timeout=None) def testExamine(self): """ L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and @@ -927,7 +995,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.examinedArgs = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def examine(): def examined(args): @@ -945,7 +1013,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('TEST-MAILBOX-E') + mbox = self.server.theAccount.getMailbox('test-mailbox-e') self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) self.assertEqual(self.examinedArgs, { 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, @@ -954,15 +1022,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'READ-WRITE': False}) def _listSetup(self, f): - SimpleLEAPServer.theAccount.addMailbox('root/subthingl', - creation_ts=42) - SimpleLEAPServer.theAccount.addMailbox('root/another-thing', - creation_ts=42) - SimpleLEAPServer.theAccount.addMailbox('non-root/subthing', - creation_ts=42) + LeapIMAPServer.theAccount.addMailbox('root/subthingl', + creation_ts=42) + LeapIMAPServer.theAccount.addMailbox('root/another-thing', + creation_ts=42) + LeapIMAPServer.theAccount.addMailbox('non-root/subthing', + creation_ts=42) def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def listed(answers): self.listed = answers @@ -975,6 +1043,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d2 = self.loopback() return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed) + @deferred(timeout=None) def testList(self): """ Test List command @@ -985,38 +1054,37 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL"), - (SoledadMailbox.INIT_FLAGS, "/", "ROOT/ANOTHER-THING") + (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"), + (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") ]) )) return d - # XXX implement subscriptions - ''' + @deferred(timeout=None) def testLSub(self): """ Test LSub command """ - SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL') + LeapIMAPServer.theAccount.subscribe('root/subthingl2') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL")]) + [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")]) return d - ''' + @deferred(timeout=None) def testStatus(self): """ Test Status command """ - SimpleLEAPServer.theAccount.addMailbox('root/subthings') + LeapIMAPServer.theAccount.addMailbox('root/subthings') # XXX FIXME ---- should populate this a little bit, # with unseen etc... def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def status(): return self.client.status( @@ -1038,12 +1106,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): )) return d + @deferred(timeout=None) def testFailedStatus(self): """ Test failed status command with a non-existent mailbox """ def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def status(): return self.client.status( @@ -1077,22 +1146,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # messages # + @deferred(timeout=None) def testFullAppend(self): """ Test appending a full message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) - SimpleLEAPServer.theAccount.addMailbox('root/subthing') + LeapIMAPServer.theAccount.addMailbox('root/subthing') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def append(): return self.client.append( 'root/subthing', message, - ['\\SEEN', '\\DELETED'], + ('\\SEEN', '\\DELETED'), 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', ) @@ -1104,29 +1174,39 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestFullAppend, infile) def _cbTestFullAppend(self, ignored, infile): - mb = SimpleLEAPServer.theAccount.getMailbox('ROOT/SUBTHING') + mb = LeapIMAPServer.theAccount.getMailbox('root/subthing') self.assertEqual(1, len(mb.messages)) + msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ['\\SEEN', '\\DELETED'], - mb.messages[1].content['flags']) + set(('\\Recent', '\\SEEN', '\\DELETED')), + set(msg.getFlags())) self.assertEqual( 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - mb.messages[1].content['date']) + msg.getInternalDate()) - self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + parsed = self.parser.parse(open(infile)) + body = parsed.get_payload() + headers = dict(parsed.items()) + self.assertEqual( + body, + msg.getBodyFile().read()) + gotheaders = msg.getHeaders(True) + + self.assertItemsEqual( + headers, gotheaders) + @deferred(timeout=None) def testPartialAppend(self): """ Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - message = open(infile) - SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def append(): message = file(infile) @@ -1143,27 +1223,32 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestPartialAppend, infile) + return d.addCallback( + self._cbTestPartialAppend, infile) def _cbTestPartialAppend(self, ignored, infile): - mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') + mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') self.assertEqual(1, len(mb.messages)) + msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ['\\SEEN',], - mb.messages[1].content['flags'] + set(('\\SEEN', '\\Recent')), + set(msg.getFlags()) ) + parsed = self.parser.parse(open(infile)) + body = parsed.get_payload() self.assertEqual( - 'Right now', mb.messages[1].content['date']) - self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + body, + msg.getBodyFile().read()) + @deferred(timeout=None) def testCheck(self): """ Test check command """ - SimpleLEAPServer.theAccount.addMailbox('root/subthing') + LeapIMAPServer.theAccount.addMailbox('root/subthing') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def select(): return self.client.select('root/subthing') @@ -1179,6 +1264,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # Okay, that was fun + @deferred(timeout=5) def testClose(self): """ Test closing the mailbox. We expect to get deleted all messages flagged @@ -1187,23 +1273,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): name = 'mailbox-close' self.server.theAccount.addMailbox(name) - m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) - m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) + m = LeapIMAPServer.theAccount.getMailbox(name) def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def select(): return self.client.select(name) + def add_messages(): + d1 = m.messages.add_msg( + 'test 1', uid=1, subject="Message 1", + flags=('\\Deleted', 'AnotherFlag')) + d2 = m.messages.add_msg( + 'test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + d3 = m.messages.add_msg( + 'test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) + d = defer.gatherResults([d1, d2, d3]) + return d + def close(): return self.client.close() d = self.connected.addCallback(strip(login)) d.addCallbacks(strip(select), self._ebGeneral) + d.addCallbacks(strip(add_messages), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -1211,30 +1307,42 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestClose(self, ignored, m): self.assertEqual(len(m.messages), 1) + + msg = m.messages.get_msg_by_uid(2) + self.assertFalse(msg is None) self.assertEqual( - m.messages[1].content['subject'], + msg._hdoc.content['subject'], 'Message 2') - self.failUnless(m.closed) + @deferred(timeout=5) def testExpunge(self): """ Test expunge command """ name = 'mailbox-expunge' - SimpleLEAPServer.theAccount.addMailbox(name) - m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) - m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) + self.server.theAccount.addMailbox(name) + m = LeapIMAPServer.theAccount.getMailbox(name) def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def select(): return self.client.select('mailbox-expunge') + def add_messages(): + d1 = m.messages.add_msg( + 'test 1', uid=1, subject="Message 1", + flags=('\\Deleted', 'AnotherFlag')) + d2 = m.messages.add_msg( + 'test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + d3 = m.messages.add_msg( + 'test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) + d = defer.gatherResults([d1, d2, d3]) + return d + def expunge(): return self.client.expunge() @@ -1245,6 +1353,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallbacks(strip(add_messages), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -1253,17 +1362,96 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExpunge, m) def _cbTestExpunge(self, ignored, m): + # we only left 1 mssage with no deleted flag self.assertEqual(len(m.messages), 1) + + msg = m.messages.get_msg_by_uid(2) self.assertEqual( - m.messages[1].content['subject'], + msg._hdoc.content['subject'], 'Message 2') - self.assertEqual(self.results, [0, 1]) - # XXX fix this thing with the indexes... + # the uids of the deleted messages + self.assertItemsEqual(self.results, [1, 3]) + + +class StoreAndFetchTestCase(unittest.TestCase, IMAP4HelperMixin): + """ + Several tests to check that the internal storage representation + is able to render the message structures as we expect them. + """ + # TODO get rid of the fucking sleeps with a proper defer + # management. + + def setUp(self): + IMAP4HelperMixin.setUp(self) + MBOX_NAME = "multipart/SIGNED" + self.received_messages = self.received_uid = None + self.result = None + + self.server.state = 'select' + + infile = util.sibpath(__file__, 'rfc822.multi-signed.message') + raw = open(infile).read() + + self.server.theAccount.addMailbox(MBOX_NAME) + mbox = self.server.theAccount.getMailbox(MBOX_NAME) + time.sleep(1) + self.server.mbox = mbox + self.server.mbox.messages.add_msg(raw, uid=1) + time.sleep(1) + + def addListener(self, x): + pass + + def removeListener(self, x): + pass + + def _fetchWork(self, uids): + + def result(R): + self.result = R + + self.connected.addCallback( + lambda _: self.function( + uids, uid=1) # do NOT use seq numbers! + ).addCallback(result).addCallback( + self._cbStopClient).addErrback(self._ebGeneral) + + d = loopback.loopbackTCP(self.server, self.client, noisy=False) + d.addCallback(lambda x: self.assertEqual(self.result, self.expected)) + return d + + @deferred(timeout=None) + def testMultiBody(self): + """ + Test that a multipart signed message is retrieved the same + as we stored it. + """ + time.sleep(1) + self.function = self.client.fetchBody + messages = '1' + + # XXX review. This probably should give everything? + + self.expected = {1: { + 'RFC822.TEXT': 'This is an example of a signed message,\n' + 'with attachments.\n\n\n--=20\n' + 'Nihil sine chao! =E2=88=B4\n', + 'UID': '1'}} + print "test multi: fetch uid", messages + return self._fetchWork(messages) class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): + """ - Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}. + Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ # XXX coming soon to your screens! pass + + +def tearDownModule(): + """ + Tear down functions for module level + """ + stop_reactor() diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py new file mode 100644 index 0000000..695f487 --- /dev/null +++ b/src/leap/mail/imap/tests/walktree.py @@ -0,0 +1,127 @@ +#t -*- coding: utf-8 -*- +# walktree.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the walktree module. +""" +import os +import sys +from email import parser + +from leap.mail import walk as W + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + + +p = parser.Parser() + +# TODO pass an argument of the type of message + +################################################## +# Input from hell + +if len(sys.argv) > 1: + FILENAME = sys.argv[1] +else: + FILENAME = "rfc822.multi-signed.message" + +""" +FILENAME = "rfc822.plain.message" +FILENAME = "rfc822.multi-minimal.message" +""" + +msg = p.parse(open(FILENAME)) +DO_CHECK = False +################################################# + +parts = W.get_parts(msg) + +if DEBUG: + def trim(item): + item = item[:10] + [trim(part["phash"]) for part in parts if part.get('phash', None)] + +raw_docs = list(W.get_raw_docs(msg, parts)) + +body_phash_fun = [W.get_body_phash_simple, + W.get_body_phash_multi][int(msg.is_multipart())] +body_phash = body_phash_fun(W.get_payloads(msg)) +parts_map = W.walk_msg_tree(parts, body_phash=body_phash) + + +# TODO add missing headers! +expected = { + 'body': '1ddfa80485', + 'multi': True, + 'part_map': { + 1: { + 'headers': {'Content-Disposition': 'inline', + 'Content-Type': 'multipart/mixed; ' + 'boundary="z0eOaCaDLjvTGF2l"'}, + 'multi': True, + 'part_map': {1: {'ctype': 'text/plain', + 'headers': [ + ('Content-Type', + 'text/plain; charset=utf-8'), + ('Content-Disposition', + 'inline'), + ('Content-Transfer-Encoding', + 'quoted-printable')], + 'multi': False, + 'parts': 1, + 'phash': '1ddfa80485', + 'size': 206}, + 2: {'ctype': 'text/plain', + 'headers': [('Content-Type', + 'text/plain; charset=us-ascii'), + ('Content-Disposition', + 'attachment; ' + 'filename="attach.txt"')], + 'multi': False, + 'parts': 1, + 'phash': '7a94e4d769', + 'size': 133}, + 3: {'ctype': 'application/octet-stream', + 'headers': [('Content-Type', + 'application/octet-stream'), + ('Content-Disposition', + 'attachment; filename="hack.ico"'), + ('Content-Transfer-Encoding', + 'base64')], + 'multi': False, + 'parts': 1, + 'phash': 'c42cccebbd', + 'size': 12736}}}, + 2: {'ctype': 'application/pgp-signature', + 'headers': [('Content-Type', 'application/pgp-signature')], + 'multi': False, + 'parts': 1, + 'phash': '8f49fbf749', + 'size': 877}}} + +if DEBUG and DO_CHECK: + # TODO turn this into a proper unittest + assert(parts_map == expected) + print "Structure: OK" + + +import pprint +print +print "RAW DOCS" +pprint.pprint(raw_docs) +print +print "PARTS MAP" +pprint.pprint(parts_map) diff --git a/src/leap/mail/load_tests.py b/src/leap/mail/load_tests.py index ee89fcc..be65b8d 100644 --- a/src/leap/mail/load_tests.py +++ b/src/leap/mail/load_tests.py @@ -14,12 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - - """ Provide a function for loading tests. """ - import unittest diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py index 21f6d62..c8f224c 100644 --- a/src/leap/mail/messageflow.py +++ b/src/leap/mail/messageflow.py @@ -25,12 +25,15 @@ from zope.interface import Interface, implements class IMessageConsumer(Interface): + """ + I consume messages from a queue. + """ - def consume(self, item): + def consume(self, queue): """ Consumes the passed item. - :param item: an object to be consumed. + :param item: a queue where we put the object to be consumed. :type item: object """ # TODO we could add an optional type to be passed @@ -40,15 +43,43 @@ class IMessageConsumer(Interface): # the queue, maybe wrapped in an object with a retries attribute. +class IMessageProducer(Interface): + """ + I produce messages and put them in a store to be consumed by other + entities. + """ + + def push(self, item, state=None): + """ + Push a new item in the queue. + """ + + def start(self): + """ + Start producing items. + """ + + def stop(self): + """ + Stop producing items. + """ + + def flush(self): + """ + Flush queued messages to consumer. + """ + + class DummyMsgConsumer(object): implements(IMessageConsumer) - def consume(self, item): + def consume(self, queue): """ Just prints the passed item. """ - print "got item %s" % item + if not queue.empty(): + print "got item %s" % queue.get() class MessageProducer(object): @@ -61,6 +92,8 @@ class MessageProducer(object): deferred chain and leave further processing detached from the calling loop, as in the case of smtp. """ + implements(IMessageProducer) + # TODO this can be seen as a first step towards properly implementing # components that implement IPushProducer / IConsumer interfaces. # However, I need to think more about how to pause the streaming. @@ -68,6 +101,10 @@ class MessageProducer(object): # and consumption is not likely (?) to consume huge amounts of memory in # our current settings, so the need to pause the stream is not urgent now. + # TODO use enum + STATE_NEW = 1 + STATE_DIRTY = 2 + def __init__(self, consumer, queue=Queue.Queue, period=1): """ Initializes the MessageProducer @@ -82,7 +119,8 @@ class MessageProducer(object): # it should implement a `consume` method self._consumer = consumer - self._queue = queue() + self._queue_new = queue() + self._queue_dirty = queue() self._period = period self._loop = LoopingCall(self._check_for_new) @@ -91,43 +129,50 @@ class MessageProducer(object): def _check_for_new(self): """ - Checks for new items in the internal queue, and calls the consume + Check for new items in the internal queue, and calls the consume method in the consumer. If the queue is found empty, the loop is stopped. It will be started again after the addition of new items. """ - # XXX right now I'm assuming that the period is good enough to allow - # a right pace of processing. but we could also pass the queue object - # to the consumer and let it choose whether process a new item or not. - - if self._queue.empty(): + self._consumer.consume((self._queue_new, self._queue_dirty)) + if self.is_queue_empty(): self.stop() - else: - self._consumer.consume(self._queue.get()) - # public methods + def is_queue_empty(self): + """ + Return True if queue is empty, False otherwise. + """ + new = self._queue_new + dirty = self._queue_dirty + return new.empty() and dirty.empty() + + # public methods: IMessageProducer - def put(self, item): + def push(self, item, state=None): """ - Puts a new item in the queue. + Push a new item in the queue. If the queue was empty, we will start the loop again. """ - was_empty = self._queue.empty() - # XXX this might raise if the queue does not accept any new # items. what to do then? - self._queue.put(item) - if was_empty: - self.start() + queue = self._queue_new + + if state == self.STATE_NEW: + queue = self._queue_new + if state == self.STATE_DIRTY: + queue = self._queue_dirty + + queue.put(item) + self.start() def start(self): """ - Starts polling for new items. + Start polling for new items. """ if not self._loop.running: - self._loop.start(self._period) + self._loop.start(self._period, now=True) def stop(self): """ @@ -136,6 +181,12 @@ class MessageProducer(object): if self._loop.running: self._loop.stop() + def flush(self): + """ + Flush queued messages to consumer. + """ + self._check_for_new() + if __name__ == "__main__": from twisted.internet import reactor diff --git a/src/leap/mail/size.py b/src/leap/mail/size.py new file mode 100644 index 0000000..c9eaabd --- /dev/null +++ b/src/leap/mail/size.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# size.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Recursively get size of objects. +""" +from gc import collect +from itertools import chain +from sys import getsizeof + + +def _get_size(item, seen): + known_types = {dict: lambda d: chain.from_iterable(d.items())} + default_size = getsizeof(0) + + def size_walk(item): + if id(item) in seen: + return 0 + seen.add(id(item)) + s = getsizeof(item, default_size) + for _type, fun in known_types.iteritems(): + if isinstance(item, _type): + s += sum(map(size_walk, fun(item))) + break + return s + + return size_walk(item) + + +def get_size(item): + """ + Return the cumulative size of a given object. + + Currently it supports only dictionaries, and seemingly leaks + some memory, so use with care. + + :param item: the item which size wants to be computed + :rtype: int + """ + seen = set() + size = _get_size(item, seen) + del seen + collect() + return size diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index d3eb9e8..bbd4064 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -30,7 +30,7 @@ from leap.mail.smtp.gateway import SMTPFactory def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, - smtp_cert, smtp_key, encrypted_only): + smtp_cert, smtp_key, encrypted_only): """ Setup SMTP gateway to run with Twisted. @@ -52,8 +52,8 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, :type smtp_cert: str :param smtp_key: The client key for authentication. :type smtp_key: str - :param encrypted_only: Whether the SMTP gateway should send unencrypted mail - or not. + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. :type encrypted_only: bool :returns: tuple of SMTPFactory, twisted.internet.tcp.Port diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index a78bd55..ef398d1 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -52,6 +52,7 @@ from leap.common.events import proto, signal from leap.keymanager import KeyManager from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound +from leap.mail import __version__ from leap.mail.smtp.rfc3156 import ( MultipartSigned, MultipartEncrypted, @@ -492,7 +493,7 @@ class EncryptedMessage(object): heloFallback=True, requireAuthentication=False, requireTransportSecurity=True) - factory.domain = LOCAL_FQDN + factory.domain = __version__ signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) reactor.connectSSL( self._host, self._port, factory, @@ -599,13 +600,16 @@ class EncryptedMessage(object): self._msg = self._origmsg return - # add a nice footer to the outgoing message from_address = validate_address(self._fromAddress.addrstr) username, domain = from_address.split('@') - self.lines.append('--') - self.lines.append('%s - https://%s/key/%s.' % - (self.FOOTER_STRING, domain, username)) - self.lines.append('') + + # add a nice footer to the outgoing message + if self._origmsg.get_content_type() == 'text/plain': + self.lines.append('--') + self.lines.append('%s - https://%s/key/%s' % + (self.FOOTER_STRING, domain, username)) + self.lines.append('') + self._origmsg = self.parseMessage() # get sender and recipient data diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py index dd48475..62a0675 100644 --- a/src/leap/mail/smtp/rfc3156.py +++ b/src/leap/mail/smtp/rfc3156.py @@ -24,6 +24,7 @@ import base64 from abc import ABCMeta, abstractmethod from StringIO import StringIO +from twisted.python import log from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email import errors @@ -145,14 +146,26 @@ def encode_base64(msg): :param msg: The non-multipart message to be encoded. :type msg: email.message.Message """ - orig = msg.get_payload() - encdata = _bencode(orig) - msg.set_payload(encdata) - # replace or set the Content-Transfer-Encoding header. - try: - msg.replace_header('Content-Transfer-Encoding', 'base64') - except KeyError: - msg['Content-Transfer-Encoding'] = 'base64' + encoding = msg.get('Content-Transfer-Encoding', None) + if encoding is not None: + encoding = encoding.lower() + # XXX Python's email module can only decode quoted-printable, base64 and + # uuencoded data, so we might have to implement other decoding schemes in + # order to support RFC 3156 properly and correctly calculate signatures + # for multipart attachments (eg. 7bit or 8bit encoded attachments). For + # now, if content is already encoded as base64 or if it is encoded with + # some unknown encoding, we just pass. + if encoding in [None, 'quoted-printable', 'x-uuencode', 'uue', 'x-uue']: + orig = msg.get_payload(decode=True) + encdata = _bencode(orig) + msg.set_payload(encdata) + # replace or set the Content-Transfer-Encoding header. + try: + msg.replace_header('Content-Transfer-Encoding', 'base64') + except KeyError: + msg['Content-Transfer-Encoding'] = 'base64' + elif encoding is not 'base64': + log.err('Unknown content-transfer-encoding: %s' % encoding) def encode_base64_rec(msg): @@ -361,7 +374,7 @@ class PGPSignature(MIMEApplication): """ def __init__(self, _data, name='signature.asc'): MIMEApplication.__init__(self, _data, 'pgp-signature', - _encoder=lambda x: x, name=name) + _encoder=lambda x: x, name=name) self.add_header('Content-Description', 'OpenPGP Digital Signature') diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index 4c2f04f..88ee5f7 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -101,10 +101,16 @@ class TestSmtpGateway(TestCaseWithKeyManager): '250 Sender address accepted', '250 Recipient address accepted', '354 Continue'] - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], + + # XXX this bit can be refactored away in a helper + # method... + proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key'], self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + # snip... transport = proto_helpers.StringTransport() proto.makeConnection(transport) for i, line in enumerate(self.EMAIL_DATA): @@ -118,8 +124,10 @@ class TestSmtpGateway(TestCaseWithKeyManager): """ Test if message gets encrypted to destination email. """ - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], + proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key'], self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) fromAddr = Address(ADDRESS_2) @@ -129,7 +137,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): self._config['port'], self._config['cert'], self._config['key']) for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) - m.eomReceived() + #m.eomReceived() # this includes a defer, so we avoid calling it here + m.lines.append('') # add a trailing newline # we need to call the following explicitelly because it was deferred # inside the previous method m._maybe_encrypt_and_sign() @@ -149,7 +158,7 @@ class TestSmtpGateway(TestCaseWithKeyManager): m._msg.get_payload(1).get_payload(), privkey) self.assertEqual( '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n', + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', decrypted, 'Decrypted text differs from plaintext.') @@ -158,8 +167,10 @@ class TestSmtpGateway(TestCaseWithKeyManager): Test if message gets encrypted to destination email and signed with sender key. """ - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], + proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key'], self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) @@ -170,7 +181,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) # trigger encryption and signing - m.eomReceived() + #m.eomReceived() # this includes a defer, so we avoid calling it here + m.lines.append('') # add a trailing newline # we need to call the following explicitelly because it was deferred # inside the previous method m._maybe_encrypt_and_sign() @@ -192,7 +204,7 @@ class TestSmtpGateway(TestCaseWithKeyManager): m._msg.get_payload(1).get_payload(), privkey, verify=pubkey) self.assertEqual( '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n', + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', decrypted, 'Decrypted text differs from plaintext.') @@ -202,11 +214,14 @@ class TestSmtpGateway(TestCaseWithKeyManager): """ # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], + proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key'], self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - user = User('ihavenopubkey@nonleap.se', 'gateway.leap.se', proto, ADDRESS) + user = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', proto, ADDRESS) fromAddr = Address(ADDRESS_2) m = EncryptedMessage( fromAddr, user, self._km, self._config['host'], @@ -214,7 +229,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): for line in self.EMAIL_DATA[4:12]: m.lineReceived(line) # trigger signing - m.eomReceived() + #m.eomReceived() # this includes a defer, so we avoid calling it here + m.lines.append('') # add a trailing newline # we need to call the following explicitelly because it was deferred # inside the previous method m._maybe_encrypt_and_sign() @@ -226,8 +242,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): self.assertEqual('pgp-sha512', m._msg.get_param('micalg')) # assert content of message self.assertEqual( - '\r\n'.join(self.EMAIL_DATA[9:13])+'\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n', + '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' + + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', m._msg.get_payload(0).get_payload(decode=True)) # assert content of signature self.assertTrue( @@ -262,8 +278,10 @@ class TestSmtpGateway(TestCaseWithKeyManager): # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) # prepare the SMTP factory - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], + proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key'], self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() @@ -291,8 +309,10 @@ class TestSmtpGateway(TestCaseWithKeyManager): # mock the key fetching self._km.fetch_keys_from_server = Mock(return_value=[]) # prepare the SMTP factory with encrypted only equal to false - proto = SMTPFactory(u'anotheruser@leap.se', - self._km, self._config['host'], self._config['port'], + proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, self._config['host'], + self._config['port'], self._config['cert'], self._config['key'], False).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py new file mode 100644 index 0000000..fed24b3 --- /dev/null +++ b/src/leap/mail/utils.py @@ -0,0 +1,327 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Mail utilities. +""" +import json +import re +import traceback +import Queue + +from leap.soledad.common.document import SoledadDocument + + +CHARSET_PATTERN = r"""charset=([\w-]+)""" +CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE) + + +def first(things): + """ + Return the head of a collection. + """ + try: + return things[0] + except (IndexError, TypeError): + return None + + +def empty(thing): + """ + Return True if a thing is None or its length is zero. + """ + if thing is None: + return True + if isinstance(thing, SoledadDocument): + thing = thing.content + try: + return len(thing) == 0 + except (ReferenceError, TypeError): + return True + + +def maybe_call(thing): + """ + Return the same thing, or the result of its invocation if it is a + callable. + """ + return thing() if callable(thing) else thing + + +def find_charset(thing, default=None): + """ + Looks into the object 'thing' for a charset specification. + It searchs into the object's `repr`. + + :param thing: the object to look into. + :type thing: object + :param default: the dafault charset to return if no charset is found. + :type default: str + + :return: the charset or 'default' + :rtype: str or None + """ + charset = first(CHARSET_RE.findall(repr(thing))) + if charset is None: + charset = default + return charset + + +def lowerdict(_dict): + """ + Return a dict with the keys in lowercase. + + :param _dict: the dict to convert + :rtype: dict + """ + # TODO should properly implement a CaseInsensitive dict. + # Look into requests code. + return dict((key.lower(), value) + for key, value in _dict.items()) + + +PART_MAP = "part_map" +PHASH = "phash" + + +def _str_dict(d, k): + """ + Convert the dictionary key to string if it was a string. + + :param d: the dict + :type d: dict + :param k: the key + :type k: object + """ + if isinstance(k, int): + val = d[k] + d[str(k)] = val + del(d[k]) + + +def stringify_parts_map(d): + """ + Modify a dictionary making all the nested dicts under "part_map" keys + having strings as keys. + + :param d: the dictionary to modify + :type d: dictionary + :rtype: dictionary + """ + for k in d: + if k == PART_MAP: + pmap = d[k] + for kk in pmap.keys(): + _str_dict(d[k], kk) + for kk in pmap.keys(): + stringify_parts_map(d[k][str(kk)]) + return d + + +def phash_iter(d): + """ + A recursive generator that extracts all the payload-hashes + from an arbitrary nested parts-map dictionary. + + :param d: the dictionary to walk + :type d: dictionary + :return: a list of all the phashes found + :rtype: list + """ + if PHASH in d: + yield d[PHASH] + if PART_MAP in d: + for key in d[PART_MAP]: + for phash in phash_iter(d[PART_MAP][key]): + yield phash + + +def accumulator(fun, lim): + """ + A simple accumulator that uses a closure and a mutable + object to collect items. + When the count of items is greater than `lim`, the + collection is flushed after invoking a map of the function `fun` + over it. + + The returned accumulator can also be flushed at any moment + by passing a boolean as a second parameter. + + :param fun: the function to call over the collection + when its size is greater than `lim` + :type fun: callable + :param lim: the turning point for the collection + :type lim: int + :rtype: function + + >>> from pprint import pprint + >>> acc = accumulator(pprint, 2) + >>> acc(1) + >>> acc(2) + [1, 2] + >>> acc(3) + >>> acc(4) + [3, 4] + >>> acc = accumulator(pprint, 5) + >>> acc(1) + >>> acc(2) + >>> acc(3) + >>> acc(None, flush=True) + [1,2,3] + """ + KEY = "items" + _o = {KEY: []} + + def _accumulator(item, flush=False): + collection = _o[KEY] + collection.append(item) + if len(collection) >= lim or flush: + map(fun, filter(None, collection)) + _o[KEY] = [] + + return _accumulator + + +def accumulator_queue(fun, lim): + """ + A version of the accumulator that uses a queue. + + When the count of items is greater than `lim`, the + queue is flushed after invoking the function `fun` + over its items. + + The returned accumulator can also be flushed at any moment + by passing a boolean as a second parameter. + + :param fun: the function to call over the collection + when its size is greater than `lim` + :type fun: callable + :param lim: the turning point for the collection + :type lim: int + :rtype: function + """ + _q = Queue.Queue() + + def _accumulator(item, flush=False): + _q.put(item) + if _q.qsize() >= lim or flush: + collection = [_q.get() for i in range(_q.qsize())] + map(fun, filter(None, collection)) + + return _accumulator + + +# +# String manipulation +# + +class CustomJsonScanner(object): + """ + This class is a context manager definition used to monkey patch the default + json string parsing behavior. + The emails can have more than one encoding, so the `str` objects have more + than one encoding and json does not support direct work with `str` + (only `unicode`). + """ + + def _parse_string_str(self, s, idx, *args, **kwargs): + """ + Parses the string "s" starting at the point idx and returns an `str` + object. Which basically means it works exactly the same as the regular + JSON string parsing, except that it doesn't try to decode utf8. + We need this because mail raw strings might have bytes in multiple + encodings. + + :param s: the string we want to parse + :type s: str + :param idx: the starting point for parsing + :type idx: int + + :returns: the parsed string and the index where the + string ends. + :rtype: tuple (str, int) + """ + # NOTE: we just want to use this monkey patched version if we are + # calling the loads from our custom method. Otherwise, we use the + # json's default parser. + monkey_patched = False + for i in traceback.extract_stack(): + # look for json_loads method in the call stack + if i[2] == json_loads.__name__: + monkey_patched = True + break + + if not monkey_patched: + return self._orig_scanstring(s, idx, *args, **kwargs) + + # TODO profile to see if a compiled regex can get us some + # benefit here. + found = False + end = s.find("\"", idx) + while not found: + try: + if s[end-1] != "\\": + found = True + else: + end = s.find("\"", end+1) + except Exception: + found = True + return s[idx:end].decode("string-escape"), end+1 + + def __enter__(self): + """ + Replace the json methods with the needed ones. + Also make a backup to restore them later. + """ + # backup original values + self._orig_make_scanner = json.scanner.make_scanner + self._orig_scanstring = json.decoder.scanstring + + # We need the make_scanner function to be the python one so we can + # monkey_patch the json string parsing + json.scanner.make_scanner = json.scanner.py_make_scanner + + # And now we monkey patch the money method + json.decoder.scanstring = self._parse_string_str + + def __exit__(self, exc_type, exc_value, traceback): + """ + Restores the backuped methods. + """ + # restore original values + json.scanner.make_scanner = self._orig_make_scanner + json.decoder.scanstring = self._orig_scanstring + + +def json_loads(data): + """ + It works as json.loads but supporting multiple encodings in the same + string and accepting an `str` parameter that won't be converted to unicode. + + :param data: the string to load the objects from + :type data: str + + :returns: the corresponding python object result of parsing 'data', this + behaves similarly as json.loads, with the exception of that + returns always `str` instead of `unicode`. + """ + obj = None + with CustomJsonScanner(): + # We need to use the cls parameter in order to trigger the code + # that will let us control the string parsing method. + obj = json.loads(data, cls=json.JSONDecoder) + + return obj diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py new file mode 100644 index 0000000..f747377 --- /dev/null +++ b/src/leap/mail/walk.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +# walk.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Utilities for walking along a message tree. +""" +import os + +from pycryptopp.hash import sha256 + +from leap.mail.utils import first + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + +if DEBUG: + get_hash = lambda s: sha256.SHA256(s).hexdigest()[:10] +else: + get_hash = lambda s: sha256.SHA256(s).hexdigest() + + +""" +Get interesting message parts +""" +get_parts = lambda msg: [ + {'multi': part.is_multipart(), + 'ctype': part.get_content_type(), + 'size': len(part.as_string()), + 'parts': len(part.get_payload()) + if isinstance(part.get_payload(), list) + else 1, + 'headers': part.items(), + 'phash': get_hash(part.get_payload()) + if not part.is_multipart() else None} + for part in msg.walk()] + +""" +Utility lambda functions for getting the parts vector and the +payloads from the original message. +""" + +get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts) +get_payloads = lambda msg: ((x.get_payload(), + dict(((str.lower(k), v) for k, v in (x.items())))) + for x in msg.walk()) + +get_body_phash_simple = lambda payloads: first( + [get_hash(payload) for payload, headers in payloads + if payloads]) + +get_body_phash_multi = lambda payloads: (first( + [get_hash(payload) for payload, headers in payloads + if payloads + and "text/plain" in headers.get('content-type', '')]) + or get_body_phash_simple(payloads)) + +""" +On getting the raw docs, we get also some of the headers to be able to +index the content. Here we remove any mutable part, as the the filename +in the content disposition. +""" + +get_raw_docs = lambda msg, parts: ( + {"type": "cnt", # type content they'll be + "raw": payload if not DEBUG else payload[:100], + "phash": get_hash(payload), + "content-disposition": first(headers.get( + 'content-disposition', '').split(';')), + "content-type": headers.get( + 'content-type', ''), + "content-transfer-encoding": headers.get( + 'content-transfer-type', '')} + for payload, headers in get_payloads(msg) + if not isinstance(payload, list)) + + +def walk_msg_tree(parts, body_phash=None): + """ + Take a list of interesting items of a message subparts structure, + and return a dict of dicts almost ready to be written to the content + documents that will be stored in Soledad. + + It walks down the subparts in the parsed message tree, and collapses + the leaf docuents into a wrapper document until no multipart submessages + are left. To achieve this, it iteratively calculates a wrapper vector of + all documents in the sequence that have more than one part and have unitary + documents to their right. To collapse a multipart, take as many + unitary documents as parts the submessage contains, and replace the object + in the sequence with the new wrapper document. + + :param parts: A list of dicts containing the interesting properties for + the message structure. Normally this has been generated by + doing a message walk. + :type parts: list of dicts. + :param body_phash: the payload hash of the body part, to be included + in the outer content doc for convenience. + :type body_phash: basestring or None + """ + PART_MAP = "part_map" + MULTI = "multi" + HEADERS = "headers" + PHASH = "phash" + BODY = "body" + + # parts vector + pv = list(get_parts_vector(parts)) + + inner_headers = parts[1].get(HEADERS, None) if ( + len(parts) == 2) else None + + if DEBUG: + print "parts vector: ", pv + print + + # wrappers vector + getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False + for i in range(len(pv) - 1)] + wv = getwv(pv) + + # do until no wrapper document is left + while any(wv): + wind = wv.index(True) # wrapper index + nsub = pv[wind] # number of subparts to pick + slic = parts[wind + 1:wind + 1 + nsub] # slice with subparts + + cwra = { + MULTI: True, + PART_MAP: dict((index + 1, part) # content wrapper + for index, part in enumerate(slic)), + HEADERS: dict(parts[wind][HEADERS]) + } + + # remove subparts and substitue wrapper + map(lambda i: parts.remove(i), slic) + parts[wind] = cwra + + # refresh vectors for this iteration + pv = list(get_parts_vector(parts)) + wv = getwv(pv) + + if all(x == 1 for x in pv): + # special case in the rightmost element + main_pmap = parts[0].get(PART_MAP, None) + if main_pmap is not None: + last_part = max(main_pmap.keys()) + main_pmap[last_part][PART_MAP] = {} + for partind in range(len(pv) - 1): + print partind+1, len(parts) + main_pmap[last_part][PART_MAP][partind] = parts[partind + 1] + + outer = parts[0] + outer.pop(HEADERS) + if not PART_MAP in outer: + # we have a multipart with 1 part only, so kind of fix it + # although it would be prettier if I take this special case at + # the beginning of the walk. + pdoc = {MULTI: True, + PART_MAP: {1: outer}} + pdoc[PART_MAP][1][MULTI] = False + if not pdoc[PART_MAP][1].get(PHASH, None): + pdoc[PART_MAP][1][PHASH] = body_phash + if inner_headers: + pdoc[PART_MAP][1][HEADERS] = inner_headers + else: + pdoc = outer + pdoc[BODY] = body_phash + return pdoc + +""" +Groucho Marx: Now pay particular attention to this first clause, because it's + most important. There's the party of the first part shall be + known in this contract as the party of the first part. How do you + like that, that's pretty neat eh? + +Chico Marx: No, that's no good. +Groucho Marx: What's the matter with it? + +Chico Marx: I don't know, let's hear it again. +Groucho Marx: So the party of the first part shall be known in this contract as + the party of the first part. + +Chico Marx: Well it sounds a little better this time. +Groucho Marx: Well, it grows on you. Would you like to hear it once more? + +Chico Marx: Just the first part. +Groucho Marx: All right. It says the first part of the party of the first part + shall be known in this contract as the first part of the party of + the first part, shall be known in this contract - look, why + should we quarrel about a thing like this, we'll take it right + out, eh? + +Chico Marx: Yes, it's too long anyhow. Now what have we got left? +Groucho Marx: Well I've got about a foot and a half. Now what's the matter? + +Chico Marx: I don't like the second party either. +""" + +""" +I feel you deserved it after reading the above and try to debug your problem ;) +""" diff --git a/versioneer.py b/versioneer.py index 34e4807..4e2c0a5 100644 --- a/versioneer.py +++ b/versioneer.py @@ -115,7 +115,7 @@ import sys def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.cmd on windows, not just git + # remember shell=False, so use git.exe on windows, not just git p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) except EnvironmentError: e = sys.exc_info()[1] @@ -230,7 +230,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): GIT = "git" if sys.platform == "win32": - GIT = "git.cmd" + GIT = "git.exe" stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: @@ -305,7 +305,7 @@ import sys def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.cmd on windows, not just git + # remember shell=False, so use git.exe on windows, not just git p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) except EnvironmentError: e = sys.exc_info()[1] @@ -420,7 +420,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): GIT = "git" if sys.platform == "win32": - GIT = "git.cmd" + GIT = "git.exe" stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: @@ -476,7 +476,7 @@ import sys def do_vcs_install(versionfile_source, ipy): GIT = "git" if sys.platform == "win32": - GIT = "git.cmd" + GIT = "git.exe" run_command([GIT, "add", "versioneer.py"]) run_command([GIT, "add", versionfile_source]) run_command([GIT, "add", ipy]) @@ -489,13 +489,13 @@ def do_vcs_install(versionfile_source, ipy): present = True f.close() except EnvironmentError: - pass + pass if not present: f = open(".gitattributes", "a+") f.write("%s export-subst\n" % versionfile_source) f.close() run_command([GIT, "add", ".gitattributes"]) - + SHORT_VERSION_PY = """ # This file was generated by 'versioneer.py' (0.7+) from |