summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTomás Touceda <chiiph@leap.se>2014-01-09 00:22:21 -0300
committerTomás Touceda <chiiph@leap.se>2014-01-09 00:22:21 -0300
commitb92e0962d4ab08f450700dc9fabf2d7eab203aa7 (patch)
treeec235eb8df5451e1cf9566321cf36a1718441315
parentf7a37ad5994026b37428aaee57b841c23e86952d (diff)
parent174670ab0217d4f86d7b3d12ca6b38db20a2a8d9 (diff)
Merge remote-tracking branch 'refs/remotes/kali/feature/split_message_docs' into develop
-rw-r--r--mail/changes/feature_split_message_docs7
-rw-r--r--mail/src/leap/mail/decorators.py148
-rw-r--r--mail/src/leap/mail/imap/account.py426
-rw-r--r--mail/src/leap/mail/imap/fetch.py234
-rw-r--r--mail/src/leap/mail/imap/fields.py151
-rw-r--r--mail/src/leap/mail/imap/index.py69
-rw-r--r--mail/src/leap/mail/imap/mailbox.py666
-rw-r--r--mail/src/leap/mail/imap/messages.py1346
-rw-r--r--mail/src/leap/mail/imap/parser.py113
-rw-r--r--mail/src/leap/mail/imap/server.py1807
-rw-r--r--mail/src/leap/mail/imap/service/imap.py4
-rwxr-xr-xmail/src/leap/mail/imap/tests/getmail282
-rw-r--r--mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message16
-rw-r--r--mail/src/leap/mail/imap/tests/rfc822.multi-signed.message238
-rw-r--r--mail/src/leap/mail/imap/tests/rfc822.multi.message96
-rw-r--r--mail/src/leap/mail/imap/tests/rfc822.plain.message66
-rw-r--r--mail/src/leap/mail/imap/tests/test_imap.py274
-rw-r--r--mail/src/leap/mail/imap/tests/walktree.py117
-rw-r--r--mail/src/leap/mail/utils.py29
-rw-r--r--mail/src/leap/mail/walk.py160
20 files changed, 4250 insertions, 1999 deletions
diff --git a/mail/changes/feature_split_message_docs b/mail/changes/feature_split_message_docs
new file mode 100644
index 0000000..0109501
--- /dev/null
+++ b/mail/changes/feature_split_message_docs
@@ -0,0 +1,7 @@
+ o Defer costly operations to a pool of threads.
+ o Split the internal representation of messages into three distinct documents:
+ 1) Flags 2) Headers 3) Content.
+ o Make use of the Twisted MIME interface.
+ o Add deduplication ability to the save operation, for body and attachments.
+ o Add IMessageCopier interface to mailbox implementation, so bulk moves
+ are costless. Closes: #4654
diff --git a/mail/src/leap/mail/decorators.py b/mail/src/leap/mail/decorators.py
new file mode 100644
index 0000000..d5eac97
--- /dev/null
+++ b/mail/src/leap/mail/decorators.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# decorators.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Useful decorators for mail package.
+"""
+import logging
+import os
+
+from functools import wraps
+
+from twisted.internet.threads import deferToThread
+
+logger = logging.getLogger(__name__)
+
+
+# TODO
+# Should write a helper to be able to pass a timeout argument.
+# See this answer: http://stackoverflow.com/a/19019648/1157664
+# And the notes by glyph and jpcalderone
+
+def deferred(f):
+ """
+ Decorator, for deferring methods to Threads.
+
+ It will do a deferToThread of the decorated method
+ unless the environment variable LEAPMAIL_DEBUG is set.
+
+ It uses a descriptor to delay the definition of the
+ method wrapper.
+ """
+ class descript(object):
+ """
+ The class to be used as decorator.
+
+ It takes any method as the passed object.
+ """
+
+ def __init__(self, f):
+ """
+ Initializes the decorator object.
+
+ :param f: the decorated function
+ :type f: callable
+ """
+ self.f = f
+
+ def __get__(self, instance, klass):
+ """
+ Descriptor implementation.
+
+ At creation time, the decorated `method` is unbound.
+
+ It will dispatch the make_unbound method if we still do not
+ have an instance available, and the make_bound method when the
+ method has already been bound to the instance.
+
+ :param instance: the instance of the class, or None if not exist.
+ :type instance: instantiated class or None.
+ """
+ if instance is None:
+ # Class method was requested
+ return self.make_unbound(klass)
+ return self.make_bound(instance)
+
+ def _errback(self, failure):
+ """
+ Errorback that logs the exception catched.
+
+ :param failure: a twisted failure
+ :type failure: Failure
+ """
+ logger.warning('Error in method: %s' % (self.f.__name__))
+ logger.exception(failure.getTraceback())
+
+ def make_unbound(self, klass):
+ """
+ Return a wrapped function with the unbound call, during the
+ early access to the decortad method. This gets passed
+ only the class (not the instance since it does not yet exist).
+
+ :param klass: the class to which the still unbound method belongs
+ :type klass: type
+ """
+
+ @wraps(self.f)
+ def wrapper(*args, **kwargs):
+ """
+ We're temporarily wrapping the decorated method, but this
+ should not be called, since our application should use
+ the bound-wrapped method after this decorator class has been
+ used.
+
+ This documentation will vanish at runtime.
+ """
+ raise TypeError(
+ 'unbound method {}() must be called with {} instance '
+ 'as first argument (got nothing instead)'.format(
+ self.f.__name__,
+ klass.__name__)
+ )
+ return wrapper
+
+ def make_bound(self, instance):
+ """
+ Return a function that wraps the bound method call,
+ after we are able to access the instance object.
+
+ :param instance: an instance of the class the decorated method,
+ now bound, belongs to.
+ :type instance: object
+ """
+
+ @wraps(self.f)
+ def wrapper(*args, **kwargs):
+ """
+ Do a proper function wrapper that defers the decorated method
+ call to a separated thread if the LEAPMAIL_DEBUG
+ environment variable is set.
+
+ This documentation will vanish at runtime.
+ """
+ if not os.environ.get('LEAPMAIL_DEBUG'):
+ d = deferToThread(self.f, instance, *args, **kwargs)
+ d.addErrback(self._errback)
+ return d
+ else:
+ return self.f(instance, *args, **kwargs)
+
+ # This instance does not need the descriptor anymore,
+ # let it find the wrapper directly next time:
+ setattr(instance, self.f.__name__, wrapper)
+ return wrapper
+
+ return descript(f)
diff --git a/mail/src/leap/mail/imap/account.py b/mail/src/leap/mail/imap/account.py
new file mode 100644
index 0000000..fd861e7
--- /dev/null
+++ b/mail/src/leap/mail/imap/account.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+# account.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Backed Account.
+"""
+import copy
+import time
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.imap.index import IndexedDB
+from leap.mail.imap.fields import WithMsgFields
+from leap.mail.imap.parser import MBoxParser
+from leap.mail.imap.mailbox import SoledadMailbox
+from leap.soledad.client import Soledad
+
+
+#######################################
+# Soledad Account
+#######################################
+
+
+class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+ """
+ An implementation of IAccount and INamespacePresenteer
+ that is backed by Soledad Encrypted Documents.
+ """
+
+ implements(imap4.IAccount, imap4.INamespacePresenter)
+
+ _soledad = None
+ selected = None
+
+ def __init__(self, account_name, soledad=None):
+ """
+ Creates a SoledadAccountIndex that keeps track of the mailboxes
+ and subscriptions handled by this account.
+
+ :param acct_name: The name of the account (user id).
+ :type acct_name: str
+
+ :param soledad: a Soledad instance.
+ :param soledad: Soledad
+ """
+ leap_assert(soledad, "Need a soledad instance to initialize")
+ leap_assert_type(soledad, Soledad)
+
+ # XXX SHOULD assert too that the name matches the user/uuid with which
+ # soledad has been initialized.
+
+ self._account_name = self._parse_mailbox_name(account_name)
+ self._soledad = soledad
+
+ self.initialize_db()
+
+ # every user should have the right to an inbox folder
+ # at least, so let's make one!
+
+ if not self.mailboxes:
+ self.addMailbox(self.INBOX_NAME)
+
+ def _get_empty_mailbox(self):
+ """
+ Returns an empty mailbox.
+
+ :rtype: dict
+ """
+ return copy.deepcopy(self.EMPTY_MBOX)
+
+ def _get_mailbox_by_name(self, name):
+ """
+ Return an mbox document by name.
+
+ :param name: the name of the mailbox
+ :type name: str
+
+ :rtype: SoledadDocument
+ """
+ doc = self._soledad.get_from_index(
+ self.TYPE_MBOX_IDX, self.MBOX_KEY,
+ self._parse_mailbox_name(name))
+ return doc[0] if doc else None
+
+ @property
+ def mailboxes(self):
+ """
+ A list of the current mailboxes for this account.
+ """
+ return [doc.content[self.MBOX_KEY]
+ for doc in self._soledad.get_from_index(
+ self.TYPE_IDX, self.MBOX_KEY)]
+
+ @property
+ def subscriptions(self):
+ """
+ A list of the current subscriptions for this account.
+ """
+ return [doc.content[self.MBOX_KEY]
+ for doc in self._soledad.get_from_index(
+ self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
+
+ def getMailbox(self, name):
+ """
+ Returns a Mailbox with that name, without selecting it.
+
+ :param name: name of the mailbox
+ :type name: str
+
+ :returns: a a SoledadMailbox instance
+ :rtype: SoledadMailbox
+ """
+ name = self._parse_mailbox_name(name)
+
+ if name not in self.mailboxes:
+ raise imap4.MailboxException("No such mailbox")
+
+ return SoledadMailbox(name, soledad=self._soledad)
+
+ ##
+ ## IAccount
+ ##
+
+ def addMailbox(self, name, creation_ts=None):
+ """
+ Add a mailbox to the account.
+
+ :param name: the name of the mailbox
+ :type name: str
+
+ :param creation_ts: an optional creation timestamp to be used as
+ mailbox id. A timestamp will be used if no
+ one is provided.
+ :type creation_ts: int
+
+ :returns: True if successful
+ :rtype: bool
+ """
+ name = self._parse_mailbox_name(name)
+
+ if name in self.mailboxes:
+ raise imap4.MailboxCollision, name
+
+ if not creation_ts:
+ # by default, we pass an int value
+ # taken from the current time
+ # we make sure to take enough decimals to get a unique
+ # mailbox-uidvalidity.
+ creation_ts = int(time.time() * 10E2)
+
+ mbox = self._get_empty_mailbox()
+ mbox[self.MBOX_KEY] = name
+ mbox[self.CREATED_KEY] = creation_ts
+
+ doc = self._soledad.create_doc(mbox)
+ return bool(doc)
+
+ def create(self, pathspec):
+ """
+ Create a new mailbox from the given hierarchical name.
+
+ :param pathspec: The full hierarchical name of a new mailbox to create.
+ If any of the inferior hierarchical names to this one
+ do not exist, they are created as well.
+ :type pathspec: str
+
+ :return: A true value if the creation succeeds.
+ :rtype: bool
+
+ :raise MailboxException: Raised if this mailbox cannot be added.
+ """
+ # TODO raise MailboxException
+ paths = filter(
+ None,
+ self._parse_mailbox_name(pathspec).split('/'))
+ for accum in range(1, len(paths)):
+ try:
+ self.addMailbox('/'.join(paths[:accum]))
+ except imap4.MailboxCollision:
+ pass
+ try:
+ self.addMailbox('/'.join(paths))
+ except imap4.MailboxCollision:
+ if not pathspec.endswith('/'):
+ return False
+ return True
+
+ def select(self, name, readwrite=1):
+ """
+ Selects a mailbox.
+
+ :param name: the mailbox to select
+ :type name: str
+
+ :param readwrite: 1 for readwrite permissions.
+ :type readwrite: int
+
+ :rtype: bool
+ """
+ name = self._parse_mailbox_name(name)
+
+ if name not in self.mailboxes:
+ return None
+
+ self.selected = name
+
+ return SoledadMailbox(
+ name, rw=readwrite,
+ soledad=self._soledad)
+
+ def delete(self, name, force=False):
+ """
+ Deletes a mailbox.
+
+ Right now it does not purge the messages, but just removes the mailbox
+ name from the mailboxes list!!!
+
+ :param name: the mailbox to be deleted
+ :type name: str
+
+ :param force: if True, it will not check for noselect flag or inferior
+ names. use with care.
+ :type force: bool
+ """
+ name = self._parse_mailbox_name(name)
+
+ if not name in self.mailboxes:
+ raise imap4.MailboxException("No such mailbox")
+
+ mbox = self.getMailbox(name)
+
+ if force is False:
+ # See if this box is flagged \Noselect
+ # XXX use mbox.flags instead?
+ if self.NOSELECT_FLAG in mbox.getFlags():
+ # Check for hierarchically inferior mailboxes with this one
+ # as part of their root.
+ for others in self.mailboxes:
+ if others != name and others.startswith(name):
+ raise imap4.MailboxException, (
+ "Hierarchically inferior mailboxes "
+ "exist and \\Noselect is set")
+ mbox.destroy()
+
+ # XXX FIXME --- not honoring the inferior names...
+
+ # if there are no hierarchically inferior names, we will
+ # delete it from our ken.
+ #if self._inferiorNames(name) > 1:
+ # ??! -- can this be rite?
+ #self._index.removeMailbox(name)
+
+ def rename(self, oldname, newname):
+ """
+ Renames a mailbox.
+
+ :param oldname: old name of the mailbox
+ :type oldname: str
+
+ :param newname: new name of the mailbox
+ :type newname: str
+ """
+ oldname = self._parse_mailbox_name(oldname)
+ newname = self._parse_mailbox_name(newname)
+
+ if oldname not in self.mailboxes:
+ raise imap4.NoSuchMailbox, oldname
+
+ inferiors = self._inferiorNames(oldname)
+ inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
+
+ for (old, new) in inferiors:
+ if new in self.mailboxes:
+ raise imap4.MailboxCollision, new
+
+ for (old, new) in inferiors:
+ mbox = self._get_mailbox_by_name(old)
+ mbox.content[self.MBOX_KEY] = new
+ self._soledad.put_doc(mbox)
+
+ # XXX ---- FIXME!!!! ------------------------------------
+ # until here we just renamed the index...
+ # We have to rename also the occurrence of this
+ # mailbox on ALL the messages that are contained in it!!!
+ # ... we maybe could use a reference to the doc_id
+ # in each msg, instead of the "mbox" field in msgs
+ # -------------------------------------------------------
+
+ def _inferiorNames(self, name):
+ """
+ Return hierarchically inferior mailboxes.
+
+ :param name: name of the mailbox
+ :rtype: list
+ """
+ # XXX use wildcard query instead
+ inferiors = []
+ for infname in self.mailboxes:
+ if infname.startswith(name):
+ inferiors.append(infname)
+ return inferiors
+
+ def isSubscribed(self, name):
+ """
+ Returns True if user is subscribed to this mailbox.
+
+ :param name: the mailbox to be checked.
+ :type name: str
+
+ :rtype: bool
+ """
+ mbox = self._get_mailbox_by_name(name)
+ return mbox.content.get('subscribed', False)
+
+ def _set_subscription(self, name, value):
+ """
+ Sets the subscription value for a given mailbox
+
+ :param name: the mailbox
+ :type name: str
+
+ :param value: the boolean value
+ :type value: bool
+ """
+ # maybe we should store subscriptions in another
+ # document...
+ if not name in self.mailboxes:
+ self.addMailbox(name)
+ mbox = self._get_mailbox_by_name(name)
+
+ if mbox:
+ mbox.content[self.SUBSCRIBED_KEY] = value
+ self._soledad.put_doc(mbox)
+
+ def subscribe(self, name):
+ """
+ Subscribe to this mailbox
+
+ :param name: name of the mailbox
+ :type name: str
+ """
+ name = self._parse_mailbox_name(name)
+ if name not in self.subscriptions:
+ self._set_subscription(name, True)
+
+ def unsubscribe(self, name):
+ """
+ Unsubscribe from this mailbox
+
+ :param name: name of the mailbox
+ :type name: str
+ """
+ name = self._parse_mailbox_name(name)
+ if name not in self.subscriptions:
+ raise imap4.MailboxException, "Not currently subscribed to " + name
+ self._set_subscription(name, False)
+
+ def listMailboxes(self, ref, wildcard):
+ """
+ List the mailboxes.
+
+ from rfc 3501:
+ returns a subset of names from the complete set
+ of all names available to the client. Zero or more untagged LIST
+ replies are returned, containing the name attributes, hierarchy
+ delimiter, and name.
+
+ :param ref: reference name
+ :type ref: str
+
+ :param wildcard: mailbox name with possible wildcards
+ :type wildcard: str
+ """
+ # XXX use wildcard in index query
+ ref = self._inferiorNames(
+ self._parse_mailbox_name(ref))
+ wildcard = imap4.wildcardToRegexp(wildcard, '/')
+ return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+
+ ##
+ ## INamespacePresenter
+ ##
+
+ def getPersonalNamespaces(self):
+ return [["", "/"]]
+
+ def getSharedNamespaces(self):
+ return None
+
+ def getOtherNamespaces(self):
+ return None
+
+ # extra, for convenience
+
+ def deleteAllMessages(self, iknowhatiamdoing=False):
+ """
+ Deletes all messages from all mailboxes.
+ Danger! high voltage!
+
+ :param iknowhatiamdoing: confirmation parameter, needs to be True
+ to proceed.
+ """
+ if iknowhatiamdoing is True:
+ for mbox in self.mailboxes:
+ self.delete(mbox, force=True)
+
+ def __repr__(self):
+ """
+ Representation string for this object.
+ """
+ return "<SoledadBackedAccount (%s)>" % self._account_name
diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index b1c34ba..604a2ea 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -17,21 +17,24 @@
"""
Incoming mail fetcher.
"""
-import logging
+import copy
import json
-import ssl
+import logging
+#import ssl
import threading
import time
-import copy
-from StringIO import StringIO
+import sys
+import traceback
from email.parser import Parser
from email.generator import Generator
from email.utils import parseaddr
+from StringIO import StringIO
from twisted.python import log
+from twisted.internet import defer
from twisted.internet.task import LoopingCall
-from twisted.internet.threads import deferToThread
+#from twisted.internet.threads import deferToThread
from zope.proxy import sameProxiedObjects
from leap.common import events as leap_events
@@ -45,12 +48,18 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
from leap.common.mail import get_email_charset
from leap.keymanager import errors as keymanager_errors
from leap.keymanager.openpgp import OpenPGPKey
+from leap.mail.decorators import deferred
from leap.soledad.client import Soledad
from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
logger = logging.getLogger(__name__)
+MULTIPART_ENCRYPTED = "multipart/encrypted"
+MULTIPART_SIGNED = "multipart/signed"
+PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
+PGP_END = "-----END PGP MESSAGE-----"
+
class MalformedMessage(Exception):
"""
@@ -125,6 +134,9 @@ class LeapIncomingMail(object):
self._create_soledad_indexes()
+ # initialize a mail parser only once
+ self._parser = Parser()
+
def _create_soledad_indexes(self):
"""
Create needed indexes on soledad.
@@ -152,9 +164,10 @@ class LeapIncomingMail(object):
logger.debug("fetching mail for: %s %s" % (
self._soledad.uuid, self._userid))
if not self.fetching_lock.locked():
- d = deferToThread(self._sync_soledad)
- d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error)
- d.addCallbacks(self._process_doclist, self._sync_soledad_error)
+ d1 = self._sync_soledad()
+ d = defer.gatherResults([d1], consumeErrors=True)
+ d.addCallbacks(self._signal_fetch_to_ui, self._errback)
+ d.addCallbacks(self._signal_unread_to_ui, self._errback)
return d
else:
logger.debug("Already fetching mail.")
@@ -184,6 +197,11 @@ class LeapIncomingMail(object):
# synchronize incoming mail
+ def _errback(self, failure):
+ logger.exception(failure.value)
+ traceback.print_tb(*sys.exc_info())
+
+ @deferred
def _sync_soledad(self):
"""
Synchronizes with remote soledad.
@@ -196,10 +214,9 @@ class LeapIncomingMail(object):
self._soledad.sync()
log.msg('soledad synced.')
doclist = self._soledad.get_from_index("just-mail", "*")
+ self._process_doclist(doclist)
- return doclist
-
- def _signal_unread_to_ui(self):
+ def _signal_unread_to_ui(self, *args):
"""
Sends unread event to ui.
"""
@@ -215,53 +232,18 @@ class LeapIncomingMail(object):
:returns: doclist
:rtype: iterable
"""
+ doclist = doclist[0] # gatherResults pass us a list
fetched_ts = time.mktime(time.gmtime())
- num_mails = len(doclist)
- log.msg("there are %s mails" % (num_mails,))
+ num_mails = len(doclist) if doclist is not None else 0
+ if num_mails != 0:
+ log.msg("there are %s mails" % (num_mails,))
leap_events.signal(
IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts))
- self._signal_unread_to_ui()
return doclist
- def _sync_soledad_error(self, failure):
- """
- Errback for sync errors.
- """
- # XXX should signal unrecoverable maybe.
- err = failure.value
- logger.error("error syncing soledad: %s" % (err,))
- if failure.check(ssl.SSLError):
- logger.warning('SSL Error while '
- 'syncing soledad: %r' % (err,))
- elif failure.check(Exception):
- logger.warning('Unknown error while '
- 'syncing soledad: %r' % (err,))
-
- def _log_err(self, failure):
- """
- Generic errback
- """
- err = failure.value
- logger.exception("error!: %r" % (err,))
-
- def _decryption_error(self, failure):
- """
- Errback for decryption errors.
- """
- # XXX should signal unrecoverable maybe.
- err = failure.value
- logger.error("error decrypting msg: %s" % (err,))
-
- def _saving_error(self, failure):
- """
- Errback for local save errors.
- """
- # XXX should signal unrecoverable maybe.
- err = failure.value
- logger.error("error saving msg locally: %s" % (err,))
-
# process incoming mail.
+ @defer.inlineCallbacks
def _process_doclist(self, doclist):
"""
Iterates through the doclist, checks if each doc
@@ -278,7 +260,6 @@ class LeapIncomingMail(object):
return
num_mails = len(doclist)
- docs_cb = []
for index, doc in enumerate(doclist):
logger.debug("processing doc %d of %d" % (index + 1, num_mails))
leap_events.signal(
@@ -287,35 +268,18 @@ class LeapIncomingMail(object):
if self._is_msg(keys):
# Ok, this looks like a legit msg.
# Let's process it!
- # Deferred chain for individual messages
-
- # XXX use an IConsumer instead... ?
- d = deferToThread(self._decrypt_doc, doc)
- d.addCallback(self._process_decrypted_doc)
- d.addErrback(self._log_err)
- d.addCallback(self._add_message_locally)
- d.addErrback(self._log_err)
- docs_cb.append(d)
+ decrypted = list(self._decrypt_doc(doc))[0]
+ res = self._add_message_locally(decrypted)
+ yield res
+
else:
# Ooops, this does not.
logger.debug('This does not look like a proper msg.')
- return docs_cb
#
# operations on individual messages
#
- def _is_msg(self, keys):
- """
- Checks if the keys of a dictionary match the signature
- of the document type we use for messages.
-
- :param keys: iterable containing the strings to match.
- :type keys: iterable of strings.
- :rtype: bool
- """
- return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
-
def _decrypt_doc(self, doc):
"""
Decrypt the contents of a document.
@@ -339,7 +303,9 @@ class LeapIncomingMail(object):
logger.error("Error while decrypting msg: %r" % (exc,))
decrdata = ""
leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0")
- return doc, decrdata
+
+ data = list(self._process_decrypted_doc((doc, decrdata)))
+ yield (doc, data)
def _process_decrypted_doc(self, msgtuple):
"""
@@ -357,16 +323,15 @@ class LeapIncomingMail(object):
doc, data = msgtuple
msg = json.loads(data)
if not isinstance(msg, dict):
- return False
+ defer.returnValue(False)
if not msg.get(self.INCOMING_KEY, False):
- return False
+ defer.returnValue(False)
# ok, this is an incoming message
rawmsg = msg.get(self.CONTENT_KEY, None)
if not rawmsg:
return False
- data = self._maybe_decrypt_msg(rawmsg)
- return doc, data
+ return self._maybe_decrypt_msg(rawmsg)
def _maybe_decrypt_msg(self, data):
"""
@@ -381,17 +346,16 @@ class LeapIncomingMail(object):
leap_assert_type(data, unicode)
# parse the original message
- parser = Parser()
encoding = get_email_charset(data)
data = data.encode(encoding)
- msg = parser.parsestr(data)
+ msg = self._parser.parsestr(data)
# try to obtain sender public key
senderPubkey = None
fromHeader = msg.get('from', None)
- if fromHeader is not None \
- and (msg.get_content_type() == 'multipart/encrypted' \
- or msg.get_content_type() == 'multipart/signed'):
+ if (fromHeader is not None
+ and (msg.get_content_type() == MULTIPART_ENCRYPTED
+ or msg.get_content_type() == MULTIPART_SIGNED)):
_, senderAddress = parseaddr(fromHeader)
try:
senderPubkey = self._keymanager.get_key_from_cache(
@@ -400,11 +364,14 @@ class LeapIncomingMail(object):
pass
valid_sig = False # we will add a header saying if sig is valid
- if msg.get_content_type() == 'multipart/encrypted':
- decrmsg, valid_sig = self._decrypt_multipart_encrypted_msg(
+ decrypt_multi = self._decrypt_multipart_encrypted_msg
+ decrypt_inline = self._maybe_decrypt_inline_encrypted_msg
+
+ if msg.get_content_type() == MULTIPART_ENCRYPTED:
+ decrmsg, valid_sig = decrypt_multi(
msg, encoding, senderPubkey)
else:
- decrmsg, valid_sig = self._maybe_decrypt_inline_encrypted_msg(
+ decrmsg, valid_sig = decrypt_inline(
msg, encoding, senderPubkey)
# add x-leap-signature header
@@ -419,7 +386,7 @@ class LeapIncomingMail(object):
self.LEAP_SIGNATURE_INVALID,
pubkey=senderPubkey.key_id)
- return decrmsg.as_string()
+ yield decrmsg.as_string()
def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey):
"""
@@ -437,25 +404,12 @@ class LeapIncomingMail(object):
"""
log.msg('decrypting multipart encrypted msg')
msg = copy.deepcopy(msg)
- # sanity check
- payload = msg.get_payload()
- if len(payload) != 2:
- raise MalformedMessage(
- 'Multipart/encrypted messages should have exactly 2 body '
- 'parts (instead of %d).' % len(payload))
- if payload[0].get_content_type() != 'application/pgp-encrypted':
- raise MalformedMessage(
- "Multipart/encrypted messages' first body part should "
- "have content type equal to 'application/pgp-encrypted' "
- "(instead of %s)." % payload[0].get_content_type())
- if payload[1].get_content_type() != 'application/octet-stream':
- raise MalformedMessage(
- "Multipart/encrypted messages' second body part should "
- "have content type equal to 'octet-stream' (instead of "
- "%s)." % payload[1].get_content_type())
+ self._msg_multipart_sanity_check(msg)
+
# parse message and get encrypted content
pgpencmsg = msg.get_payload()[1]
encdata = pgpencmsg.get_payload()
+
# decrypt or fail gracefully
try:
decrdata, valid_sig = self._decrypt_and_verify_data(
@@ -463,17 +417,20 @@ class LeapIncomingMail(object):
except keymanager_errors.DecryptError as e:
logger.warning('Failed to decrypt encrypted message (%s). '
'Storing message without modifications.' % str(e))
- return msg, False # return original message
+ # Bailing out!
+ return (msg, False)
+
# decrypted successully, now fix encoding and parse
try:
decrdata = decrdata.encode(encoding)
except (UnicodeEncodeError, UnicodeDecodeError) as e:
logger.error("Unicode error {0}".format(e))
decrdata = decrdata.encode(encoding, 'replace')
- parser = Parser()
- decrmsg = parser.parsestr(decrdata)
+
+ decrmsg = self._parser.parsestr(decrdata)
# remove original message's multipart/encrypted content-type
del(msg['content-type'])
+
# replace headers back in original message
for hkey, hval in decrmsg.items():
try:
@@ -481,9 +438,10 @@ class LeapIncomingMail(object):
msg.replace_header(hkey, hval)
except KeyError:
msg[hkey] = hval
- # replace payload by unencrypted payload
+
+ # all ok, replace payload by unencrypted payload
msg.set_payload(decrmsg.get_payload())
- return msg, valid_sig
+ return (msg, valid_sig)
def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
senderPubkey):
@@ -497,8 +455,9 @@ class LeapIncomingMail(object):
:param senderPubkey: The key of the sender of the message.
:type senderPubkey: OpenPGPKey
- :return: A unitary tuple containing a decrypted message.
- :rtype: (Message)
+ :return: A tuple containing a decrypted message and
+ a bool indicating whether the signature is valid.
+ :rtype: (Message, bool)
"""
log.msg('maybe decrypting inline encrypted msg')
# serialize the original message
@@ -507,8 +466,6 @@ class LeapIncomingMail(object):
g.flatten(origmsg)
data = buf.getvalue()
# handle exactly one inline PGP message
- PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
- PGP_END = "-----END PGP MESSAGE-----"
valid_sig = False
if PGP_BEGIN in data:
begin = data.find(PGP_BEGIN)
@@ -522,11 +479,11 @@ class LeapIncomingMail(object):
except keymanager_errors.DecryptError:
logger.warning('Failed to decrypt potential inline encrypted '
'message. Storing message as is...')
+
# if message is not encrypted, return raw data
if isinstance(data, unicode):
data = data.encode(encoding, 'replace')
- parser = Parser()
- return parser.parsestr(data), valid_sig
+ return (self._parser.parsestr(data), valid_sig)
def _decrypt_and_verify_data(self, data, senderPubkey):
"""
@@ -555,7 +512,7 @@ class LeapIncomingMail(object):
except keymanager_errors.InvalidSignature:
decrdata = self._keymanager.decrypt(
data, self._pkey)
- return decrdata, valid_sig
+ return (decrdata, valid_sig)
def _add_message_locally(self, msgtuple):
"""
@@ -570,10 +527,55 @@ class LeapIncomingMail(object):
"""
log.msg('adding message to local db')
doc, data = msgtuple
- self._inbox.addMessage(data, (self.RECENT_FLAG,))
+
+ if isinstance(data, list):
+ data = data[0]
+
+ self._inbox.addMessage(data, flags=(self.RECENT_FLAG,))
+
leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
doc_id = doc.doc_id
self._soledad.delete_doc(doc)
log.msg("deleted doc %s from incoming" % doc_id)
leap_events.signal(IMAP_MSG_DELETED_INCOMING)
self._signal_unread_to_ui()
+ return True
+
+ #
+ # helpers
+ #
+
+ def _msg_multipart_sanity_check(self, msg):
+ """
+ Performs a sanity check against a multipart encrypted msg
+
+ :param msg: The original encrypted message.
+ :type msg: Message
+ """
+ # sanity check
+ payload = msg.get_payload()
+ if len(payload) != 2:
+ raise MalformedMessage(
+ 'Multipart/encrypted messages should have exactly 2 body '
+ 'parts (instead of %d).' % len(payload))
+ if payload[0].get_content_type() != 'application/pgp-encrypted':
+ raise MalformedMessage(
+ "Multipart/encrypted messages' first body part should "
+ "have content type equal to 'application/pgp-encrypted' "
+ "(instead of %s)." % payload[0].get_content_type())
+ if payload[1].get_content_type() != 'application/octet-stream':
+ raise MalformedMessage(
+ "Multipart/encrypted messages' second body part should "
+ "have content type equal to 'octet-stream' (instead of "
+ "%s)." % payload[1].get_content_type())
+
+ def _is_msg(self, keys):
+ """
+ Checks if the keys of a dictionary match the signature
+ of the document type we use for messages.
+
+ :param keys: iterable containing the strings to match.
+ :type keys: iterable of strings.
+ :rtype: bool
+ """
+ return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
diff --git a/mail/src/leap/mail/imap/fields.py b/mail/src/leap/mail/imap/fields.py
new file mode 100644
index 0000000..2545adf
--- /dev/null
+++ b/mail/src/leap/mail/imap/fields.py
@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+# fields.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Fields for Mailbox and Message.
+"""
+from leap.mail.imap.parser import MBoxParser
+
+
+class WithMsgFields(object):
+ """
+ Container class for class-attributes to be shared by
+ several message-related classes.
+ """
+ # indexing
+ CONTENT_HASH_KEY = "chash"
+ PAYLOAD_HASH_KEY = "phash"
+
+ # Internal representation of Message
+
+ # flags doc
+ UID_KEY = "uid"
+ MBOX_KEY = "mbox"
+ SEEN_KEY = "seen"
+ DEL_KEY = "deleted"
+ RECENT_KEY = "recent"
+ FLAGS_KEY = "flags"
+ MULTIPART_KEY = "multi"
+ SIZE_KEY = "size"
+
+ # headers
+ HEADERS_KEY = "headers"
+ DATE_KEY = "date"
+ SUBJECT_KEY = "subject"
+ # XXX DELETE-ME
+ #NUM_PARTS_KEY = "numparts" # not needed?!
+ PARTS_MAP_KEY = "part_map"
+ BODY_KEY = "body" # link to phash of body
+
+ # content
+ LINKED_FROM_KEY = "lkf"
+ RAW_KEY = "raw"
+ CTYPE_KEY = "ctype"
+
+ # Mailbox specific keys
+ CLOSED_KEY = "closed"
+ CREATED_KEY = "created"
+ SUBSCRIBED_KEY = "subscribed"
+ RW_KEY = "rw"
+ LAST_UID_KEY = "lastuid"
+
+ # Document Type, for indexing
+ TYPE_KEY = "type"
+ TYPE_MBOX_VAL = "mbox"
+ TYPE_FLAGS_VAL = "flags"
+ TYPE_HEADERS_VAL = "head"
+ TYPE_CONTENT_VAL = "cnt"
+
+ # XXX DEPRECATE
+ #TYPE_MESSAGE_VAL = "msg"
+ #TYPE_ATTACHMENT_VAL = "attach"
+
+ INBOX_VAL = "inbox"
+
+ # Flags in Mailbox and Message
+ SEEN_FLAG = "\\Seen"
+ RECENT_FLAG = "\\Recent"
+ ANSWERED_FLAG = "\\Answered"
+ FLAGGED_FLAG = "\\Flagged" # yo dawg
+ DELETED_FLAG = "\\Deleted"
+ DRAFT_FLAG = "\\Draft"
+ NOSELECT_FLAG = "\\Noselect"
+ LIST_FLAG = "List" # is this OK? (no \. ie, no system flag)
+
+ # Fields in mail object
+ SUBJECT_FIELD = "Subject"
+ DATE_FIELD = "Date"
+
+ # Index types
+ # --------------
+
+ TYPE_IDX = 'by-type'
+ TYPE_MBOX_IDX = 'by-type-and-mbox'
+ TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
+ TYPE_SUBS_IDX = 'by-type-and-subscribed'
+ TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+ TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+ TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+ TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+ TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+ TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
+
+ # Tomas created the `recent and seen index`, but the semantic is not too
+ # correct since the recent flag is volatile.
+ TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
+
+ KTYPE = TYPE_KEY
+ MBOX_VAL = TYPE_MBOX_VAL
+ CHASH_VAL = CONTENT_HASH_KEY
+ PHASH_VAL = PAYLOAD_HASH_KEY
+
+ INDEXES = {
+ # generic
+ TYPE_IDX: [KTYPE],
+ TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
+ TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
+
+ # mailboxes
+ TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
+
+ # content, headers doc
+ TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
+
+ # attachment payload dedup
+ TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
+
+ # messages
+ TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
+ TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
+ TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
+ TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
+ 'bool(recent)', 'bool(seen)'],
+ }
+
+ MBOX_KEY = MBOX_VAL
+
+ EMPTY_MBOX = {
+ TYPE_KEY: MBOX_KEY,
+ TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
+ SUBJECT_KEY: "",
+ FLAGS_KEY: [],
+ CLOSED_KEY: False,
+ SUBSCRIBED_KEY: False,
+ RW_KEY: 1,
+ LAST_UID_KEY: 0
+ }
+
+fields = WithMsgFields # alias for convenience
diff --git a/mail/src/leap/mail/imap/index.py b/mail/src/leap/mail/imap/index.py
new file mode 100644
index 0000000..5f0919a
--- /dev/null
+++ b/mail/src/leap/mail/imap/index.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# index.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Index for SoledadBackedAccount, Mailbox and Messages.
+"""
+import logging
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail.imap.fields import fields
+
+
+logger = logging.getLogger(__name__)
+
+
+class IndexedDB(object):
+ """
+ Methods dealing with the index.
+
+ This is a MixIn that needs access to the soledad instance,
+ and also assumes that a INDEXES attribute is accessible to the instance.
+
+ INDEXES must be a dictionary of type:
+ {'index-name': ['field1', 'field2']}
+ """
+ # TODO we might want to move this to soledad itself, check
+
+ def initialize_db(self):
+ """
+ Initialize the database.
+ """
+ leap_assert(self._soledad,
+ "Need a soledad attribute accesible in the instance")
+ leap_assert_type(self.INDEXES, dict)
+
+ # Ask the database for currently existing indexes.
+ if not self._soledad:
+ logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
+ return
+ db_indexes = dict()
+ if self._soledad is not None:
+ db_indexes = dict(self._soledad.list_indexes())
+ for name, expression in fields.INDEXES.items():
+ if name not in db_indexes:
+ # The index does not yet exist.
+ self._soledad.create_index(name, *expression)
+ continue
+
+ if expression == db_indexes[name]:
+ # The index exists and is up to date.
+ continue
+ # The index exists but the definition is not what expected, so we
+ # delete it and add the proper index expression.
+ self._soledad.delete_index(name)
+ self._soledad.create_index(name, *expression)
diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py
new file mode 100644
index 0000000..7c01490
--- /dev/null
+++ b/mail/src/leap/mail/imap/mailbox.py
@@ -0,0 +1,666 @@
+# *- coding: utf-8 -*-
+# mailbox.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Mailbox.
+"""
+import copy
+import threading
+import logging
+import time
+import StringIO
+import cStringIO
+
+from collections import defaultdict
+
+from twisted.internet import defer
+from twisted.python import log
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common import events as leap_events
+from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
+from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.decorators import deferred
+from leap.mail.imap.fields import WithMsgFields, fields
+from leap.mail.imap.messages import MessageCollection
+from leap.mail.imap.parser import MBoxParser
+
+logger = logging.getLogger(__name__)
+
+
+class SoledadMailbox(WithMsgFields, MBoxParser):
+ """
+ A Soledad-backed IMAP mailbox.
+
+ Implements the high-level method needed for the Mailbox interfaces.
+ The low-level database methods are contained in MessageCollection class,
+ which we instantiate and make accessible in the `messages` attribute.
+ """
+ implements(
+ imap4.IMailbox,
+ imap4.IMailboxInfo,
+ imap4.ICloseableMailbox,
+ imap4.IMessageCopier)
+
+ # XXX should finish the implementation of IMailboxListener
+ # XXX should implement ISearchableMailbox too
+
+ messages = None
+ _closed = False
+
+ INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
+ WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
+ WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
+ WithMsgFields.LIST_FLAG)
+ flags = None
+
+ CMD_MSG = "MESSAGES"
+ CMD_RECENT = "RECENT"
+ CMD_UIDNEXT = "UIDNEXT"
+ CMD_UIDVALIDITY = "UIDVALIDITY"
+ CMD_UNSEEN = "UNSEEN"
+
+ _listeners = defaultdict(set)
+ next_uid_lock = threading.Lock()
+
+ def __init__(self, mbox, soledad=None, rw=1):
+ """
+ SoledadMailbox constructor. Needs to get passed a name, plus a
+ Soledad instance.
+
+ :param mbox: the mailbox name
+ :type mbox: str
+
+ :param soledad: a Soledad instance.
+ :type soledad: Soledad
+
+ :param rw: read-and-write flags
+ :type rw: int
+ """
+ leap_assert(mbox, "Need a mailbox name to initialize")
+ leap_assert(soledad, "Need a soledad instance to initialize")
+
+ # XXX should move to wrapper
+ #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
+ #"soledad._db must be an instance of SQLCipherDatabase")
+
+ self.mbox = self._parse_mailbox_name(mbox)
+ self.rw = rw
+
+ self._soledad = soledad
+
+ self.messages = MessageCollection(
+ mbox=mbox, soledad=self._soledad)
+
+ if not self.getFlags():
+ self.setFlags(self.INIT_FLAGS)
+
+ @property
+ def listeners(self):
+ """
+ Returns listeners for this mbox.
+
+ The server itself is a listener to the mailbox.
+ so we can notify it (and should!) after changes in flags
+ and number of messages.
+
+ :rtype: set
+ """
+ return self._listeners[self.mbox]
+
+ def addListener(self, listener):
+ """
+ Adds a listener to the listeners queue.
+ The server adds itself as a listener when there is a SELECT,
+ so it can send EXIST commands.
+
+ :param listener: listener to add
+ :type listener: an object that implements IMailboxListener
+ """
+ logger.debug('adding mailbox listener: %s' % listener)
+ self.listeners.add(listener)
+
+ def removeListener(self, listener):
+ """
+ Removes a listener from the listeners queue.
+
+ :param listener: listener to remove
+ :type listener: an object that implements IMailboxListener
+ """
+ self.listeners.remove(listener)
+
+ def _get_mbox(self):
+ """
+ Returns mailbox document.
+
+ :return: A SoledadDocument containing this mailbox, or None if
+ the query failed.
+ :rtype: SoledadDocument or None.
+ """
+ try:
+ query = self._soledad.get_from_index(
+ fields.TYPE_MBOX_IDX,
+ fields.TYPE_MBOX_VAL, self.mbox)
+ if query:
+ return query.pop()
+ except Exception as exc:
+ logger.error("Unhandled error %r" % exc)
+
+ def getFlags(self):
+ """
+ Returns the flags defined for this mailbox.
+
+ :returns: tuple of flags for this mailbox
+ :rtype: tuple of str
+ """
+ mbox = self._get_mbox()
+ if not mbox:
+ return None
+ flags = mbox.content.get(self.FLAGS_KEY, [])
+ return map(str, flags)
+
+ def setFlags(self, flags):
+ """
+ Sets flags for this mailbox.
+
+ :param flags: a tuple with the flags
+ :type flags: tuple of str
+ """
+ leap_assert(isinstance(flags, tuple),
+ "flags expected to be a tuple")
+ mbox = self._get_mbox()
+ if not mbox:
+ return None
+ mbox.content[self.FLAGS_KEY] = map(str, flags)
+ self._soledad.put_doc(mbox)
+
+ # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
+
+ def _get_closed(self):
+ """
+ Return the closed attribute for this mailbox.
+
+ :return: True if the mailbox is closed
+ :rtype: bool
+ """
+ mbox = self._get_mbox()
+ return mbox.content.get(self.CLOSED_KEY, False)
+
+ def _set_closed(self, closed):
+ """
+ Set the closed attribute for this mailbox.
+
+ :param closed: the state to be set
+ :type closed: bool
+ """
+ leap_assert(isinstance(closed, bool), "closed needs to be boolean")
+ mbox = self._get_mbox()
+ mbox.content[self.CLOSED_KEY] = closed
+ self._soledad.put_doc(mbox)
+
+ closed = property(
+ _get_closed, _set_closed, doc="Closed attribute.")
+
+ def _get_last_uid(self):
+ """
+ Return the last uid for this mailbox.
+
+ :return: the last uid for messages in this mailbox
+ :rtype: bool
+ """
+ mbox = self._get_mbox()
+ return mbox.content.get(self.LAST_UID_KEY, 1)
+
+ def _set_last_uid(self, uid):
+ """
+ Sets the last uid for this mailbox.
+
+ :param uid: the uid to be set
+ :type uid: int
+ """
+ leap_assert(isinstance(uid, int), "uid has to be int")
+ mbox = self._get_mbox()
+ key = self.LAST_UID_KEY
+
+ count = self.getMessageCount()
+
+ # XXX safety-catch. If we do get duplicates,
+ # we want to avoid further duplication.
+
+ if uid >= count:
+ value = uid
+ else:
+ # something is wrong,
+ # just set the last uid
+ # beyond the max msg count.
+ logger.debug("WRONG uid < count. Setting last uid to %s", count)
+ value = count
+
+ mbox.content[key] = value
+ self._soledad.put_doc(mbox)
+
+ last_uid = property(
+ _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
+
+ def getUIDValidity(self):
+ """
+ Return the unique validity identifier for this mailbox.
+
+ :return: unique validity identifier
+ :rtype: int
+ """
+ mbox = self._get_mbox()
+ return mbox.content.get(self.CREATED_KEY, 1)
+
+ def getUID(self, message):
+ """
+ Return the UID of a message in the mailbox
+
+ .. note:: this implementation does not make much sense RIGHT NOW,
+ but in the future will be useful to get absolute UIDs from
+ message sequence numbers.
+
+ :param message: the message uid
+ :type message: int
+
+ :rtype: int
+ """
+ msg = self.messages.get_msg_by_uid(message)
+ return msg.getUID()
+
+ def getUIDNext(self):
+ """
+ Return the likely UID for the next message added to this
+ mailbox. Currently it returns the higher UID incremented by
+ one.
+
+ We increment the next uid *each* time this function gets called.
+ In this way, there will be gaps if the message with the allocated
+ uid cannot be saved. But that is preferable to having race conditions
+ if we get to parallel message adding.
+
+ :rtype: int
+ """
+ with self.next_uid_lock:
+ self.last_uid += 1
+ return self.last_uid
+
+ def getMessageCount(self):
+ """
+ Returns the total count of messages in this mailbox.
+
+ :rtype: int
+ """
+ return self.messages.count()
+
+ def getUnseenCount(self):
+ """
+ Returns the number of messages with the 'Unseen' flag.
+
+ :return: count of messages flagged `unseen`
+ :rtype: int
+ """
+ return self.messages.count_unseen()
+
+ def getRecentCount(self):
+ """
+ Returns the number of messages with the 'Recent' flag.
+
+ :return: count of messages flagged `recent`
+ :rtype: int
+ """
+ return self.messages.count_recent()
+
+ def isWriteable(self):
+ """
+ Get the read/write status of the mailbox.
+
+ :return: 1 if mailbox is read-writeable, 0 otherwise.
+ :rtype: int
+ """
+ return self.rw
+
+ def getHierarchicalDelimiter(self):
+ """
+ Returns the character used to delimite hierarchies in mailboxes.
+
+ :rtype: str
+ """
+ return '/'
+
+ def requestStatus(self, names):
+ """
+ Handles a status request by gathering the output of the different
+ status commands.
+
+ :param names: a list of strings containing the status commands
+ :type names: iter
+ """
+ r = {}
+ if self.CMD_MSG in names:
+ r[self.CMD_MSG] = self.getMessageCount()
+ if self.CMD_RECENT in names:
+ r[self.CMD_RECENT] = self.getRecentCount()
+ if self.CMD_UIDNEXT in names:
+ r[self.CMD_UIDNEXT] = self.last_uid + 1
+ if self.CMD_UIDVALIDITY in names:
+ r[self.CMD_UIDVALIDITY] = self.getUID()
+ if self.CMD_UNSEEN in names:
+ r[self.CMD_UNSEEN] = self.getUnseenCount()
+ return defer.succeed(r)
+
+ def addMessage(self, message, flags, date=None):
+ """
+ Adds a message to this mailbox.
+
+ :param message: the raw message
+ :type message: str
+
+ :param flags: flag list
+ :type flags: list of str
+
+ :param date: timestamp
+ :type date: str
+
+ :return: a deferred that evals to None
+ """
+ if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)):
+ message = message.getvalue()
+ # XXX we should treat the message as an IMessage from here
+ leap_assert_type(message, basestring)
+ uid_next = self.getUIDNext()
+ logger.debug('Adding msg with UID :%s' % uid_next)
+ if flags is None:
+ flags = tuple()
+ else:
+ flags = tuple(str(flag) for flag in flags)
+
+ d = self._do_add_message(message, flags=flags, date=date, uid=uid_next)
+ d.addCallback(self._notify_new)
+ return d
+
+ @deferred
+ def _do_add_message(self, message, flags, date, uid):
+ """
+ Calls to the messageCollection add_msg method (deferred to thread).
+ Invoked from addMessage.
+ """
+ self.messages.add_msg(message, flags=flags, date=date, uid=uid)
+
+ def _notify_new(self, *args):
+ """
+ Notify of new messages to all the listeners.
+
+ :param args: ignored.
+ """
+ exists = self.getMessageCount()
+ recent = self.getRecentCount()
+ logger.debug("NOTIFY: there are %s messages, %s recent" % (
+ exists,
+ recent))
+
+ logger.debug("listeners: %s", str(self.listeners))
+ for l in self.listeners:
+ logger.debug('notifying...')
+ l.newMessages(exists, recent)
+
+ # commands, do not rename methods
+
+ def destroy(self):
+ """
+ Called before this mailbox is permanently deleted.
+
+ Should cleanup resources, and set the \\Noselect flag
+ on the mailbox.
+ """
+ self.setFlags((self.NOSELECT_FLAG,))
+ self.deleteAllDocs()
+
+ # XXX removing the mailbox in situ for now,
+ # we should postpone the removal
+ self._soledad.delete_doc(self._get_mbox())
+
+ def _close_cb(self, result):
+ self.closed = True
+
+ def close(self):
+ """
+ Expunge and mark as closed
+ """
+ d = self.expunge()
+ d.addCallback(self._close_cb)
+ return d
+
+ def _expunge_cb(self, result):
+ return result
+
+ def expunge(self):
+ """
+ Remove all messages flagged \\Deleted
+ """
+ if not self.isWriteable():
+ raise imap4.ReadOnlyMailbox
+ d = self.messages.remove_all_deleted()
+ d.addCallback(self._expunge_cb)
+ return d
+
+ @deferred
+ def fetch(self, messages, uid):
+ """
+ Retrieve one or more messages in this mailbox.
+
+ from rfc 3501: The data items to be fetched can be either a single atom
+ or a parenthesized list.
+
+ :param messages: IDs of the messages to retrieve information about
+ :type messages: MessageSet
+
+ :param uid: If true, the IDs are UIDs. They are message sequence IDs
+ otherwise.
+ :type uid: bool
+
+ :rtype: A tuple of two-tuples of message sequence numbers and
+ LeapMessage
+ """
+ result = []
+
+ # For the moment our UID is sequential, so we
+ # can treat them all the same.
+ # Change this to the flag that twisted expects when we
+ # switch to content-hash based index + local UID table.
+
+ sequence = False
+ #sequence = True if uid == 0 else False
+
+ if not messages.last:
+ try:
+ iter(messages)
+ except TypeError:
+ # looks like we cannot iterate
+ messages.last = self.last_uid
+
+ # for sequence numbers (uid = 0)
+ if sequence:
+ logger.debug("Getting msg by index: INEFFICIENT call!")
+ raise NotImplementedError
+
+ else:
+ for msg_id in messages:
+ msg = self.messages.get_msg_by_uid(msg_id)
+ if msg:
+ result.append((msg_id, msg))
+ else:
+ logger.debug("fetch %s, no msg found!!!" % msg_id)
+
+ if self.isWriteable():
+ self._unset_recent_flag()
+ self._signal_unread_to_ui()
+
+ # XXX workaround for hangs in thunderbird
+ #return tuple(result[:100]) # --- doesn't show all!!
+ return tuple(result)
+
+ @deferred
+ def _unset_recent_flag(self):
+ """
+ Unsets `Recent` flag from a tuple of messages.
+ Called from fetch.
+
+ From RFC, about `Recent`:
+
+ Message is "recently" arrived in this mailbox. This session
+ is the first session to have been notified about this
+ message; if the session is read-write, subsequent sessions
+ will not see \Recent set for this message. This flag can not
+ be altered by the client.
+
+ If it is not possible to determine whether or not this
+ session is the first session to be notified about a message,
+ then that message SHOULD be considered recent.
+ """
+ # TODO this fucker, for the sake of correctness, is messing with
+ # the whole collection of flag docs.
+
+ # Possible ways of action:
+ # 1. Ignore it, we want fun.
+ # 2. Trigger it with a delay
+ # 3. Route it through a queue with lesser priority than the
+ # regularar writer.
+
+ # hmm let's try 2. in a quickndirty way...
+ time.sleep(1)
+ log.msg('unsetting recent flags...')
+ for msg in self.messages.get_recent():
+ msg.removeFlags((fields.RECENT_FLAG,))
+ self._signal_unread_to_ui()
+
+ @deferred
+ def _signal_unread_to_ui(self):
+ """
+ Sends unread event to ui.
+ """
+ unseen = self.getUnseenCount()
+ leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
+
+ @deferred
+ def store(self, messages, flags, mode, uid):
+ """
+ Sets the flags of one or more messages.
+
+ :param messages: The identifiers of the messages to set the flags
+ :type messages: A MessageSet object with the list of messages requested
+
+ :param flags: The flags to set, unset, or add.
+ :type flags: sequence of str
+
+ :param mode: If mode is -1, these flags should be removed from the
+ specified messages. If mode is 1, these flags should be
+ added to the specified messages. If mode is 0, all
+ existing flags should be cleared and these flags should be
+ added.
+ :type mode: -1, 0, or 1
+
+ :param uid: If true, the IDs specified in the query are UIDs;
+ otherwise they are message sequence IDs.
+ :type uid: bool
+
+ :return: A dict mapping message sequence numbers to sequences of
+ str representing the flags set on the message after this
+ operation has been performed.
+ :rtype: dict
+
+ :raise ReadOnlyMailbox: Raised if this mailbox is not open for
+ read-write.
+ """
+ # XXX implement also sequence (uid = 0)
+ # XXX we should prevent cclient from setting Recent flag.
+ leap_assert(not isinstance(flags, basestring),
+ "flags cannot be a string")
+ flags = tuple(flags)
+
+ if not self.isWriteable():
+ log.msg('read only mailbox!')
+ raise imap4.ReadOnlyMailbox
+
+ if not messages.last:
+ messages.last = self.messages.count()
+
+ result = {}
+ for msg_id in messages:
+ log.msg("MSG ID = %s" % msg_id)
+ msg = self.messages.get_msg_by_uid(msg_id)
+ if not msg:
+ return result
+ if mode == 1:
+ msg.addFlags(flags)
+ elif mode == -1:
+ msg.removeFlags(flags)
+ elif mode == 0:
+ msg.setFlags(flags)
+ result[msg_id] = msg.getFlags()
+
+ self._signal_unread_to_ui()
+ return result
+
+ # IMessageCopier
+
+ @deferred
+ def copy(self, messageObject):
+ """
+ Copy the given message object into this mailbox.
+ """
+ uid_next = self.getUIDNext()
+ msg = messageObject
+
+ # XXX should use a public api instead
+ fdoc = msg._fdoc
+ if not fdoc:
+ logger.debug("Tried to copy a MSG with no fdoc")
+ return
+
+ new_fdoc = copy.deepcopy(fdoc.content)
+ new_fdoc[self.UID_KEY] = uid_next
+ new_fdoc[self.MBOX_KEY] = self.mbox
+
+ d = self._do_add_doc(new_fdoc)
+ d.addCallback(self._notify_new)
+
+ @deferred
+ def _do_add_doc(self, doc):
+ """
+ Defers the adding of a new doc.
+ :param doc: document to be created in soledad.
+ """
+ self._soledad.create_doc(doc)
+
+ # convenience fun
+
+ def deleteAllDocs(self):
+ """
+ Deletes all docs in this mailbox
+ """
+ docs = self.messages.get_all_docs()
+ for doc in docs:
+ self.messages._soledad.delete_doc(doc)
+
+ def __repr__(self):
+ """
+ Representation string for this mailbox.
+ """
+ return u"<SoledadMailbox: mbox '%s' (%s)>" % (
+ self.mbox, self.messages.count())
diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py
new file mode 100644
index 0000000..37e4311
--- /dev/null
+++ b/mail/src/leap/mail/imap/messages.py
@@ -0,0 +1,1346 @@
+# -*- coding: utf-8 -*-
+# messages.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+LeapMessage and MessageCollection.
+"""
+import copy
+import logging
+import StringIO
+
+from collections import defaultdict, namedtuple
+
+from twisted.mail import imap4
+from twisted.internet import defer
+from twisted.python import log
+from u1db import errors as u1db_errors
+from zope.interface import implements
+from zope.proxy import sameProxiedObjects
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.decorators import memoized_method
+from leap.common.mail import get_email_charset
+from leap.mail import walk
+from leap.mail.utils import first
+from leap.mail.decorators import deferred
+from leap.mail.imap.index import IndexedDB
+from leap.mail.imap.fields import fields, WithMsgFields
+from leap.mail.imap.parser import MailParser, MBoxParser
+from leap.mail.messageflow import IMessageConsumer, MessageProducer
+
+logger = logging.getLogger(__name__)
+
+
+# TODO ------------------------------------------------------------
+
+# [ ] Add linked-from info.
+# [ ] Delete incoming mail only after successful write!
+# [ ] Remove UID from syncable db. Store only those indexes locally.
+# [ ] Send patch to twisted for bug in imap4.py:5717 (content-type can be
+# none? lower-case?)
+
+def lowerdict(_dict):
+ """
+ Return a dict with the keys in lowercase.
+
+ :param _dict: the dict to convert
+ :rtype: dict
+ """
+ return dict((key.lower(), value)
+ for key, value in _dict.items())
+
+
+class MessagePart(object):
+ """
+ IMessagePart implementor.
+ It takes a subpart message and is able to find
+ the inner parts.
+
+ Excusatio non petita: see the interface documentation.
+ """
+
+ implements(imap4.IMessagePart)
+
+ def __init__(self, soledad, part_map):
+ """
+ Initializes the MessagePart.
+
+ :param part_map: a dictionary containing the parts map for this
+ message
+ :type part_map: dict
+ """
+ # TODO
+ # It would be good to pass the uid/mailbox also
+ # for references while debugging.
+
+ # We have a problem on bulk moves, and is
+ # that when the fetch on the new mailbox is done
+ # the parts maybe are not complete.
+ # So we should be able to fail with empty
+ # docs until we solve that. The ideal would be
+ # to gather the results of the deferred operations
+ # to signal the operation is complete.
+ #leap_assert(part_map, "part map dict cannot be null")
+ self._soledad = soledad
+ self._pmap = part_map
+
+ def getSize(self):
+ """
+ Return the total size, in octets, of this message part.
+
+ :return: size of the message, in octets
+ :rtype: int
+ """
+ if not self._pmap:
+ return 0
+ size = self._pmap.get('size', None)
+ if not size:
+ logger.error("Message part cannot find size in the partmap")
+ return size
+
+ def getBodyFile(self):
+ """
+ Retrieve a file object containing only the body of this message.
+
+ :return: file-like object opened for reading
+ :rtype: StringIO
+ """
+ fd = StringIO.StringIO()
+ if self._pmap:
+ multi = self._pmap.get('multi')
+ if not multi:
+ phash = self._pmap.get("phash", None)
+ else:
+ pmap = self._pmap.get('part_map')
+ first_part = pmap.get('1', None)
+ if first_part:
+ phash = first_part['phash']
+
+ if not phash:
+ logger.warning("Could not find phash for this subpart!")
+ payload = str("")
+ else:
+ payload = self._get_payload_from_document(phash)
+
+ else:
+ logger.warning("Message with no part_map!")
+ payload = str("")
+
+ if payload:
+ #headers = self.getHeaders(True)
+ #headers = lowerdict(headers)
+ #content_type = headers.get('content-type', "")
+ content_type = self._get_ctype_from_document(phash)
+ charset_split = content_type.split('charset=')
+ # XXX fuck all this, use a regex!
+ if len(charset_split) > 1:
+ charset = charset_split[1]
+ if charset:
+ charset = charset.strip()
+ else:
+ charset = None
+ if not charset:
+ charset = self._get_charset(payload)
+ try:
+ payload = payload.encode(charset)
+ except (UnicodeEncodeError, UnicodeDecodeError) as e:
+ logger.error("Unicode error {0}".format(e))
+ payload = payload.encode(charset, 'replace')
+
+ fd.write(payload)
+ fd.seek(0)
+ return fd
+
+ # TODO cache the phash retrieval
+ def _get_payload_from_document(self, phash):
+ """
+ Gets the message payload from the content document.
+
+ :param phash: the payload hash to retrieve by.
+ :type phash: basestring
+ """
+ cdocs = self._soledad.get_from_index(
+ fields.TYPE_P_HASH_IDX,
+ fields.TYPE_CONTENT_VAL, str(phash))
+
+ cdoc = first(cdocs)
+ if not cdoc:
+ logger.warning(
+ "Could not find the content doc "
+ "for phash %s" % (phash,))
+ payload = cdoc.content.get(fields.RAW_KEY, "")
+ return payload
+
+ # TODO cache the pahash retrieval
+ def _get_ctype_from_document(self, phash):
+ """
+ Gets the content-type from the content document.
+
+ :param phash: the payload hash to retrieve by.
+ :type phash: basestring
+ """
+ cdocs = self._soledad.get_from_index(
+ fields.TYPE_P_HASH_IDX,
+ fields.TYPE_CONTENT_VAL, str(phash))
+
+ cdoc = first(cdocs)
+ if not cdoc:
+ logger.warning(
+ "Could not find the content doc "
+ "for phash %s" % (phash,))
+ ctype = cdoc.content.get('ctype', "")
+ return ctype
+
+ @memoized_method
+ def _get_charset(self, stuff):
+ # TODO put in a common class with LeapMessage
+ """
+ Gets (guesses?) the charset of a payload.
+
+ :param stuff: the stuff to guess about.
+ :type stuff: basestring
+ :returns: charset
+ """
+ # XXX existential doubt 2. shouldn't we make the scope
+ # of the decorator somewhat more persistent?
+ # ah! yes! and put memory bounds.
+ return get_email_charset(unicode(stuff))
+
+ def getHeaders(self, negate, *names):
+ """
+ Retrieve a group of message headers.
+
+ :param names: The names of the headers to retrieve or omit.
+ :type names: tuple of str
+
+ :param negate: If True, indicates that the headers listed in names
+ should be omitted from the return value, rather
+ than included.
+ :type negate: bool
+
+ :return: A mapping of header field names to header field values
+ :rtype: dict
+ """
+ if not self._pmap:
+ logger.warning("No pmap in Subpart!")
+ return {}
+ headers = dict(self._pmap.get("headers", []))
+
+ # twisted imap server expects *some* headers to be lowercase
+ # We could use a CaseInsensitiveDict here...
+ headers = dict(
+ (str(key), str(value)) if key.lower() != "content-type"
+ else (str(key.lower()), str(value))
+ for (key, value) in headers.items())
+
+ names = map(lambda s: s.upper(), names)
+ if negate:
+ cond = lambda key: key.upper() not in names
+ else:
+ cond = lambda key: key.upper() in names
+
+ # unpack and filter original dict by negate-condition
+ filter_by_cond = [
+ map(str, (key, val)) for
+ key, val in headers.items()
+ if cond(key)]
+ filtered = dict(filter_by_cond)
+ return filtered
+
+ def isMultipart(self):
+ """
+ Return True if this message is multipart.
+ """
+ if not self._pmap:
+ logger.warning("Could not get part map!")
+ return False
+ multi = self._pmap.get("multi", False)
+ return multi
+
+ def getSubPart(self, part):
+ """
+ Retrieve a MIME submessage
+
+ :type part: C{int}
+ :param part: The number of the part to retrieve, indexed from 0.
+ :raise IndexError: Raised if the specified part does not exist.
+ :raise TypeError: Raised if this message is not multipart.
+ :rtype: Any object implementing C{IMessagePart}.
+ :return: The specified sub-part.
+ """
+ if not self.isMultipart():
+ raise TypeError
+ sub_pmap = self._pmap.get("part_map", {})
+ try:
+ part_map = sub_pmap[str(part + 1)]
+ except KeyError:
+ logger.debug("getSubpart for %s: KeyError" % (part,))
+ raise IndexError
+
+ # XXX check for validity
+ return MessagePart(self._soledad, part_map)
+
+
+class LeapMessage(fields, MailParser, MBoxParser):
+ """
+ The main representation of a message.
+
+ It indexes the messages in one mailbox by a combination
+ of uid+mailbox name.
+ """
+
+ # TODO this has to change.
+ # Should index primarily by chash, and keep a local-lonly
+ # UID table.
+
+ implements(imap4.IMessage)
+
+ def __init__(self, soledad, uid, mbox):
+ """
+ Initializes a LeapMessage.
+
+ :param soledad: a Soledad instance
+ :type soledad: Soledad
+ :param uid: the UID for the message.
+ :type uid: int or basestring
+ :param mbox: the mbox this message belongs to
+ :type mbox: basestring
+ """
+ MailParser.__init__(self)
+ self._soledad = soledad
+ self._uid = int(uid)
+ self._mbox = self._parse_mailbox_name(mbox)
+
+ self.__chash = None
+ self.__bdoc = None
+
+ @property
+ def _fdoc(self):
+ """
+ An accessor to the flags document.
+ """
+ if all(map(bool, (self._uid, self._mbox))):
+ fdoc = self._get_flags_doc()
+ if fdoc:
+ self.__chash = fdoc.content.get(
+ fields.CONTENT_HASH_KEY, None)
+ return fdoc
+
+ @property
+ def _chash(self):
+ """
+ An accessor to the content hash for this message.
+ """
+ if not self._fdoc:
+ return None
+ if not self.__chash and self._fdoc:
+ self.__chash = self._fdoc.content.get(
+ fields.CONTENT_HASH_KEY, None)
+ return self.__chash
+
+ @property
+ def _hdoc(self):
+ """
+ An accessor to the headers document.
+ """
+ return self._get_headers_doc()
+
+ @property
+ def _bdoc(self):
+ """
+ An accessor to the body document.
+ """
+ if not self._hdoc:
+ return None
+ if not self.__bdoc:
+ self.__bdoc = self._get_body_doc()
+ return self.__bdoc
+
+ # IMessage implementation
+
+ def getUID(self):
+ """
+ Retrieve the unique identifier associated with this message
+
+ :return: uid for this message
+ :rtype: int
+ """
+ return self._uid
+
+ def getFlags(self):
+ """
+ Retrieve the flags associated with this message
+
+ :return: The flags, represented as strings
+ :rtype: tuple
+ """
+ if self._uid is None:
+ return []
+
+ flags = []
+ fdoc = self._fdoc
+ if fdoc:
+ flags = fdoc.content.get(self.FLAGS_KEY, None)
+ if flags:
+ flags = map(str, flags)
+ return tuple(flags)
+
+ # setFlags, addFlags, removeFlags are not in the interface spec
+ # but we use them with store command.
+
+ def setFlags(self, flags):
+ """
+ Sets the flags for this message
+
+ Returns a SoledadDocument that needs to be updated by the caller.
+
+ :param flags: the flags to update in the message.
+ :type flags: tuple of str
+
+ :return: a SoledadDocument instance
+ :rtype: SoledadDocument
+ """
+ leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+ log.msg('setting flags: %s' % (self._uid))
+
+ doc = self._fdoc
+ if not doc:
+ logger.warning(
+ "Could not find FDOC for %s:%s while setting flags!" %
+ (self._mbox, self._uid))
+ return
+ doc.content[self.FLAGS_KEY] = flags
+ doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
+ doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
+ doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags
+ self._soledad.put_doc(doc)
+
+ def addFlags(self, flags):
+ """
+ Adds flags to this message.
+
+ Returns a SoledadDocument that needs to be updated by the caller.
+
+ :param flags: the flags to add to the message.
+ :type flags: tuple of str
+
+ :return: a SoledadDocument instance
+ :rtype: SoledadDocument
+ """
+ leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+ oldflags = self.getFlags()
+ self.setFlags(tuple(set(flags + oldflags)))
+
+ def removeFlags(self, flags):
+ """
+ Remove flags from this message.
+
+ Returns a SoledadDocument that needs to be updated by the caller.
+
+ :param flags: the flags to be removed from the message.
+ :type flags: tuple of str
+
+ :return: a SoledadDocument instance
+ :rtype: SoledadDocument
+ """
+ leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
+ oldflags = self.getFlags()
+ self.setFlags(tuple(set(oldflags) - set(flags)))
+
+ def getInternalDate(self):
+ """
+ Retrieve the date internally associated with this message
+
+ :rtype: C{str}
+ :return: An RFC822-formatted date string.
+ """
+ return str(self._hdoc.content.get(self.DATE_KEY, ''))
+
+ #
+ # IMessagePart
+ #
+
+ # XXX we should implement this interface too for the subparts
+ # so we allow nested parts...
+
+ def getBodyFile(self):
+ """
+ Retrieve a file object containing only the body of this message.
+
+ :return: file-like object opened for reading
+ :rtype: StringIO
+ """
+ fd = StringIO.StringIO()
+ bdoc = self._bdoc
+ if bdoc:
+ body = str(self._bdoc.content.get(self.RAW_KEY, ""))
+ else:
+ logger.warning("No BDOC found for message.")
+ body = str("")
+
+ # XXX not needed, isn't it? ---- ivan?
+ #if bdoc:
+ #content_type = bdoc.content.get('content-type', "")
+ #charset = content_type.split('charset=')[1]
+ #if charset:
+ #charset = charset.strip()
+ #if not charset:
+ #charset = self._get_charset(body)
+ #try:
+ #body = str(body.encode(charset))
+ #except (UnicodeEncodeError, UnicodeDecodeError) as e:
+ #logger.error("Unicode error {0}".format(e))
+ #body = str(body.encode(charset, 'replace'))
+
+ fd.write(body)
+ fd.seek(0)
+ return fd
+
+ @memoized_method
+ def _get_charset(self, stuff):
+ """
+ Gets (guesses?) the charset of a payload.
+
+ :param stuff: the stuff to guess about.
+ :type stuff: basestring
+ :returns: charset
+ """
+ # TODO get from subpart headers
+ # XXX existential doubt 2. shouldn't we make the scope
+ # of the decorator somewhat more persistent?
+ # ah! yes! and put memory bounds.
+ return get_email_charset(unicode(stuff))
+
+ def getSize(self):
+ """
+ Return the total size, in octets, of this message.
+
+ :return: size of the message, in octets
+ :rtype: int
+ """
+ size = None
+ if self._fdoc:
+ size = self._fdoc.content.get(self.SIZE_KEY, False)
+ else:
+ logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
+ self._uid))
+ if not size:
+ # XXX fallback, should remove when all migrated.
+ size = self.getBodyFile().len
+ return size
+
+ def getHeaders(self, negate, *names):
+ """
+ Retrieve a group of message headers.
+
+ :param names: The names of the headers to retrieve or omit.
+ :type names: tuple of str
+
+ :param negate: If True, indicates that the headers listed in names
+ should be omitted from the return value, rather
+ than included.
+ :type negate: bool
+
+ :return: A mapping of header field names to header field values
+ :rtype: dict
+ """
+ # TODO split in smaller methods
+ headers = self._get_headers()
+ if not headers:
+ logger.warning("No headers found")
+ return {str('content-type'): str('')}
+
+ names = map(lambda s: s.upper(), names)
+ if negate:
+ cond = lambda key: key.upper() not in names
+ else:
+ cond = lambda key: key.upper() in names
+
+ if isinstance(headers, list):
+ headers = dict(headers)
+
+ # twisted imap server expects *some* headers to be lowercase
+ # XXX refactor together with MessagePart method
+ headers = dict(
+ (str(key), str(value)) if key.lower() != "content-type"
+ else (str(key.lower()), str(value))
+ for (key, value) in headers.items())
+
+ # unpack and filter original dict by negate-condition
+ filter_by_cond = [(key, val) for key, val
+ in headers.items() if cond(key)]
+
+ return dict(filter_by_cond)
+
+ def _get_headers(self):
+ """
+ Return the headers dict for this message.
+ """
+ if self._hdoc is not None:
+ headers = self._hdoc.content.get(self.HEADERS_KEY, {})
+ return headers
+
+ else:
+ logger.warning(
+ "No HEADERS doc for msg %s:%s" % (
+ self._mbox,
+ self._uid))
+
+ def isMultipart(self):
+ """
+ Return True if this message is multipart.
+ """
+ if self._fdoc:
+ is_multipart = self._fdoc.content.get(self.MULTIPART_KEY, False)
+ return is_multipart
+ else:
+ logger.warning(
+ "No FLAGS doc for msg %s:%s" % (
+ self._mbox,
+ self._uid))
+
+ def getSubPart(self, part):
+ """
+ Retrieve a MIME submessage
+
+ :type part: C{int}
+ :param part: The number of the part to retrieve, indexed from 0.
+ :raise IndexError: Raised if the specified part does not exist.
+ :raise TypeError: Raised if this message is not multipart.
+ :rtype: Any object implementing C{IMessagePart}.
+ :return: The specified sub-part.
+ """
+ if not self.isMultipart():
+ raise TypeError
+ try:
+ pmap_dict = self._get_part_from_parts_map(part + 1)
+ except KeyError:
+ logger.debug("getSubpart for %s: KeyError" % (part,))
+ raise IndexError
+ return MessagePart(self._soledad, pmap_dict)
+
+ #
+ # accessors
+ #
+
+ def _get_part_from_parts_map(self, part):
+ """
+ Get a part map from the headers doc
+
+ :raises: KeyError if key does not exist
+ :rtype: dict
+ """
+ if not self._hdoc:
+ logger.warning("Tried to get part but no HDOC found!")
+ return None
+
+ pmap = self._hdoc.content.get(fields.PARTS_MAP_KEY, {})
+ return pmap[str(part)]
+
+ def _get_flags_doc(self):
+ """
+ Return the document that keeps the flags for this
+ message.
+ """
+ flag_docs = self._soledad.get_from_index(
+ fields.TYPE_MBOX_UID_IDX,
+ fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+ return first(flag_docs)
+
+ def _get_headers_doc(self):
+ """
+ Return the document that keeps the headers for this
+ message.
+ """
+ head_docs = self._soledad.get_from_index(
+ fields.TYPE_C_HASH_IDX,
+ fields.TYPE_HEADERS_VAL, str(self._chash))
+ return first(head_docs)
+
+ def _get_body_doc(self):
+ """
+ Return the document that keeps the body for this
+ message.
+ """
+ body_phash = self._hdoc.content.get(
+ fields.BODY_KEY, None)
+ if not body_phash:
+ logger.warning("No body phash for this document!")
+ return None
+ body_docs = self._soledad.get_from_index(
+ fields.TYPE_P_HASH_IDX,
+ fields.TYPE_CONTENT_VAL, str(body_phash))
+
+ return first(body_docs)
+
+ def __getitem__(self, key):
+ """
+ Return an item from the content of the flags document,
+ for convenience.
+
+ :param key: The key
+ :type key: str
+
+ :return: The content value indexed by C{key} or None
+ :rtype: str
+ """
+ return self._fdoc.content.get(key, None)
+
+ # setters
+
+ # XXX to be used in the messagecopier interface?!
+
+ def set_uid(self, uid):
+ """
+ Set new uid for this message.
+
+ :param uid: the new uid
+ :type uid: basestring
+ """
+ # XXX dangerous! lock?
+ self._uid = uid
+ d = self._fdoc
+ d.content[self.UID_KEY] = uid
+ self._soledad.put_doc(d)
+
+ def set_mbox(self, mbox):
+ """
+ Set new mbox for this message.
+
+ :param mbox: the new mbox
+ :type mbox: basestring
+ """
+ # XXX dangerous! lock?
+ self._mbox = mbox
+ d = self._fdoc
+ d.content[self.MBOX_KEY] = mbox
+ self._soledad.put_doc(d)
+
+ # destructor
+
+ @deferred
+ def remove(self):
+ """
+ Remove all docs associated with this message.
+ """
+ # XXX For the moment we are only removing the flags and headers
+ # docs. The rest we leave there polluting your hard disk,
+ # until we think about a good way of deorphaning.
+ # Maybe a crawler of unreferenced docs.
+
+ # XXX implement elijah's idea of using a PUT document as a
+ # token to ensure consistency in the removal.
+
+ uid = self._uid
+
+ fd = self._get_flags_doc()
+ #hd = self._get_headers_doc()
+ #bd = self._get_body_doc()
+ #docs = [fd, hd, bd]
+
+ docs = [fd]
+
+ for d in filter(None, docs):
+ try:
+ self._soledad.delete_doc(d)
+ except Exception as exc:
+ logger.error(exc)
+ return uid
+
+ def does_exist(self):
+ """
+ Return True if there is actually a flags message for this
+ UID and mbox.
+ """
+ return self._fdoc is not None
+
+
+SoledadWriterPayload = namedtuple(
+ 'SoledadWriterPayload', ['mode', 'payload'])
+
+# TODO we could consider using enum here:
+# https://pypi.python.org/pypi/enum
+
+SoledadWriterPayload.CREATE = 1
+SoledadWriterPayload.PUT = 2
+SoledadWriterPayload.CONTENT_CREATE = 3
+
+
+class SoledadDocWriter(object):
+ """
+ This writer will create docs serially in the local soledad database.
+ """
+
+ implements(IMessageConsumer)
+
+ def __init__(self, soledad):
+ """
+ Initialize the writer.
+
+ :param soledad: the soledad instance
+ :type soledad: Soledad
+ """
+ self._soledad = soledad
+
+ def _get_call_for_item(self, item):
+ """
+ Return the proper call type for a given item.
+
+ :param item: one of the types defined under the
+ attributes of SoledadWriterPayload
+ :type item: int
+ """
+ call = None
+ payload = item.payload
+
+ if item.mode == SoledadWriterPayload.CREATE:
+ call = self._soledad.create_doc
+ elif (item.mode == SoledadWriterPayload.CONTENT_CREATE
+ and not self._content_does_exist(payload)):
+ call = self._soledad.create_doc
+ elif item.mode == SoledadWriterPayload.PUT:
+ call = self._soledad.put_doc
+ return call
+
+ def _process(self, queue):
+ """
+ Return the item and the proper call type for the next
+ item in the queue if any.
+
+ :param queue: the queue from where we'll pick item.
+ :type queue: Queue
+ """
+ item = queue.get()
+ call = self._get_call_for_item(item)
+ return item, call
+
+ def consume(self, queue):
+ """
+ Creates a new document in soledad db.
+
+ :param queue: queue to get item from, with content of the document
+ to be inserted.
+ :type queue: Queue
+ """
+ empty = queue.empty()
+ while not empty:
+ item, call = self._process(queue)
+
+ if call:
+ # XXX should handle the delete case
+ # should handle errors
+ try:
+ call(item.payload)
+ except u1db_errors.RevisionConflict as exc:
+ logger.error("Error: %r" % (exc,))
+ raise exc
+
+ empty = queue.empty()
+
+ """
+ Message deduplication.
+
+ We do a query for the content hashes before writing to our beloved
+ sqlcipher backend of Soledad. This means, by now, that:
+
+ 1. We will not store the same attachment twice, only the hash of it.
+ 2. We will not store the same message body twice, only the hash of it.
+
+ The first case is useful if you are always receiving the same old memes
+ from unwary friends that still have not discovered that 4chan is the
+ generator of the internet. The second will save your day if you have
+ initiated session with the same account in two different machines. I also
+ wonder why would you do that, but let's respect each other choices, like
+ with the religious celebrations, and assume that one day we'll be able
+ to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
+ Stack.
+ """
+
+ def _content_does_exist(self, doc):
+ """
+ Check whether we already have a content document for a payload
+ with this hash in our database.
+
+ :param doc: tentative body document
+ :type doc: dict
+ :returns: True if that happens, False otherwise.
+ """
+ if not doc:
+ return False
+ phash = doc[fields.PAYLOAD_HASH_KEY]
+ attach_docs = self._soledad.get_from_index(
+ fields.TYPE_P_HASH_IDX,
+ fields.TYPE_CONTENT_VAL, str(phash))
+ if not attach_docs:
+ return False
+
+ if len(attach_docs) != 1:
+ logger.warning("Found more than one copy of phash %s!"
+ % (phash,))
+ logger.debug("Found attachment doc with that hash! Skipping save!")
+ return True
+
+
+class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
+ """
+ A collection of messages, surprisingly.
+
+ It is tied to a selected mailbox name that is passed to constructor.
+ Implements a filter query over the messages contained in a soledad
+ database.
+ """
+ # XXX this should be able to produce a MessageSet methinks
+ # could validate these kinds of objects turning them
+ # into a template for the class.
+ FLAGS_DOC = "FLAGS"
+ HEADERS_DOC = "HEADERS"
+ CONTENT_DOC = "CONTENT"
+
+ templates = {
+
+ FLAGS_DOC: {
+ fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
+ fields.UID_KEY: 1, # XXX moe to a local table
+ fields.MBOX_KEY: fields.INBOX_VAL,
+ fields.CONTENT_HASH_KEY: "",
+
+ fields.SEEN_KEY: False,
+ fields.RECENT_KEY: True,
+ fields.DEL_KEY: False,
+ fields.FLAGS_KEY: [],
+ fields.MULTIPART_KEY: False,
+ fields.SIZE_KEY: 0
+ },
+
+ HEADERS_DOC: {
+ fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
+ fields.CONTENT_HASH_KEY: "",
+
+ fields.DATE_KEY: "",
+ fields.SUBJECT_KEY: "",
+
+ fields.HEADERS_KEY: {},
+ fields.PARTS_MAP_KEY: {},
+ },
+
+ CONTENT_DOC: {
+ fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
+ fields.PAYLOAD_HASH_KEY: "",
+ fields.LINKED_FROM_KEY: [],
+ fields.CTYPE_KEY: "", # should index by this too
+
+ # should only get inmutable headers parts
+ # (for indexing)
+ fields.HEADERS_KEY: {},
+ fields.RAW_KEY: "",
+ fields.PARTS_MAP_KEY: {},
+ fields.HEADERS_KEY: {},
+ fields.MULTIPART_KEY: False,
+ },
+
+ }
+
+ def __init__(self, mbox=None, soledad=None):
+ """
+ Constructor for MessageCollection.
+
+ :param mbox: the name of the mailbox. It is the name
+ with which we filter the query over the
+ messages database
+ :type mbox: str
+
+ :param soledad: Soledad database
+ :type soledad: Soledad instance
+ """
+ MailParser.__init__(self)
+ leap_assert(mbox, "Need a mailbox name to initialize")
+ leap_assert(mbox.strip() != "", "mbox cannot be blank space")
+ leap_assert(isinstance(mbox, (str, unicode)),
+ "mbox needs to be a string")
+ leap_assert(soledad, "Need a soledad instance to initialize")
+
+ # okay, all in order, keep going...
+ self.mbox = self._parse_mailbox_name(mbox)
+ self._soledad = soledad
+ self.initialize_db()
+
+ # I think of someone like nietzsche when reading this
+
+ # this will be the producer that will enqueue the content
+ # to be processed serially by the consumer (the writer). We just
+ # need to `put` the new material on its plate.
+
+ self.soledad_writer = MessageProducer(
+ SoledadDocWriter(soledad),
+ period=0.02)
+
+ def _get_empty_doc(self, _type=FLAGS_DOC):
+ """
+ Returns an empty doc for storing different message parts.
+ Defaults to returning a template for a flags document.
+ :return: a dict with the template
+ :rtype: dict
+ """
+ if not _type in self.templates.keys():
+ raise TypeError("Improper type passed to _get_empty_doc")
+ return copy.deepcopy(self.templates[_type])
+
+ def _do_parse(self, raw):
+ """
+ Parse raw message and return it along with
+ relevant information about its outer level.
+
+ :param raw: the raw message
+ :type raw: StringIO or basestring
+ :return: msg, chash, size, multi
+ :rtype: tuple
+ """
+ msg = self._get_parsed_msg(raw)
+ chash = self._get_hash(msg)
+ size = len(msg.as_string())
+ multi = msg.is_multipart()
+ return msg, chash, size, multi
+
+ def _populate_flags(self, flags, uid, chash, size, multi):
+ """
+ Return a flags doc.
+
+ XXX Missing DOC -----------
+ """
+ fd = self._get_empty_doc(self.FLAGS_DOC)
+
+ fd[self.MBOX_KEY] = self.mbox
+ fd[self.UID_KEY] = uid
+ fd[self.CONTENT_HASH_KEY] = chash
+ fd[self.SIZE_KEY] = size
+ fd[self.MULTIPART_KEY] = multi
+ if flags:
+ fd[self.FLAGS_KEY] = map(self._stringify, flags)
+ fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
+ fd[self.DEL_KEY] = self.DELETED_FLAG in flags
+ fd[self.RECENT_KEY] = True # set always by default
+ return fd
+
+ def _populate_headr(self, msg, chash, subject, date):
+ """
+ Return a headers doc.
+
+ XXX Missing DOC -----------
+ """
+ headers = defaultdict(list)
+ for k, v in msg.items():
+ headers[k].append(v)
+
+ # "fix" for repeated headers.
+ for k, v in headers.items():
+ newline = "\n%s: " % (k,)
+ headers[k] = newline.join(v)
+
+ hd = self._get_empty_doc(self.HEADERS_DOC)
+ hd[self.CONTENT_HASH_KEY] = chash
+ hd[self.HEADERS_KEY] = headers
+
+ if not subject and self.SUBJECT_FIELD in headers:
+ hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD])
+ else:
+ hd[self.SUBJECT_KEY] = subject
+
+ if not date and self.DATE_FIELD in headers:
+ hd[self.DATE_KEY] = first(headers[self.DATE_FIELD])
+ else:
+ hd[self.DATE_KEY] = date
+ return hd
+
+ @deferred
+ def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
+ """
+ Creates a new message document.
+
+ :param raw: the raw message
+ :type raw: str
+
+ :param subject: subject of the message.
+ :type subject: str
+
+ :param flags: flags
+ :type flags: list
+
+ :param date: the received date for the message
+ :type date: str
+
+ :param uid: the message uid for this mailbox
+ :type uid: int
+ """
+ # TODO signal that we can delete the original message!-----
+ # when all the processing is done.
+
+ # TODO add the linked-from info !
+
+ logger.debug('adding message')
+ if flags is None:
+ flags = tuple()
+ leap_assert_type(flags, tuple)
+
+ # parse
+ msg, chash, size, multi = self._do_parse(raw)
+
+ fd = self._populate_flags(flags, uid, chash, size, multi)
+ hd = self._populate_headr(msg, chash, subject, date)
+
+ parts = walk.get_parts(msg)
+ body_phash_fun = [walk.get_body_phash_simple,
+ walk.get_body_phash_multi][int(multi)]
+ body_phash = body_phash_fun(walk.get_payloads(msg))
+ parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+
+ # add parts map to header doc
+ # (body, multi, part_map)
+ for key in parts_map:
+ hd[key] = parts_map[key]
+ del parts_map
+
+ docs = [fd, hd]
+ cdocs = walk.get_raw_docs(msg, parts)
+
+ # Saving
+ logger.debug('enqueuing message docs for write')
+ ptuple = SoledadWriterPayload
+
+ # first, regular docs: flags and headers
+ for doc in docs:
+ self.soledad_writer.put(ptuple(
+ mode=ptuple.CREATE, payload=doc))
+
+ # and last, but not least, try to create
+ # content docs if not already there.
+ for cd in cdocs:
+ self.soledad_writer.put(ptuple(
+ mode=ptuple.CONTENT_CREATE, payload=cd))
+
+ def _remove_cb(self, result):
+ return result
+
+ def remove_all_deleted(self):
+ """
+ Removes all messages flagged as deleted.
+ """
+ delete_deferl = []
+ for msg in self.get_deleted():
+ delete_deferl.append(msg.remove())
+ d1 = defer.gatherResults(delete_deferl, consumeErrors=True)
+ d1.addCallback(self._remove_cb)
+ return d1
+
+ def remove(self, msg):
+ """
+ Remove a given msg.
+ :param msg: the message to be removed
+ :type msg: LeapMessage
+ """
+ d = msg.remove()
+ d.addCallback(self._remove_cb)
+ return d
+
+ # getters
+
+ def get_msg_by_uid(self, uid):
+ """
+ Retrieves a LeapMessage by UID.
+
+ :param uid: the message uid to query by
+ :type uid: int
+
+ :return: A LeapMessage instance matching the query,
+ or None if not found.
+ :rtype: LeapMessage
+ """
+ msg = LeapMessage(self._soledad, uid, self.mbox)
+ if not msg.does_exist():
+ return None
+ return msg
+
+ def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
+ """
+ Get all documents for the selected mailbox of the
+ passed type. By default, it returns the flag docs.
+
+ If you want acess to the content, use __iter__ instead
+
+ :return: a list of u1db documents
+ :rtype: list of SoledadDocument
+ """
+ if _type not in fields.__dict__.values():
+ raise TypeError("Wrong type passed to get_all_docs")
+
+ if sameProxiedObjects(self._soledad, None):
+ logger.warning('Tried to get messages but soledad is None!')
+ return []
+
+ all_docs = [doc for doc in self._soledad.get_from_index(
+ fields.TYPE_MBOX_IDX,
+ _type, self.mbox)]
+
+ # inneficient, but first let's grok it and then
+ # let's worry about efficiency.
+ # XXX FIXINDEX -- should implement order by in soledad
+ return sorted(all_docs, key=lambda item: item.content['uid'])
+
+ def all_msg_iter(self):
+ """
+ Return an iterator trhough the UIDs of all messages, sorted in
+ ascending order.
+ """
+ all_uids = (doc.content[self.UID_KEY] for doc in
+ self._soledad.get_from_index(
+ fields.TYPE_MBOX_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox))
+ return (u for u in sorted(all_uids))
+
+ def count(self):
+ """
+ Return the count of messages for this mailbox.
+
+ :rtype: int
+ """
+ count = self._soledad.get_count_from_index(
+ fields.TYPE_MBOX_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox)
+ return count
+
+ # unseen messages
+
+ def unseen_iter(self):
+ """
+ Get an iterator for the message UIDs with no `seen` flag
+ for this mailbox.
+
+ :return: iterator through unseen message doc UIDs
+ :rtype: iterable
+ """
+ return (doc.content[self.UID_KEY] for doc in
+ self._soledad.get_from_index(
+ fields.TYPE_MBOX_SEEN_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox, '0'))
+
+ def count_unseen(self):
+ """
+ Count all messages with the `Unseen` flag.
+
+ :returns: count
+ :rtype: int
+ """
+ count = self._soledad.get_count_from_index(
+ fields.TYPE_MBOX_SEEN_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox, '0')
+ return count
+
+ def get_unseen(self):
+ """
+ Get all messages with the `Unseen` flag
+
+ :returns: a list of LeapMessages
+ :rtype: list
+ """
+ return [LeapMessage(self._soledad, docid, self.mbox)
+ for docid in self.unseen_iter()]
+
+ # recent messages
+
+ def recent_iter(self):
+ """
+ Get an iterator for the message UIDs with `recent` flag.
+
+ :return: iterator through recent message docs
+ :rtype: iterable
+ """
+ return (doc.content[self.UID_KEY] for doc in
+ self._soledad.get_from_index(
+ fields.TYPE_MBOX_RECT_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox, '1'))
+
+ def get_recent(self):
+ """
+ Get all messages with the `Recent` flag.
+
+ :returns: a list of LeapMessages
+ :rtype: list
+ """
+ return [LeapMessage(self._soledad, docid, self.mbox)
+ for docid in self.recent_iter()]
+
+ def count_recent(self):
+ """
+ Count all messages with the `Recent` flag.
+
+ :returns: count
+ :rtype: int
+ """
+ count = self._soledad.get_count_from_index(
+ fields.TYPE_MBOX_RECT_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox, '1')
+ return count
+
+ # deleted messages
+
+ def deleted_iter(self):
+ """
+ Get an iterator for the message UIDs with `deleted` flag.
+
+ :return: iterator through deleted message docs
+ :rtype: iterable
+ """
+ return (doc.content[self.UID_KEY] for doc in
+ self._soledad.get_from_index(
+ fields.TYPE_MBOX_DEL_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox, '1'))
+
+ def get_deleted(self):
+ """
+ Get all messages with the `Deleted` flag.
+
+ :returns: a generator of LeapMessages
+ :rtype: generator
+ """
+ return (LeapMessage(self._soledad, docid, self.mbox)
+ for docid in self.deleted_iter())
+
+ def __len__(self):
+ """
+ Returns the number of messages on this mailbox.
+
+ :rtype: int
+ """
+ return self.count()
+
+ def __iter__(self):
+ """
+ Returns an iterator over all messages.
+
+ :returns: iterator of dicts with content for all messages.
+ :rtype: iterable
+ """
+ return (LeapMessage(self._soledad, docuid, self.mbox)
+ for docuid in self.all_msg_iter())
+
+ def __repr__(self):
+ """
+ Representation string for this object.
+ """
+ return u"<MessageCollection: mbox '%s' (%s)>" % (
+ self.mbox, self.count())
+
+ # XXX should implement __eq__ also !!!
+ # --- use the content hash for that, will be used for dedup.
diff --git a/mail/src/leap/mail/imap/parser.py b/mail/src/leap/mail/imap/parser.py
new file mode 100644
index 0000000..306dcf0
--- /dev/null
+++ b/mail/src/leap/mail/imap/parser.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# parser.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Mail parser mixins.
+"""
+import cStringIO
+import StringIO
+import hashlib
+import re
+
+from email.message import Message
+from email.parser import Parser
+
+from leap.common.check import leap_assert_type
+
+
+class MailParser(object):
+ """
+ Mixin with utility methods to parse raw messages.
+ """
+ def __init__(self):
+ """
+ Initializes the mail parser.
+ """
+ self._parser = Parser()
+
+ def _get_parsed_msg(self, raw, headersonly=False):
+ """
+ Return a parsed Message.
+
+ :param raw: the raw string to parse
+ :type raw: basestring, or StringIO object
+
+ :param headersonly: True for parsing only the headers.
+ :type headersonly: bool
+ """
+ msg = self._get_parser_fun(raw)(raw, headersonly=headersonly)
+ return msg
+
+ def _get_hash(self, msg):
+ """
+ Returns a hash of the string representation of the raw message,
+ suitable for indexing the inmutable pieces.
+
+ :param msg: a Message object
+ :type msg: Message
+ """
+ leap_assert_type(msg, Message)
+ return hashlib.sha256(msg.as_string()).hexdigest()
+
+ def _get_parser_fun(self, o):
+ """
+ Retunn the proper parser function for an object.
+
+ :param o: object
+ :type o: object
+ :param parser: an instance of email.parser.Parser
+ :type parser: email.parser.Parser
+ """
+ if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+ return self._parser.parse
+ if isinstance(o, basestring):
+ return self._parser.parsestr
+ # fallback
+ return self._parser.parsestr
+
+ def _stringify(self, o):
+ """
+ Return a string object.
+
+ :param o: object
+ :type o: object
+ """
+ # XXX Maybe we don't need no more, we're using
+ # msg.as_string()
+ if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
+ return o.getvalue()
+ else:
+ return o
+
+
+class MBoxParser(object):
+ """
+ Utility function to parse mailbox names.
+ """
+ INBOX_NAME = "INBOX"
+ INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
+ def _parse_mailbox_name(self, name):
+ """
+ :param name: the name of the mailbox
+ :type name: unicode
+
+ :rtype: unicode
+ """
+ if self.INBOX_RE.match(name):
+ # ensure inital INBOX is uppercase
+ return self.INBOX_NAME + name[len(self.INBOX_NAME):]
+ return name
diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py
deleted file mode 100644
index b9b72d0..0000000
--- a/mail/src/leap/mail/imap/server.py
+++ /dev/null
@@ -1,1807 +0,0 @@
-# -*- coding: utf-8 -*-
-# server.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Soledad-backed IMAP Server.
-"""
-import copy
-import logging
-import StringIO
-import cStringIO
-import time
-import re
-
-from collections import defaultdict
-from email.parser import Parser
-
-from zope.interface import implements
-from zope.proxy import sameProxiedObjects
-
-from twisted.mail import imap4
-from twisted.internet import defer
-from twisted.internet.threads import deferToThread
-from twisted.python import log
-
-
-from leap.common import events as leap_events
-from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.mail import get_email_charset
-from leap.mail.messageflow import IMessageConsumer, MessageProducer
-from leap.soledad.client import Soledad
-
-logger = logging.getLogger(__name__)
-
-
-class MissingIndexError(Exception):
- """
- Raises when tried to access a non existent index document.
- """
-
-
-class BadIndexError(Exception):
- """
- Raises when index is malformed or has the wrong cardinality.
- """
-
-
-class WithMsgFields(object):
- """
- Container class for class-attributes to be shared by
- several message-related classes.
- """
- # Internal representation of Message
- DATE_KEY = "date"
- HEADERS_KEY = "headers"
- FLAGS_KEY = "flags"
- MBOX_KEY = "mbox"
- RAW_KEY = "raw"
- SUBJECT_KEY = "subject"
- UID_KEY = "uid"
-
- # Mailbox specific keys
- CLOSED_KEY = "closed"
- CREATED_KEY = "created"
- SUBSCRIBED_KEY = "subscribed"
- RW_KEY = "rw"
- LAST_UID_KEY = "lastuid"
-
- # Document Type, for indexing
- TYPE_KEY = "type"
- TYPE_MESSAGE_VAL = "msg"
- TYPE_MBOX_VAL = "mbox"
-
- INBOX_VAL = "inbox"
-
- # Flags for SoledadDocument for indexing.
- SEEN_KEY = "seen"
- RECENT_KEY = "recent"
-
- # Flags in Mailbox and Message
- SEEN_FLAG = "\\Seen"
- RECENT_FLAG = "\\Recent"
- ANSWERED_FLAG = "\\Answered"
- FLAGGED_FLAG = "\\Flagged" # yo dawg
- DELETED_FLAG = "\\Deleted"
- DRAFT_FLAG = "\\Draft"
- NOSELECT_FLAG = "\\Noselect"
- LIST_FLAG = "List" # is this OK? (no \. ie, no system flag)
-
- # Fields in mail object
- SUBJECT_FIELD = "Subject"
- DATE_FIELD = "Date"
-
-
-class IndexedDB(object):
- """
- Methods dealing with the index.
-
- This is a MixIn that needs access to the soledad instance,
- and also assumes that a INDEXES attribute is accessible to the instance.
-
- INDEXES must be a dictionary of type:
- {'index-name': ['field1', 'field2']}
- """
- # TODO we might want to move this to soledad itself, check
-
- def initialize_db(self):
- """
- Initialize the database.
- """
- leap_assert(self._soledad,
- "Need a soledad attribute accesible in the instance")
- leap_assert_type(self.INDEXES, dict)
-
- # Ask the database for currently existing indexes.
- if not self._soledad:
- logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
- return
- db_indexes = dict()
- if self._soledad is not None:
- db_indexes = dict(self._soledad.list_indexes())
- for name, expression in SoledadBackedAccount.INDEXES.items():
- if name not in db_indexes:
- # The index does not yet exist.
- self._soledad.create_index(name, *expression)
- continue
-
- if expression == db_indexes[name]:
- # The index exists and is up to date.
- continue
- # The index exists but the definition is not what expected, so we
- # delete it and add the proper index expression.
- self._soledad.delete_index(name)
- self._soledad.create_index(name, *expression)
-
-
-#######################################
-# Soledad Account
-#######################################
-
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB):
- """
- An implementation of IAccount and INamespacePresenteer
- that is backed by Soledad Encrypted Documents.
- """
-
- implements(imap4.IAccount, imap4.INamespacePresenter)
-
- _soledad = None
- selected = None
-
- TYPE_IDX = 'by-type'
- TYPE_MBOX_IDX = 'by-type-and-mbox'
- TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
- TYPE_SUBS_IDX = 'by-type-and-subscribed'
- TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
- TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
- # Tomas created the `recent and seen index`, but the semantic is not too
- # correct since the recent flag is volatile.
- TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
- KTYPE = WithMsgFields.TYPE_KEY
- MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL
-
- INDEXES = {
- # generic
- TYPE_IDX: [KTYPE],
- TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
- TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY],
-
- # mailboxes
- TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
- # messages
- TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
- TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
- TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
- 'bool(recent)', 'bool(seen)'],
- }
-
- INBOX_NAME = "INBOX"
- MBOX_KEY = MBOX_VAL
-
- EMPTY_MBOX = {
- WithMsgFields.TYPE_KEY: MBOX_KEY,
- WithMsgFields.TYPE_MBOX_VAL: INBOX_NAME,
- WithMsgFields.SUBJECT_KEY: "",
- WithMsgFields.FLAGS_KEY: [],
- WithMsgFields.CLOSED_KEY: False,
- WithMsgFields.SUBSCRIBED_KEY: False,
- WithMsgFields.RW_KEY: 1,
- WithMsgFields.LAST_UID_KEY: 0
- }
-
- INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
- def __init__(self, account_name, soledad=None):
- """
- Creates a SoledadAccountIndex that keeps track of the mailboxes
- and subscriptions handled by this account.
-
- :param acct_name: The name of the account (user id).
- :type acct_name: str
-
- :param soledad: a Soledad instance.
- :param soledad: Soledad
- """
- leap_assert(soledad, "Need a soledad instance to initialize")
- leap_assert_type(soledad, Soledad)
-
- # XXX SHOULD assert too that the name matches the user/uuid with which
- # soledad has been initialized.
-
- self._account_name = self._parse_mailbox_name(account_name)
- self._soledad = soledad
-
- self.initialize_db()
-
- # every user should have the right to an inbox folder
- # at least, so let's make one!
-
- if not self.mailboxes:
- self.addMailbox(self.INBOX_NAME)
-
- def _get_empty_mailbox(self):
- """
- Returns an empty mailbox.
-
- :rtype: dict
- """
- return copy.deepcopy(self.EMPTY_MBOX)
-
- def _parse_mailbox_name(self, name):
- """
- :param name: the name of the mailbox
- :type name: unicode
-
- :rtype: unicode
- """
- if self.INBOX_RE.match(name):
- # ensure inital INBOX is uppercase
- return self.INBOX_NAME + name[len(self.INBOX_NAME):]
- return name
-
- def _get_mailbox_by_name(self, name):
- """
- Return an mbox document by name.
-
- :param name: the name of the mailbox
- :type name: str
-
- :rtype: SoledadDocument
- """
- doc = self._soledad.get_from_index(
- self.TYPE_MBOX_IDX, self.MBOX_KEY,
- self._parse_mailbox_name(name))
- return doc[0] if doc else None
-
- @property
- def mailboxes(self):
- """
- A list of the current mailboxes for this account.
- """
- return [doc.content[self.MBOX_KEY]
- for doc in self._soledad.get_from_index(
- self.TYPE_IDX, self.MBOX_KEY)]
-
- @property
- def subscriptions(self):
- """
- A list of the current subscriptions for this account.
- """
- return [doc.content[self.MBOX_KEY]
- for doc in self._soledad.get_from_index(
- self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
-
- def getMailbox(self, name):
- """
- Returns a Mailbox with that name, without selecting it.
-
- :param name: name of the mailbox
- :type name: str
-
- :returns: a a SoledadMailbox instance
- :rtype: SoledadMailbox
- """
- name = self._parse_mailbox_name(name)
-
- if name not in self.mailboxes:
- raise imap4.MailboxException("No such mailbox")
-
- return SoledadMailbox(name, soledad=self._soledad)
-
- ##
- ## IAccount
- ##
-
- def addMailbox(self, name, creation_ts=None):
- """
- Add a mailbox to the account.
-
- :param name: the name of the mailbox
- :type name: str
-
- :param creation_ts: an optional creation timestamp to be used as
- mailbox id. A timestamp will be used if no
- one is provided.
- :type creation_ts: int
-
- :returns: True if successful
- :rtype: bool
- """
- name = self._parse_mailbox_name(name)
-
- if name in self.mailboxes:
- raise imap4.MailboxCollision, name
-
- if not creation_ts:
- # by default, we pass an int value
- # taken from the current time
- # we make sure to take enough decimals to get a unique
- # mailbox-uidvalidity.
- creation_ts = int(time.time() * 10E2)
-
- mbox = self._get_empty_mailbox()
- mbox[self.MBOX_KEY] = name
- mbox[self.CREATED_KEY] = creation_ts
-
- doc = self._soledad.create_doc(mbox)
- return bool(doc)
-
- def create(self, pathspec):
- """
- Create a new mailbox from the given hierarchical name.
-
- :param pathspec: The full hierarchical name of a new mailbox to create.
- If any of the inferior hierarchical names to this one
- do not exist, they are created as well.
- :type pathspec: str
-
- :return: A true value if the creation succeeds.
- :rtype: bool
-
- :raise MailboxException: Raised if this mailbox cannot be added.
- """
- # TODO raise MailboxException
- paths = filter(None,
- self._parse_mailbox_name(pathspec).split('/'))
- for accum in range(1, len(paths)):
- try:
- self.addMailbox('/'.join(paths[:accum]))
- except imap4.MailboxCollision:
- pass
- try:
- self.addMailbox('/'.join(paths))
- except imap4.MailboxCollision:
- if not pathspec.endswith('/'):
- return False
- return True
-
- def select(self, name, readwrite=1):
- """
- Selects a mailbox.
-
- :param name: the mailbox to select
- :type name: str
-
- :param readwrite: 1 for readwrite permissions.
- :type readwrite: int
-
- :rtype: bool
- """
- name = self._parse_mailbox_name(name)
-
- if name not in self.mailboxes:
- return None
-
- self.selected = name
-
- return SoledadMailbox(
- name, rw=readwrite,
- soledad=self._soledad)
-
- def delete(self, name, force=False):
- """
- Deletes a mailbox.
-
- Right now it does not purge the messages, but just removes the mailbox
- name from the mailboxes list!!!
-
- :param name: the mailbox to be deleted
- :type name: str
-
- :param force: if True, it will not check for noselect flag or inferior
- names. use with care.
- :type force: bool
- """
- name = self._parse_mailbox_name(name)
-
- if not name in self.mailboxes:
- raise imap4.MailboxException("No such mailbox")
-
- mbox = self.getMailbox(name)
-
- if force is False:
- # See if this box is flagged \Noselect
- # XXX use mbox.flags instead?
- if self.NOSELECT_FLAG in mbox.getFlags():
- # Check for hierarchically inferior mailboxes with this one
- # as part of their root.
- for others in self.mailboxes:
- if others != name and others.startswith(name):
- raise imap4.MailboxException, (
- "Hierarchically inferior mailboxes "
- "exist and \\Noselect is set")
- mbox.destroy()
-
- # XXX FIXME --- not honoring the inferior names...
-
- # if there are no hierarchically inferior names, we will
- # delete it from our ken.
- #if self._inferiorNames(name) > 1:
- # ??! -- can this be rite?
- #self._index.removeMailbox(name)
-
- def rename(self, oldname, newname):
- """
- Renames a mailbox.
-
- :param oldname: old name of the mailbox
- :type oldname: str
-
- :param newname: new name of the mailbox
- :type newname: str
- """
- oldname = self._parse_mailbox_name(oldname)
- newname = self._parse_mailbox_name(newname)
-
- if oldname not in self.mailboxes:
- raise imap4.NoSuchMailbox, oldname
-
- inferiors = self._inferiorNames(oldname)
- inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
-
- for (old, new) in inferiors:
- if new in self.mailboxes:
- raise imap4.MailboxCollision, new
-
- for (old, new) in inferiors:
- mbox = self._get_mailbox_by_name(old)
- mbox.content[self.MBOX_KEY] = new
- self._soledad.put_doc(mbox)
-
- # XXX ---- FIXME!!!! ------------------------------------
- # until here we just renamed the index...
- # We have to rename also the occurrence of this
- # mailbox on ALL the messages that are contained in it!!!
- # ... we maybe could use a reference to the doc_id
- # in each msg, instead of the "mbox" field in msgs
- # -------------------------------------------------------
-
- def _inferiorNames(self, name):
- """
- Return hierarchically inferior mailboxes.
-
- :param name: name of the mailbox
- :rtype: list
- """
- # XXX use wildcard query instead
- inferiors = []
- for infname in self.mailboxes:
- if infname.startswith(name):
- inferiors.append(infname)
- return inferiors
-
- def isSubscribed(self, name):
- """
- Returns True if user is subscribed to this mailbox.
-
- :param name: the mailbox to be checked.
- :type name: str
-
- :rtype: bool
- """
- mbox = self._get_mailbox_by_name(name)
- return mbox.content.get('subscribed', False)
-
- def _set_subscription(self, name, value):
- """
- Sets the subscription value for a given mailbox
-
- :param name: the mailbox
- :type name: str
-
- :param value: the boolean value
- :type value: bool
- """
- # maybe we should store subscriptions in another
- # document...
- if not name in self.mailboxes:
- self.addMailbox(name)
- mbox = self._get_mailbox_by_name(name)
-
- if mbox:
- mbox.content[self.SUBSCRIBED_KEY] = value
- self._soledad.put_doc(mbox)
-
- def subscribe(self, name):
- """
- Subscribe to this mailbox
-
- :param name: name of the mailbox
- :type name: str
- """
- name = self._parse_mailbox_name(name)
- if name not in self.subscriptions:
- self._set_subscription(name, True)
-
- def unsubscribe(self, name):
- """
- Unsubscribe from this mailbox
-
- :param name: name of the mailbox
- :type name: str
- """
- name = self._parse_mailbox_name(name)
- if name not in self.subscriptions:
- raise imap4.MailboxException, "Not currently subscribed to " + name
- self._set_subscription(name, False)
-
- def listMailboxes(self, ref, wildcard):
- """
- List the mailboxes.
-
- from rfc 3501:
- returns a subset of names from the complete set
- of all names available to the client. Zero or more untagged LIST
- replies are returned, containing the name attributes, hierarchy
- delimiter, and name.
-
- :param ref: reference name
- :type ref: str
-
- :param wildcard: mailbox name with possible wildcards
- :type wildcard: str
- """
- # XXX use wildcard in index query
- ref = self._inferiorNames(
- self._parse_mailbox_name(ref))
- wildcard = imap4.wildcardToRegexp(wildcard, '/')
- return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
-
- ##
- ## INamespacePresenter
- ##
-
- def getPersonalNamespaces(self):
- return [["", "/"]]
-
- def getSharedNamespaces(self):
- return None
-
- def getOtherNamespaces(self):
- return None
-
- # extra, for convenience
-
- def deleteAllMessages(self, iknowhatiamdoing=False):
- """
- Deletes all messages from all mailboxes.
- Danger! high voltage!
-
- :param iknowhatiamdoing: confirmation parameter, needs to be True
- to proceed.
- """
- if iknowhatiamdoing is True:
- for mbox in self.mailboxes:
- self.delete(mbox, force=True)
-
- def __repr__(self):
- """
- Representation string for this object.
- """
- return "<SoledadBackedAccount (%s)>" % self._account_name
-
-#######################################
-# LeapMessage, MessageCollection
-# and Mailbox
-#######################################
-
-
-class LeapMessage(WithMsgFields):
-
- implements(imap4.IMessage, imap4.IMessageFile)
-
- def __init__(self, doc):
- """
- Initializes a LeapMessage.
-
- :param doc: A SoledadDocument containing the internal
- representation of the message
- :type doc: SoledadDocument
- """
- self._doc = doc
-
- def getUID(self):
- """
- Retrieve the unique identifier associated with this message
-
- :return: uid for this message
- :rtype: int
- """
- # XXX debug, to remove after a while...
- if not self._doc:
- log.msg('BUG!!! ---- message has no doc!')
- return
- return self._doc.content[self.UID_KEY]
-
- def getFlags(self):
- """
- Retrieve the flags associated with this message
-
- :return: The flags, represented as strings
- :rtype: tuple
- """
- if self._doc is None:
- return []
- flags = self._doc.content.get(self.FLAGS_KEY, None)
- if flags:
- flags = map(str, flags)
- return tuple(flags)
-
- # setFlags, addFlags, removeFlags are not in the interface spec
- # but we use them with store command.
-
- def setFlags(self, flags):
- """
- Sets the flags for this message
-
- Returns a SoledadDocument that needs to be updated by the caller.
-
- :param flags: the flags to update in the message.
- :type flags: tuple of str
-
- :return: a SoledadDocument instance
- :rtype: SoledadDocument
- """
- leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
- log.msg('setting flags')
- doc = self._doc
- doc.content[self.FLAGS_KEY] = flags
- doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags
- doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags
- return doc
-
- def addFlags(self, flags):
- """
- Adds flags to this message.
-
- Returns a SoledadDocument that needs to be updated by the caller.
-
- :param flags: the flags to add to the message.
- :type flags: tuple of str
-
- :return: a SoledadDocument instance
- :rtype: SoledadDocument
- """
- leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
- oldflags = self.getFlags()
- return self.setFlags(tuple(set(flags + oldflags)))
-
- def removeFlags(self, flags):
- """
- Remove flags from this message.
-
- Returns a SoledadDocument that needs to be updated by the caller.
-
- :param flags: the flags to be removed from the message.
- :type flags: tuple of str
-
- :return: a SoledadDocument instance
- :rtype: SoledadDocument
- """
- leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
- oldflags = self.getFlags()
- return self.setFlags(tuple(set(oldflags) - set(flags)))
-
- def getInternalDate(self):
- """
- Retrieve the date internally associated with this message
-
- :rtype: C{str}
- :return: An RFC822-formatted date string.
- """
- return str(self._doc.content.get(self.DATE_KEY, ''))
-
- #
- # IMessageFile
- #
-
- """
- Optional message interface for representing messages as files.
-
- If provided by message objects, this interface will be used instead
- the more complex MIME-based interface.
- """
-
- def open(self):
- """
- Return an file-like object opened for reading.
-
- Reading from the returned file will return all the bytes
- of which this message consists.
-
- :return: file-like object opened fore reading.
- :rtype: StringIO
- """
- fd = cStringIO.StringIO()
- content = self._doc.content.get(self.RAW_KEY, '')
- charset = get_email_charset(
- unicode(self._doc.content.get(self.RAW_KEY, '')))
- try:
- content = content.encode(charset)
- except (UnicodeEncodeError, UnicodeDecodeError) as e:
- logger.error("Unicode error {0}".format(e))
- content = content.encode(charset, 'replace')
- fd.write(content)
- fd.seek(0)
- return fd
-
- #
- # IMessagePart
- #
-
- # XXX should implement the rest of IMessagePart interface:
- # (and do not use the open above)
-
- def getBodyFile(self):
- """
- Retrieve a file object containing only the body of this message.
-
- :return: file-like object opened for reading
- :rtype: StringIO
- """
- fd = StringIO.StringIO()
- content = self._doc.content.get(self.RAW_KEY, '')
- charset = get_email_charset(
- unicode(self._doc.content.get(self.RAW_KEY, '')))
- try:
- content = content.encode(charset)
- except (UnicodeEncodeError, UnicodeDecodeError) as e:
- logger.error("Unicode error {0}".format(e))
- content = content.encode(charset, 'replace')
- fd.write(content)
- # XXX SHOULD use a separate BODY FIELD ...
- fd.seek(0)
- return fd
-
- def getSize(self):
- """
- Return the total size, in octets, of this message.
-
- :return: size of the message, in octets
- :rtype: int
- """
- return self.getBodyFile().len
-
- def _get_headers(self):
- """
- Return the headers dict stored in this message document.
- """
- return self._doc.content.get(self.HEADERS_KEY, {})
-
- def getHeaders(self, negate, *names):
- """
- Retrieve a group of message headers.
-
- :param names: The names of the headers to retrieve or omit.
- :type names: tuple of str
-
- :param negate: If True, indicates that the headers listed in names
- should be omitted from the return value, rather
- than included.
- :type negate: bool
-
- :return: A mapping of header field names to header field values
- :rtype: dict
- """
- headers = self._get_headers()
- names = map(lambda s: s.upper(), names)
- if negate:
- cond = lambda key: key.upper() not in names
- else:
- cond = lambda key: key.upper() in names
-
- # unpack and filter original dict by negate-condition
- filter_by_cond = [
- map(str, (key, val)) for
- key, val in headers.items()
- if cond(key)]
- return dict(filter_by_cond)
-
- # --- no multipart for now
- # XXX Fix MULTIPART SUPPORT!
-
- def isMultipart(self):
- return False
-
- def getSubPart(part):
- return None
-
- #
- # accessors
- #
-
- def __getitem__(self, key):
- """
- Return the content of the message document.
-
- @param key: The key
- @type key: str
-
- @return: The content value indexed by C{key} or None
- @rtype: str
- """
- return self._doc.content.get(key, None)
-
-
-class SoledadDocWriter(object):
- """
- This writer will create docs serially in the local soledad database.
- """
-
- implements(IMessageConsumer)
-
- def __init__(self, soledad):
- """
- Initialize the writer.
-
- :param soledad: the soledad instance
- :type soledad: Soledad
- """
- self._soledad = soledad
-
- def consume(self, queue):
- """
- Creates a new document in soledad db.
-
- :param queue: queue to get item from, with content of the document
- to be inserted.
- :type queue: Queue
- """
- empty = queue.empty()
- while not empty:
- item = queue.get()
- payload = item['payload']
- mode = item['mode']
- if mode == "create":
- self._soledad.create_doc(payload)
- elif mode == "put":
- self._soledad.put_doc(payload)
- empty = queue.empty()
-
-
-class MessageCollection(WithMsgFields, IndexedDB):
- """
- A collection of messages, surprisingly.
-
- It is tied to a selected mailbox name that is passed to constructor.
- Implements a filter query over the messages contained in a soledad
- database.
- """
- # XXX this should be able to produce a MessageSet methinks
-
- EMPTY_MSG = {
- WithMsgFields.TYPE_KEY: WithMsgFields.TYPE_MESSAGE_VAL,
- WithMsgFields.UID_KEY: 1,
- WithMsgFields.MBOX_KEY: WithMsgFields.INBOX_VAL,
- WithMsgFields.SUBJECT_KEY: "",
- WithMsgFields.DATE_KEY: "",
- WithMsgFields.SEEN_KEY: False,
- WithMsgFields.RECENT_KEY: True,
- WithMsgFields.FLAGS_KEY: [],
- WithMsgFields.HEADERS_KEY: {},
- WithMsgFields.RAW_KEY: "",
- }
-
- # get from SoledadBackedAccount the needed index-related constants
- INDEXES = SoledadBackedAccount.INDEXES
- TYPE_IDX = SoledadBackedAccount.TYPE_IDX
-
- def __init__(self, mbox=None, soledad=None):
- """
- Constructor for MessageCollection.
-
- :param mbox: the name of the mailbox. It is the name
- with which we filter the query over the
- messages database
- :type mbox: str
-
- :param soledad: Soledad database
- :type soledad: Soledad instance
- """
- # XXX pass soledad directly
-
- leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(mbox.strip() != "", "mbox cannot be blank space")
- leap_assert(isinstance(mbox, (str, unicode)),
- "mbox needs to be a string")
- leap_assert(soledad, "Need a soledad instance to initialize")
-
- # This is a wrapper now!...
- # should move assertion there...
- #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
- #"soledad._db must be an instance of SQLCipherDatabase")
-
- # okay, all in order, keep going...
-
- self.mbox = mbox.upper()
- self._soledad = soledad
- self.initialize_db()
- self._parser = Parser()
-
- # I think of someone like nietzsche when reading this
-
- # this will be the producer that will enqueue the content
- # to be processed serially by the consumer (the writer). We just
- # need to `put` the new material on its plate.
-
- self.soledad_writer = MessageProducer(
- SoledadDocWriter(soledad),
- period=0.1)
-
- def _get_empty_msg(self):
- """
- Returns an empty message.
-
- :return: a dict containing a default empty message
- :rtype: dict
- """
- return copy.deepcopy(self.EMPTY_MSG)
-
- def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
- """
- Creates a new message document.
-
- :param raw: the raw message
- :type raw: str
-
- :param subject: subject of the message.
- :type subject: str
-
- :param flags: flags
- :type flags: list
-
- :param date: the received date for the message
- :type date: str
-
- :param uid: the message uid for this mailbox
- :type uid: int
- """
- logger.debug('adding message')
- if flags is None:
- flags = tuple()
- leap_assert_type(flags, tuple)
-
- def stringify(o):
- if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
- return o.getvalue()
- else:
- return o
-
- content = self._get_empty_msg()
- content[self.MBOX_KEY] = self.mbox
-
- if flags:
- content[self.FLAGS_KEY] = map(stringify, flags)
- content[self.SEEN_KEY] = self.SEEN_FLAG in flags
-
- def _get_parser_fun(o):
- if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)):
- return self._parser.parse
- if isinstance(o, (str, unicode)):
- return self._parser.parsestr
-
- msg = _get_parser_fun(raw)(raw, True)
- headers = dict(msg)
-
- # XXX get lower case for keys?
- content[self.HEADERS_KEY] = headers
- # set subject based on message headers and eventually replace by
- # subject given as param
- if self.SUBJECT_FIELD in headers:
- content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
- if subject is not None:
- content[self.SUBJECT_KEY] = subject
- content[self.RAW_KEY] = stringify(raw)
-
- if not date and self.DATE_FIELD in headers:
- content[self.DATE_KEY] = headers[self.DATE_FIELD]
- else:
- content[self.DATE_KEY] = date
-
- # ...should get a sanity check here.
- content[self.UID_KEY] = uid
-
- logger.debug('enqueuing message for write')
-
- # XXX create namedtuple
- self.soledad_writer.put({"mode": "create",
- "payload": content})
- # XXX have to decide what shall we do with errors with this change...
- #return self._soledad.create_doc(content)
-
- def remove(self, msg):
- """
- Removes a message.
-
- :param msg: a u1db doc containing the message
- :type msg: SoledadDocument
- """
- self._soledad.delete_doc(msg)
-
- # getters
-
- def get_by_uid(self, uid):
- """
- Retrieves a message document by UID.
-
- :param uid: the message uid to query by
- :type uid: int
-
- :return: A SoledadDocument instance matching the query,
- or None if not found.
- :rtype: SoledadDocument
- """
- docs = self._soledad.get_from_index(
- SoledadBackedAccount.TYPE_MBOX_UID_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox, str(uid))
-
- return docs[0] if docs else None
-
- def get_msg_by_uid(self, uid):
- """
- Retrieves a LeapMessage by UID.
-
- :param uid: the message uid to query by
- :type uid: int
-
- :return: A LeapMessage instance matching the query,
- or None if not found.
- :rtype: LeapMessage
- """
- doc = self.get_by_uid(uid)
- if doc:
- return LeapMessage(doc)
-
- def get_by_index(self, index):
- """
- Retrieves a mesage document by mailbox index.
-
- :param index: the index of the sequence (zero-indexed)
- :type index: int
- """
- # XXX inneficient! ---- we should keep an index document
- # with uid -- doc_uuid :)
- try:
- return self.get_all()[index]
- except IndexError:
- return None
-
- def get_msg_by_index(self, index):
- """
- Retrieves a LeapMessage by sequence index.
-
- :param index: the index of the sequence (zero-indexed)
- :type index: int
- """
- doc = self.get_by_index(index)
- if doc:
- return LeapMessage(doc)
-
- def is_deleted(self, doc):
- """
- Returns whether a given doc is deleted or not.
-
- :param doc: the document to check
- :rtype: bool
- """
- return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
-
- def get_all(self):
- """
- Get all message documents for the selected mailbox.
- If you want acess to the content, use __iter__ instead
-
- :return: a list of u1db documents
- :rtype: list of SoledadDocument
- """
- if sameProxiedObjects(self._soledad, None):
- logger.warning('Tried to get messages but soledad is None!')
- return []
-
- #f XXX this should return LeapMessage instances
- all_docs = [doc for doc in self._soledad.get_from_index(
- SoledadBackedAccount.TYPE_MBOX_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox)]
- # highly inneficient, but first let's grok it and then
- # let's worry about efficiency.
-
- # XXX FIXINDEX
- return sorted(all_docs, key=lambda item: item.content['uid'])
-
- def count(self):
- """
- Return the count of messages for this mailbox.
-
- :rtype: int
- """
- count = self._soledad.get_count_from_index(
- SoledadBackedAccount.TYPE_MBOX_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox)
- return count
-
- # unseen messages
-
- def unseen_iter(self):
- """
- Get an iterator for the message docs with no `seen` flag
-
- :return: iterator through unseen message docs
- :rtype: iterable
- """
- return (doc for doc in
- self._soledad.get_from_index(
- SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox, '0'))
-
- def count_unseen(self):
- """
- Count all messages with the `Unseen` flag.
-
- :returns: count
- :rtype: int
- """
- count = self._soledad.get_count_from_index(
- SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox, '0')
- return count
-
- def get_unseen(self):
- """
- Get all messages with the `Unseen` flag
-
- :returns: a list of LeapMessages
- :rtype: list
- """
- return [LeapMessage(doc) for doc in self.unseen_iter()]
-
- # recent messages
-
- def recent_iter(self):
- """
- Get an iterator for the message docs with `recent` flag.
-
- :return: iterator through recent message docs
- :rtype: iterable
- """
- return (doc for doc in
- self._soledad.get_from_index(
- SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox, '1'))
-
- def get_recent(self):
- """
- Get all messages with the `Recent` flag.
-
- :returns: a list of LeapMessages
- :rtype: list
- """
- return [LeapMessage(doc) for doc in self.recent_iter()]
-
- def count_recent(self):
- """
- Count all messages with the `Recent` flag.
-
- :returns: count
- :rtype: int
- """
- count = self._soledad.get_count_from_index(
- SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox, '1')
- return count
-
- def __len__(self):
- """
- Returns the number of messages on this mailbox.
-
- :rtype: int
- """
- return self.count()
-
- def __iter__(self):
- """
- Returns an iterator over all messages.
-
- :returns: iterator of dicts with content for all messages.
- :rtype: iterable
- """
- # XXX return LeapMessage instead?! (change accordingly)
- return (m.content for m in self.get_all())
-
- def __getitem__(self, uid):
- """
- Allows indexing as a list, with msg uid as the index.
-
- :param uid: an integer index
- :type uid: int
-
- :return: LeapMessage or None if not found.
- :rtype: LeapMessage
- """
- # XXX FIXME inneficcient, we are evaulating.
- try:
- return [doc
- for doc in self.get_all()][uid - 1]
- except IndexError:
- return None
-
- def __repr__(self):
- """
- Representation string for this object.
- """
- return u"<MessageCollection: mbox '%s' (%s)>" % (
- self.mbox, self.count())
-
- # XXX should implement __eq__ also
-
-
-class SoledadMailbox(WithMsgFields):
- """
- A Soledad-backed IMAP mailbox.
-
- Implements the high-level method needed for the Mailbox interfaces.
- The low-level database methods are contained in MessageCollection class,
- which we instantiate and make accessible in the `messages` attribute.
- """
- implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox)
- # XXX should finish the implementation of IMailboxListener
-
- messages = None
- _closed = False
-
- INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
- WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
- WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
- WithMsgFields.LIST_FLAG)
- flags = None
-
- CMD_MSG = "MESSAGES"
- CMD_RECENT = "RECENT"
- CMD_UIDNEXT = "UIDNEXT"
- CMD_UIDVALIDITY = "UIDVALIDITY"
- CMD_UNSEEN = "UNSEEN"
-
- _listeners = defaultdict(set)
-
- def __init__(self, mbox, soledad=None, rw=1):
- """
- SoledadMailbox constructor. Needs to get passed a name, plus a
- Soledad instance.
-
- :param mbox: the mailbox name
- :type mbox: str
-
- :param soledad: a Soledad instance.
- :type soledad: Soledad
-
- :param rw: read-and-write flags
- :type rw: int
- """
- leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(soledad, "Need a soledad instance to initialize")
-
- # XXX should move to wrapper
- #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
- #"soledad._db must be an instance of SQLCipherDatabase")
-
- self.mbox = mbox
- self.rw = rw
-
- self._soledad = soledad
-
- self.messages = MessageCollection(
- mbox=mbox, soledad=self._soledad)
-
- if not self.getFlags():
- self.setFlags(self.INIT_FLAGS)
-
- @property
- def listeners(self):
- """
- Returns listeners for this mbox.
-
- The server itself is a listener to the mailbox.
- so we can notify it (and should!) after changes in flags
- and number of messages.
-
- :rtype: set
- """
- return self._listeners[self.mbox]
-
- def addListener(self, listener):
- """
- Rdds a listener to the listeners queue.
-
- :param listener: listener to add
- :type listener: an object that implements IMailboxListener
- """
- logger.debug('adding mailbox listener: %s' % listener)
- self.listeners.add(listener)
-
- def removeListener(self, listener):
- """
- Removes a listener from the listeners queue.
-
- :param listener: listener to remove
- :type listener: an object that implements IMailboxListener
- """
- self.listeners.remove(listener)
-
- def _get_mbox(self):
- """
- Returns mailbox document.
-
- :return: A SoledadDocument containing this mailbox, or None if
- the query failed.
- :rtype: SoledadDocument or None.
- """
- try:
- query = self._soledad.get_from_index(
- SoledadBackedAccount.TYPE_MBOX_IDX,
- self.TYPE_MBOX_VAL, self.mbox)
- if query:
- return query.pop()
- except Exception as exc:
- logger.error("Unhandled error %r" % exc)
-
- def getFlags(self):
- """
- Returns the flags defined for this mailbox.
-
- :returns: tuple of flags for this mailbox
- :rtype: tuple of str
- """
- #return map(str, self.INIT_FLAGS)
-
- # XXX CHECK against thunderbird XXX
- # XXX I think this is slightly broken.. :/
-
- mbox = self._get_mbox()
- if not mbox:
- return None
- flags = mbox.content.get(self.FLAGS_KEY, [])
- return map(str, flags)
-
- def setFlags(self, flags):
- """
- Sets flags for this mailbox.
-
- :param flags: a tuple with the flags
- :type flags: tuple of str
- """
- # TODO -- fix also getFlags
- leap_assert(isinstance(flags, tuple),
- "flags expected to be a tuple")
- mbox = self._get_mbox()
- if not mbox:
- return None
- mbox.content[self.FLAGS_KEY] = map(str, flags)
- self._soledad.put_doc(mbox)
-
- # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
-
- def _get_closed(self):
- """
- Return the closed attribute for this mailbox.
-
- :return: True if the mailbox is closed
- :rtype: bool
- """
- mbox = self._get_mbox()
- return mbox.content.get(self.CLOSED_KEY, False)
-
- def _set_closed(self, closed):
- """
- Set the closed attribute for this mailbox.
-
- :param closed: the state to be set
- :type closed: bool
- """
- leap_assert(isinstance(closed, bool), "closed needs to be boolean")
- mbox = self._get_mbox()
- mbox.content[self.CLOSED_KEY] = closed
- self._soledad.put_doc(mbox)
-
- closed = property(
- _get_closed, _set_closed, doc="Closed attribute.")
-
- def _get_last_uid(self):
- """
- Return the last uid for this mailbox.
-
- :return: the last uid for messages in this mailbox
- :rtype: bool
- """
- mbox = self._get_mbox()
- return mbox.content.get(self.LAST_UID_KEY, 1)
-
- def _set_last_uid(self, uid):
- """
- Sets the last uid for this mailbox.
-
- :param uid: the uid to be set
- :type uid: int
- """
- leap_assert(isinstance(uid, int), "uid has to be int")
- mbox = self._get_mbox()
- key = self.LAST_UID_KEY
-
- count = self.getMessageCount()
-
- # XXX safety-catch. If we do get duplicates,
- # we want to avoid further duplication.
-
- if uid >= count:
- value = uid
- else:
- # something is wrong,
- # just set the last uid
- # beyond the max msg count.
- logger.debug("WRONG uid < count. Setting last uid to ", count)
- value = count
-
- mbox.content[key] = value
- self._soledad.put_doc(mbox)
-
- last_uid = property(
- _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
-
- def getUIDValidity(self):
- """
- Return the unique validity identifier for this mailbox.
-
- :return: unique validity identifier
- :rtype: int
- """
- mbox = self._get_mbox()
- return mbox.content.get(self.CREATED_KEY, 1)
-
- def getUID(self, message):
- """
- Return the UID of a message in the mailbox
-
- .. note:: this implementation does not make much sense RIGHT NOW,
- but in the future will be useful to get absolute UIDs from
- message sequence numbers.
-
- :param message: the message uid
- :type message: int
-
- :rtype: int
- """
- msg = self.messages.get_msg_by_uid(message)
- return msg.getUID()
-
- def getUIDNext(self):
- """
- Return the likely UID for the next message added to this
- mailbox. Currently it returns the higher UID incremented by
- one.
-
- We increment the next uid *each* time this function gets called.
- In this way, there will be gaps if the message with the allocated
- uid cannot be saved. But that is preferable to having race conditions
- if we get to parallel message adding.
-
- :rtype: int
- """
- self.last_uid += 1
- return self.last_uid
-
- def getMessageCount(self):
- """
- Returns the total count of messages in this mailbox.
-
- :rtype: int
- """
- return self.messages.count()
-
- def getUnseenCount(self):
- """
- Returns the number of messages with the 'Unseen' flag.
-
- :return: count of messages flagged `unseen`
- :rtype: int
- """
- return self.messages.count_unseen()
-
- def getRecentCount(self):
- """
- Returns the number of messages with the 'Recent' flag.
-
- :return: count of messages flagged `recent`
- :rtype: int
- """
- return self.messages.count_recent()
-
- def isWriteable(self):
- """
- Get the read/write status of the mailbox.
-
- :return: 1 if mailbox is read-writeable, 0 otherwise.
- :rtype: int
- """
- return self.rw
-
- def getHierarchicalDelimiter(self):
- """
- Returns the character used to delimite hierarchies in mailboxes.
-
- :rtype: str
- """
- return '/'
-
- def requestStatus(self, names):
- """
- Handles a status request by gathering the output of the different
- status commands.
-
- :param names: a list of strings containing the status commands
- :type names: iter
- """
- r = {}
- if self.CMD_MSG in names:
- r[self.CMD_MSG] = self.getMessageCount()
- if self.CMD_RECENT in names:
- r[self.CMD_RECENT] = self.getRecentCount()
- if self.CMD_UIDNEXT in names:
- r[self.CMD_UIDNEXT] = self.getMessageCount() + 1
- if self.CMD_UIDVALIDITY in names:
- r[self.CMD_UIDVALIDITY] = self.getUID()
- if self.CMD_UNSEEN in names:
- r[self.CMD_UNSEEN] = self.getUnseenCount()
- return defer.succeed(r)
-
- def addMessage(self, message, flags, date=None):
- """
- Adds a message to this mailbox.
-
- :param message: the raw message
- :type message: str
-
- :param flags: flag list
- :type flags: list of str
-
- :param date: timestamp
- :type date: str
-
- :return: a deferred that evals to None
- """
- # XXX we should treat the message as an IMessage from here
- uid_next = self.getUIDNext()
- logger.debug('Adding msg with UID :%s' % uid_next)
- if flags is None:
- flags = tuple()
- else:
- flags = tuple(str(flag) for flag in flags)
-
- self.messages.add_msg(message, flags=flags, date=date,
- uid=uid_next)
-
- exists = self.getMessageCount()
- recent = self.getRecentCount()
- logger.debug("there are %s messages, %s recent" % (
- exists,
- recent))
- for listener in self.listeners:
- listener.newMessages(exists, recent)
- return defer.succeed(None)
-
- # commands, do not rename methods
-
- def destroy(self):
- """
- Called before this mailbox is permanently deleted.
-
- Should cleanup resources, and set the \\Noselect flag
- on the mailbox.
- """
- self.setFlags((self.NOSELECT_FLAG,))
- self.deleteAllDocs()
-
- # XXX removing the mailbox in situ for now,
- # we should postpone the removal
- self._soledad.delete_doc(self._get_mbox())
-
- def expunge(self):
- """
- Remove all messages flagged \\Deleted
- """
- if not self.isWriteable():
- raise imap4.ReadOnlyMailbox
- delete = []
- deleted = []
-
- for m in self.messages.get_all():
- if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
- delete.append(m)
- for m in delete:
- deleted.append(m.content)
- self.messages.remove(m)
-
- # XXX should return the UIDs of the deleted messages
- # more generically
- return [x for x in range(len(deleted))]
-
- def fetch(self, messages, uid):
- """
- Retrieve one or more messages in this mailbox.
-
- from rfc 3501: The data items to be fetched can be either a single atom
- or a parenthesized list.
-
- :param messages: IDs of the messages to retrieve information about
- :type messages: MessageSet
-
- :param uid: If true, the IDs are UIDs. They are message sequence IDs
- otherwise.
- :type uid: bool
-
- :rtype: A tuple of two-tuples of message sequence numbers and
- LeapMessage
- """
- result = []
- sequence = True if uid == 0 else False
-
- if not messages.last:
- try:
- iter(messages)
- except TypeError:
- # looks like we cannot iterate
- messages.last = self.last_uid
-
- # for sequence numbers (uid = 0)
- if sequence:
- for msg_id in messages:
- msg = self.messages.get_msg_by_index(msg_id - 1)
- if msg:
- result.append((msg.getUID(), msg))
- else:
- print "fetch %s, no msg found!!!" % msg_id
-
- else:
- for msg_id in messages:
- msg = self.messages.get_msg_by_uid(msg_id)
- if msg:
- result.append((msg_id, msg))
- else:
- print "fetch %s, no msg found!!!" % msg_id
-
- if self.isWriteable():
- self._unset_recent_flag()
-
- return tuple(result[:100])
-
- def _unset_recent_flag(self):
- """
- Unsets `Recent` flag from a tuple of messages.
- Called from fetch.
-
- From RFC, about `Recent`:
-
- Message is "recently" arrived in this mailbox. This session
- is the first session to have been notified about this
- message; if the session is read-write, subsequent sessions
- will not see \Recent set for this message. This flag can not
- be altered by the client.
-
- If it is not possible to determine whether or not this
- session is the first session to be notified about a message,
- then that message SHOULD be considered recent.
- """
- log.msg('unsetting recent flags...')
- for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
- newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
- self._update(newflags)
-
- def _signal_unread_to_ui(self):
- """
- Sends unread event to ui.
- """
- unseen = self.getUnseenCount()
- leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
-
- def store(self, messages, flags, mode, uid):
- """
- Sets the flags of one or more messages.
-
- :param messages: The identifiers of the messages to set the flags
- :type messages: A MessageSet object with the list of messages requested
-
- :param flags: The flags to set, unset, or add.
- :type flags: sequence of str
-
- :param mode: If mode is -1, these flags should be removed from the
- specified messages. If mode is 1, these flags should be
- added to the specified messages. If mode is 0, all
- existing flags should be cleared and these flags should be
- added.
- :type mode: -1, 0, or 1
-
- :param uid: If true, the IDs specified in the query are UIDs;
- otherwise they are message sequence IDs.
- :type uid: bool
-
- :return: A dict mapping message sequence numbers to sequences of
- str representing the flags set on the message after this
- operation has been performed.
- :rtype: dict
-
- :raise ReadOnlyMailbox: Raised if this mailbox is not open for
- read-write.
- """
- # XXX implement also sequence (uid = 0)
- # XXX we should prevent cclient from setting Recent flag.
- leap_assert(not isinstance(flags, basestring),
- "flags cannot be a string")
- flags = tuple(flags)
-
- if not self.isWriteable():
- log.msg('read only mailbox!')
- raise imap4.ReadOnlyMailbox
-
- if not messages.last:
- messages.last = self.messages.count()
-
- result = {}
- for msg_id in messages:
- print "MSG ID = %s" % msg_id
- msg = self.messages.get_msg_by_uid(msg_id)
- if mode == 1:
- self._update(msg.addFlags(flags))
- elif mode == -1:
- self._update(msg.removeFlags(flags))
- elif mode == 0:
- self._update(msg.setFlags(flags))
- result[msg_id] = msg.getFlags()
-
- self._signal_unread_to_ui()
- return result
-
- def close(self):
- """
- Expunge and mark as closed
- """
- self.expunge()
- self.closed = True
-
- # convenience fun
-
- def deleteAllDocs(self):
- """
- Deletes all docs in this mailbox
- """
- docs = self.messages.get_all()
- for doc in docs:
- self.messages._soledad.delete_doc(doc)
-
- def _update(self, doc):
- """
- Updates document in u1db database
- """
- # XXX create namedtuple
- self.messages.soledad_writer.put({"mode": "put",
- "payload": doc})
-
- def __repr__(self):
- """
- Representation string for this mailbox.
- """
- return u"<SoledadMailbox: mbox '%s' (%s)>" % (
- self.mbox, self.messages.count())
diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py
index 8756ddc..234996d 100644
--- a/mail/src/leap/mail/imap/service/imap.py
+++ b/mail/src/leap/mail/imap/service/imap.py
@@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
from leap.common import events as leap_events
from leap.common.check import leap_assert, leap_assert_type, leap_check
from leap.keymanager import KeyManager
-from leap.mail.imap.server import SoledadBackedAccount
+from leap.mail.imap.account import SoledadBackedAccount
from leap.mail.imap.fetch import LeapIncomingMail
from leap.soledad.client import Soledad
@@ -87,6 +87,8 @@ class LeapIMAPServer(imap4.IMAP4Server):
:param line: the line from the server, without the line delimiter.
:type line: str
"""
+ print "RECV: STATE (%s)" % self.state
+
if "login" in line.lower():
# avoid to log the pass, even though we are using a dummy auth
# by now.
diff --git a/mail/src/leap/mail/imap/tests/getmail b/mail/src/leap/mail/imap/tests/getmail
new file mode 100755
index 0000000..17e195c
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/getmail
@@ -0,0 +1,282 @@
+#!/usr/bin/env python
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE in twisted for details.
+
+# Modifications by LEAP Developers 2014 to fit
+# Bitmask configuration settings.
+
+
+"""
+Simple IMAP4 client which displays the subjects of all messages in a
+particular mailbox.
+"""
+
+import sys
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+
+class TrivialPrompter(basic.LineReceiver):
+ from os import linesep as delimiter
+
+ promptDeferred = None
+
+ def prompt(self, msg):
+ assert self.promptDeferred is None
+ self.display(msg)
+ self.promptDeferred = defer.Deferred()
+ return self.promptDeferred
+
+ def display(self, msg):
+ self.transport.write(msg)
+
+ def lineReceived(self, line):
+ if self.promptDeferred is None:
+ return
+ d, self.promptDeferred = self.promptDeferred, None
+ d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+ """
+ A client with callbacks for greeting messages from an IMAP server.
+ """
+ greetDeferred = None
+
+ def serverGreeting(self, caps):
+ self.serverCapabilities = caps
+ if self.greetDeferred is not None:
+ d, self.greetDeferred = self.greetDeferred, None
+ d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+ usedUp = False
+
+ protocol = SimpleIMAP4Client
+
+ def __init__(self, username, onConn):
+ self.ctx = ssl.ClientContextFactory()
+
+ self.username = username
+ self.onConn = onConn
+
+ def buildProtocol(self, addr):
+ """
+ Initiate the protocol instance. Since we are building a simple IMAP
+ client, we don't bother checking what capabilities the server has. We
+ just add all the authenticators twisted.mail has. Note: Gmail no
+ longer uses any of the methods below, it's been using XOAUTH since
+ 2010.
+ """
+ assert not self.usedUp
+ self.usedUp = True
+
+ p = self.protocol(self.ctx)
+ p.factory = self
+ p.greetDeferred = self.onConn
+
+ p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+ p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+ p.registerAuthenticator(
+ imap4.CramMD5ClientAuthenticator(self.username))
+
+ return p
+
+ def clientConnectionFailed(self, connector, reason):
+ d, self.onConn = self.onConn, None
+ d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+ """
+ Initial callback - invoked after the server sends us its greet message.
+ """
+ # Hook up stdio
+ tp = TrivialPrompter()
+ stdio.StandardIO(tp)
+
+ # And make it easily accessible
+ proto.prompt = tp.prompt
+ proto.display = tp.display
+
+ # Try to authenticate securely
+ return proto.authenticate(
+ password).addCallback(
+ cbAuthentication,
+ proto).addErrback(
+ ebAuthentication, proto, username, password
+ )
+
+
+def ebConnection(reason):
+ """
+ Fallback error-handler. If anything goes wrong, log it and quit.
+ """
+ log.startLogging(sys.stdout)
+ log.err(reason)
+ return reason
+
+
+def cbAuthentication(result, proto):
+ """
+ Callback after authentication has succeeded.
+
+ Lists a bunch of mailboxes.
+ """
+ return proto.list("", "*"
+ ).addCallback(cbMailboxList, proto
+ )
+
+
+def ebAuthentication(failure, proto, username, password):
+ """
+ Errback invoked when authentication fails.
+
+ If it failed because no SASL mechanisms match, offer the user the choice
+ of logging in insecurely.
+
+ If you are trying to connect to your Gmail account, you will be here!
+ """
+ failure.trap(imap4.NoSupportedAuthentication)
+ return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+ """
+ insecure-login.
+ """
+ return proto.login(username, password
+ ).addCallback(cbAuthentication, proto
+ )
+
+
+def cbMailboxList(result, proto):
+ """
+ Callback invoked when a list of mailboxes has been retrieved.
+ """
+ result = [e[2] for e in result]
+ s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)])
+ if not s:
+ return defer.fail(Exception("No mailboxes exist on server!"))
+ return proto.prompt(s + "\nWhich mailbox? [1] "
+ ).addCallback(cbPickMailbox, proto, result
+ )
+
+
+def cbPickMailbox(result, proto, mboxes):
+ """
+ When the user selects a mailbox, "examine" it.
+ """
+ mbox = mboxes[int(result or '1') - 1]
+ return proto.examine(mbox
+ ).addCallback(cbExamineMbox, proto
+ )
+
+
+def cbExamineMbox(result, proto):
+ """
+ Callback invoked when examine command completes.
+
+ Retrieve the subject header of every message in the mailbox.
+ """
+ return proto.fetchSpecific('1:*',
+ headerType='HEADER.FIELDS',
+ headerArgs=['SUBJECT'],
+ ).addCallback(cbFetch, proto,
+ )
+
+
+def cbFetch(result, proto):
+ """
+ Display headers.
+ """
+ if result:
+ keys = result.keys()
+ keys.sort()
+ for k in keys:
+ proto.display('%s %s' % (k, result[k][0][2]))
+ else:
+ print "Hey, an empty mailbox!"
+
+ return proto.prompt("\nWhich message? [1] (Q quits) "
+ ).addCallback(cbPickMessage, proto)
+
+
+def cbPickMessage(result, proto):
+ """
+ Pick a message.
+ """
+ if result == "Q":
+ print "Bye!"
+ return proto.logout()
+
+ return proto.fetchSpecific(
+ '%s' % result,
+ headerType='',
+ headerArgs=['BODY.PEEK[]'],
+ ).addCallback(cbShowmessage, proto)
+
+
+def cbShowmessage(result, proto):
+ """
+ Display message.
+ """
+ if result:
+ keys = result.keys()
+ keys.sort()
+ for k in keys:
+ proto.display('%s %s' % (k, result[k][0][2]))
+ else:
+ print "Hey, an empty message!"
+
+ return proto.logout()
+
+
+def cbClose(result):
+ """
+ Close the connection when we finish everything.
+ """
+ from twisted.internet import reactor
+ reactor.stop()
+
+
+def main():
+ import sys
+
+ if len(sys.argv) != 3:
+ print "Usage: getmail <user> <pass>"
+ sys.exit()
+
+ hostname = "localhost"
+ port = "1984"
+ username = sys.argv[1]
+ password = sys.argv[2]
+
+ onConn = defer.Deferred(
+ ).addCallback(cbServerGreeting, username, password
+ ).addErrback(ebConnection
+ ).addBoth(cbClose)
+
+ factory = SimpleIMAP4ClientFactory(username, onConn)
+
+ from twisted.internet import reactor
+ if port == '993':
+ reactor.connectSSL(
+ hostname, int(port), factory, ssl.ClientContextFactory())
+ else:
+ if not port:
+ port = 143
+ reactor.connectTCP(hostname, int(port), factory)
+ reactor.run()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message
new file mode 100644
index 0000000..582297c
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message
@@ -0,0 +1,16 @@
+Content-Type: multipart/mixed; boundary="===============6203542367371144092=="
+MIME-Version: 1.0
+Subject: [TEST] 010 - Inceptos cum lorem risus congue
+From: testmailbitmaskspam@gmail.com
+To: test_c5@dev.bitmask.net
+
+--===============6203542367371144092==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+Howdy from python!
+The subject: [TEST] 010 - Inceptos cum lorem risus congue
+Current date & time: Wed Jan 8 16:36:21 2014
+Trying to attach: []
+--===============6203542367371144092==--
diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message
new file mode 100644
index 0000000..9907c2d
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message
@@ -0,0 +1,238 @@
+Date: Mon, 6 Jan 2014 04:40:47 -0400
+From: Kali Kaneko <kali@leap.se>
+To: penguin@example.com
+Subject: signed message
+Message-ID: <20140106084047.GA21317@samsara.lan>
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha1;
+ protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.21 (2012-12-30)
+
+
+--z9ECzHErBrwFF8sy
+Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l"
+Content-Disposition: inline
+
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+
+This is an example of a signed message,
+with attachments.
+
+
+--=20
+Nihil sine chao! =E2=88=B4
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: attachment; filename="attach.txt"
+
+this is attachment in plain text.
+
+--z0eOaCaDLjvTGF2l
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="hack.ico"
+Content-Transfer-Encoding: base64
+
+AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA
+KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG
+RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA
+PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl
+5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA
+/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ
+yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A
+Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK
+ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK
+LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP
+QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy
+AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs
+AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA
+AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA
+gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d
+HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA
+x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7
++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA
+AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5
++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA
+OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK
+igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA
+JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra
+2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA
+xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj
+owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB
+AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA
+AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d
+XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d
+XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA
+AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB
+AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm
+X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC
+AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B
+bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ
+S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu
+J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y
+AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N
+KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB
+XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A
+AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA
+AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d
+XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA
+AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr
+RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA
+AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A
+Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI
+yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA
+CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys
+rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA
+vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d
+HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA
+urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx
+cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA
+CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo
+6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA
+2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7
+OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA
+UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp
+qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA
+lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa
+WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB
+AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA
+ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA
+AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB
+AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB
+AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA
+tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA
+AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB
+AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB
+AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA
+AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd
+AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB
+AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB
+AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8
+ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2
+NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF
+RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB
+lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA
+AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa
+WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA
+AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX
+AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB
+AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA
+AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA
+AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA
+AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA
+AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB
+AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB
+AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA
+ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA
+AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7
+LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA
+AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA
+ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4
+RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi
+JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2
+NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK
+T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN
+UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA
+AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA
+W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA
+AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB
+l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB
+AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+
+WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA
+AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv
+RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA
+AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj
+AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB
+AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA
+AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA
+AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA
+dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A
+AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB
+AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW
+pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+
+--z0eOaCaDLjvTGF2l--
+
+--z9ECzHErBrwFF8sy
+Content-Type: application/pgp-signature
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv
+kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl
+vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK
+PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC
+w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw
+sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr
+BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN
+QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt
+mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ
+jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8
+gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X
+sSdfcAhT7Tno7PB/Acoh
+=+okv
+-----END PGP SIGNATURE-----
+
+--z9ECzHErBrwFF8sy--
diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi.message b/mail/src/leap/mail/imap/tests/rfc822.multi.message
new file mode 100644
index 0000000..30f74e5
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.multi.message
@@ -0,0 +1,96 @@
+Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
+From: Doug Sauder <doug@penguin.example.com>
+To: Joe Blow <blow@example.com>
+Subject: Test message from PINE
+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
+MIME-Version: 1.0
+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
+
+ This message is in MIME format. The first part should be readable text,
+ while the remaining parts are likely unreadable without MIME-aware tools.
+ Send mail to mime@docserver.cac.washington.edu for more info.
+
+---1463757054-952513540-958744548=:8452
+Content-Type: TEXT/PLAIN; charset=US-ASCII
+
+This is a test message from PINE MUA.
+
+
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="redball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="redball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
+vAAAAABJRU5ErkJggg==
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="blueball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="blueball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
+Z3VldDZzO7wAAAAASUVORK5CYII=
+---1463757054-952513540-958744548=:8452--
diff --git a/mail/src/leap/mail/imap/tests/rfc822.plain.message b/mail/src/leap/mail/imap/tests/rfc822.plain.message
new file mode 100644
index 0000000..fc627c3
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/rfc822.plain.message
@@ -0,0 +1,66 @@
+From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014
+Return-Path: <pyar-bounces@python.org.ar>
+X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net
+X-Spam-Level: **
+X-Spam-Pyzor: Reported 0 times.
+X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE,
+ CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP,
+ NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled
+ version=3.3.2
+Delivered-To: kali@leap.se
+Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33])
+ (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))
+ (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified))
+ by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F
+ for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC)
+Received: from pyar.usla.org.ar (unknown [190.228.30.157])
+ by mx1.riseup.net (Postfix) with ESMTP id F244C533F4
+ for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST)
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+ by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F
+ for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART)
+MIME-Version: 1.0
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+From: pyar-request@python.org.ar
+To: kali@leap.se
+Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+Reply-To: pyar-request@python.org.ar
+Auto-Submitted: auto-replied
+Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar>
+Date: Wed, 08 Jan 2014 15:45:59 -0300
+Precedence: bulk
+X-BeenThere: pyar@python.org.ar
+X-Mailman-Version: 2.1.15
+List-Id: Python Argentina <pyar.python.org.ar>
+X-List-Administrivia: yes
+Errors-To: pyar-bounces@python.org.ar
+Sender: "pyar" <pyar-bounces@python.org.ar>
+X-Virus-Scanned: clamav-milter 0.97.8 at mx1
+X-Virus-Status: Clean
+
+Mailing list subscription confirmation notice for mailing list pyar
+
+We have received a request de kaliyuga@riseup.net for subscription of
+your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar
+mailing list. To confirm that you want to be added to this mailing
+list, simply reply to this message, keeping the Subject: header
+intact. Or visit this web page:
+
+ http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b=
+3377148ac2
+
+
+Or include the following line -- and only the following line -- in a
+message to pyar-request@python.org.ar:
+
+ confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+
+Note that simply sending a `reply' to this message should work from
+most mail readers, since that usually leaves the Subject: line in the
+right form (additional "Re:" text in the Subject: is okay).
+
+If you do not wish to be subscribed to this list, please simply
+disregard this message. If you think you are being maliciously
+subscribed to the list, or have any other questions, send them to
+pyar-owner@python.org.ar.
diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py
index ea75854..8c1cf20 100644
--- a/mail/src/leap/mail/imap/tests/test_imap.py
+++ b/mail/src/leap/mail/imap/tests/test_imap.py
@@ -25,7 +25,7 @@ XXX add authors from the original twisted tests.
@license: GPLv3, see included LICENSE file
"""
# XXX review license of the original tests!!!
-from nose.twistedtools import deferred
+from email import parser
try:
from cStringIO import StringIO
@@ -36,9 +36,13 @@ import os
import types
import tempfile
import shutil
+import time
+
+from itertools import chain
from mock import Mock
+from nose.twistedtools import deferred, stop_reactor
from twisted.mail import imap4
@@ -58,9 +62,9 @@ import twisted.cred.portal
# import u1db
from leap.common.testing.basetest import BaseLeapTest
-from leap.mail.imap.server import SoledadMailbox
-from leap.mail.imap.server import SoledadBackedAccount
-from leap.mail.imap.server import MessageCollection
+from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.messages import MessageCollection
from leap.soledad.client import Soledad
from leap.soledad.client import SoledadCrypto
@@ -321,6 +325,9 @@ class IMAP4HelperMixin(BaseLeapTest):
for mb in self.server.theAccount.mailboxes:
self.server.theAccount.delete(mb)
+ # email parser
+ self.parser = parser.Parser()
+
def tearDown(self):
"""
tearDown method called after each test.
@@ -350,11 +357,11 @@ class IMAP4HelperMixin(BaseLeapTest):
# XXX we also should put this in a mailbox!
- self._soledad.messages.add_msg('', subject="test1")
- self._soledad.messages.add_msg('', subject="test2")
- self._soledad.messages.add_msg('', subject="test3")
+ self._soledad.messages.add_msg('', uid=1, subject="test1")
+ self._soledad.messages.add_msg('', uid=2, subject="test2")
+ self._soledad.messages.add_msg('', uid=3, subject="test3")
# XXX should change Flags too
- self._soledad.messages.add_msg('', subject="test4")
+ self._soledad.messages.add_msg('', uid=4, subject="test4")
def delete_all_docs(self):
"""
@@ -389,6 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Tests for the MessageCollection class
"""
+ count = 0
def setUp(self):
"""
@@ -396,34 +404,35 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
We override mixin method since we are only testing
MessageCollection interface in this particular TestCase
"""
- self.messages = MessageCollection("testmbox", self._soledad)
- for m in self.messages.get_all():
- self.messages.remove(m)
+ self.messages = MessageCollection("testmbox%s" % (self.count,),
+ self._soledad)
+ MessageCollectionTestCase.count += 1
def tearDown(self):
"""
tearDown method for each test
- Delete the message collection
"""
del self.messages
+ def wait(self):
+ time.sleep(2)
+
def testEmptyMessage(self):
"""
Test empty message and collection
"""
- em = self.messages._get_empty_msg()
+ em = self.messages._get_empty_doc()
self.assertEqual(
em,
{
- "date": '',
"flags": [],
- "headers": {},
"mbox": "inbox",
- "raw": "",
"recent": True,
"seen": False,
- "subject": "",
- "type": "msg",
+ "deleted": False,
+ "multi": False,
+ "size": 0,
+ "type": "flags",
"uid": 1,
})
self.assertEqual(self.messages.count(), 0)
@@ -432,23 +441,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Add multiple messages
"""
+ # TODO really profile addition
mc = self.messages
+ print "messages", self.messages
self.assertEqual(self.messages.count(), 0)
- mc.add_msg('Stuff', subject="test1")
- self.assertEqual(self.messages.count(), 1)
- mc.add_msg('Stuff', subject="test2")
- self.assertEqual(self.messages.count(), 2)
- mc.add_msg('Stuff', subject="test3")
- self.assertEqual(self.messages.count(), 3)
- mc.add_msg('Stuff', subject="test4")
+ mc.add_msg('Stuff', uid=1, subject="test1")
+ mc.add_msg('Stuff', uid=2, subject="test2")
+ mc.add_msg('Stuff', uid=3, subject="test3")
+ mc.add_msg('Stuff', uid=4, subject="test4")
+ self.wait()
self.assertEqual(self.messages.count(), 4)
- mc.add_msg('Stuff', subject="test5")
- mc.add_msg('Stuff', subject="test6")
- mc.add_msg('Stuff', subject="test7")
- mc.add_msg('Stuff', subject="test8")
- mc.add_msg('Stuff', subject="test9")
- mc.add_msg('Stuff', subject="test10")
- self.assertEqual(self.messages.count(), 10)
+ mc.add_msg('Stuff', uid=5, subject="test5")
+ mc.add_msg('Stuff', uid=6, subject="test6")
+ mc.add_msg('Stuff', uid=7, subject="test7")
+ self.wait()
+ self.assertEqual(self.messages.count(), 7)
+ self.wait()
def testRecentCount(self):
"""
@@ -456,45 +464,48 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
mc = self.messages
self.assertEqual(self.messages.count_recent(), 0)
- mc.add_msg('Stuff', subject="test1", uid=1)
+ mc.add_msg('Stuff', uid=1, subject="test1")
# For the semantics defined in the RFC, we auto-add the
# recent flag by default.
+ self.wait()
self.assertEqual(self.messages.count_recent(), 1)
- mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',))
+ mc.add_msg('Stuff', subject="test2", uid=2,
+ flags=('\\Deleted',))
+ self.wait()
self.assertEqual(self.messages.count_recent(), 2)
- mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',))
+ mc.add_msg('Stuff', subject="test3", uid=3,
+ flags=('\\Recent',))
+ self.wait()
self.assertEqual(self.messages.count_recent(), 3)
mc.add_msg('Stuff', subject="test4", uid=4,
flags=('\\Deleted', '\\Recent'))
+ self.wait()
self.assertEqual(self.messages.count_recent(), 4)
- for m in mc:
- msg = self.messages.get_msg_by_uid(m.get('uid'))
- msg_newflags = msg.removeFlags(('\\Recent',))
- self._soledad.put_doc(msg_newflags)
-
+ for msg in mc:
+ msg.removeFlags(('\\Recent',))
self.assertEqual(mc.count_recent(), 0)
def testFilterByMailbox(self):
"""
Test that queries filter by selected mailbox
"""
+ def wait():
+ time.sleep(1)
+
mc = self.messages
self.assertEqual(self.messages.count(), 0)
- mc.add_msg('', subject="test1")
- self.assertEqual(self.messages.count(), 1)
- mc.add_msg('', subject="test2")
- self.assertEqual(self.messages.count(), 2)
- mc.add_msg('', subject="test3")
+ mc.add_msg('', uid=1, subject="test1")
+ mc.add_msg('', uid=2, subject="test2")
+ mc.add_msg('', uid=3, subject="test3")
+ wait()
self.assertEqual(self.messages.count(), 3)
-
- newmsg = mc._get_empty_msg()
+ newmsg = mc._get_empty_doc()
newmsg['mailbox'] = "mailbox/foo"
- newmsg['subject'] = "test another mailbox"
mc._soledad.create_doc(newmsg)
self.assertEqual(mc.count(), 3)
self.assertEqual(
- len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4)
+ len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
@@ -1174,16 +1185,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def login():
return self.client.login('testuser', 'password-test')
+ def wait():
+ time.sleep(0.5)
+
def append():
return self.client.append(
'root/subthing',
message,
- ['\\SEEN', '\\DELETED'],
+ ('\\SEEN', '\\DELETED'),
'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
)
d1 = self.connected.addCallback(strip(login))
d1.addCallbacks(strip(append), self._ebGeneral)
+ d1.addCallbacks(strip(wait), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
@@ -1191,17 +1206,31 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def _cbTestFullAppend(self, ignored, infile):
mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing')
+ time.sleep(0.5)
self.assertEqual(1, len(mb.messages))
+ msg = mb.messages.get_msg_by_uid(1)
self.assertEqual(
- ['\\SEEN', '\\DELETED'],
- mb.messages[1].content['flags'])
+ ('\\SEEN', '\\DELETED'),
+ msg.getFlags())
self.assertEqual(
'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
- mb.messages[1].content['date'])
+ msg.getInternalDate())
+
+ parsed = self.parser.parse(open(infile))
+ body = parsed.get_payload()
+ headers = parsed.items()
+ self.assertEqual(
+ body,
+ msg.getBodyFile().read())
+
+ msg_headers = msg.getHeaders(True, "",)
+ gotheaders = list(chain(
+ *[[(k, item) for item in v] for (k, v) in msg_headers.items()]))
- self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+ self.assertItemsEqual(
+ headers, gotheaders)
@deferred(timeout=None)
def testPartialAppend(self):
@@ -1209,12 +1238,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
Test partially appending a message to the mailbox
"""
infile = util.sibpath(__file__, 'rfc822.message')
- message = open(infile)
SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
def login():
return self.client.login('testuser', 'password-test')
+ def wait():
+ time.sleep(1)
+
def append():
message = file(infile)
return self.client.sendCommand(
@@ -1226,6 +1257,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
)
)
d1 = self.connected.addCallback(strip(login))
+ d1.addCallbacks(strip(wait), self._ebGeneral)
d1.addCallbacks(strip(append), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
@@ -1235,15 +1267,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def _cbTestPartialAppend(self, ignored, infile):
mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
-
+ time.sleep(1)
self.assertEqual(1, len(mb.messages))
+ msg = mb.messages.get_msg_by_uid(1)
self.assertEqual(
- ['\\SEEN', ],
- mb.messages[1].content['flags']
+ ('\\SEEN', ),
+ msg.getFlags()
)
+ #self.assertEqual(
+ #'Right now', msg.getInternalDate())
+ parsed = self.parser.parse(open(infile))
+ body = parsed.get_payload()
self.assertEqual(
- 'Right now', mb.messages[1].content['date'])
- self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+ body,
+ msg.getBodyFile().read())
@deferred(timeout=None)
def testCheck(self):
@@ -1279,14 +1316,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.server.theAccount.addMailbox(name)
m = SimpleLEAPServer.theAccount.getMailbox(name)
- m.messages.add_msg('', subject="Message 1",
+ m.messages.add_msg('test 1', uid=1, subject="Message 1",
flags=('\\Deleted', 'AnotherFlag'))
- m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
- m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
+ m.messages.add_msg('test 2', uid=2, subject="Message 2",
+ flags=('AnotherFlag',))
+ m.messages.add_msg('test 3', uid=3, subject="Message 3",
+ flags=('\\Deleted',))
def login():
return self.client.login('testuser', 'password-test')
+ def wait():
+ time.sleep(1)
+
def select():
return self.client.select(name)
@@ -1294,6 +1336,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return self.client.close()
d = self.connected.addCallback(strip(login))
+ d.addCallbacks(strip(wait), self._ebGeneral)
d.addCallbacks(strip(select), self._ebGeneral)
d.addCallbacks(strip(close), self._ebGeneral)
d.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -1302,8 +1345,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def _cbTestClose(self, ignored, m):
self.assertEqual(len(m.messages), 1)
+ messages = [msg for msg in m.messages]
+ self.assertFalse(messages[0] is None)
self.assertEqual(
- m.messages[1].content['subject'],
+ messages[0]._hdoc.content['subject'],
'Message 2')
self.failUnless(m.closed)
@@ -1315,17 +1360,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
name = 'mailbox-expunge'
SimpleLEAPServer.theAccount.addMailbox(name)
m = SimpleLEAPServer.theAccount.getMailbox(name)
- m.messages.add_msg('', subject="Message 1",
+ m.messages.add_msg('test 1', uid=1, subject="Message 1",
flags=('\\Deleted', 'AnotherFlag'))
- self.failUnless(m.messages.count() == 1)
- m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
- self.failUnless(m.messages.count() == 2)
- m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
- self.failUnless(m.messages.count() == 3)
+ m.messages.add_msg('test 2', uid=2, subject="Message 2",
+ flags=('AnotherFlag',))
+ m.messages.add_msg('test 3', uid=3, subject="Message 3",
+ flags=('\\Deleted',))
def login():
return self.client.login('testuser', 'password-test')
+ def wait():
+ time.sleep(2)
+
def select():
return self.client.select('mailbox-expunge')
@@ -1338,6 +1385,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.results = None
d1 = self.connected.addCallback(strip(login))
+ d1.addCallbacks(strip(wait), self._ebGeneral)
d1.addCallbacks(strip(select), self._ebGeneral)
d1.addCallbacks(strip(expunge), self._ebGeneral)
d1.addCallbacks(expunged, self._ebGeneral)
@@ -1348,18 +1396,94 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def _cbTestExpunge(self, ignored, m):
# we only left 1 mssage with no deleted flag
- self.assertEqual(m.messages.count(), 1)
+ self.assertEqual(len(m.messages), 1)
+ messages = [msg for msg in m.messages]
self.assertEqual(
- m.messages[1].content['subject'],
+ messages[0]._hdoc.content['subject'],
'Message 2')
- self.assertEqual(self.results, [0, 1])
- # XXX fix this thing with the indexes...
+ # the uids of the deleted messages
+ self.assertItemsEqual(self.results, [1, 3])
+
+
+class StoreAndFetchTestCase(unittest.TestCase, IMAP4HelperMixin):
+ """
+ Several tests to check that the internal storage representation
+ is able to render the message structures as we expect them.
+ """
+ # TODO get rid of the fucking sleeps with a proper defer
+ # management.
+
+ def setUp(self):
+ IMAP4HelperMixin.setUp(self)
+ MBOX_NAME = "multipart/SIGNED"
+ self.received_messages = self.received_uid = None
+ self.result = None
+
+ self.server.state = 'select'
+
+ infile = util.sibpath(__file__, 'rfc822.multi-signed.message')
+ raw = open(infile).read()
+
+ self.server.theAccount.addMailbox(MBOX_NAME)
+ mbox = self.server.theAccount.getMailbox(MBOX_NAME)
+ time.sleep(1)
+ self.server.mbox = mbox
+ self.server.mbox.messages.add_msg(raw, uid=1)
+ time.sleep(1)
+
+ def addListener(self, x):
+ pass
+
+ def removeListener(self, x):
+ pass
+
+ def _fetchWork(self, uids):
+
+ def result(R):
+ self.result = R
+
+ self.connected.addCallback(
+ lambda _: self.function(
+ uids, uid=1) # do NOT use seq numbers!
+ ).addCallback(result).addCallback(
+ self._cbStopClient).addErrback(self._ebGeneral)
+
+ d = loopback.loopbackTCP(self.server, self.client, noisy=False)
+ d.addCallback(lambda x: self.assertEqual(self.result, self.expected))
+ return d
+
+ @deferred(timeout=None)
+ def testMultiBody(self):
+ """
+ Test that a multipart signed message is retrieved the same
+ as we stored it.
+ """
+ time.sleep(1)
+ self.function = self.client.fetchBody
+ messages = '1'
+
+ # XXX review. This probably should give everything?
+
+ self.expected = {1: {
+ 'RFC822.TEXT': 'This is an example of a signed message,\n'
+ 'with attachments.\n\n\n--=20\n'
+ 'Nihil sine chao! =E2=88=B4\n',
+ 'UID': '1'}}
+ print "test multi: fetch uid", messages
+ return self._fetchWork(messages)
class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
- Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}.
+ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}.
"""
# XXX coming soon to your screens!
pass
+
+
+def tearDownModule():
+ """
+ Tear down functions for module level
+ """
+ stop_reactor()
diff --git a/mail/src/leap/mail/imap/tests/walktree.py b/mail/src/leap/mail/imap/tests/walktree.py
new file mode 100644
index 0000000..1626f65
--- /dev/null
+++ b/mail/src/leap/mail/imap/tests/walktree.py
@@ -0,0 +1,117 @@
+#t -*- coding: utf-8 -*-
+# walktree.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the walktree module.
+"""
+import os
+from email import parser
+
+from leap.mail import walk as W
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+p = parser.Parser()
+
+# TODO pass an argument of the type of message
+
+##################################################
+# Input from hell
+
+#msg = p.parse(open('rfc822.multi-signed.message'))
+#msg = p.parse(open('rfc822.plain.message'))
+msg = p.parse(open('rfc822.multi-minimal.message'))
+DO_CHECK = False
+#################################################
+
+parts = W.get_parts(msg)
+
+if DEBUG:
+ def trim(item):
+ item = item[:10]
+ [trim(part["phash"]) for part in parts if part.get('phash', None)]
+
+raw_docs = list(W.get_raw_docs(msg, parts))
+
+body_phash_fun = [W.get_body_phash_simple,
+ W.get_body_phash_multi][int(msg.is_multipart())]
+body_phash = body_phash_fun(W.get_payloads(msg))
+parts_map = W.walk_msg_tree(parts, body_phash=body_phash)
+
+
+# TODO add missing headers!
+expected = {
+ 'body': '1ddfa80485',
+ 'multi': True,
+ 'part_map': {
+ 1: {
+ 'headers': {'Content-Disposition': 'inline',
+ 'Content-Type': 'multipart/mixed; '
+ 'boundary="z0eOaCaDLjvTGF2l"'},
+ 'multi': True,
+ 'part_map': {1: {'ctype': 'text/plain',
+ 'headers': [
+ ('Content-Type',
+ 'text/plain; charset=utf-8'),
+ ('Content-Disposition',
+ 'inline'),
+ ('Content-Transfer-Encoding',
+ 'quoted-printable')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': '1ddfa80485',
+ 'size': 206},
+ 2: {'ctype': 'text/plain',
+ 'headers': [('Content-Type',
+ 'text/plain; charset=us-ascii'),
+ ('Content-Disposition',
+ 'attachment; '
+ 'filename="attach.txt"')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': '7a94e4d769',
+ 'size': 133},
+ 3: {'ctype': 'application/octet-stream',
+ 'headers': [('Content-Type',
+ 'application/octet-stream'),
+ ('Content-Disposition',
+ 'attachment; filename="hack.ico"'),
+ ('Content-Transfer-Encoding',
+ 'base64')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': 'c42cccebbd',
+ 'size': 12736}}},
+ 2: {'ctype': 'application/pgp-signature',
+ 'headers': [('Content-Type', 'application/pgp-signature')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': '8f49fbf749',
+ 'size': 877}}}
+
+if DEBUG and DO_CHECK:
+ # TODO turn this into a proper unittest
+ assert(parts_map == expected)
+ print "Structure: OK"
+
+
+import pprint
+print
+print "RAW DOCS"
+pprint.pprint(raw_docs)
+print
+print "PARTS MAP"
+pprint.pprint(parts_map)
diff --git a/mail/src/leap/mail/utils.py b/mail/src/leap/mail/utils.py
new file mode 100644
index 0000000..2480efc
--- /dev/null
+++ b/mail/src/leap/mail/utils.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Small utilities.
+"""
+
+
+def first(things):
+ """
+ Return the head of a collection.
+ """
+ try:
+ return things[0]
+ except (IndexError, TypeError):
+ return None
diff --git a/mail/src/leap/mail/walk.py b/mail/src/leap/mail/walk.py
new file mode 100644
index 0000000..820b8c7
--- /dev/null
+++ b/mail/src/leap/mail/walk.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# walk.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for walking along a message tree.
+"""
+import hashlib
+import os
+
+from leap.mail.utils import first
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+if DEBUG:
+ get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10]
+else:
+ get_hash = lambda s: hashlib.sha256(s).hexdigest()
+
+
+"""
+Get interesting message parts
+"""
+get_parts = lambda msg: [
+ {'multi': part.is_multipart(),
+ 'ctype': part.get_content_type(),
+ 'size': len(part.as_string()),
+ 'parts': len(part.get_payload())
+ if isinstance(part.get_payload(), list)
+ else 1,
+ 'headers': part.items(),
+ 'phash': get_hash(part.get_payload())
+ if not part.is_multipart() else None}
+ for part in msg.walk()]
+
+"""
+Utility lambda functions for getting the parts vector and the
+payloads from the original message.
+"""
+
+get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts)
+get_payloads = lambda msg: ((x.get_payload(),
+ dict(((str.lower(k), v) for k, v in (x.items()))))
+ for x in msg.walk())
+
+get_body_phash_simple = lambda payloads: first(
+ [get_hash(payload) for payload, headers in payloads
+ if "text/plain" in headers.get('content-type')])
+
+get_body_phash_multi = lambda payloads: (first(
+ [get_hash(payload) for payload, headers in payloads
+ if "text/plain" in headers.get('content-type')])
+ or get_body_phash_simple(payloads))
+
+"""
+On getting the raw docs, we get also some of the headers to be able to
+index the content. Here we remove any mutable part, as the the filename
+in the content disposition.
+"""
+
+get_raw_docs = lambda msg, parts: (
+ {"type": "cnt", # type content they'll be
+ "raw": payload if not DEBUG else payload[:100],
+ "phash": get_hash(payload),
+ "content-disposition": first(headers.get(
+ 'content-disposition', '').split(';')),
+ "content-type": headers.get(
+ 'content-type', ''),
+ "content-transfer-encoding": headers.get(
+ 'content-transfer-type', '')}
+ for payload, headers in get_payloads(msg)
+ if not isinstance(payload, list))
+
+
+def walk_msg_tree(parts, body_phash=None):
+ """
+ Take a list of interesting items of a message subparts structure,
+ and return a dict of dicts almost ready to be written to the content
+ documents that will be stored in Soledad.
+
+ It walks down the subparts in the parsed message tree, and collapses
+ the leaf docuents into a wrapper document until no multipart submessages
+ are left. To achieve this, it iteratively calculates a wrapper vector of
+ all documents in the sequence that have more than one part and have unitary
+ documents to their right. To collapse a multipart, take as many
+ unitary documents as parts the submessage contains, and replace the object
+ in the sequence with the new wrapper document.
+
+ :param parts: A list of dicts containing the interesting properties for
+ the message structure. Normally this has been generated by
+ doing a message walk.
+ :type parts: list of dicts.
+ :param body_phash: the payload hash of the body part, to be included
+ in the outer content doc for convenience.
+ :type body_phash: basestring or None
+ """
+ # parts vector
+ pv = list(get_parts_vector(parts))
+
+ if len(parts) == 2:
+ inner_headers = parts[1].get("headers", None)
+
+ if DEBUG:
+ print "parts vector: ", pv
+ print
+
+ # wrappers vector
+ getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False
+ for i in range(len(pv) - 1)]
+ wv = getwv(pv)
+
+ # do until no wrapper document is left
+ while any(wv):
+ wind = wv.index(True) # wrapper index
+ nsub = pv[wind] # number of subparts to pick
+ slic = parts[wind + 1:wind + 1 + nsub] # slice with subparts
+
+ cwra = {
+ "multi": True,
+ "part_map": dict((index + 1, part) # content wrapper
+ for index, part in enumerate(slic)),
+ "headers": dict(parts[wind]['headers'])
+ }
+
+ # remove subparts and substitue wrapper
+ map(lambda i: parts.remove(i), slic)
+ parts[wind] = cwra
+
+ # refresh vectors for this iteration
+ pv = list(get_parts_vector(parts))
+ wv = getwv(pv)
+
+ outer = parts[0]
+ outer.pop('headers')
+ if not "part_map" in outer:
+ # we have a multipart with 1 part only, so kind of fix it
+ # although it would be prettier if I take this special case at
+ # the beginning of the walk.
+ pdoc = {"multi": True,
+ "part_map": {1: outer}}
+ pdoc["part_map"][1]["multi"] = False
+ if not pdoc["part_map"][1].get("phash", None):
+ pdoc["part_map"][1]["phash"] = body_phash
+ pdoc["part_map"][1]["headers"] = inner_headers
+ else:
+ pdoc = outer
+ pdoc["body"] = body_phash
+ return pdoc