From a65aba63cf90efcf036ed7ba0d515865c5e8457e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 24 Dec 2013 20:28:58 -0400 Subject: defer costly operations --- src/leap/mail/imap/server.py | 88 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b9b72d0..e97ed2a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -21,11 +21,13 @@ import copy import logging import StringIO import cStringIO +import os import time import re from collections import defaultdict from email.parser import Parser +from functools import wraps from zope.interface import implements from zope.proxy import sameProxiedObjects @@ -35,6 +37,7 @@ from twisted.internet import defer from twisted.internet.threads import deferToThread from twisted.python import log +from u1db import errors as u1db_errors from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL @@ -46,6 +49,65 @@ from leap.soledad.client import Soledad logger = logging.getLogger(__name__) +def deferred(f): + ''' + Decorator, for deferring methods to Threads. + + It will do a deferToThread of the decorated method + unless the environment variable LEAPMAIL_DEBUG is set. + + It uses a descriptor to delay the definition of the + method wrapper. + ''' + class descript(object): + def __init__(self, f): + self.f = f + + def __get__(self, instance, klass): + if instance is None: + # Class method was requested + return self.make_unbound(klass) + return self.make_bound(instance) + + def _errback(self, failure): + err = failure.value + #logger.error(err) + log.err(err) + + def make_unbound(self, klass): + + @wraps(self.f) + def wrapper(*args, **kwargs): + '''This documentation will vanish :)''' + raise TypeError( + 'unbound method {}() must be called with {} instance ' + 'as first argument (got nothing instead)'.format( + self.f.__name__, + klass.__name__) + ) + return wrapper + + def make_bound(self, instance): + + @wraps(self.f) + def wrapper(*args, **kwargs): + '''This documentation will disapear :)''' + + if not os.environ.get('LEAPMAIL_DEBUG'): + d = deferToThread(self.f, instance, *args, **kwargs) + d.addErrback(self._errback) + return d + else: + return self.f(instance, *args, **kwargs) + + # This instance does not need the descriptor anymore, + # let it find the wrapper directly next time: + setattr(instance, self.f.__name__, wrapper) + return wrapper + + return descript(f) + + class MissingIndexError(Exception): """ Raises when tried to access a non existent index document. @@ -870,9 +932,19 @@ class SoledadDocWriter(object): payload = item['payload'] mode = item['mode'] if mode == "create": - self._soledad.create_doc(payload) + call = self._soledad.create_doc elif mode == "put": - self._soledad.put_doc(payload) + call = self._soledad.put_doc + + # should handle errors + try: + call(payload) + except u1db_errors.RevisionConflict as exc: + logger.error("Error: %r" % (exc,)) + # XXX DEBUG -- remove-me + #logger.debug("conflicting doc: %s" % payload) + raise exc + empty = queue.empty() @@ -954,6 +1026,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MSG) + @deferred def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): """ Creates a new message document. @@ -1639,6 +1712,7 @@ class SoledadMailbox(WithMsgFields): # more generically return [x for x in range(len(deleted))] + @deferred def fetch(self, messages, uid): """ Retrieve one or more messages in this mailbox. @@ -1668,6 +1742,7 @@ class SoledadMailbox(WithMsgFields): # for sequence numbers (uid = 0) if sequence: + logger.debug("Getting msg by index: INEFFICIENT call!") for msg_id in messages: msg = self.messages.get_msg_by_index(msg_id - 1) if msg: @@ -1686,8 +1761,11 @@ class SoledadMailbox(WithMsgFields): if self.isWriteable(): self._unset_recent_flag() - return tuple(result[:100]) + # XXX workaround for hangs in thunderbird + #return tuple(result[:100]) + return tuple(result) + @deferred def _unset_recent_flag(self): """ Unsets `Recent` flag from a tuple of messages. @@ -1706,10 +1784,12 @@ class SoledadMailbox(WithMsgFields): then that message SHOULD be considered recent. """ log.msg('unsetting recent flags...') + for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()): newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,)) self._update(newflags) + @deferred def _signal_unread_to_ui(self): """ Sends unread event to ui. @@ -1717,6 +1797,7 @@ class SoledadMailbox(WithMsgFields): unseen = self.getUnseenCount() leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + @deferred def store(self, messages, flags, mode, uid): """ Sets the flags of one or more messages. @@ -1774,6 +1855,7 @@ class SoledadMailbox(WithMsgFields): self._signal_unread_to_ui() return result + @deferred def close(self): """ Expunge and mark as closed -- cgit v1.2.3 From 70fdb37740b0737202ad08b4637bf23c12dab87e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 25 Dec 2013 02:46:27 -0400 Subject: Move flags and other metadata to a separate doc. This change will allow for quicker access times, and smaller syncs since the fields that change more often will fall in a pretty small document. For the big raw message, we only need to sync once. Also, implemented multipart interface for messages. This will need additional migration helper in --repair-mailboxes. --- src/leap/mail/imap/server.py | 612 ++++++++++++++++++++++++------------------- 1 file changed, 338 insertions(+), 274 deletions(-) (limited to 'src') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index e97ed2a..8758dcb 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -25,7 +25,7 @@ import os import time import re -from collections import defaultdict +from collections import defaultdict, namedtuple from email.parser import Parser from functools import wraps @@ -71,7 +71,7 @@ def deferred(f): def _errback(self, failure): err = failure.value - #logger.error(err) + logger.warning('error in method: %s' % (self.f.__name__)) log.err(err) def make_unbound(self, klass): @@ -133,6 +133,8 @@ class WithMsgFields(object): RAW_KEY = "raw" SUBJECT_KEY = "subject" UID_KEY = "uid" + MULTIPART_KEY = "multi" + SIZE_KEY = "size" # Mailbox specific keys CLOSED_KEY = "closed" @@ -145,6 +147,8 @@ class WithMsgFields(object): TYPE_KEY = "type" TYPE_MESSAGE_VAL = "msg" TYPE_MBOX_VAL = "mbox" + TYPE_FLAGS_VAL = "flags" + # should add also a headers val INBOX_VAL = "inbox" @@ -166,6 +170,8 @@ class WithMsgFields(object): SUBJECT_FIELD = "Subject" DATE_FIELD = "Date" +fields = WithMsgFields # alias for convenience + class IndexedDB(object): """ @@ -209,12 +215,79 @@ class IndexedDB(object): self._soledad.create_index(name, *expression) +class MailParser(object): + """ + Mixin with utility methods to parse raw messages. + """ + def __init__(self): + """ + Initializes the mail parser. + """ + self._parser = Parser() + + def _get_parsed_msg(self, raw): + """ + Return a parsed Message. + + :param raw: the raw string to parse + :type raw: basestring, or StringIO object + """ + msg = self._get_parser_fun(raw)(raw, True) + return msg + + def _get_parser_fun(self, o): + """ + Retunn the proper parser function for an object. + + :param o: object + :type o: object + :param parser: an instance of email.parser.Parser + :type parser: email.parser.Parser + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return self._parser.parse + if isinstance(o, basestring): + return self._parser.parsestr + + def _stringify(self, o): + """ + Return a string object. + + :param o: object + :type o: object + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return o.getvalue() + else: + return o + + +class MBoxParser(object): + """ + Utility function to parse mailbox names. + """ + INBOX_NAME = "INBOX" + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + def _parse_mailbox_name(self, name): + """ + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name + + ####################################### # Soledad Account ####################################### -class SoledadBackedAccount(WithMsgFields, IndexedDB): +class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): """ An implementation of IAccount and INamespacePresenteer that is backed by Soledad Encrypted Documents. @@ -254,12 +327,11 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): 'bool(recent)', 'bool(seen)'], } - INBOX_NAME = "INBOX" MBOX_KEY = MBOX_VAL EMPTY_MBOX = { WithMsgFields.TYPE_KEY: MBOX_KEY, - WithMsgFields.TYPE_MBOX_VAL: INBOX_NAME, + WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, WithMsgFields.SUBJECT_KEY: "", WithMsgFields.FLAGS_KEY: [], WithMsgFields.CLOSED_KEY: False, @@ -268,8 +340,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): WithMsgFields.LAST_UID_KEY: 0 } - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - def __init__(self, account_name, soledad=None): """ Creates a SoledadAccountIndex that keeps track of the mailboxes @@ -306,18 +376,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MBOX) - def _parse_mailbox_name(self, name): - """ - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name - def _get_mailbox_by_name(self, name): """ Return an mbox document by name. @@ -420,7 +478,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - paths = filter(None, + paths = filter( + None, self._parse_mailbox_name(pathspec).split('/')) for accum in range(1, len(paths)): try: @@ -665,19 +724,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): ####################################### -class LeapMessage(WithMsgFields): +class LeapMessage(fields, MailParser, MBoxParser): - implements(imap4.IMessage, imap4.IMessageFile) + implements(imap4.IMessage) - def __init__(self, doc): + def __init__(self, soledad, uid, mbox): """ Initializes a LeapMessage. - :param doc: A SoledadDocument containing the internal - representation of the message - :type doc: SoledadDocument + :param soledad: a Soledad instance + :type soledad: Soledad + :param uid: the UID for the message. + :type uid: int or basestring + :param mbox: the mbox this message belongs to + :type mbox: basestring """ - self._doc = doc + MailParser.__init__(self) + self._soledad = soledad + self._uid = int(uid) + self._mbox = self._parse_mailbox_name(mbox) + + self.__cdoc = None + + @property + def _fdoc(self): + """ + An accessor to the flags docuemnt + """ + return self._get_flags_doc() + + @property + def _cdoc(self): + """ + An accessor to the content docuemnt + """ + if not self.__cdoc: + self.__cdoc = self._get_content_doc() + return self.__cdoc def getUID(self): """ @@ -686,11 +769,7 @@ class LeapMessage(WithMsgFields): :return: uid for this message :rtype: int """ - # XXX debug, to remove after a while... - if not self._doc: - log.msg('BUG!!! ---- message has no doc!') - return - return self._doc.content[self.UID_KEY] + return self._uid def getFlags(self): """ @@ -699,9 +778,13 @@ class LeapMessage(WithMsgFields): :return: The flags, represented as strings :rtype: tuple """ - if self._doc is None: + if self._uid is None: return [] - flags = self._doc.content.get(self.FLAGS_KEY, None) + + flags = [] + flag_doc = self._fdoc + if flag_doc: + flags = flag_doc.content.get(self.FLAGS_KEY, None) if flags: flags = map(str, flags) return tuple(flags) @@ -722,12 +805,13 @@ class LeapMessage(WithMsgFields): :rtype: SoledadDocument """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - log.msg('setting flags') - doc = self._doc + log.msg('setting flags: %s' % (self._uid)) + + doc = self._fdoc doc.content[self.FLAGS_KEY] = flags doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags - return doc + self._soledad.put_doc(doc) def addFlags(self, flags): """ @@ -743,7 +827,7 @@ class LeapMessage(WithMsgFields): """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") oldflags = self.getFlags() - return self.setFlags(tuple(set(flags + oldflags))) + self.setFlags(tuple(set(flags + oldflags))) def removeFlags(self, flags): """ @@ -759,7 +843,7 @@ class LeapMessage(WithMsgFields): """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") oldflags = self.getFlags() - return self.setFlags(tuple(set(oldflags) - set(flags))) + self.setFlags(tuple(set(oldflags) - set(flags))) def getInternalDate(self): """ @@ -768,48 +852,14 @@ class LeapMessage(WithMsgFields): :rtype: C{str} :return: An RFC822-formatted date string. """ - return str(self._doc.content.get(self.DATE_KEY, '')) - - # - # IMessageFile - # - - """ - Optional message interface for representing messages as files. - - If provided by message objects, this interface will be used instead - the more complex MIME-based interface. - """ - - def open(self): - """ - Return an file-like object opened for reading. - - Reading from the returned file will return all the bytes - of which this message consists. - - :return: file-like object opened fore reading. - :rtype: StringIO - """ - fd = cStringIO.StringIO() - content = self._doc.content.get(self.RAW_KEY, '') - charset = get_email_charset( - unicode(self._doc.content.get(self.RAW_KEY, ''))) - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - fd.write(content) - fd.seek(0) - return fd + return str(self._cdoc.content.get(self.DATE_KEY, '')) # # IMessagePart # - # XXX should implement the rest of IMessagePart interface: - # (and do not use the open above) + # XXX we should implement this interface too for the subparts + # so we allow nested parts... def getBodyFile(self): """ @@ -819,15 +869,21 @@ class LeapMessage(WithMsgFields): :rtype: StringIO """ fd = StringIO.StringIO() - content = self._doc.content.get(self.RAW_KEY, '') + + cdoc = self._cdoc + content = cdoc.content.get(self.RAW_KEY, '') charset = get_email_charset( - unicode(self._doc.content.get(self.RAW_KEY, ''))) + unicode(cdoc.content.get(self.RAW_KEY, ''))) try: content = content.encode(charset) except (UnicodeEncodeError, UnicodeDecodeError) as e: logger.error("Unicode error {0}".format(e)) content = content.encode(charset, 'replace') - fd.write(content) + + raw = self._get_raw_msg() + msg = self._get_parsed_msg(raw) + body = msg.get_payload() + fd.write(body) # XXX SHOULD use a separate BODY FIELD ... fd.seek(0) return fd @@ -839,13 +895,18 @@ class LeapMessage(WithMsgFields): :return: size of the message, in octets :rtype: int """ - return self.getBodyFile().len + size = self._cdoc.content.get(self.SIZE_KEY, False) + if not size: + # XXX fallback, should remove when all migrated. + size = self.getBodyFile().len + return size def _get_headers(self): """ Return the headers dict stored in this message document. """ - return self._doc.content.get(self.HEADERS_KEY, {}) + # XXX get from the headers doc + return self._cdoc.content.get(self.HEADERS_KEY, {}) def getHeaders(self, negate, *names): """ @@ -876,30 +937,90 @@ class LeapMessage(WithMsgFields): if cond(key)] return dict(filter_by_cond) - # --- no multipart for now - # XXX Fix MULTIPART SUPPORT! - def isMultipart(self): - return False + """ + Return True if this message is multipart. + """ + if self._cdoc: + retval = self._cdoc.content.get(self.MULTIPART_KEY, False) + print "MULTIPART? ", retval - def getSubPart(part): - return None + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + if not self.isMultipart(): + raise TypeError + + msg = self._get_parsed_msg() + # XXX should wrap IMessagePart + return msg.get_payload()[part] # # accessors # + def _get_flags_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + flag_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + flag_doc = flag_docs[0] if flag_docs else None + return flag_doc + + def _get_content_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + cont_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid)) + cont_doc = cont_docs[0] if cont_docs else None + return cont_doc + + def _get_raw_msg(self): + """ + Return the raw msg. + :rtype: basestring + """ + return self._cdoc.content.get(self.RAW_KEY, '') + def __getitem__(self, key): """ Return the content of the message document. - @param key: The key - @type key: str + :param key: The key + :type key: str - @return: The content value indexed by C{key} or None - @rtype: str + :return: The content value indexed by C{key} or None + :rtype: str """ - return self._doc.content.get(key, None) + return self._cdoc.content.get(key, None) + + def does_exist(self): + """ + Return True if there is actually a message for this + UID and mbox. + """ + return bool(self._fdoc) + + +SoledadWriterPayload = namedtuple( + 'SoledadWriterPayload', ['mode', 'payload']) + +SoledadWriterPayload.CREATE = 1 +SoledadWriterPayload.PUT = 2 class SoledadDocWriter(object): @@ -929,26 +1050,22 @@ class SoledadDocWriter(object): empty = queue.empty() while not empty: item = queue.get() - payload = item['payload'] - mode = item['mode'] - if mode == "create": + if item.mode == SoledadWriterPayload.CREATE: call = self._soledad.create_doc - elif mode == "put": + elif item.mode == SoledadWriterPayload.PUT: call = self._soledad.put_doc # should handle errors try: - call(payload) + call(item.payload) except u1db_errors.RevisionConflict as exc: logger.error("Error: %r" % (exc,)) - # XXX DEBUG -- remove-me - #logger.debug("conflicting doc: %s" % payload) raise exc empty = queue.empty() -class MessageCollection(WithMsgFields, IndexedDB): +class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): """ A collection of messages, surprisingly. @@ -959,16 +1076,27 @@ class MessageCollection(WithMsgFields, IndexedDB): # XXX this should be able to produce a MessageSet methinks EMPTY_MSG = { - WithMsgFields.TYPE_KEY: WithMsgFields.TYPE_MESSAGE_VAL, - WithMsgFields.UID_KEY: 1, - WithMsgFields.MBOX_KEY: WithMsgFields.INBOX_VAL, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.DATE_KEY: "", - WithMsgFields.SEEN_KEY: False, - WithMsgFields.RECENT_KEY: True, - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.HEADERS_KEY: {}, - WithMsgFields.RAW_KEY: "", + fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.SUBJECT_KEY: "", + fields.DATE_KEY: "", + fields.RAW_KEY: "", + + # XXX should separate headers into another doc + fields.HEADERS_KEY: {}, + } + + EMPTY_FLAGS = { + fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.FLAGS_KEY: [], + fields.SEEN_KEY: False, + fields.RECENT_KEY: True, + fields.MULTIPART_KEY: False, } # get from SoledadBackedAccount the needed index-related constants @@ -987,25 +1115,17 @@ class MessageCollection(WithMsgFields, IndexedDB): :param soledad: Soledad database :type soledad: Soledad instance """ - # XXX pass soledad directly - + MailParser.__init__(self) leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(mbox.strip() != "", "mbox cannot be blank space") leap_assert(isinstance(mbox, (str, unicode)), "mbox needs to be a string") leap_assert(soledad, "Need a soledad instance to initialize") - # This is a wrapper now!... - # should move assertion there... - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - # okay, all in order, keep going... - - self.mbox = mbox.upper() + self.mbox = self._parse_mailbox_name(mbox) self._soledad = soledad self.initialize_db() - self._parser = Parser() # I think of someone like nietzsche when reading this @@ -1015,7 +1135,7 @@ class MessageCollection(WithMsgFields, IndexedDB): self.soledad_writer = MessageProducer( SoledadDocWriter(soledad), - period=0.1) + period=0.05) def _get_empty_msg(self): """ @@ -1026,6 +1146,15 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MSG) + def _get_empty_flags_doc(self): + """ + Returns an empty doc for storing flags. + + :return: + :rtype: + """ + return copy.deepcopy(self.EMPTY_FLAGS) + @deferred def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): """ @@ -1046,58 +1175,57 @@ class MessageCollection(WithMsgFields, IndexedDB): :param uid: the message uid for this mailbox :type uid: int """ + # TODO: split in smaller methods logger.debug('adding message') if flags is None: flags = tuple() leap_assert_type(flags, tuple) - def stringify(o): - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() - else: - return o + content_doc = self._get_empty_msg() + flags_doc = self._get_empty_flags_doc() - content = self._get_empty_msg() - content[self.MBOX_KEY] = self.mbox + content_doc[self.MBOX_KEY] = self.mbox + flags_doc[self.MBOX_KEY] = self.mbox + # ...should get a sanity check here. + content_doc[self.UID_KEY] = uid + flags_doc[self.UID_KEY] = uid if flags: - content[self.FLAGS_KEY] = map(stringify, flags) - content[self.SEEN_KEY] = self.SEEN_FLAG in flags + flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) + flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags - def _get_parser_fun(o): - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, (str, unicode)): - return self._parser.parsestr - - msg = _get_parser_fun(raw)(raw, True) + msg = self._get_parsed_msg(raw) headers = dict(msg) + flags_doc[self.MULTIPART_KEY] = msg.is_multipart() # XXX get lower case for keys? - content[self.HEADERS_KEY] = headers + # XXX get headers doc + content_doc[self.HEADERS_KEY] = headers # set subject based on message headers and eventually replace by # subject given as param if self.SUBJECT_FIELD in headers: - content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] if subject is not None: - content[self.SUBJECT_KEY] = subject - content[self.RAW_KEY] = stringify(raw) + content_doc[self.SUBJECT_KEY] = subject + + # XXX could separate body into its own doc + # but should also separate multiparts + # that should be wrapped in MessagePart + content_doc[self.RAW_KEY] = self._stringify(raw) + content_doc[self.SIZE_KEY] = len(raw) if not date and self.DATE_FIELD in headers: - content[self.DATE_KEY] = headers[self.DATE_FIELD] + content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] else: - content[self.DATE_KEY] = date - - # ...should get a sanity check here. - content[self.UID_KEY] = uid + content_doc[self.DATE_KEY] = date logger.debug('enqueuing message for write') - # XXX create namedtuple - self.soledad_writer.put({"mode": "create", - "payload": content}) - # XXX have to decide what shall we do with errors with this change... - #return self._soledad.create_doc(content) + ptuple = SoledadWriterPayload + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=content_doc)) + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=flags_doc)) def remove(self, msg): """ @@ -1110,23 +1238,6 @@ class MessageCollection(WithMsgFields, IndexedDB): # getters - def get_by_uid(self, uid): - """ - Retrieves a message document by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A SoledadDocument instance matching the query, - or None if not found. - :rtype: SoledadDocument - """ - docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, str(uid)) - - return docs[0] if docs else None - def get_msg_by_uid(self, uid): """ Retrieves a LeapMessage by UID. @@ -1138,43 +1249,10 @@ class MessageCollection(WithMsgFields, IndexedDB): or None if not found. :rtype: LeapMessage """ - doc = self.get_by_uid(uid) - if doc: - return LeapMessage(doc) - - def get_by_index(self, index): - """ - Retrieves a mesage document by mailbox index. - - :param index: the index of the sequence (zero-indexed) - :type index: int - """ - # XXX inneficient! ---- we should keep an index document - # with uid -- doc_uuid :) - try: - return self.get_all()[index] - except IndexError: + msg = LeapMessage(self._soledad, uid, self.mbox) + if not msg.does_exist(): return None - - def get_msg_by_index(self, index): - """ - Retrieves a LeapMessage by sequence index. - - :param index: the index of the sequence (zero-indexed) - :type index: int - """ - doc = self.get_by_index(index) - if doc: - return LeapMessage(doc) - - def is_deleted(self, doc): - """ - Returns whether a given doc is deleted or not. - - :param doc: the document to check - :rtype: bool - """ - return self.DELETED_FLAG in doc.content[self.FLAGS_KEY] + return msg def get_all(self): """ @@ -1184,18 +1262,19 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: a list of u1db documents :rtype: list of SoledadDocument """ + # TODO change to get_all_docs and turn this + # into returning messages if sameProxiedObjects(self._soledad, None): logger.warning('Tried to get messages but soledad is None!') return [] - #f XXX this should return LeapMessage instances all_docs = [doc for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox)] - # highly inneficient, but first let's grok it and then - # let's worry about efficiency. + fields.TYPE_FLAGS_VAL, self.mbox)] - # XXX FIXINDEX + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad return sorted(all_docs, key=lambda item: item.content['uid']) def count(self): @@ -1206,7 +1285,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ count = self._soledad.get_count_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox) + fields.TYPE_FLAGS_VAL, self.mbox) return count # unseen messages @@ -1215,13 +1294,13 @@ class MessageCollection(WithMsgFields, IndexedDB): """ Get an iterator for the message docs with no `seen` flag - :return: iterator through unseen message docs + :return: iterator through unseen message doc UIDs :rtype: iterable """ - return (doc for doc in + return (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '0')) + self.TYPE_FLAGS_VAL, self.mbox, '0')) def count_unseen(self): """ @@ -1232,7 +1311,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ count = self._soledad.get_count_from_index( SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '0') + self.TYPE_FLAGS_VAL, self.mbox, '0') return count def get_unseen(self): @@ -1242,7 +1321,8 @@ class MessageCollection(WithMsgFields, IndexedDB): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(doc) for doc in self.unseen_iter()] + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.unseen_iter()] # recent messages @@ -1253,10 +1333,10 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: iterator through recent message docs :rtype: iterable """ - return (doc for doc in + return (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1')) + self.TYPE_FLAGS_VAL, self.mbox, '1')) def get_recent(self): """ @@ -1265,7 +1345,8 @@ class MessageCollection(WithMsgFields, IndexedDB): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(doc) for doc in self.recent_iter()] + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.recent_iter()] def count_recent(self): """ @@ -1276,7 +1357,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ count = self._soledad.get_count_from_index( SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1') + self.TYPE_FLAGS_VAL, self.mbox, '1') return count def __len__(self): @@ -1297,23 +1378,6 @@ class MessageCollection(WithMsgFields, IndexedDB): # XXX return LeapMessage instead?! (change accordingly) return (m.content for m in self.get_all()) - def __getitem__(self, uid): - """ - Allows indexing as a list, with msg uid as the index. - - :param uid: an integer index - :type uid: int - - :return: LeapMessage or None if not found. - :rtype: LeapMessage - """ - # XXX FIXME inneficcient, we are evaulating. - try: - return [doc - for doc in self.get_all()][uid - 1] - except IndexError: - return None - def __repr__(self): """ Representation string for this object. @@ -1321,10 +1385,11 @@ class MessageCollection(WithMsgFields, IndexedDB): return u"" % ( self.mbox, self.count()) - # XXX should implement __eq__ also + # XXX should implement __eq__ also !!! --- use a hash + # of content for that, will be used for dedup. -class SoledadMailbox(WithMsgFields): +class SoledadMailbox(WithMsgFields, MBoxParser): """ A Soledad-backed IMAP mailbox. @@ -1373,7 +1438,7 @@ class SoledadMailbox(WithMsgFields): #leap_assert(isinstance(soledad._db, SQLCipherDatabase), #"soledad._db must be an instance of SQLCipherDatabase") - self.mbox = mbox + self.mbox = self._parse_mailbox_name(mbox) self.rw = rw self._soledad = soledad @@ -1440,11 +1505,6 @@ class SoledadMailbox(WithMsgFields): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - #return map(str, self.INIT_FLAGS) - - # XXX CHECK against thunderbird XXX - # XXX I think this is slightly broken.. :/ - mbox = self._get_mbox() if not mbox: return None @@ -1458,7 +1518,6 @@ class SoledadMailbox(WithMsgFields): :param flags: a tuple with the flags :type flags: tuple of str """ - # TODO -- fix also getFlags leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") mbox = self._get_mbox() @@ -1526,7 +1585,7 @@ class SoledadMailbox(WithMsgFields): # something is wrong, # just set the last uid # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to ", count) + logger.debug("WRONG uid < count. Setting last uid to %s", count) value = count mbox.content[key] = value @@ -1634,7 +1693,7 @@ class SoledadMailbox(WithMsgFields): if self.CMD_RECENT in names: r[self.CMD_RECENT] = self.getRecentCount() if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getMessageCount() + 1 + r[self.CMD_UIDNEXT] = self.last_uid + 1 if self.CMD_UIDVALIDITY in names: r[self.CMD_UIDVALIDITY] = self.getUID() if self.CMD_UNSEEN in names: @@ -1664,17 +1723,34 @@ class SoledadMailbox(WithMsgFields): else: flags = tuple(str(flag) for flag in flags) + d = self._do_add_messages(message, flags, date, uid_next) + d.addCallback(self._notify_new) + + @deferred + def _do_add_messages(self, message, flags, date, uid_next): + """ + Calls to the messageCollection add_msg method (deferred to thread). + Invoked from addMessage. + """ self.messages.add_msg(message, flags=flags, date=date, uid=uid_next) + def _notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + :param args: ignored. + """ exists = self.getMessageCount() recent = self.getRecentCount() - logger.debug("there are %s messages, %s recent" % ( + logger.debug("NOTIFY: there are %s messages, %s recent" % ( exists, recent)) - for listener in self.listeners: - listener.newMessages(exists, recent) - return defer.succeed(None) + + logger.debug("listeners: %s", str(self.listeners)) + for l in self.listeners: + logger.debug('notifying...') + l.newMessages(exists, recent) # commands, do not rename methods @@ -1743,15 +1819,11 @@ class SoledadMailbox(WithMsgFields): # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") - for msg_id in messages: - msg = self.messages.get_msg_by_index(msg_id - 1) - if msg: - result.append((msg.getUID(), msg)) - else: - print "fetch %s, no msg found!!!" % msg_id + raise NotImplementedError else: for msg_id in messages: + print "getting msg by uid", msg_id msg = self.messages.get_msg_by_uid(msg_id) if msg: result.append((msg_id, msg)) @@ -1760,9 +1832,10 @@ class SoledadMailbox(WithMsgFields): if self.isWriteable(): self._unset_recent_flag() + self._signal_unread_to_ui() # XXX workaround for hangs in thunderbird - #return tuple(result[:100]) + #return tuple(result[:100]) # --- doesn't show all!! return tuple(result) @deferred @@ -1784,10 +1857,9 @@ class SoledadMailbox(WithMsgFields): then that message SHOULD be considered recent. """ log.msg('unsetting recent flags...') - - for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()): - newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,)) - self._update(newflags) + for msg in self.messages.get_recent(): + msg.removeFlags((fields.RECENT_FLAG,)) + self._signal_unread_to_ui() @deferred def _signal_unread_to_ui(self): @@ -1842,14 +1914,14 @@ class SoledadMailbox(WithMsgFields): result = {} for msg_id in messages: - print "MSG ID = %s" % msg_id + log.msg("MSG ID = %s" % msg_id) msg = self.messages.get_msg_by_uid(msg_id) if mode == 1: - self._update(msg.addFlags(flags)) + msg.addFlags(flags) elif mode == -1: - self._update(msg.removeFlags(flags)) + msg.removeFlags(flags) elif mode == 0: - self._update(msg.setFlags(flags)) + msg.setFlags(flags) result[msg_id] = msg.getFlags() self._signal_unread_to_ui() @@ -1873,14 +1945,6 @@ class SoledadMailbox(WithMsgFields): for doc in docs: self.messages._soledad.delete_doc(doc) - def _update(self, doc): - """ - Updates document in u1db database - """ - # XXX create namedtuple - self.messages.soledad_writer.put({"mode": "put", - "payload": doc}) - def __repr__(self): """ Representation string for this mailbox. -- cgit v1.2.3 From 59756d93f329ceee925a813bd4137c2cb34e7679 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 25 Dec 2013 11:57:42 -0400 Subject: inlineCallbacks all the things! --- src/leap/mail/decorators.py | 93 +++++++++++++++++ src/leap/mail/imap/fetch.py | 235 ++++++++++++++++++++++--------------------- src/leap/mail/imap/server.py | 78 ++------------ 3 files changed, 222 insertions(+), 184 deletions(-) create mode 100644 src/leap/mail/decorators.py (limited to 'src') diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py new file mode 100644 index 0000000..9e49605 --- /dev/null +++ b/src/leap/mail/decorators.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# decorators.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Useful decorators for mail package. +""" +import logging +import os +import sys +import traceback + +from functools import wraps + +from twisted.internet.threads import deferToThread +from twisted.python import log + +logger = logging.getLogger(__name__) + + +def deferred(f): + """ + Decorator, for deferring methods to Threads. + + It will do a deferToThread of the decorated method + unless the environment variable LEAPMAIL_DEBUG is set. + + It uses a descriptor to delay the definition of the + method wrapper. + """ + class descript(object): + def __init__(self, f): + self.f = f + + def __get__(self, instance, klass): + if instance is None: + # Class method was requested + return self.make_unbound(klass) + return self.make_bound(instance) + + def _errback(self, failure): + err = failure.value + logger.warning('error in method: %s' % (self.f.__name__)) + logger.exception(err) + log.err(err) + + def make_unbound(self, klass): + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + this doc will vanish + """ + raise TypeError( + 'unbound method {}() must be called with {} instance ' + 'as first argument (got nothing instead)'.format( + self.f.__name__, + klass.__name__) + ) + return wrapper + + def make_bound(self, instance): + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + This documentation will disapear + """ + if not os.environ.get('LEAPMAIL_DEBUG'): + d = deferToThread(self.f, instance, *args, **kwargs) + d.addErrback(self._errback) + return d + else: + return self.f(instance, *args, **kwargs) + + # This instance does not need the descriptor anymore, + # let it find the wrapper directly next time: + setattr(instance, self.f.__name__, wrapper) + return wrapper + + return descript(f) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index b1c34ba..0b31c3b 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -17,21 +17,24 @@ """ Incoming mail fetcher. """ -import logging +import copy import json -import ssl +import logging +#import ssl import threading import time -import copy -from StringIO import StringIO +import sys +import traceback from email.parser import Parser from email.generator import Generator from email.utils import parseaddr +from StringIO import StringIO from twisted.python import log +from twisted.internet import defer from twisted.internet.task import LoopingCall -from twisted.internet.threads import deferToThread +#from twisted.internet.threads import deferToThread from zope.proxy import sameProxiedObjects from leap.common import events as leap_events @@ -45,12 +48,18 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.decorators import deferred from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY logger = logging.getLogger(__name__) +MULTIPART_ENCRYPTED = "multipart/encrypted" +MULTIPART_SIGNED = "multipart/signed" +PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" +PGP_END = "-----END PGP MESSAGE-----" + class MalformedMessage(Exception): """ @@ -125,6 +134,9 @@ class LeapIncomingMail(object): self._create_soledad_indexes() + # initialize a mail parser only once + self._parser = Parser() + def _create_soledad_indexes(self): """ Create needed indexes on soledad. @@ -152,9 +164,10 @@ class LeapIncomingMail(object): logger.debug("fetching mail for: %s %s" % ( self._soledad.uuid, self._userid)) if not self.fetching_lock.locked(): - d = deferToThread(self._sync_soledad) - d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error) - d.addCallbacks(self._process_doclist, self._sync_soledad_error) + d1 = self._sync_soledad() + d = defer.gatherResults([d1], consumeErrors=True) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + d.addCallbacks(self._signal_unread_to_ui, self._errback) return d else: logger.debug("Already fetching mail.") @@ -184,6 +197,11 @@ class LeapIncomingMail(object): # synchronize incoming mail + def _errback(self, failure): + logger.exception(failure.value) + traceback.print_tb(*sys.exc_info()) + + @deferred def _sync_soledad(self): """ Synchronizes with remote soledad. @@ -196,10 +214,9 @@ class LeapIncomingMail(object): self._soledad.sync() log.msg('soledad synced.') doclist = self._soledad.get_from_index("just-mail", "*") + self._process_doclist(doclist) - return doclist - - def _signal_unread_to_ui(self): + def _signal_unread_to_ui(self, *args): """ Sends unread event to ui. """ @@ -215,53 +232,18 @@ class LeapIncomingMail(object): :returns: doclist :rtype: iterable """ + doclist = doclist[0] # gatherResults pass us a list fetched_ts = time.mktime(time.gmtime()) - num_mails = len(doclist) - log.msg("there are %s mails" % (num_mails,)) + num_mails = len(doclist) if doclist is not None else 0 + if num_mails != 0: + log.msg("there are %s mails" % (num_mails,)) leap_events.signal( IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) - self._signal_unread_to_ui() return doclist - def _sync_soledad_error(self, failure): - """ - Errback for sync errors. - """ - # XXX should signal unrecoverable maybe. - err = failure.value - logger.error("error syncing soledad: %s" % (err,)) - if failure.check(ssl.SSLError): - logger.warning('SSL Error while ' - 'syncing soledad: %r' % (err,)) - elif failure.check(Exception): - logger.warning('Unknown error while ' - 'syncing soledad: %r' % (err,)) - - def _log_err(self, failure): - """ - Generic errback - """ - err = failure.value - logger.exception("error!: %r" % (err,)) - - def _decryption_error(self, failure): - """ - Errback for decryption errors. - """ - # XXX should signal unrecoverable maybe. - err = failure.value - logger.error("error decrypting msg: %s" % (err,)) - - def _saving_error(self, failure): - """ - Errback for local save errors. - """ - # XXX should signal unrecoverable maybe. - err = failure.value - logger.error("error saving msg locally: %s" % (err,)) - # process incoming mail. + @defer.inlineCallbacks def _process_doclist(self, doclist): """ Iterates through the doclist, checks if each doc @@ -278,7 +260,6 @@ class LeapIncomingMail(object): return num_mails = len(doclist) - docs_cb = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) leap_events.signal( @@ -287,35 +268,18 @@ class LeapIncomingMail(object): if self._is_msg(keys): # Ok, this looks like a legit msg. # Let's process it! - # Deferred chain for individual messages - - # XXX use an IConsumer instead... ? - d = deferToThread(self._decrypt_doc, doc) - d.addCallback(self._process_decrypted_doc) - d.addErrback(self._log_err) - d.addCallback(self._add_message_locally) - d.addErrback(self._log_err) - docs_cb.append(d) + decrypted = list(self._decrypt_doc(doc))[0] + res = self._add_message_locally(decrypted) + yield res + else: # Ooops, this does not. logger.debug('This does not look like a proper msg.') - return docs_cb # # operations on individual messages # - def _is_msg(self, keys): - """ - Checks if the keys of a dictionary match the signature - of the document type we use for messages. - - :param keys: iterable containing the strings to match. - :type keys: iterable of strings. - :rtype: bool - """ - return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys - def _decrypt_doc(self, doc): """ Decrypt the contents of a document. @@ -339,7 +303,9 @@ class LeapIncomingMail(object): logger.error("Error while decrypting msg: %r" % (exc,)) decrdata = "" leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - return doc, decrdata + + data = list(self._process_decrypted_doc((doc, decrdata))) + yield (doc, data) def _process_decrypted_doc(self, msgtuple): """ @@ -357,16 +323,15 @@ class LeapIncomingMail(object): doc, data = msgtuple msg = json.loads(data) if not isinstance(msg, dict): - return False + defer.returnValue(False) if not msg.get(self.INCOMING_KEY, False): - return False + defer.returnValue(False) # ok, this is an incoming message rawmsg = msg.get(self.CONTENT_KEY, None) if not rawmsg: return False - data = self._maybe_decrypt_msg(rawmsg) - return doc, data + return self._maybe_decrypt_msg(rawmsg) def _maybe_decrypt_msg(self, data): """ @@ -381,17 +346,16 @@ class LeapIncomingMail(object): leap_assert_type(data, unicode) # parse the original message - parser = Parser() encoding = get_email_charset(data) data = data.encode(encoding) - msg = parser.parsestr(data) + msg = self._parser.parsestr(data) # try to obtain sender public key senderPubkey = None fromHeader = msg.get('from', None) - if fromHeader is not None \ - and (msg.get_content_type() == 'multipart/encrypted' \ - or msg.get_content_type() == 'multipart/signed'): + if (fromHeader is not None + and (msg.get_content_type() == MULTIPART_ENCRYPTED + or msg.get_content_type() == MULTIPART_SIGNED)): _, senderAddress = parseaddr(fromHeader) try: senderPubkey = self._keymanager.get_key_from_cache( @@ -400,11 +364,14 @@ class LeapIncomingMail(object): pass valid_sig = False # we will add a header saying if sig is valid - if msg.get_content_type() == 'multipart/encrypted': - decrmsg, valid_sig = self._decrypt_multipart_encrypted_msg( + decrypt_multi = self._decrypt_multipart_encrypted_msg + decrypt_inline = self._maybe_decrypt_inline_encrypted_msg + + if msg.get_content_type() == MULTIPART_ENCRYPTED: + decrmsg, valid_sig = decrypt_multi( msg, encoding, senderPubkey) else: - decrmsg, valid_sig = self._maybe_decrypt_inline_encrypted_msg( + decrmsg, valid_sig = decrypt_inline( msg, encoding, senderPubkey) # add x-leap-signature header @@ -419,7 +386,7 @@ class LeapIncomingMail(object): self.LEAP_SIGNATURE_INVALID, pubkey=senderPubkey.key_id) - return decrmsg.as_string() + yield decrmsg.as_string() def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey): """ @@ -437,43 +404,33 @@ class LeapIncomingMail(object): """ log.msg('decrypting multipart encrypted msg') msg = copy.deepcopy(msg) - # sanity check - payload = msg.get_payload() - if len(payload) != 2: - raise MalformedMessage( - 'Multipart/encrypted messages should have exactly 2 body ' - 'parts (instead of %d).' % len(payload)) - if payload[0].get_content_type() != 'application/pgp-encrypted': - raise MalformedMessage( - "Multipart/encrypted messages' first body part should " - "have content type equal to 'application/pgp-encrypted' " - "(instead of %s)." % payload[0].get_content_type()) - if payload[1].get_content_type() != 'application/octet-stream': - raise MalformedMessage( - "Multipart/encrypted messages' second body part should " - "have content type equal to 'octet-stream' (instead of " - "%s)." % payload[1].get_content_type()) + self._multipart_sanity_check(msg) + # parse message and get encrypted content pgpencmsg = msg.get_payload()[1] encdata = pgpencmsg.get_payload() + # decrypt or fail gracefully try: - decrdata, valid_sig = self._decrypt_and_verify_data( + decrdata, valid_sig = yield self._decrypt_and_verify_data( encdata, senderPubkey) except keymanager_errors.DecryptError as e: logger.warning('Failed to decrypt encrypted message (%s). ' 'Storing message without modifications.' % str(e)) - return msg, False # return original message + # Bailing out! + yield (msg, False) + # decrypted successully, now fix encoding and parse try: decrdata = decrdata.encode(encoding) except (UnicodeEncodeError, UnicodeDecodeError) as e: logger.error("Unicode error {0}".format(e)) decrdata = decrdata.encode(encoding, 'replace') - parser = Parser() - decrmsg = parser.parsestr(decrdata) + + decrmsg = self._parser.parsestr(decrdata) # remove original message's multipart/encrypted content-type del(msg['content-type']) + # replace headers back in original message for hkey, hval in decrmsg.items(): try: @@ -481,9 +438,10 @@ class LeapIncomingMail(object): msg.replace_header(hkey, hval) except KeyError: msg[hkey] = hval - # replace payload by unencrypted payload + + # all ok, replace payload by unencrypted payload msg.set_payload(decrmsg.get_payload()) - return msg, valid_sig + yield (msg, valid_sig) def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, senderPubkey): @@ -497,8 +455,9 @@ class LeapIncomingMail(object): :param senderPubkey: The key of the sender of the message. :type senderPubkey: OpenPGPKey - :return: A unitary tuple containing a decrypted message. - :rtype: (Message) + :return: A unitary tuple containing a decrypted message and + a bool indicating wether the signature is valid. + :rtype: (Message, bool) """ log.msg('maybe decrypting inline encrypted msg') # serialize the original message @@ -507,8 +466,6 @@ class LeapIncomingMail(object): g.flatten(origmsg) data = buf.getvalue() # handle exactly one inline PGP message - PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" - PGP_END = "-----END PGP MESSAGE-----" valid_sig = False if PGP_BEGIN in data: begin = data.find(PGP_BEGIN) @@ -522,11 +479,11 @@ class LeapIncomingMail(object): except keymanager_errors.DecryptError: logger.warning('Failed to decrypt potential inline encrypted ' 'message. Storing message as is...') + # if message is not encrypted, return raw data if isinstance(data, unicode): data = data.encode(encoding, 'replace') - parser = Parser() - return parser.parsestr(data), valid_sig + return (self._parser.parsestr(data), valid_sig) def _decrypt_and_verify_data(self, data, senderPubkey): """ @@ -555,7 +512,7 @@ class LeapIncomingMail(object): except keymanager_errors.InvalidSignature: decrdata = self._keymanager.decrypt( data, self._pkey) - return decrdata, valid_sig + return (decrdata, valid_sig) def _add_message_locally(self, msgtuple): """ @@ -570,10 +527,54 @@ class LeapIncomingMail(object): """ log.msg('adding message to local db') doc, data = msgtuple - self._inbox.addMessage(data, (self.RECENT_FLAG,)) + if isinstance(data, list): + data = data[0] + + self._inbox.addMessage(data, flags=(self.RECENT_FLAG,)) + leap_events.signal(IMAP_MSG_SAVED_LOCALLY) doc_id = doc.doc_id self._soledad.delete_doc(doc) log.msg("deleted doc %s from incoming" % doc_id) leap_events.signal(IMAP_MSG_DELETED_INCOMING) self._signal_unread_to_ui() + return True + + # + # helpers + # + + def _msg_multipart_sanity_check(self, msg): + """ + Performs a sanity check against a multipart encrypted msg + + :param msg: The original encrypted message. + :type msg: Message + """ + # sanity check + payload = msg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 8758dcb..57587a5 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -21,20 +21,17 @@ import copy import logging import StringIO import cStringIO -import os import time import re from collections import defaultdict, namedtuple from email.parser import Parser -from functools import wraps from zope.interface import implements from zope.proxy import sameProxiedObjects from twisted.mail import imap4 from twisted.internet import defer -from twisted.internet.threads import deferToThread from twisted.python import log from u1db import errors as u1db_errors @@ -44,70 +41,12 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.common.mail import get_email_charset from leap.mail.messageflow import IMessageConsumer, MessageProducer +from leap.mail.decorators import deferred from leap.soledad.client import Soledad logger = logging.getLogger(__name__) -def deferred(f): - ''' - Decorator, for deferring methods to Threads. - - It will do a deferToThread of the decorated method - unless the environment variable LEAPMAIL_DEBUG is set. - - It uses a descriptor to delay the definition of the - method wrapper. - ''' - class descript(object): - def __init__(self, f): - self.f = f - - def __get__(self, instance, klass): - if instance is None: - # Class method was requested - return self.make_unbound(klass) - return self.make_bound(instance) - - def _errback(self, failure): - err = failure.value - logger.warning('error in method: %s' % (self.f.__name__)) - log.err(err) - - def make_unbound(self, klass): - - @wraps(self.f) - def wrapper(*args, **kwargs): - '''This documentation will vanish :)''' - raise TypeError( - 'unbound method {}() must be called with {} instance ' - 'as first argument (got nothing instead)'.format( - self.f.__name__, - klass.__name__) - ) - return wrapper - - def make_bound(self, instance): - - @wraps(self.f) - def wrapper(*args, **kwargs): - '''This documentation will disapear :)''' - - if not os.environ.get('LEAPMAIL_DEBUG'): - d = deferToThread(self.f, instance, *args, **kwargs) - d.addErrback(self._errback) - return d - else: - return self.f(instance, *args, **kwargs) - - # This instance does not need the descriptor anymore, - # let it find the wrapper directly next time: - setattr(instance, self.f.__name__, wrapper) - return wrapper - - return descript(f) - - class MissingIndexError(Exception): """ Raises when tried to access a non existent index document. @@ -248,6 +187,8 @@ class MailParser(object): return self._parser.parse if isinstance(o, basestring): return self._parser.parsestr + # fallback + return self._parser.parsestr def _stringify(self, o): """ @@ -942,8 +883,8 @@ class LeapMessage(fields, MailParser, MBoxParser): Return True if this message is multipart. """ if self._cdoc: - retval = self._cdoc.content.get(self.MULTIPART_KEY, False) - print "MULTIPART? ", retval + retval = self._fdoc.content.get(self.MULTIPART_KEY, False) + return retval def getSubPart(self, part): """ @@ -1197,6 +1138,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): msg = self._get_parsed_msg(raw) headers = dict(msg) + logger.debug("adding. is multipart:%s" % msg.is_multipart()) flags_doc[self.MULTIPART_KEY] = msg.is_multipart() # XXX get lower case for keys? # XXX get headers doc @@ -1464,7 +1406,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def addListener(self, listener): """ - Rdds a listener to the listeners queue. + Adds a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. :param listener: listener to add :type listener: an object that implements IMailboxListener @@ -1716,6 +1660,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: a deferred that evals to None """ # XXX we should treat the message as an IMessage from here + leap_assert_type(message, basestring) uid_next = self.getUIDNext() logger.debug('Adding msg with UID :%s' % uid_next) if flags is None: @@ -1823,12 +1768,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: for msg_id in messages: - print "getting msg by uid", msg_id msg = self.messages.get_msg_by_uid(msg_id) if msg: result.append((msg_id, msg)) else: - print "fetch %s, no msg found!!!" % msg_id + logger.debug("fetch %s, no msg found!!!" % msg_id) if self.isWriteable(): self._unset_recent_flag() -- cgit v1.2.3 From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Dec 2013 14:10:14 -0400 Subject: Split the near-2k loc file into more handy modules. ...aaaand not a single fuck was given that day! --- src/leap/mail/imap/account.py | 426 ++++++++ src/leap/mail/imap/fields.py | 127 +++ src/leap/mail/imap/index.py | 69 ++ src/leap/mail/imap/mailbox.py | 617 ++++++++++++ src/leap/mail/imap/messages.py | 735 ++++++++++++++ src/leap/mail/imap/parser.py | 93 ++ src/leap/mail/imap/server.py | 1897 ------------------------------------ src/leap/mail/imap/service/imap.py | 2 +- 8 files changed, 2068 insertions(+), 1898 deletions(-) create mode 100644 src/leap/mail/imap/account.py create mode 100644 src/leap/mail/imap/fields.py create mode 100644 src/leap/mail/imap/index.py create mode 100644 src/leap/mail/imap/mailbox.py create mode 100644 src/leap/mail/imap/messages.py create mode 100644 src/leap/mail/imap/parser.py delete mode 100644 src/leap/mail/imap/server.py (limited to 'src') diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py new file mode 100644 index 0000000..fd861e7 --- /dev/null +++ b/src/leap/mail/imap/account.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- +# account.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Backed Account. +""" +import copy +import time + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.imap.index import IndexedDB +from leap.mail.imap.fields import WithMsgFields +from leap.mail.imap.parser import MBoxParser +from leap.mail.imap.mailbox import SoledadMailbox +from leap.soledad.client import Soledad + + +####################################### +# Soledad Account +####################################### + + +class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): + """ + An implementation of IAccount and INamespacePresenteer + that is backed by Soledad Encrypted Documents. + """ + + implements(imap4.IAccount, imap4.INamespacePresenter) + + _soledad = None + selected = None + + def __init__(self, account_name, soledad=None): + """ + Creates a SoledadAccountIndex that keeps track of the mailboxes + and subscriptions handled by this account. + + :param acct_name: The name of the account (user id). + :type acct_name: str + + :param soledad: a Soledad instance. + :param soledad: Soledad + """ + leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert_type(soledad, Soledad) + + # XXX SHOULD assert too that the name matches the user/uuid with which + # soledad has been initialized. + + self._account_name = self._parse_mailbox_name(account_name) + self._soledad = soledad + + self.initialize_db() + + # every user should have the right to an inbox folder + # at least, so let's make one! + + if not self.mailboxes: + self.addMailbox(self.INBOX_NAME) + + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + return copy.deepcopy(self.EMPTY_MBOX) + + def _get_mailbox_by_name(self, name): + """ + Return an mbox document by name. + + :param name: the name of the mailbox + :type name: str + + :rtype: SoledadDocument + """ + doc = self._soledad.get_from_index( + self.TYPE_MBOX_IDX, self.MBOX_KEY, + self._parse_mailbox_name(name)) + return doc[0] if doc else None + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + """ + return [doc.content[self.MBOX_KEY] + for doc in self._soledad.get_from_index( + self.TYPE_IDX, self.MBOX_KEY)] + + @property + def subscriptions(self): + """ + A list of the current subscriptions for this account. + """ + return [doc.content[self.MBOX_KEY] + for doc in self._soledad.get_from_index( + self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + + def getMailbox(self, name): + """ + Returns a Mailbox with that name, without selecting it. + + :param name: name of the mailbox + :type name: str + + :returns: a a SoledadMailbox instance + :rtype: SoledadMailbox + """ + name = self._parse_mailbox_name(name) + + if name not in self.mailboxes: + raise imap4.MailboxException("No such mailbox") + + return SoledadMailbox(name, soledad=self._soledad) + + ## + ## IAccount + ## + + def addMailbox(self, name, creation_ts=None): + """ + Add a mailbox to the account. + + :param name: the name of the mailbox + :type name: str + + :param creation_ts: an optional creation timestamp to be used as + mailbox id. A timestamp will be used if no + one is provided. + :type creation_ts: int + + :returns: True if successful + :rtype: bool + """ + name = self._parse_mailbox_name(name) + + if name in self.mailboxes: + raise imap4.MailboxCollision, name + + if not creation_ts: + # by default, we pass an int value + # taken from the current time + # we make sure to take enough decimals to get a unique + # mailbox-uidvalidity. + creation_ts = int(time.time() * 10E2) + + mbox = self._get_empty_mailbox() + mbox[self.MBOX_KEY] = name + mbox[self.CREATED_KEY] = creation_ts + + doc = self._soledad.create_doc(mbox) + return bool(doc) + + def create(self, pathspec): + """ + Create a new mailbox from the given hierarchical name. + + :param pathspec: The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. + :type pathspec: str + + :return: A true value if the creation succeeds. + :rtype: bool + + :raise MailboxException: Raised if this mailbox cannot be added. + """ + # TODO raise MailboxException + paths = filter( + None, + self._parse_mailbox_name(pathspec).split('/')) + for accum in range(1, len(paths)): + try: + self.addMailbox('/'.join(paths[:accum])) + except imap4.MailboxCollision: + pass + try: + self.addMailbox('/'.join(paths)) + except imap4.MailboxCollision: + if not pathspec.endswith('/'): + return False + return True + + def select(self, name, readwrite=1): + """ + Selects a mailbox. + + :param name: the mailbox to select + :type name: str + + :param readwrite: 1 for readwrite permissions. + :type readwrite: int + + :rtype: bool + """ + name = self._parse_mailbox_name(name) + + if name not in self.mailboxes: + return None + + self.selected = name + + return SoledadMailbox( + name, rw=readwrite, + soledad=self._soledad) + + def delete(self, name, force=False): + """ + Deletes a mailbox. + + Right now it does not purge the messages, but just removes the mailbox + name from the mailboxes list!!! + + :param name: the mailbox to be deleted + :type name: str + + :param force: if True, it will not check for noselect flag or inferior + names. use with care. + :type force: bool + """ + name = self._parse_mailbox_name(name) + + if not name in self.mailboxes: + raise imap4.MailboxException("No such mailbox") + + mbox = self.getMailbox(name) + + if force is False: + # See if this box is flagged \Noselect + # XXX use mbox.flags instead? + if self.NOSELECT_FLAG in mbox.getFlags(): + # Check for hierarchically inferior mailboxes with this one + # as part of their root. + for others in self.mailboxes: + if others != name and others.startswith(name): + raise imap4.MailboxException, ( + "Hierarchically inferior mailboxes " + "exist and \\Noselect is set") + mbox.destroy() + + # XXX FIXME --- not honoring the inferior names... + + # if there are no hierarchically inferior names, we will + # delete it from our ken. + #if self._inferiorNames(name) > 1: + # ??! -- can this be rite? + #self._index.removeMailbox(name) + + def rename(self, oldname, newname): + """ + Renames a mailbox. + + :param oldname: old name of the mailbox + :type oldname: str + + :param newname: new name of the mailbox + :type newname: str + """ + oldname = self._parse_mailbox_name(oldname) + newname = self._parse_mailbox_name(newname) + + if oldname not in self.mailboxes: + raise imap4.NoSuchMailbox, oldname + + inferiors = self._inferiorNames(oldname) + inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + + for (old, new) in inferiors: + if new in self.mailboxes: + raise imap4.MailboxCollision, new + + for (old, new) in inferiors: + mbox = self._get_mailbox_by_name(old) + mbox.content[self.MBOX_KEY] = new + self._soledad.put_doc(mbox) + + # XXX ---- FIXME!!!! ------------------------------------ + # until here we just renamed the index... + # We have to rename also the occurrence of this + # mailbox on ALL the messages that are contained in it!!! + # ... we maybe could use a reference to the doc_id + # in each msg, instead of the "mbox" field in msgs + # ------------------------------------------------------- + + def _inferiorNames(self, name): + """ + Return hierarchically inferior mailboxes. + + :param name: name of the mailbox + :rtype: list + """ + # XXX use wildcard query instead + inferiors = [] + for infname in self.mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + def isSubscribed(self, name): + """ + Returns True if user is subscribed to this mailbox. + + :param name: the mailbox to be checked. + :type name: str + + :rtype: bool + """ + mbox = self._get_mailbox_by_name(name) + return mbox.content.get('subscribed', False) + + def _set_subscription(self, name, value): + """ + Sets the subscription value for a given mailbox + + :param name: the mailbox + :type name: str + + :param value: the boolean value + :type value: bool + """ + # maybe we should store subscriptions in another + # document... + if not name in self.mailboxes: + self.addMailbox(name) + mbox = self._get_mailbox_by_name(name) + + if mbox: + mbox.content[self.SUBSCRIBED_KEY] = value + self._soledad.put_doc(mbox) + + def subscribe(self, name): + """ + Subscribe to this mailbox + + :param name: name of the mailbox + :type name: str + """ + name = self._parse_mailbox_name(name) + if name not in self.subscriptions: + self._set_subscription(name, True) + + def unsubscribe(self, name): + """ + Unsubscribe from this mailbox + + :param name: name of the mailbox + :type name: str + """ + name = self._parse_mailbox_name(name) + if name not in self.subscriptions: + raise imap4.MailboxException, "Not currently subscribed to " + name + self._set_subscription(name, False) + + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames( + self._parse_mailbox_name(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + ## + ## INamespacePresenter + ## + + def getPersonalNamespaces(self): + return [["", "/"]] + + def getSharedNamespaces(self): + return None + + def getOtherNamespaces(self): + return None + + # extra, for convenience + + def deleteAllMessages(self, iknowhatiamdoing=False): + """ + Deletes all messages from all mailboxes. + Danger! high voltage! + + :param iknowhatiamdoing: confirmation parameter, needs to be True + to proceed. + """ + if iknowhatiamdoing is True: + for mbox in self.mailboxes: + self.delete(mbox, force=True) + + def __repr__(self): + """ + Representation string for this object. + """ + return "" % self._account_name diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py new file mode 100644 index 0000000..96b937e --- /dev/null +++ b/src/leap/mail/imap/fields.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# fields.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Fields for Mailbox and Message. +""" +from leap.mail.imap.parser import MBoxParser + + +class WithMsgFields(object): + """ + Container class for class-attributes to be shared by + several message-related classes. + """ + # Internal representation of Message + DATE_KEY = "date" + HEADERS_KEY = "headers" + FLAGS_KEY = "flags" + MBOX_KEY = "mbox" + CONTENT_HASH_KEY = "chash" + RAW_KEY = "raw" + SUBJECT_KEY = "subject" + UID_KEY = "uid" + MULTIPART_KEY = "multi" + SIZE_KEY = "size" + + # Mailbox specific keys + CLOSED_KEY = "closed" + CREATED_KEY = "created" + SUBSCRIBED_KEY = "subscribed" + RW_KEY = "rw" + LAST_UID_KEY = "lastuid" + + # Document Type, for indexing + TYPE_KEY = "type" + TYPE_MBOX_VAL = "mbox" + TYPE_MESSAGE_VAL = "msg" + TYPE_FLAGS_VAL = "flags" + TYPE_HEADERS_VAL = "head" + TYPE_ATTACHMENT_VAL = "attach" + # should add also a headers val + + INBOX_VAL = "inbox" + + # Flags for SoledadDocument for indexing. + SEEN_KEY = "seen" + RECENT_KEY = "recent" + + # Flags in Mailbox and Message + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) + + # Fields in mail object + SUBJECT_FIELD = "Subject" + DATE_FIELD = "Date" + + # Index types + # -------------- + + TYPE_IDX = 'by-type' + TYPE_MBOX_IDX = 'by-type-and-mbox' + TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' + TYPE_SUBS_IDX = 'by-type-and-subscribed' + TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' + TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + TYPE_HASH_IDX = 'by-type-and-hash' + + # Tomas created the `recent and seen index`, but the semantic is not too + # correct since the recent flag is volatile. + TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + + KTYPE = TYPE_KEY + MBOX_VAL = TYPE_MBOX_VAL + HASH_VAL = CONTENT_HASH_KEY + + INDEXES = { + # generic + TYPE_IDX: [KTYPE], + TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], + TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], + + # mailboxes + TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], + + # content, headers doc + TYPE_HASH_IDX: [KTYPE, HASH_VAL], + + # messages + TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], + TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, + 'bool(recent)', 'bool(seen)'], + } + + MBOX_KEY = MBOX_VAL + + EMPTY_MBOX = { + TYPE_KEY: MBOX_KEY, + TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, + SUBJECT_KEY: "", + FLAGS_KEY: [], + CLOSED_KEY: False, + SUBSCRIBED_KEY: False, + RW_KEY: 1, + LAST_UID_KEY: 0 + } + +fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py new file mode 100644 index 0000000..2280d86 --- /dev/null +++ b/src/leap/mail/imap/index.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# index.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Index for SoledadBackedAccount, Mailbox and Messages. +""" +import logging + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail.imap.account import SoledadBackedAccount + + +logger = logging.getLogger(__name__) + + +class IndexedDB(object): + """ + Methods dealing with the index. + + This is a MixIn that needs access to the soledad instance, + and also assumes that a INDEXES attribute is accessible to the instance. + + INDEXES must be a dictionary of type: + {'index-name': ['field1', 'field2']} + """ + # TODO we might want to move this to soledad itself, check + + def initialize_db(self): + """ + Initialize the database. + """ + leap_assert(self._soledad, + "Need a soledad attribute accesible in the instance") + leap_assert_type(self.INDEXES, dict) + + # Ask the database for currently existing indexes. + if not self._soledad: + logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") + return + db_indexes = dict() + if self._soledad is not None: + db_indexes = dict(self._soledad.list_indexes()) + for name, expression in SoledadBackedAccount.INDEXES.items(): + if name not in db_indexes: + # The index does not yet exist. + self._soledad.create_index(name, *expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so we + # delete it and add the proper index expression. + self._soledad.delete_index(name) + self._soledad.create_index(name, *expression) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py new file mode 100644 index 0000000..09c06a2 --- /dev/null +++ b/src/leap/mail/imap/mailbox.py @@ -0,0 +1,617 @@ +# *- coding: utf-8 -*- +# mailbox.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Mailbox. +""" +import logging +from collections import defaultdict + +from twisted.internet import defer +from twisted.python import log + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common import events as leap_events +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.decorators import deferred +from leap.mail.imap.fields import WithMsgFields, fields +from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.parser import MBoxParser + +logger = logging.getLogger(__name__) + + +class SoledadMailbox(WithMsgFields, MBoxParser): + """ + A Soledad-backed IMAP mailbox. + + Implements the high-level method needed for the Mailbox interfaces. + The low-level database methods are contained in MessageCollection class, + which we instantiate and make accessible in the `messages` attribute. + """ + implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) + # XXX should finish the implementation of IMailboxListener + # XXX should implement IMessageCopier too + + messages = None + _closed = False + + INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, + WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, + WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, + WithMsgFields.LIST_FLAG) + flags = None + + CMD_MSG = "MESSAGES" + CMD_RECENT = "RECENT" + CMD_UIDNEXT = "UIDNEXT" + CMD_UIDVALIDITY = "UIDVALIDITY" + CMD_UNSEEN = "UNSEEN" + + _listeners = defaultdict(set) + + def __init__(self, mbox, soledad=None, rw=1): + """ + SoledadMailbox constructor. Needs to get passed a name, plus a + Soledad instance. + + :param mbox: the mailbox name + :type mbox: str + + :param soledad: a Soledad instance. + :type soledad: Soledad + + :param rw: read-and-write flags + :type rw: int + """ + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(soledad, "Need a soledad instance to initialize") + + # XXX should move to wrapper + #leap_assert(isinstance(soledad._db, SQLCipherDatabase), + #"soledad._db must be an instance of SQLCipherDatabase") + + self.mbox = self._parse_mailbox_name(mbox) + self.rw = rw + + self._soledad = soledad + + self.messages = MessageCollection( + mbox=mbox, soledad=self._soledad) + + if not self.getFlags(): + self.setFlags(self.INIT_FLAGS) + + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox] + + def addListener(self, listener): + """ + Adds a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + logger.debug('adding mailbox listener: %s' % listener) + self.listeners.add(listener) + + def removeListener(self, listener): + """ + Removes a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + self.listeners.remove(listener) + + def _get_mbox(self): + """ + Returns mailbox document. + + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. + """ + try: + query = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_MBOX_VAL, self.mbox) + if query: + return query.pop() + except Exception as exc: + logger.error("Unhandled error %r" % exc) + + def getFlags(self): + """ + Returns the flags defined for this mailbox. + + :returns: tuple of flags for this mailbox + :rtype: tuple of str + """ + mbox = self._get_mbox() + if not mbox: + return None + flags = mbox.content.get(self.FLAGS_KEY, []) + return map(str, flags) + + def setFlags(self, flags): + """ + Sets flags for this mailbox. + + :param flags: a tuple with the flags + :type flags: tuple of str + """ + leap_assert(isinstance(flags, tuple), + "flags expected to be a tuple") + mbox = self._get_mbox() + if not mbox: + return None + mbox.content[self.FLAGS_KEY] = map(str, flags) + self._soledad.put_doc(mbox) + + # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + + def _get_closed(self): + """ + Return the closed attribute for this mailbox. + + :return: True if the mailbox is closed + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.CLOSED_KEY, False) + + def _set_closed(self, closed): + """ + Set the closed attribute for this mailbox. + + :param closed: the state to be set + :type closed: bool + """ + leap_assert(isinstance(closed, bool), "closed needs to be boolean") + mbox = self._get_mbox() + mbox.content[self.CLOSED_KEY] = closed + self._soledad.put_doc(mbox) + + closed = property( + _get_closed, _set_closed, doc="Closed attribute.") + + def _get_last_uid(self): + """ + Return the last uid for this mailbox. + + :return: the last uid for messages in this mailbox + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.LAST_UID_KEY, 1) + + def _set_last_uid(self, uid): + """ + Sets the last uid for this mailbox. + + :param uid: the uid to be set + :type uid: int + """ + leap_assert(isinstance(uid, int), "uid has to be int") + mbox = self._get_mbox() + key = self.LAST_UID_KEY + + count = self.getMessageCount() + + # XXX safety-catch. If we do get duplicates, + # we want to avoid further duplication. + + if uid >= count: + value = uid + else: + # something is wrong, + # just set the last uid + # beyond the max msg count. + logger.debug("WRONG uid < count. Setting last uid to %s", count) + value = count + + mbox.content[key] = value + self._soledad.put_doc(mbox) + + last_uid = property( + _get_last_uid, _set_last_uid, doc="Last_UID attribute.") + + def getUIDValidity(self): + """ + Return the unique validity identifier for this mailbox. + + :return: unique validity identifier + :rtype: int + """ + mbox = self._get_mbox() + return mbox.content.get(self.CREATED_KEY, 1) + + def getUID(self, message): + """ + Return the UID of a message in the mailbox + + .. note:: this implementation does not make much sense RIGHT NOW, + but in the future will be useful to get absolute UIDs from + message sequence numbers. + + :param message: the message uid + :type message: int + + :rtype: int + """ + msg = self.messages.get_msg_by_uid(message) + return msg.getUID() + + def getUIDNext(self): + """ + Return the likely UID for the next message added to this + mailbox. Currently it returns the higher UID incremented by + one. + + We increment the next uid *each* time this function gets called. + In this way, there will be gaps if the message with the allocated + uid cannot be saved. But that is preferable to having race conditions + if we get to parallel message adding. + + :rtype: int + """ + self.last_uid += 1 + return self.last_uid + + def getMessageCount(self): + """ + Returns the total count of messages in this mailbox. + + :rtype: int + """ + return self.messages.count() + + def getUnseenCount(self): + """ + Returns the number of messages with the 'Unseen' flag. + + :return: count of messages flagged `unseen` + :rtype: int + """ + return self.messages.count_unseen() + + def getRecentCount(self): + """ + Returns the number of messages with the 'Recent' flag. + + :return: count of messages flagged `recent` + :rtype: int + """ + return self.messages.count_recent() + + def isWriteable(self): + """ + Get the read/write status of the mailbox. + + :return: 1 if mailbox is read-writeable, 0 otherwise. + :rtype: int + """ + return self.rw + + def getHierarchicalDelimiter(self): + """ + Returns the character used to delimite hierarchies in mailboxes. + + :rtype: str + """ + return '/' + + def requestStatus(self, names): + """ + Handles a status request by gathering the output of the different + status commands. + + :param names: a list of strings containing the status commands + :type names: iter + """ + r = {} + if self.CMD_MSG in names: + r[self.CMD_MSG] = self.getMessageCount() + if self.CMD_RECENT in names: + r[self.CMD_RECENT] = self.getRecentCount() + if self.CMD_UIDNEXT in names: + r[self.CMD_UIDNEXT] = self.last_uid + 1 + if self.CMD_UIDVALIDITY in names: + r[self.CMD_UIDVALIDITY] = self.getUID() + if self.CMD_UNSEEN in names: + r[self.CMD_UNSEEN] = self.getUnseenCount() + return defer.succeed(r) + + def addMessage(self, message, flags, date=None): + """ + Adds a message to this mailbox. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str + + :return: a deferred that evals to None + """ + # XXX we should treat the message as an IMessage from here + leap_assert_type(message, basestring) + uid_next = self.getUIDNext() + logger.debug('Adding msg with UID :%s' % uid_next) + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + d = self._do_add_messages(message, flags, date, uid_next) + d.addCallback(self._notify_new) + + @deferred + def _do_add_messages(self, message, flags, date, uid_next): + """ + Calls to the messageCollection add_msg method (deferred to thread). + Invoked from addMessage. + """ + self.messages.add_msg(message, flags=flags, date=date, + uid=uid_next) + + def _notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + :param args: ignored. + """ + exists = self.getMessageCount() + recent = self.getRecentCount() + logger.debug("NOTIFY: there are %s messages, %s recent" % ( + exists, + recent)) + + logger.debug("listeners: %s", str(self.listeners)) + for l in self.listeners: + logger.debug('notifying...') + l.newMessages(exists, recent) + + # commands, do not rename methods + + def destroy(self): + """ + Called before this mailbox is permanently deleted. + + Should cleanup resources, and set the \\Noselect flag + on the mailbox. + """ + self.setFlags((self.NOSELECT_FLAG,)) + self.deleteAllDocs() + + # XXX removing the mailbox in situ for now, + # we should postpone the removal + self._soledad.delete_doc(self._get_mbox()) + + def expunge(self): + """ + Remove all messages flagged \\Deleted + """ + if not self.isWriteable(): + raise imap4.ReadOnlyMailbox + delete = [] + deleted = [] + + for m in self.messages.get_all_docs(): + # XXX should operate with LeapMessages instead, + # so we don't expose the implementation. + # (so, iterate for m in self.messages) + if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: + delete.append(m) + for m in delete: + deleted.append(m.content) + self.messages.remove(m) + + # XXX should return the UIDs of the deleted messages + # more generically + return [x for x in range(len(deleted))] + + @deferred + def fetch(self, messages, uid): + """ + Retrieve one or more messages in this mailbox. + + from rfc 3501: The data items to be fetched can be either a single atom + or a parenthesized list. + + :param messages: IDs of the messages to retrieve information about + :type messages: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :rtype: A tuple of two-tuples of message sequence numbers and + LeapMessage + """ + result = [] + sequence = True if uid == 0 else False + + if not messages.last: + try: + iter(messages) + except TypeError: + # looks like we cannot iterate + messages.last = self.last_uid + + # for sequence numbers (uid = 0) + if sequence: + logger.debug("Getting msg by index: INEFFICIENT call!") + raise NotImplementedError + + else: + for msg_id in messages: + msg = self.messages.get_msg_by_uid(msg_id) + if msg: + result.append((msg_id, msg)) + else: + logger.debug("fetch %s, no msg found!!!" % msg_id) + + if self.isWriteable(): + self._unset_recent_flag() + self._signal_unread_to_ui() + + # XXX workaround for hangs in thunderbird + #return tuple(result[:100]) # --- doesn't show all!! + return tuple(result) + + @deferred + def _unset_recent_flag(self): + """ + Unsets `Recent` flag from a tuple of messages. + Called from fetch. + + From RFC, about `Recent`: + + Message is "recently" arrived in this mailbox. This session + is the first session to have been notified about this + message; if the session is read-write, subsequent sessions + will not see \Recent set for this message. This flag can not + be altered by the client. + + If it is not possible to determine whether or not this + session is the first session to be notified about a message, + then that message SHOULD be considered recent. + """ + log.msg('unsetting recent flags...') + for msg in self.messages.get_recent(): + msg.removeFlags((fields.RECENT_FLAG,)) + self._signal_unread_to_ui() + + @deferred + def _signal_unread_to_ui(self): + """ + Sends unread event to ui. + """ + unseen = self.getUnseenCount() + leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + + @deferred + def store(self, messages, flags, mode, uid): + """ + Sets the flags of one or more messages. + + :param messages: The identifiers of the messages to set the flags + :type messages: A MessageSet object with the list of messages requested + + :param flags: The flags to set, unset, or add. + :type flags: sequence of str + + :param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be + added to the specified messages. If mode is 0, all + existing flags should be cleared and these flags should be + added. + :type mode: -1, 0, or 1 + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A dict mapping message sequence numbers to sequences of + str representing the flags set on the message after this + operation has been performed. + :rtype: dict + + :raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + # XXX implement also sequence (uid = 0) + # XXX we should prevent cclient from setting Recent flag. + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) + + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + if not messages.last: + messages.last = self.messages.count() + + result = {} + for msg_id in messages: + log.msg("MSG ID = %s" % msg_id) + msg = self.messages.get_msg_by_uid(msg_id) + if mode == 1: + msg.addFlags(flags) + elif mode == -1: + msg.removeFlags(flags) + elif mode == 0: + msg.setFlags(flags) + result[msg_id] = msg.getFlags() + + self._signal_unread_to_ui() + return result + + @deferred + def close(self): + """ + Expunge and mark as closed + """ + self.expunge() + self.closed = True + + #@deferred + #def copy(self, messageObject): + #""" + #Copy the given message object into this mailbox. + #""" + # XXX should just: + # 1. Get the message._fdoc + # 2. Change the UID to UIDNext for this mailbox + # 3. Add implements IMessageCopier + + # convenience fun + + def deleteAllDocs(self): + """ + Deletes all docs in this mailbox + """ + docs = self.messages.get_all_docs() + for doc in docs: + self.messages._soledad.delete_doc(doc) + + def __repr__(self): + """ + Representation string for this mailbox. + """ + return u"" % ( + self.mbox, self.messages.count()) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py new file mode 100644 index 0000000..b0d5da2 --- /dev/null +++ b/src/leap/mail/imap/messages.py @@ -0,0 +1,735 @@ +# -*- coding: utf-8 -*- +# messages.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +LeapMessage and MessageCollection. +""" +import copy +import logging +import StringIO +from collections import namedtuple + +from twisted.mail import imap4 +from twisted.python import log +from u1db import errors as u1db_errors +from zope.interface import implements +from zope.proxy import sameProxiedObjects + +from leap.common.check import leap_assert, leap_assert_type +from leap.common.mail import get_email_charset +from leap.mail.decorators import deferred +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.index import IndexedDB +from leap.mail.imap.fields import fields, WithMsgFields +from leap.mail.imap.parser import MailParser, MBoxParser +from leap.mail.messageflow import IMessageConsumer, MessageProducer + +logger = logging.getLogger(__name__) + + +class LeapMessage(fields, MailParser, MBoxParser): + + implements(imap4.IMessage) + + def __init__(self, soledad, uid, mbox): + """ + Initializes a LeapMessage. + + :param soledad: a Soledad instance + :type soledad: Soledad + :param uid: the UID for the message. + :type uid: int or basestring + :param mbox: the mbox this message belongs to + :type mbox: basestring + """ + MailParser.__init__(self) + self._soledad = soledad + self._uid = int(uid) + self._mbox = self._parse_mailbox_name(mbox) + self._chash = None + + self.__cdoc = None + + @property + def _fdoc(self): + """ + An accessor to the flags document. + """ + return self._get_flags_doc() + + @property + def _cdoc(self): + """ + An accessor to the content document. + """ + if not self.__cdoc: + self.__cdoc = self._get_content_doc() + return self.__cdoc + + @property + def _chash(self): + """ + An accessor to the content hash for this message. + """ + if not self._fdoc: + return None + return self._fdoc.content.get(fields.CONTENT_HASH_KEY, None) + + # IMessage implementation + + def getUID(self): + """ + Retrieve the unique identifier associated with this message + + :return: uid for this message + :rtype: int + """ + return self._uid + + def getFlags(self): + """ + Retrieve the flags associated with this message + + :return: The flags, represented as strings + :rtype: tuple + """ + if self._uid is None: + return [] + + flags = [] + flag_doc = self._fdoc + if flag_doc: + flags = flag_doc.content.get(self.FLAGS_KEY, None) + if flags: + flags = map(str, flags) + return tuple(flags) + + # setFlags, addFlags, removeFlags are not in the interface spec + # but we use them with store command. + + def setFlags(self, flags): + """ + Sets the flags for this message + + Returns a SoledadDocument that needs to be updated by the caller. + + :param flags: the flags to update in the message. + :type flags: tuple of str + + :return: a SoledadDocument instance + :rtype: SoledadDocument + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + log.msg('setting flags: %s' % (self._uid)) + + doc = self._fdoc + doc.content[self.FLAGS_KEY] = flags + doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags + doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags + self._soledad.put_doc(doc) + + def addFlags(self, flags): + """ + Adds flags to this message. + + Returns a SoledadDocument that needs to be updated by the caller. + + :param flags: the flags to add to the message. + :type flags: tuple of str + + :return: a SoledadDocument instance + :rtype: SoledadDocument + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + oldflags = self.getFlags() + self.setFlags(tuple(set(flags + oldflags))) + + def removeFlags(self, flags): + """ + Remove flags from this message. + + Returns a SoledadDocument that needs to be updated by the caller. + + :param flags: the flags to be removed from the message. + :type flags: tuple of str + + :return: a SoledadDocument instance + :rtype: SoledadDocument + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + oldflags = self.getFlags() + self.setFlags(tuple(set(oldflags) - set(flags))) + + def getInternalDate(self): + """ + Retrieve the date internally associated with this message + + :rtype: C{str} + :return: An RFC822-formatted date string. + """ + return str(self._cdoc.content.get(self.DATE_KEY, '')) + + # + # IMessagePart + # + + # XXX we should implement this interface too for the subparts + # so we allow nested parts... + + def getBodyFile(self): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: StringIO + """ + fd = StringIO.StringIO() + + cdoc = self._cdoc + content = cdoc.content.get(self.RAW_KEY, '') + charset = get_email_charset( + unicode(cdoc.content.get(self.RAW_KEY, ''))) + try: + content = content.encode(charset) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + content = content.encode(charset, 'replace') + + raw = self._get_raw_msg() + msg = self._get_parsed_msg(raw) + body = msg.get_payload() + fd.write(body) + # XXX SHOULD use a separate BODY FIELD ... + fd.seek(0) + return fd + + def getSize(self): + """ + Return the total size, in octets, of this message. + + :return: size of the message, in octets + :rtype: int + """ + size = self._cdoc.content.get(self.SIZE_KEY, False) + if not size: + # XXX fallback, should remove when all migrated. + size = self.getBodyFile().len + return size + + def _get_headers(self): + """ + Return the headers dict stored in this message document. + """ + # XXX get from the headers doc + return self._cdoc.content.get(self.HEADERS_KEY, {}) + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + headers = self._get_headers() + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + # unpack and filter original dict by negate-condition + filter_by_cond = [ + map(str, (key, val)) for + key, val in headers.items() + if cond(key)] + return dict(filter_by_cond) + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + if self._cdoc: + retval = self._fdoc.content.get(self.MULTIPART_KEY, False) + return retval + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + if not self.isMultipart(): + raise TypeError + + msg = self._get_parsed_msg() + # XXX should wrap IMessagePart + return msg.get_payload()[part] + + # + # accessors + # + + def _get_flags_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + flag_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + flag_doc = flag_docs[0] if flag_docs else None + return flag_doc + + def _get_content_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + cont_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_HASH_IDX, + fields.TYPE_MESSAGE_VAL, self._content_hash, str(self._uid)) + cont_doc = cont_docs[0] if cont_docs else None + return cont_doc + + def _get_raw_msg(self): + """ + Return the raw msg. + :rtype: basestring + """ + return self._cdoc.content.get(self.RAW_KEY, '') + + def __getitem__(self, key): + """ + Return the content of the message document. + + :param key: The key + :type key: str + + :return: The content value indexed by C{key} or None + :rtype: str + """ + return self._cdoc.content.get(key, None) + + def does_exist(self): + """ + Return True if there is actually a message for this + UID and mbox. + """ + return bool(self._fdoc) + + +SoledadWriterPayload = namedtuple( + 'SoledadWriterPayload', ['mode', 'payload']) + +SoledadWriterPayload.CREATE = 1 +SoledadWriterPayload.PUT = 2 + + +class SoledadDocWriter(object): + """ + This writer will create docs serially in the local soledad database. + """ + + implements(IMessageConsumer) + + def __init__(self, soledad): + """ + Initialize the writer. + + :param soledad: the soledad instance + :type soledad: Soledad + """ + self._soledad = soledad + + def consume(self, queue): + """ + Creates a new document in soledad db. + + :param queue: queue to get item from, with content of the document + to be inserted. + :type queue: Queue + """ + empty = queue.empty() + while not empty: + item = queue.get() + if item.mode == SoledadWriterPayload.CREATE: + call = self._soledad.create_doc + elif item.mode == SoledadWriterPayload.PUT: + call = self._soledad.put_doc + + # should handle errors + try: + call(item.payload) + except u1db_errors.RevisionConflict as exc: + logger.error("Error: %r" % (exc,)) + raise exc + + empty = queue.empty() + + +class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): + """ + A collection of messages, surprisingly. + + It is tied to a selected mailbox name that is passed to constructor. + Implements a filter query over the messages contained in a soledad + database. + """ + # XXX this should be able to produce a MessageSet methinks + + EMPTY_MSG = { + fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.SUBJECT_KEY: "", + fields.DATE_KEY: "", + fields.RAW_KEY: "", + + # XXX should separate headers into another doc + fields.HEADERS_KEY: {}, + } + + EMPTY_FLAGS = { + fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.FLAGS_KEY: [], + fields.SEEN_KEY: False, + fields.RECENT_KEY: True, + fields.MULTIPART_KEY: False, + } + + # get from SoledadBackedAccount the needed index-related constants + INDEXES = SoledadBackedAccount.INDEXES + TYPE_IDX = SoledadBackedAccount.TYPE_IDX + + def __init__(self, mbox=None, soledad=None): + """ + Constructor for MessageCollection. + + :param mbox: the name of the mailbox. It is the name + with which we filter the query over the + messages database + :type mbox: str + + :param soledad: Soledad database + :type soledad: Soledad instance + """ + MailParser.__init__(self) + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(mbox.strip() != "", "mbox cannot be blank space") + leap_assert(isinstance(mbox, (str, unicode)), + "mbox needs to be a string") + leap_assert(soledad, "Need a soledad instance to initialize") + + # okay, all in order, keep going... + self.mbox = self._parse_mailbox_name(mbox) + self._soledad = soledad + self.initialize_db() + + # I think of someone like nietzsche when reading this + + # this will be the producer that will enqueue the content + # to be processed serially by the consumer (the writer). We just + # need to `put` the new material on its plate. + + self.soledad_writer = MessageProducer( + SoledadDocWriter(soledad), + period=0.05) + + def _get_empty_msg(self): + """ + Returns an empty message. + + :return: a dict containing a default empty message + :rtype: dict + """ + return copy.deepcopy(self.EMPTY_MSG) + + def _get_empty_flags_doc(self): + """ + Returns an empty doc for storing flags. + + :return: + :rtype: + """ + return copy.deepcopy(self.EMPTY_FLAGS) + + @deferred + def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): + """ + Creates a new message document. + + :param raw: the raw message + :type raw: str + + :param subject: subject of the message. + :type subject: str + + :param flags: flags + :type flags: list + + :param date: the received date for the message + :type date: str + + :param uid: the message uid for this mailbox + :type uid: int + """ + # TODO: split in smaller methods + logger.debug('adding message') + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + + content_doc = self._get_empty_msg() + flags_doc = self._get_empty_flags_doc() + + content_doc[self.MBOX_KEY] = self.mbox + flags_doc[self.MBOX_KEY] = self.mbox + # ...should get a sanity check here. + content_doc[self.UID_KEY] = uid + flags_doc[self.UID_KEY] = uid + + if flags: + flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) + flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags + + msg = self._get_parsed_msg(raw) + headers = dict(msg) + + logger.debug("adding. is multipart:%s" % msg.is_multipart()) + flags_doc[self.MULTIPART_KEY] = msg.is_multipart() + # XXX get lower case for keys? + # XXX get headers doc + content_doc[self.HEADERS_KEY] = headers + # set subject based on message headers and eventually replace by + # subject given as param + if self.SUBJECT_FIELD in headers: + content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + if subject is not None: + content_doc[self.SUBJECT_KEY] = subject + + # XXX could separate body into its own doc + # but should also separate multiparts + # that should be wrapped in MessagePart + content_doc[self.RAW_KEY] = self._stringify(raw) + content_doc[self.SIZE_KEY] = len(raw) + + if not date and self.DATE_FIELD in headers: + content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] + else: + content_doc[self.DATE_KEY] = date + + logger.debug('enqueuing message for write') + + ptuple = SoledadWriterPayload + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=content_doc)) + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=flags_doc)) + + def remove(self, msg): + """ + Removes a message. + + :param msg: a Leapmessage instance + :type msg: LeapMessage + """ + # XXX remove + #self._soledad.delete_doc(msg) + msg.remove() + + # getters + + def get_msg_by_uid(self, uid): + """ + Retrieves a LeapMessage by UID. + + :param uid: the message uid to query by + :type uid: int + + :return: A LeapMessage instance matching the query, + or None if not found. + :rtype: LeapMessage + """ + msg = LeapMessage(self._soledad, uid, self.mbox) + if not msg.does_exist(): + return None + return msg + + def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + """ + Get all documents for the selected mailbox of the + passed type. By default, it returns the flag docs. + + If you want acess to the content, use __iter__ instead + + :return: a list of u1db documents + :rtype: list of SoledadDocument + """ + if _type not in fields.__dict__.values(): + raise TypeError("Wrong type passed to get_all") + + if sameProxiedObjects(self._soledad, None): + logger.warning('Tried to get messages but soledad is None!') + return [] + + all_docs = [doc for doc in self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + _type, self.mbox)] + + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad + return sorted(all_docs, key=lambda item: item.content['uid']) + + def all_msg_iter(self): + """ + Return an iterator trhough the UIDs of all messages, sorted in + ascending order. + """ + all_uids = (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + self.TYPE_FLAGS_VAL, self.mbox)) + return (u for u in sorted(all_uids)) + + def count(self): + """ + Return the count of messages for this mailbox. + + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + return count + + # unseen messages + + def unseen_iter(self): + """ + Get an iterator for the message UIDs with no `seen` flag + for this mailbox. + + :return: iterator through unseen message doc UIDs + :rtype: iterable + """ + return (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '0')) + + def count_unseen(self): + """ + Count all messages with the `Unseen` flag. + + :returns: count + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '0') + return count + + def get_unseen(self): + """ + Get all messages with the `Unseen` flag + + :returns: a list of LeapMessages + :rtype: list + """ + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.unseen_iter()] + + # recent messages + + def recent_iter(self): + """ + Get an iterator for the message docs with `recent` flag. + + :return: iterator through recent message docs + :rtype: iterable + """ + return (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_RECT_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '1')) + + def get_recent(self): + """ + Get all messages with the `Recent` flag. + + :returns: a list of LeapMessages + :rtype: list + """ + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.recent_iter()] + + def count_recent(self): + """ + Count all messages with the `Recent` flag. + + :returns: count + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_RECT_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '1') + return count + + def __len__(self): + """ + Returns the number of messages on this mailbox. + + :rtype: int + """ + return self.count() + + def __iter__(self): + """ + Returns an iterator over all messages. + + :returns: iterator of dicts with content for all messages. + :rtype: iterable + """ + return (LeapMessage(self._soledad, docuid, self.mbox) + for docuid in self.all_msg_iter()) + + def __repr__(self): + """ + Representation string for this object. + """ + return u"" % ( + self.mbox, self.count()) + + # XXX should implement __eq__ also !!! --- use a hash + # of content for that, will be used for dedup. diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py new file mode 100644 index 0000000..1ae19c0 --- /dev/null +++ b/src/leap/mail/imap/parser.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# parser.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Mail parser mixins. +""" +import cStringIO +import StringIO +import re + +from email.parser import Parser + + +class MailParser(object): + """ + Mixin with utility methods to parse raw messages. + """ + def __init__(self): + """ + Initializes the mail parser. + """ + self._parser = Parser() + + def _get_parsed_msg(self, raw): + """ + Return a parsed Message. + + :param raw: the raw string to parse + :type raw: basestring, or StringIO object + """ + msg = self._get_parser_fun(raw)(raw, True) + return msg + + def _get_parser_fun(self, o): + """ + Retunn the proper parser function for an object. + + :param o: object + :type o: object + :param parser: an instance of email.parser.Parser + :type parser: email.parser.Parser + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return self._parser.parse + if isinstance(o, basestring): + return self._parser.parsestr + # fallback + return self._parser.parsestr + + def _stringify(self, o): + """ + Return a string object. + + :param o: object + :type o: object + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return o.getvalue() + else: + return o + + +class MBoxParser(object): + """ + Utility function to parse mailbox names. + """ + INBOX_NAME = "INBOX" + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + def _parse_mailbox_name(self, name): + """ + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py deleted file mode 100644 index 57587a5..0000000 --- a/src/leap/mail/imap/server.py +++ /dev/null @@ -1,1897 +0,0 @@ -# -*- coding: utf-8 -*- -# server.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Soledad-backed IMAP Server. -""" -import copy -import logging -import StringIO -import cStringIO -import time -import re - -from collections import defaultdict, namedtuple -from email.parser import Parser - -from zope.interface import implements -from zope.proxy import sameProxiedObjects - -from twisted.mail import imap4 -from twisted.internet import defer -from twisted.python import log - -from u1db import errors as u1db_errors - -from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.check import leap_assert, leap_assert_type -from leap.common.mail import get_email_charset -from leap.mail.messageflow import IMessageConsumer, MessageProducer -from leap.mail.decorators import deferred -from leap.soledad.client import Soledad - -logger = logging.getLogger(__name__) - - -class MissingIndexError(Exception): - """ - Raises when tried to access a non existent index document. - """ - - -class BadIndexError(Exception): - """ - Raises when index is malformed or has the wrong cardinality. - """ - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # Internal representation of Message - DATE_KEY = "date" - HEADERS_KEY = "headers" - FLAGS_KEY = "flags" - MBOX_KEY = "mbox" - RAW_KEY = "raw" - SUBJECT_KEY = "subject" - UID_KEY = "uid" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" - - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MESSAGE_VAL = "msg" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - # should add also a headers val - - INBOX_VAL = "inbox" - - # Flags for SoledadDocument for indexing. - SEEN_KEY = "seen" - RECENT_KEY = "recent" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - -fields = WithMsgFields # alias for convenience - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - db_indexes = dict() - if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in SoledadBackedAccount.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) - - -class MailParser(object): - """ - Mixin with utility methods to parse raw messages. - """ - def __init__(self): - """ - Initializes the mail parser. - """ - self._parser = Parser() - - def _get_parsed_msg(self, raw): - """ - Return a parsed Message. - - :param raw: the raw string to parse - :type raw: basestring, or StringIO object - """ - msg = self._get_parser_fun(raw)(raw, True) - return msg - - def _get_parser_fun(self, o): - """ - Retunn the proper parser function for an object. - - :param o: object - :type o: object - :param parser: an instance of email.parser.Parser - :type parser: email.parser.Parser - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, basestring): - return self._parser.parsestr - # fallback - return self._parser.parsestr - - def _stringify(self, o): - """ - Return a string object. - - :param o: object - :type o: object - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() - else: - return o - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name - - -####################################### -# Soledad Account -####################################### - - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): - """ - An implementation of IAccount and INamespacePresenteer - that is backed by Soledad Encrypted Documents. - """ - - implements(imap4.IAccount, imap4.INamespacePresenter) - - _soledad = None - selected = None - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - KTYPE = WithMsgFields.TYPE_KEY - MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - WithMsgFields.TYPE_KEY: MBOX_KEY, - WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.CLOSED_KEY: False, - WithMsgFields.SUBSCRIBED_KEY: False, - WithMsgFields.RW_KEY: 1, - WithMsgFields.LAST_UID_KEY: 0 - } - - def __init__(self, account_name, soledad=None): - """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. - - :param acct_name: The name of the account (user id). - :type acct_name: str - - :param soledad: a Soledad instance. - :param soledad: Soledad - """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) - - # XXX SHOULD assert too that the name matches the user/uuid with which - # soledad has been initialized. - - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - - self.initialize_db() - - # every user should have the right to an inbox folder - # at least, so let's make one! - - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - - def _get_mailbox_by_name(self, name): - """ - Return an mbox document by name. - - :param name: the name of the mailbox - :type name: str - - :rtype: SoledadDocument - """ - doc = self._soledad.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) - return doc[0] if doc else None - - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)] - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] - - def getMailbox(self, name): - """ - Returns a Mailbox with that name, without selecting it. - - :param name: name of the mailbox - :type name: str - - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox - """ - name = self._parse_mailbox_name(name) - - if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - return SoledadMailbox(name, soledad=self._soledad) - - ## - ## IAccount - ## - - def addMailbox(self, name, creation_ts=None): - """ - Add a mailbox to the account. - - :param name: the name of the mailbox - :type name: str - - :param creation_ts: an optional creation timestamp to be used as - mailbox id. A timestamp will be used if no - one is provided. - :type creation_ts: int - - :returns: True if successful - :rtype: bool - """ - name = self._parse_mailbox_name(name) - - if name in self.mailboxes: - raise imap4.MailboxCollision, name - - if not creation_ts: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # mailbox-uidvalidity. - creation_ts = int(time.time() * 10E2) - - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - doc = self._soledad.create_doc(mbox) - return bool(doc) - - def create(self, pathspec): - """ - Create a new mailbox from the given hierarchical name. - - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. - :type pathspec: str - - :return: A true value if the creation succeeds. - :rtype: bool - - :raise MailboxException: Raised if this mailbox cannot be added. - """ - # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) - for accum in range(1, len(paths)): - try: - self.addMailbox('/'.join(paths[:accum])) - except imap4.MailboxCollision: - pass - try: - self.addMailbox('/'.join(paths)) - except imap4.MailboxCollision: - if not pathspec.endswith('/'): - return False - return True - - def select(self, name, readwrite=1): - """ - Selects a mailbox. - - :param name: the mailbox to select - :type name: str - - :param readwrite: 1 for readwrite permissions. - :type readwrite: int - - :rtype: bool - """ - name = self._parse_mailbox_name(name) - - if name not in self.mailboxes: - return None - - self.selected = name - - return SoledadMailbox( - name, rw=readwrite, - soledad=self._soledad) - - def delete(self, name, force=False): - """ - Deletes a mailbox. - - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - - :param name: the mailbox to be deleted - :type name: str - - :param force: if True, it will not check for noselect flag or inferior - names. use with care. - :type force: bool - """ - name = self._parse_mailbox_name(name) - - if not name in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - mbox = self.getMailbox(name) - - if force is False: - # See if this box is flagged \Noselect - # XXX use mbox.flags instead? - if self.NOSELECT_FLAG in mbox.getFlags(): - # Check for hierarchically inferior mailboxes with this one - # as part of their root. - for others in self.mailboxes: - if others != name and others.startswith(name): - raise imap4.MailboxException, ( - "Hierarchically inferior mailboxes " - "exist and \\Noselect is set") - mbox.destroy() - - # XXX FIXME --- not honoring the inferior names... - - # if there are no hierarchically inferior names, we will - # delete it from our ken. - #if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - #self._index.removeMailbox(name) - - def rename(self, oldname, newname): - """ - Renames a mailbox. - - :param oldname: old name of the mailbox - :type oldname: str - - :param newname: new name of the mailbox - :type newname: str - """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox, oldname - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] - - for (old, new) in inferiors: - if new in self.mailboxes: - raise imap4.MailboxCollision, new - - for (old, new) in inferiors: - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new - self._soledad.put_doc(mbox) - - # XXX ---- FIXME!!!! ------------------------------------ - # until here we just renamed the index... - # We have to rename also the occurrence of this - # mailbox on ALL the messages that are contained in it!!! - # ... we maybe could use a reference to the doc_id - # in each msg, instead of the "mbox" field in msgs - # ------------------------------------------------------- - - def _inferiorNames(self, name): - """ - Return hierarchically inferior mailboxes. - - :param name: name of the mailbox - :rtype: list - """ - # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors - - def isSubscribed(self, name): - """ - Returns True if user is subscribed to this mailbox. - - :param name: the mailbox to be checked. - :type name: str - - :rtype: bool - """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) - - def _set_subscription(self, name, value): - """ - Sets the subscription value for a given mailbox - - :param name: the mailbox - :type name: str - - :param value: the boolean value - :type value: bool - """ - # maybe we should store subscriptions in another - # document... - if not name in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) - - def subscribe(self, name): - """ - Subscribe to this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) - - def unsubscribe(self, name): - """ - Unsubscribe from this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException, "Not currently subscribed to " + name - self._set_subscription(name, False) - - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - - ## - ## INamespacePresenter - ## - - def getPersonalNamespaces(self): - return [["", "/"]] - - def getSharedNamespaces(self): - return None - - def getOtherNamespaces(self): - return None - - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - - def __repr__(self): - """ - Representation string for this object. - """ - return "" % self._account_name - -####################################### -# LeapMessage, MessageCollection -# and Mailbox -####################################### - - -class LeapMessage(fields, MailParser, MBoxParser): - - implements(imap4.IMessage) - - def __init__(self, soledad, uid, mbox): - """ - Initializes a LeapMessage. - - :param soledad: a Soledad instance - :type soledad: Soledad - :param uid: the UID for the message. - :type uid: int or basestring - :param mbox: the mbox this message belongs to - :type mbox: basestring - """ - MailParser.__init__(self) - self._soledad = soledad - self._uid = int(uid) - self._mbox = self._parse_mailbox_name(mbox) - - self.__cdoc = None - - @property - def _fdoc(self): - """ - An accessor to the flags docuemnt - """ - return self._get_flags_doc() - - @property - def _cdoc(self): - """ - An accessor to the content docuemnt - """ - if not self.__cdoc: - self.__cdoc = self._get_content_doc() - return self.__cdoc - - def getUID(self): - """ - Retrieve the unique identifier associated with this message - - :return: uid for this message - :rtype: int - """ - return self._uid - - def getFlags(self): - """ - Retrieve the flags associated with this message - - :return: The flags, represented as strings - :rtype: tuple - """ - if self._uid is None: - return [] - - flags = [] - flag_doc = self._fdoc - if flag_doc: - flags = flag_doc.content.get(self.FLAGS_KEY, None) - if flags: - flags = map(str, flags) - return tuple(flags) - - # setFlags, addFlags, removeFlags are not in the interface spec - # but we use them with store command. - - def setFlags(self, flags): - """ - Sets the flags for this message - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to update in the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - log.msg('setting flags: %s' % (self._uid)) - - doc = self._fdoc - doc.content[self.FLAGS_KEY] = flags - doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags - doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags - self._soledad.put_doc(doc) - - def addFlags(self, flags): - """ - Adds flags to this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to add to the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - oldflags = self.getFlags() - self.setFlags(tuple(set(flags + oldflags))) - - def removeFlags(self, flags): - """ - Remove flags from this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to be removed from the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - oldflags = self.getFlags() - self.setFlags(tuple(set(oldflags) - set(flags))) - - def getInternalDate(self): - """ - Retrieve the date internally associated with this message - - :rtype: C{str} - :return: An RFC822-formatted date string. - """ - return str(self._cdoc.content.get(self.DATE_KEY, '')) - - # - # IMessagePart - # - - # XXX we should implement this interface too for the subparts - # so we allow nested parts... - - def getBodyFile(self): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO - """ - fd = StringIO.StringIO() - - cdoc = self._cdoc - content = cdoc.content.get(self.RAW_KEY, '') - charset = get_email_charset( - unicode(cdoc.content.get(self.RAW_KEY, ''))) - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - - raw = self._get_raw_msg() - msg = self._get_parsed_msg(raw) - body = msg.get_payload() - fd.write(body) - # XXX SHOULD use a separate BODY FIELD ... - fd.seek(0) - return fd - - def getSize(self): - """ - Return the total size, in octets, of this message. - - :return: size of the message, in octets - :rtype: int - """ - size = self._cdoc.content.get(self.SIZE_KEY, False) - if not size: - # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len - return size - - def _get_headers(self): - """ - Return the headers dict stored in this message document. - """ - # XXX get from the headers doc - return self._cdoc.content.get(self.HEADERS_KEY, {}) - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - headers = self._get_headers() - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # unpack and filter original dict by negate-condition - filter_by_cond = [ - map(str, (key, val)) for - key, val in headers.items() - if cond(key)] - return dict(filter_by_cond) - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - if self._cdoc: - retval = self._fdoc.content.get(self.MULTIPART_KEY, False) - return retval - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - if not self.isMultipart(): - raise TypeError - - msg = self._get_parsed_msg() - # XXX should wrap IMessagePart - return msg.get_payload()[part] - - # - # accessors - # - - def _get_flags_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - flag_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - flag_doc = flag_docs[0] if flag_docs else None - return flag_doc - - def _get_content_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - cont_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid)) - cont_doc = cont_docs[0] if cont_docs else None - return cont_doc - - def _get_raw_msg(self): - """ - Return the raw msg. - :rtype: basestring - """ - return self._cdoc.content.get(self.RAW_KEY, '') - - def __getitem__(self, key): - """ - Return the content of the message document. - - :param key: The key - :type key: str - - :return: The content value indexed by C{key} or None - :rtype: str - """ - return self._cdoc.content.get(key, None) - - def does_exist(self): - """ - Return True if there is actually a message for this - UID and mbox. - """ - return bool(self._fdoc) - - -SoledadWriterPayload = namedtuple( - 'SoledadWriterPayload', ['mode', 'payload']) - -SoledadWriterPayload.CREATE = 1 -SoledadWriterPayload.PUT = 2 - - -class SoledadDocWriter(object): - """ - This writer will create docs serially in the local soledad database. - """ - - implements(IMessageConsumer) - - def __init__(self, soledad): - """ - Initialize the writer. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - self._soledad = soledad - - def consume(self, queue): - """ - Creates a new document in soledad db. - - :param queue: queue to get item from, with content of the document - to be inserted. - :type queue: Queue - """ - empty = queue.empty() - while not empty: - item = queue.get() - if item.mode == SoledadWriterPayload.CREATE: - call = self._soledad.create_doc - elif item.mode == SoledadWriterPayload.PUT: - call = self._soledad.put_doc - - # should handle errors - try: - call(item.payload) - except u1db_errors.RevisionConflict as exc: - logger.error("Error: %r" % (exc,)) - raise exc - - empty = queue.empty() - - -class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): - """ - A collection of messages, surprisingly. - - It is tied to a selected mailbox name that is passed to constructor. - Implements a filter query over the messages contained in a soledad - database. - """ - # XXX this should be able to produce a MessageSet methinks - - EMPTY_MSG = { - fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.SUBJECT_KEY: "", - fields.DATE_KEY: "", - fields.RAW_KEY: "", - - # XXX should separate headers into another doc - fields.HEADERS_KEY: {}, - } - - EMPTY_FLAGS = { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.FLAGS_KEY: [], - fields.SEEN_KEY: False, - fields.RECENT_KEY: True, - fields.MULTIPART_KEY: False, - } - - # get from SoledadBackedAccount the needed index-related constants - INDEXES = SoledadBackedAccount.INDEXES - TYPE_IDX = SoledadBackedAccount.TYPE_IDX - - def __init__(self, mbox=None, soledad=None): - """ - Constructor for MessageCollection. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database - :type mbox: str - - :param soledad: Soledad database - :type soledad: Soledad instance - """ - MailParser.__init__(self) - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") - - # okay, all in order, keep going... - self.mbox = self._parse_mailbox_name(mbox) - self._soledad = soledad - self.initialize_db() - - # I think of someone like nietzsche when reading this - - # this will be the producer that will enqueue the content - # to be processed serially by the consumer (the writer). We just - # need to `put` the new material on its plate. - - self.soledad_writer = MessageProducer( - SoledadDocWriter(soledad), - period=0.05) - - def _get_empty_msg(self): - """ - Returns an empty message. - - :return: a dict containing a default empty message - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MSG) - - def _get_empty_flags_doc(self): - """ - Returns an empty doc for storing flags. - - :return: - :rtype: - """ - return copy.deepcopy(self.EMPTY_FLAGS) - - @deferred - def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): - """ - Creates a new message document. - - :param raw: the raw message - :type raw: str - - :param subject: subject of the message. - :type subject: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :param uid: the message uid for this mailbox - :type uid: int - """ - # TODO: split in smaller methods - logger.debug('adding message') - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - - content_doc = self._get_empty_msg() - flags_doc = self._get_empty_flags_doc() - - content_doc[self.MBOX_KEY] = self.mbox - flags_doc[self.MBOX_KEY] = self.mbox - # ...should get a sanity check here. - content_doc[self.UID_KEY] = uid - flags_doc[self.UID_KEY] = uid - - if flags: - flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) - flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags - - msg = self._get_parsed_msg(raw) - headers = dict(msg) - - logger.debug("adding. is multipart:%s" % msg.is_multipart()) - flags_doc[self.MULTIPART_KEY] = msg.is_multipart() - # XXX get lower case for keys? - # XXX get headers doc - content_doc[self.HEADERS_KEY] = headers - # set subject based on message headers and eventually replace by - # subject given as param - if self.SUBJECT_FIELD in headers: - content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - if subject is not None: - content_doc[self.SUBJECT_KEY] = subject - - # XXX could separate body into its own doc - # but should also separate multiparts - # that should be wrapped in MessagePart - content_doc[self.RAW_KEY] = self._stringify(raw) - content_doc[self.SIZE_KEY] = len(raw) - - if not date and self.DATE_FIELD in headers: - content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - content_doc[self.DATE_KEY] = date - - logger.debug('enqueuing message for write') - - ptuple = SoledadWriterPayload - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=content_doc)) - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=flags_doc)) - - def remove(self, msg): - """ - Removes a message. - - :param msg: a u1db doc containing the message - :type msg: SoledadDocument - """ - self._soledad.delete_doc(msg) - - # getters - - def get_msg_by_uid(self, uid): - """ - Retrieves a LeapMessage by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - msg = LeapMessage(self._soledad, uid, self.mbox) - if not msg.does_exist(): - return None - return msg - - def get_all(self): - """ - Get all message documents for the selected mailbox. - If you want acess to the content, use __iter__ instead - - :return: a list of u1db documents - :rtype: list of SoledadDocument - """ - # TODO change to get_all_docs and turn this - # into returning messages - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - all_docs = [doc for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox)] - - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - return sorted(all_docs, key=lambda item: item.content['uid']) - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - return count - - # unseen messages - - def unseen_iter(self): - """ - Get an iterator for the message docs with no `seen` flag - - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - return (doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0')) - - def count_unseen(self): - """ - Count all messages with the `Unseen` flag. - - :returns: count - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0') - return count - - def get_unseen(self): - """ - Get all messages with the `Unseen` flag - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox) - for docid in self.unseen_iter()] - - # recent messages - - def recent_iter(self): - """ - Get an iterator for the message docs with `recent` flag. - - :return: iterator through recent message docs - :rtype: iterable - """ - return (doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1')) - - def get_recent(self): - """ - Get all messages with the `Recent` flag. - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox) - for docid in self.recent_iter()] - - def count_recent(self): - """ - Count all messages with the `Recent` flag. - - :returns: count - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1') - return count - - def __len__(self): - """ - Returns the number of messages on this mailbox. - - :rtype: int - """ - return self.count() - - def __iter__(self): - """ - Returns an iterator over all messages. - - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - # XXX return LeapMessage instead?! (change accordingly) - return (m.content for m in self.get_all()) - - def __repr__(self): - """ - Representation string for this object. - """ - return u"" % ( - self.mbox, self.count()) - - # XXX should implement __eq__ also !!! --- use a hash - # of content for that, will be used for dedup. - - -class SoledadMailbox(WithMsgFields, MBoxParser): - """ - A Soledad-backed IMAP mailbox. - - Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. - """ - implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) - # XXX should finish the implementation of IMailboxListener - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None - - CMD_MSG = "MESSAGES" - CMD_RECENT = "RECENT" - CMD_UIDNEXT = "UIDNEXT" - CMD_UIDVALIDITY = "UIDVALIDITY" - CMD_UNSEEN = "UNSEEN" - - _listeners = defaultdict(set) - - def __init__(self, mbox, soledad=None, rw=1): - """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. - - :param mbox: the mailbox name - :type mbox: str - - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param rw: read-and-write flags - :type rw: int - """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") - - # XXX should move to wrapper - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - - self.mbox = self._parse_mailbox_name(mbox) - self.rw = rw - - self._soledad = soledad - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad) - - if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - @property - def listeners(self): - """ - Returns listeners for this mbox. - - The server itself is a listener to the mailbox. - so we can notify it (and should!) after changes in flags - and number of messages. - - :rtype: set - """ - return self._listeners[self.mbox] - - def addListener(self, listener): - """ - Adds a listener to the listeners queue. - The server adds itself as a listener when there is a SELECT, - so it can send EXIST commands. - - :param listener: listener to add - :type listener: an object that implements IMailboxListener - """ - logger.debug('adding mailbox listener: %s' % listener) - self.listeners.add(listener) - - def removeListener(self, listener): - """ - Removes a listener from the listeners queue. - - :param listener: listener to remove - :type listener: an object that implements IMailboxListener - """ - self.listeners.remove(listener) - - def _get_mbox(self): - """ - Returns mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - try: - query = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() - except Exception as exc: - logger.error("Unhandled error %r" % exc) - - def getFlags(self): - """ - Returns the flags defined for this mailbox. - - :returns: tuple of flags for this mailbox - :rtype: tuple of str - """ - mbox = self._get_mbox() - if not mbox: - return None - flags = mbox.content.get(self.FLAGS_KEY, []) - return map(str, flags) - - def setFlags(self, flags): - """ - Sets flags for this mailbox. - - :param flags: a tuple with the flags - :type flags: tuple of str - """ - leap_assert(isinstance(flags, tuple), - "flags expected to be a tuple") - mbox = self._get_mbox() - if not mbox: - return None - mbox.content[self.FLAGS_KEY] = map(str, flags) - self._soledad.put_doc(mbox) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. - - def _get_closed(self): - """ - Return the closed attribute for this mailbox. - - :return: True if the mailbox is closed - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.CLOSED_KEY, False) - - def _set_closed(self, closed): - """ - Set the closed attribute for this mailbox. - - :param closed: the state to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - mbox = self._get_mbox() - mbox.content[self.CLOSED_KEY] = closed - self._soledad.put_doc(mbox) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - - :return: the last uid for messages in this mailbox - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.LAST_UID_KEY, 1) - - def _set_last_uid(self, uid): - """ - Sets the last uid for this mailbox. - - :param uid: the uid to be set - :type uid: int - """ - leap_assert(isinstance(uid, int), "uid has to be int") - mbox = self._get_mbox() - key = self.LAST_UID_KEY - - count = self.getMessageCount() - - # XXX safety-catch. If we do get duplicates, - # we want to avoid further duplication. - - if uid >= count: - value = uid - else: - # something is wrong, - # just set the last uid - # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to %s", count) - value = count - - mbox.content[key] = value - self._soledad.put_doc(mbox) - - last_uid = property( - _get_last_uid, _set_last_uid, doc="Last_UID attribute.") - - def getUIDValidity(self): - """ - Return the unique validity identifier for this mailbox. - - :return: unique validity identifier - :rtype: int - """ - mbox = self._get_mbox() - return mbox.content.get(self.CREATED_KEY, 1) - - def getUID(self, message): - """ - Return the UID of a message in the mailbox - - .. note:: this implementation does not make much sense RIGHT NOW, - but in the future will be useful to get absolute UIDs from - message sequence numbers. - - :param message: the message uid - :type message: int - - :rtype: int - """ - msg = self.messages.get_msg_by_uid(message) - return msg.getUID() - - def getUIDNext(self): - """ - Return the likely UID for the next message added to this - mailbox. Currently it returns the higher UID incremented by - one. - - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int - """ - self.last_uid += 1 - return self.last_uid - - def getMessageCount(self): - """ - Returns the total count of messages in this mailbox. - - :rtype: int - """ - return self.messages.count() - - def getUnseenCount(self): - """ - Returns the number of messages with the 'Unseen' flag. - - :return: count of messages flagged `unseen` - :rtype: int - """ - return self.messages.count_unseen() - - def getRecentCount(self): - """ - Returns the number of messages with the 'Recent' flag. - - :return: count of messages flagged `recent` - :rtype: int - """ - return self.messages.count_recent() - - def isWriteable(self): - """ - Get the read/write status of the mailbox. - - :return: 1 if mailbox is read-writeable, 0 otherwise. - :rtype: int - """ - return self.rw - - def getHierarchicalDelimiter(self): - """ - Returns the character used to delimite hierarchies in mailboxes. - - :rtype: str - """ - return '/' - - def requestStatus(self, names): - """ - Handles a status request by gathering the output of the different - status commands. - - :param names: a list of strings containing the status commands - :type names: iter - """ - r = {} - if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() - if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() - if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 - if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUID() - if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) - - def addMessage(self, message, flags, date=None): - """ - Adds a message to this mailbox. - - :param message: the raw message - :type message: str - - :param flags: flag list - :type flags: list of str - - :param date: timestamp - :type date: str - - :return: a deferred that evals to None - """ - # XXX we should treat the message as an IMessage from here - leap_assert_type(message, basestring) - uid_next = self.getUIDNext() - logger.debug('Adding msg with UID :%s' % uid_next) - if flags is None: - flags = tuple() - else: - flags = tuple(str(flag) for flag in flags) - - d = self._do_add_messages(message, flags, date, uid_next) - d.addCallback(self._notify_new) - - @deferred - def _do_add_messages(self, message, flags, date, uid_next): - """ - Calls to the messageCollection add_msg method (deferred to thread). - Invoked from addMessage. - """ - self.messages.add_msg(message, flags=flags, date=date, - uid=uid_next) - - def _notify_new(self, *args): - """ - Notify of new messages to all the listeners. - - :param args: ignored. - """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY: there are %s messages, %s recent" % ( - exists, - recent)) - - logger.debug("listeners: %s", str(self.listeners)) - for l in self.listeners: - logger.debug('notifying...') - l.newMessages(exists, recent) - - # commands, do not rename methods - - def destroy(self): - """ - Called before this mailbox is permanently deleted. - - Should cleanup resources, and set the \\Noselect flag - on the mailbox. - """ - self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - self._soledad.delete_doc(self._get_mbox()) - - def expunge(self): - """ - Remove all messages flagged \\Deleted - """ - if not self.isWriteable(): - raise imap4.ReadOnlyMailbox - delete = [] - deleted = [] - - for m in self.messages.get_all(): - if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: - delete.append(m) - for m in delete: - deleted.append(m.content) - self.messages.remove(m) - - # XXX should return the UIDs of the deleted messages - # more generically - return [x for x in range(len(deleted))] - - @deferred - def fetch(self, messages, uid): - """ - Retrieve one or more messages in this mailbox. - - from rfc 3501: The data items to be fetched can be either a single atom - or a parenthesized list. - - :param messages: IDs of the messages to retrieve information about - :type messages: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ - result = [] - sequence = True if uid == 0 else False - - if not messages.last: - try: - iter(messages) - except TypeError: - # looks like we cannot iterate - messages.last = self.last_uid - - # for sequence numbers (uid = 0) - if sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") - raise NotImplementedError - - else: - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) - else: - logger.debug("fetch %s, no msg found!!!" % msg_id) - - if self.isWriteable(): - self._unset_recent_flag() - self._signal_unread_to_ui() - - # XXX workaround for hangs in thunderbird - #return tuple(result[:100]) # --- doesn't show all!! - return tuple(result) - - @deferred - def _unset_recent_flag(self): - """ - Unsets `Recent` flag from a tuple of messages. - Called from fetch. - - From RFC, about `Recent`: - - Message is "recently" arrived in this mailbox. This session - is the first session to have been notified about this - message; if the session is read-write, subsequent sessions - will not see \Recent set for this message. This flag can not - be altered by the client. - - If it is not possible to determine whether or not this - session is the first session to be notified about a message, - then that message SHOULD be considered recent. - """ - log.msg('unsetting recent flags...') - for msg in self.messages.get_recent(): - msg.removeFlags((fields.RECENT_FLAG,)) - self._signal_unread_to_ui() - - @deferred - def _signal_unread_to_ui(self): - """ - Sends unread event to ui. - """ - unseen = self.getUnseenCount() - leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) - - @deferred - def store(self, messages, flags, mode, uid): - """ - Sets the flags of one or more messages. - - :param messages: The identifiers of the messages to set the flags - :type messages: A MessageSet object with the list of messages requested - - :param flags: The flags to set, unset, or add. - :type flags: sequence of str - - :param mode: If mode is -1, these flags should be removed from the - specified messages. If mode is 1, these flags should be - added to the specified messages. If mode is 0, all - existing flags should be cleared and these flags should be - added. - :type mode: -1, 0, or 1 - - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool - - :return: A dict mapping message sequence numbers to sequences of - str representing the flags set on the message after this - operation has been performed. - :rtype: dict - - :raise ReadOnlyMailbox: Raised if this mailbox is not open for - read-write. - """ - # XXX implement also sequence (uid = 0) - # XXX we should prevent cclient from setting Recent flag. - leap_assert(not isinstance(flags, basestring), - "flags cannot be a string") - flags = tuple(flags) - - if not self.isWriteable(): - log.msg('read only mailbox!') - raise imap4.ReadOnlyMailbox - - if not messages.last: - messages.last = self.messages.count() - - result = {} - for msg_id in messages: - log.msg("MSG ID = %s" % msg_id) - msg = self.messages.get_msg_by_uid(msg_id) - if mode == 1: - msg.addFlags(flags) - elif mode == -1: - msg.removeFlags(flags) - elif mode == 0: - msg.setFlags(flags) - result[msg_id] = msg.getFlags() - - self._signal_unread_to_ui() - return result - - @deferred - def close(self): - """ - Expunge and mark as closed - """ - self.expunge() - self.closed = True - - # convenience fun - - def deleteAllDocs(self): - """ - Deletes all docs in this mailbox - """ - docs = self.messages.get_all() - for doc in docs: - self.messages._soledad.delete_doc(doc) - - def __repr__(self): - """ - Representation string for this mailbox. - """ - return u"" % ( - self.mbox, self.messages.count()) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8756ddc..26e14c3 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.server import SoledadBackedAccount +from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail from leap.soledad.client import Soledad -- cgit v1.2.3 From 25a0aea875fd0d67238beed1237f7239474673ec Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 27 Dec 2013 02:06:44 -0400 Subject: First stage of the storage schema rewrite. * Separates between flags, docs, body and attachment docs. * Implement IMessageCopier interface: move and have fun! This little change is known to push forward our beloved architect emotional rollercoster. * Message deduplication. * It also fixes a hidden bug that was rendering the multipart mime interface useless (yes, the "True" parameter in the parsestr method). * Does not handle well nested attachs, includes dirty workaround that flattens them. * Includes chiiph's patch for rc2: * return deferred from addMessage * convert StringIO types to string * remove unneeded yields from the chain of deferreds in fetcher --- src/leap/mail/imap/fetch.py | 7 +- src/leap/mail/imap/fields.py | 49 ++- src/leap/mail/imap/index.py | 4 +- src/leap/mail/imap/mailbox.py | 103 +++-- src/leap/mail/imap/messages.py | 831 +++++++++++++++++++++++++++++++++-------- src/leap/mail/imap/parser.py | 24 +- 6 files changed, 802 insertions(+), 216 deletions(-) (limited to 'src') diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 0b31c3b..fdf1412 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -412,13 +412,13 @@ class LeapIncomingMail(object): # decrypt or fail gracefully try: - decrdata, valid_sig = yield self._decrypt_and_verify_data( + decrdata, valid_sig = self._decrypt_and_verify_data( encdata, senderPubkey) except keymanager_errors.DecryptError as e: logger.warning('Failed to decrypt encrypted message (%s). ' 'Storing message without modifications.' % str(e)) # Bailing out! - yield (msg, False) + return (msg, False) # decrypted successully, now fix encoding and parse try: @@ -441,7 +441,7 @@ class LeapIncomingMail(object): # all ok, replace payload by unencrypted payload msg.set_payload(decrmsg.get_payload()) - yield (msg, valid_sig) + return (msg, valid_sig) def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, senderPubkey): @@ -527,6 +527,7 @@ class LeapIncomingMail(object): """ log.msg('adding message to local db') doc, data = msgtuple + if isinstance(data, list): data = data[0] diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 96b937e..40817cd 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -25,18 +25,35 @@ class WithMsgFields(object): Container class for class-attributes to be shared by several message-related classes. """ - # Internal representation of Message - DATE_KEY = "date" - HEADERS_KEY = "headers" - FLAGS_KEY = "flags" - MBOX_KEY = "mbox" + # indexing CONTENT_HASH_KEY = "chash" - RAW_KEY = "raw" - SUBJECT_KEY = "subject" + PAYLOAD_HASH_KEY = "phash" + + # Internal representation of Message + + # flags doc UID_KEY = "uid" + MBOX_KEY = "mbox" + SEEN_KEY = "seen" + RECENT_KEY = "recent" + FLAGS_KEY = "flags" MULTIPART_KEY = "multi" SIZE_KEY = "size" + # headers + HEADERS_KEY = "headers" + NUM_PARTS_KEY = "numparts" + PARTS_MAP_KEY = "partmap" + DATE_KEY = "date" + SUBJECT_KEY = "subject" + + # attachment + PART_NUMBER_KEY = "part" + RAW_KEY = "raw" + + # content + BODY_KEY = "body" + # Mailbox specific keys CLOSED_KEY = "closed" CREATED_KEY = "created" @@ -55,10 +72,6 @@ class WithMsgFields(object): INBOX_VAL = "inbox" - # Flags for SoledadDocument for indexing. - SEEN_KEY = "seen" - RECENT_KEY = "recent" - # Flags in Mailbox and Message SEEN_FLAG = "\\Seen" RECENT_FLAG = "\\Recent" @@ -82,7 +95,9 @@ class WithMsgFields(object): TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_HASH_IDX = 'by-type-and-hash' + TYPE_C_HASH_IDX = 'by-type-and-contenthash' + TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' + TYPE_P_HASH_IDX = 'by-type-and-payloadhash' # Tomas created the `recent and seen index`, but the semantic is not too # correct since the recent flag is volatile. @@ -90,7 +105,9 @@ class WithMsgFields(object): KTYPE = TYPE_KEY MBOX_VAL = TYPE_MBOX_VAL - HASH_VAL = CONTENT_HASH_KEY + CHASH_VAL = CONTENT_HASH_KEY + PHASH_VAL = PAYLOAD_HASH_KEY + PART_VAL = PART_NUMBER_KEY INDEXES = { # generic @@ -102,7 +119,11 @@ class WithMsgFields(object): TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], # content, headers doc - TYPE_HASH_IDX: [KTYPE, HASH_VAL], + TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], + # attachment docs + TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL], + # attachment payload dedup + TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], # messages TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py index 2280d86..5f0919a 100644 --- a/src/leap/mail/imap/index.py +++ b/src/leap/mail/imap/index.py @@ -21,7 +21,7 @@ import logging from leap.common.check import leap_assert, leap_assert_type -from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.fields import fields logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ class IndexedDB(object): db_indexes = dict() if self._soledad is not None: db_indexes = dict(self._soledad.list_indexes()) - for name, expression in SoledadBackedAccount.INDEXES.items(): + for name, expression in fields.INDEXES.items(): if name not in db_indexes: # The index does not yet exist. self._soledad.create_index(name, *expression) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 09c06a2..5ea6f55 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -17,7 +17,13 @@ """ Soledad Mailbox. """ +import copy +import threading import logging +import time +import StringIO +import cStringIO + from collections import defaultdict from twisted.internet import defer @@ -45,9 +51,14 @@ class SoledadMailbox(WithMsgFields, MBoxParser): The low-level database methods are contained in MessageCollection class, which we instantiate and make accessible in the `messages` attribute. """ - implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) + implements( + imap4.IMailbox, + imap4.IMailboxInfo, + imap4.ICloseableMailbox, + imap4.IMessageCopier) + # XXX should finish the implementation of IMailboxListener - # XXX should implement IMessageCopier too + # XXX should implement ISearchableMailbox too messages = None _closed = False @@ -65,6 +76,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): CMD_UNSEEN = "UNSEEN" _listeners = defaultdict(set) + next_uid_lock = threading.Lock() def __init__(self, mbox, soledad=None, rw=1): """ @@ -284,8 +296,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: int """ - self.last_uid += 1 - return self.last_uid + with self.next_uid_lock: + self.last_uid += 1 + return self.last_uid def getMessageCount(self): """ @@ -366,6 +379,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: a deferred that evals to None """ + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() # XXX we should treat the message as an IMessage from here leap_assert_type(message, basestring) uid_next = self.getUIDNext() @@ -375,11 +390,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_messages(message, flags, date, uid_next) + d = self._do_add_message(message, flags, date, uid_next) d.addCallback(self._notify_new) + return d @deferred - def _do_add_messages(self, message, flags, date, uid_next): + def _do_add_message(self, message, flags, date, uid_next): """ Calls to the messageCollection add_msg method (deferred to thread). Invoked from addMessage. @@ -420,28 +436,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # we should postpone the removal self._soledad.delete_doc(self._get_mbox()) + @deferred def expunge(self): """ Remove all messages flagged \\Deleted """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - delete = [] deleted = [] - - for m in self.messages.get_all_docs(): - # XXX should operate with LeapMessages instead, - # so we don't expose the implementation. - # (so, iterate for m in self.messages) - if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: - delete.append(m) - for m in delete: - deleted.append(m.content) - self.messages.remove(m) - - # XXX should return the UIDs of the deleted messages - # more generically - return [x for x in range(len(deleted))] + for m in self.messages: + if self.DELETED_FLAG in m.getFlags(): + self.messages.remove(m) + # XXX this would ve more efficient if we can just pass + # a sequence of uids. + deleted.append(m.getUID()) + return deleted @deferred def fetch(self, messages, uid): @@ -510,6 +519,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): session is the first session to be notified about a message, then that message SHOULD be considered recent. """ + # TODO this fucker, for the sake of correctness, is messing with + # the whole collection of flag docs. + + # Possible ways of action: + # 1. Ignore it, we want fun. + # 2. Trigger it with a delay + # 3. Route it through a queue with lesser priority than the + # regularar writer. + + # hmm let's try 2. in a quickndirty way... + time.sleep(1) log.msg('unsetting recent flags...') for msg in self.messages.get_recent(): msg.removeFlags((fields.RECENT_FLAG,)) @@ -570,6 +590,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): for msg_id in messages: log.msg("MSG ID = %s" % msg_id) msg = self.messages.get_msg_by_uid(msg_id) + if not msg: + return result if mode == 1: msg.addFlags(flags) elif mode == -1: @@ -589,15 +611,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.expunge() self.closed = True - #@deferred - #def copy(self, messageObject): - #""" - #Copy the given message object into this mailbox. - #""" - # XXX should just: - # 1. Get the message._fdoc - # 2. Change the UID to UIDNext for this mailbox - # 3. Add implements IMessageCopier + # IMessageCopier + + @deferred + def copy(self, messageObject): + """ + Copy the given message object into this mailbox. + """ + uid_next = self.getUIDNext() + msg = messageObject + + # XXX should use a public api instead + fdoc = msg._fdoc + if not fdoc: + logger.debug("Tried to copy a MSG with no fdoc") + return + + new_fdoc = copy.deepcopy(fdoc.content) + new_fdoc[self.UID_KEY] = uid_next + new_fdoc[self.MBOX_KEY] = self.mbox + + d = self._do_add_doc(new_fdoc) + d.addCallback(self._notify_new) + + @deferred + def _do_add_doc(self, doc): + """ + Defers the adding of a new doc. + :param doc: document to be created in soledad. + """ + self._soledad.create_doc(doc) # convenience fun diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index b0d5da2..c69c023 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -29,9 +29,9 @@ from zope.interface import implements from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type +from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset from leap.mail.decorators import deferred -from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.index import IndexedDB from leap.mail.imap.fields import fields, WithMsgFields from leap.mail.imap.parser import MailParser, MBoxParser @@ -40,6 +40,181 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer logger = logging.getLogger(__name__) +def first(things): + """ + Return the head of a collection. + """ + try: + return things[0] + except (IndexError, TypeError): + return None + + +class MessageBody(object): + """ + IMessagePart implementor for the main + body of a multipart message. + + Excusatio non petita: see the interface documentation. + """ + + implements(imap4.IMessagePart) + + def __init__(self, fdoc, bdoc): + self._fdoc = fdoc + self._bdoc = bdoc + + def getSize(self): + return len(self._bdoc.content[fields.BODY_KEY]) + + def getBodyFile(self): + fd = StringIO.StringIO() + + if self._bdoc: + body = self._bdoc.content[fields.BODY_KEY] + else: + body = "" + charset = self._get_charset(body) + try: + body = body.encode(charset) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + body = body.encode(charset, 'replace') + fd.write(body) + fd.seek(0) + return fd + + @memoized_method + def _get_charset(self, stuff): + return get_email_charset(unicode(stuff)) + + def getHeaders(self, negate, *names): + return {} + + def isMultipart(self): + return False + + def getSubPart(self, part): + return None + + +class MessageAttachment(object): + + implements(imap4.IMessagePart) + + def __init__(self, msg): + """ + Initializes the messagepart with a Message instance. + :param msg: a message instance + :type msg: Message + """ + self._msg = msg + + def getSize(self): + """ + Return the total size, in octets, of this message part. + + :return: size of the message, in octets + :rtype: int + """ + if not self._msg: + return 0 + return len(self._msg.as_string()) + + def getBodyFile(self): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: StringIO + """ + fd = StringIO.StringIO() + if self._msg: + body = self._msg.get_payload() + else: + logger.debug("Empty message!") + body = "" + + # XXX should only do the dance if we're sure it's + # content/text-plain!!! + #charset = self._get_charset(body) + #try: + #body = body.encode(charset) + #except (UnicodeEncodeError, UnicodeDecodeError) as e: + #logger.error("Unicode error {0}".format(e)) + #body = body.encode(charset, 'replace') + fd.write(body) + fd.seek(0) + return fd + + @memoized_method + def _get_charset(self, stuff): + # TODO put in a common class with LeapMessage + """ + Gets (guesses?) the charset of a payload. + + :param stuff: the stuff to guess about. + :type stuff: basestring + :returns: charset + """ + # XXX existential doubt 1. wouldn't be smarter to + # peek into the mail headers? + # XXX existential doubt 2. shouldn't we make the scope + # of the decorator somewhat more persistent? + # ah! yes! and put memory bounds. + return get_email_charset(unicode(stuff)) + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + if not self._msg: + return {} + headers = dict(self._msg.items()) + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + # unpack and filter original dict by negate-condition + filter_by_cond = [ + map(str, (key, val)) for + key, val in headers.items() + if cond(key)] + return dict(filter_by_cond) + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + return self._msg.is_multipart() + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + return self._msg.get_payload() + + class LeapMessage(fields, MailParser, MBoxParser): implements(imap4.IMessage) @@ -59,25 +234,21 @@ class LeapMessage(fields, MailParser, MBoxParser): self._soledad = soledad self._uid = int(uid) self._mbox = self._parse_mailbox_name(mbox) - self._chash = None - self.__cdoc = None + self.__chash = None + self.__bdoc = None @property def _fdoc(self): """ An accessor to the flags document. """ - return self._get_flags_doc() - - @property - def _cdoc(self): - """ - An accessor to the content document. - """ - if not self.__cdoc: - self.__cdoc = self._get_content_doc() - return self.__cdoc + if all(map(bool, (self._uid, self._mbox))): + fdoc = self._get_flags_doc() + if fdoc: + self.__chash = fdoc.content.get( + fields.CONTENT_HASH_KEY, None) + return fdoc @property def _chash(self): @@ -86,7 +257,26 @@ class LeapMessage(fields, MailParser, MBoxParser): """ if not self._fdoc: return None - return self._fdoc.content.get(fields.CONTENT_HASH_KEY, None) + if not self.__chash and self._fdoc: + self.__chash = self._fdoc.content.get( + fields.CONTENT_HASH_KEY, None) + return self.__chash + + @property + def _hdoc(self): + """ + An accessor to the headers document. + """ + return self._get_headers_doc() + + @property + def _bdoc(self): + """ + An accessor to the body document. + """ + if not self.__bdoc: + self.__bdoc = self._get_body_doc() + return self.__bdoc # IMessage implementation @@ -110,9 +300,9 @@ class LeapMessage(fields, MailParser, MBoxParser): return [] flags = [] - flag_doc = self._fdoc - if flag_doc: - flags = flag_doc.content.get(self.FLAGS_KEY, None) + fdoc = self._fdoc + if fdoc: + flags = fdoc.content.get(self.FLAGS_KEY, None) if flags: flags = map(str, flags) return tuple(flags) @@ -180,7 +370,7 @@ class LeapMessage(fields, MailParser, MBoxParser): :rtype: C{str} :return: An RFC822-formatted date string. """ - return str(self._cdoc.content.get(self.DATE_KEY, '')) + return str(self._hdoc.content.get(self.DATE_KEY, '')) # # IMessagePart @@ -197,25 +387,38 @@ class LeapMessage(fields, MailParser, MBoxParser): :rtype: StringIO """ fd = StringIO.StringIO() + bdoc = self._bdoc + if bdoc: + body = self._bdoc.content.get(self.BODY_KEY, "") + else: + body = "" - cdoc = self._cdoc - content = cdoc.content.get(self.RAW_KEY, '') - charset = get_email_charset( - unicode(cdoc.content.get(self.RAW_KEY, ''))) + charset = self._get_charset(body) try: - content = content.encode(charset) + body = body.encode(charset) except (UnicodeEncodeError, UnicodeDecodeError) as e: logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - - raw = self._get_raw_msg() - msg = self._get_parsed_msg(raw) - body = msg.get_payload() + body = body.encode(charset, 'replace') fd.write(body) - # XXX SHOULD use a separate BODY FIELD ... fd.seek(0) return fd + @memoized_method + def _get_charset(self, stuff): + """ + Gets (guesses?) the charset of a payload. + + :param stuff: the stuff to guess about. + :type stuff: basestring + :returns: charset + """ + # XXX existential doubt 1. wouldn't be smarter to + # peek into the mail headers? + # XXX existential doubt 2. shouldn't we make the scope + # of the decorator somewhat more persistent? + # ah! yes! and put memory bounds. + return get_email_charset(unicode(stuff)) + def getSize(self): """ Return the total size, in octets, of this message. @@ -223,19 +426,17 @@ class LeapMessage(fields, MailParser, MBoxParser): :return: size of the message, in octets :rtype: int """ - size = self._cdoc.content.get(self.SIZE_KEY, False) + size = None + if self._fdoc: + size = self._fdoc.content.get(self.SIZE_KEY, False) + else: + logger.warning("No FLAGS doc for %s:%s" % (self._mbox, + self._uid)) if not size: # XXX fallback, should remove when all migrated. size = self.getBodyFile().len return size - def _get_headers(self): - """ - Return the headers dict stored in this message document. - """ - # XXX get from the headers doc - return self._cdoc.content.get(self.HEADERS_KEY, {}) - def getHeaders(self, negate, *names): """ Retrieve a group of message headers. @@ -252,26 +453,49 @@ class LeapMessage(fields, MailParser, MBoxParser): :rtype: dict """ headers = self._get_headers() + if not headers: + return {'content-type': ''} names = map(lambda s: s.upper(), names) if negate: cond = lambda key: key.upper() not in names else: cond = lambda key: key.upper() in names + head = copy.deepcopy(dict(headers.items())) + + # twisted imap server expects headers to be lowercase + head = dict( + map(str, (key, value)) if key.lower() != "content-type" + else map(str, (key.lower(), value)) + for (key, value) in head.items()) + # unpack and filter original dict by negate-condition - filter_by_cond = [ - map(str, (key, val)) for - key, val in headers.items() - if cond(key)] + filter_by_cond = [(key, val) for key, val in head.items() if cond(key)] return dict(filter_by_cond) + def _get_headers(self): + """ + Return the headers dict for this message. + """ + if self._hdoc is not None: + return self._hdoc.content.get(self.HEADERS_KEY, {}) + else: + logger.warning( + "No HEADERS doc for msg %s:%s" % ( + self._mbox, + self._uid)) + def isMultipart(self): """ Return True if this message is multipart. """ - if self._cdoc: - retval = self._fdoc.content.get(self.MULTIPART_KEY, False) - return retval + if self._fdoc: + return self._fdoc.content.get(self.MULTIPART_KEY, False) + else: + logger.warning( + "No FLAGS doc for msg %s:%s" % ( + self.mbox, + self.uid)) def getSubPart(self, part): """ @@ -284,12 +508,22 @@ class LeapMessage(fields, MailParser, MBoxParser): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ + logger.debug("Getting subpart: %s" % part) if not self.isMultipart(): raise TypeError - msg = self._get_parsed_msg() - # XXX should wrap IMessagePart - return msg.get_payload()[part] + if part == 0: + # Let's get the first part, which + # is really the body. + return MessageBody(self._fdoc, self._bdoc) + + attach_doc = self._get_attachment_doc(part) + if not attach_doc: + # so long and thanks for all the fish + logger.debug("...not today") + raise IndexError + msg_part = self._get_parsed_msg(attach_doc.content[self.RAW_KEY]) + return MessageAttachment(msg_part) # # accessors @@ -301,32 +535,87 @@ class LeapMessage(fields, MailParser, MBoxParser): message. """ flag_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_MBOX_UID_IDX, fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - flag_doc = flag_docs[0] if flag_docs else None - return flag_doc + return first(flag_docs) - def _get_content_doc(self): + def _get_headers_doc(self): """ - Return the document that keeps the flags for this + Return the document that keeps the headers for this + message. + """ + head_docs = self._soledad.get_from_index( + fields.TYPE_C_HASH_IDX, + fields.TYPE_HEADERS_VAL, str(self._chash)) + return first(head_docs) + + def _get_body_doc(self): + """ + Return the document that keeps the body for this message. """ - cont_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_HASH_IDX, - fields.TYPE_MESSAGE_VAL, self._content_hash, str(self._uid)) - cont_doc = cont_docs[0] if cont_docs else None - return cont_doc + body_docs = self._soledad.get_from_index( + fields.TYPE_C_HASH_IDX, + fields.TYPE_MESSAGE_VAL, str(self._chash)) + return first(body_docs) + + def _get_num_parts(self): + """ + Return the number of parts for a multipart message. + """ + if not self.isMultipart(): + raise TypeError( + "Tried to get num parts in a non-multipart message") + if not self._hdoc: + return None + return self._hdoc.content.get(fields.NUM_PARTS_KEY, 2) + + def _get_attachment_doc(self, part): + """ + Return the document that keeps the headers for this + message. + + :param part: the part number for the multipart message. + :type part: int + """ + if not self._hdoc: + return None + try: + phash = self._hdoc.content[self.PARTS_MAP_KEY][str(part)] + except KeyError: + # this is the remnant of a debug session until + # I found that the index is actually a string... + # It should be safe to just raise the KeyError now, + # but leaving it here while the blood is fresh... + logger.warning("We expected a phash in the " + "index %s, but noone found" % (part, )) + logger.debug(self._hdoc.content[self.PARTS_MAP_KEY]) + return None + attach_docs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_ATTACHMENT_VAL, str(phash)) + + # The following is true for the fist owner. + # We could use this relationship to flag the "owner" + # and orphan when we delete it. + + #attach_docs = self._soledad.get_from_index( + #fields.TYPE_C_HASH_PART_IDX, + #fields.TYPE_ATTACHMENT_VAL, str(self._chash), str(part)) + return first(attach_docs) def _get_raw_msg(self): """ Return the raw msg. :rtype: basestring """ - return self._cdoc.content.get(self.RAW_KEY, '') + # TODO deprecate this. + return self._bdoc.content.get(self.RAW_KEY, '') def __getitem__(self, key): """ - Return the content of the message document. + Return an item from the content of the flags document, + for convenience. :param key: The key :type key: str @@ -334,14 +623,73 @@ class LeapMessage(fields, MailParser, MBoxParser): :return: The content value indexed by C{key} or None :rtype: str """ - return self._cdoc.content.get(key, None) + return self._fdoc.content.get(key, None) + + # setters + + # XXX to be used in the messagecopier interface?! + + def set_uid(self, uid): + """ + Set new uid for this message. + + :param uid: the new uid + :type uid: basestring + """ + # XXX dangerous! lock? + self._uid = uid + d = self._fdoc + d.content[self.UID_KEY] = uid + self._soledad.put_doc(d) + + def set_mbox(self, mbox): + """ + Set new mbox for this message. + + :param mbox: the new mbox + :type mbox: basestring + """ + # XXX dangerous! lock? + self._mbox = mbox + d = self._fdoc + d.content[self.MBOX_KEY] = mbox + self._soledad.put_doc(d) + + # destructor + + @deferred + def remove(self): + """ + Remove all docs associated with this message. + """ + # XXX this would ve more efficient if we can just pass + # a sequence of uids. + + # XXX For the moment we are only removing the flags and headers + # docs. The rest we leave there polluting your hard disk, + # until we think about a good way of deorphaning. + # Maybe a crawler of unreferenced docs. + + fd = self._get_flags_doc() + hd = self._get_headers_doc() + #bd = self._get_body_doc() + #docs = [fd, hd, bd] + + docs = [fd, hd] + + #for pn in range(self._get_num_parts()[1:]): + #ad = self._get_attachment_doc(pn) + #docs.append(ad) + + for d in filter(None, docs): + self._soledad.delete_doc(d) def does_exist(self): """ - Return True if there is actually a message for this + Return True if there is actually a flags message for this UID and mbox. """ - return bool(self._fdoc) + return self._fdoc is not None SoledadWriterPayload = namedtuple( @@ -349,6 +697,8 @@ SoledadWriterPayload = namedtuple( SoledadWriterPayload.CREATE = 1 SoledadWriterPayload.PUT = 2 +SoledadWriterPayload.BODY_CREATE = 3 +SoledadWriterPayload.ATTACHMENT_CREATE = 4 class SoledadDocWriter(object): @@ -378,20 +728,98 @@ class SoledadDocWriter(object): empty = queue.empty() while not empty: item = queue.get() + call = None + payload = item.payload + if item.mode == SoledadWriterPayload.CREATE: call = self._soledad.create_doc + elif item.mode == SoledadWriterPayload.BODY_CREATE: + if not self._body_does_exist(payload): + call = self._soledad.create_doc + elif item.mode == SoledadWriterPayload.ATTACHMENT_CREATE: + if not self._attachment_does_exist(payload): + call = self._soledad.create_doc elif item.mode == SoledadWriterPayload.PUT: call = self._soledad.put_doc - # should handle errors - try: - call(item.payload) - except u1db_errors.RevisionConflict as exc: - logger.error("Error: %r" % (exc,)) - raise exc + # XXX delete? + + if call: + # should handle errors + try: + call(item.payload) + except u1db_errors.RevisionConflict as exc: + logger.error("Error: %r" % (exc,)) + raise exc empty = queue.empty() + """ + Message deduplication. + + We do a query for the content hashes before writing to our beloved + slcipher backend of Soledad. This means, by now, that: + + 1. We will not store the same attachment twice, only the hash of it. + 2. We will not store the same message body twice, only the hash of it. + + The first case is useful if you are always receiving the same old memes + from unwary friends that still have not discovered that 4chan is the + generator of the internet. The second will save your day if you have + initiated session with the same account in two different machines. I also + wonder why would you do that, but let's respect each other choices, like + with the religious celebrations, and assume that one day we'll be able + to run Bitmask in completely free phones. Yes, I mean that, the whole GSM + Stack. + """ + + def _body_does_exist(self, doc): + """ + Check whether we already have a body payload with this hash in our + database. + + :param doc: tentative body document + :type doc: dict + :returns: True if that happens, False otherwise. + """ + if not doc: + return False + chash = doc[fields.CONTENT_HASH_KEY] + body_docs = self._soledad.get_from_index( + fields.TYPE_C_HASH_IDX, + fields.TYPE_MESSAGE_VAL, str(chash)) + if not body_docs: + return False + if len(body_docs) != 1: + logger.warning("Found more than one copy of chash %s!" + % (chash,)) + logger.debug("Found body doc with that hash! Skipping save!") + return True + + def _attachment_does_exist(self, doc): + """ + Check whether we already have an attachment payload with this hash + in our database. + + :param doc: tentative body document + :type doc: dict + :returns: True if that happens, False otherwise. + """ + if not doc: + return False + phash = doc[fields.PAYLOAD_HASH_KEY] + attach_docs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_ATTACHMENT_VAL, str(phash)) + if not attach_docs: + return False + + if len(attach_docs) != 1: + logger.warning("Found more than one copy of phash %s!" + % (phash,)) + logger.debug("Found attachment doc with that hash! Skipping save!") + return True + class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): """ @@ -402,35 +830,62 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): database. """ # XXX this should be able to produce a MessageSet methinks - - EMPTY_MSG = { - fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.SUBJECT_KEY: "", - fields.DATE_KEY: "", - fields.RAW_KEY: "", - - # XXX should separate headers into another doc - fields.HEADERS_KEY: {}, + # could validate these kinds of objects turning them + # into a template for the class. + FLAGS_DOC = "FLAGS" + HEADERS_DOC = "HEADERS" + ATTACHMENT_DOC = "ATTACHMENT" + BODY_DOC = "BODY" + + templates = { + + FLAGS_DOC: { + fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.SEEN_KEY: False, + fields.RECENT_KEY: True, + fields.FLAGS_KEY: [], + fields.MULTIPART_KEY: False, + fields.SIZE_KEY: 0 + }, + + HEADERS_DOC: { + fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, + fields.CONTENT_HASH_KEY: "", + + fields.HEADERS_KEY: {}, + fields.NUM_PARTS_KEY: 0, + fields.PARTS_MAP_KEY: {}, + fields.DATE_KEY: "", + fields.SUBJECT_KEY: "" + }, + + ATTACHMENT_DOC: { + fields.TYPE_KEY: fields.TYPE_ATTACHMENT_VAL, + fields.PART_NUMBER_KEY: 0, + fields.CONTENT_HASH_KEY: "", + fields.PAYLOAD_HASH_KEY: "", + + fields.RAW_KEY: "" + }, + + BODY_DOC: { + fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, + fields.CONTENT_HASH_KEY: "", + + fields.BODY_KEY: "", + + # this should not be needed, + # but let's keep the raw msg for some time + # until we are sure we can reconstruct + # the original msg from our disection. + fields.RAW_KEY: "", + + } } - EMPTY_FLAGS = { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.FLAGS_KEY: [], - fields.SEEN_KEY: False, - fields.RECENT_KEY: True, - fields.MULTIPART_KEY: False, - } - - # get from SoledadBackedAccount the needed index-related constants - INDEXES = SoledadBackedAccount.INDEXES - TYPE_IDX = SoledadBackedAccount.TYPE_IDX - def __init__(self, mbox=None, soledad=None): """ Constructor for MessageCollection. @@ -465,23 +920,16 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): SoledadDocWriter(soledad), period=0.05) - def _get_empty_msg(self): + def _get_empty_doc(self, _type=FLAGS_DOC): """ - Returns an empty message. - - :return: a dict containing a default empty message + Returns an empty doc for storing different message parts. + Defaults to returning a template for a flags document. + :return: a dict with the template :rtype: dict """ - return copy.deepcopy(self.EMPTY_MSG) - - def _get_empty_flags_doc(self): - """ - Returns an empty doc for storing flags. - - :return: - :rtype: - """ - return copy.deepcopy(self.EMPTY_FLAGS) + if not _type in self.templates.keys(): + raise TypeError("Improper type passed to _get_empty_doc") + return copy.deepcopy(self.templates[_type]) @deferred def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): @@ -509,52 +957,107 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) - content_doc = self._get_empty_msg() - flags_doc = self._get_empty_flags_doc() - - content_doc[self.MBOX_KEY] = self.mbox - flags_doc[self.MBOX_KEY] = self.mbox - # ...should get a sanity check here. - content_doc[self.UID_KEY] = uid - flags_doc[self.UID_KEY] = uid - - if flags: - flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) - flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags + # docs for flags, headers, and body + fd, hd, bd = map( + lambda t: self._get_empty_doc(t), + (self.FLAGS_DOC, self.HEADERS_DOC, self.BODY_DOC)) msg = self._get_parsed_msg(raw) headers = dict(msg) - - logger.debug("adding. is multipart:%s" % msg.is_multipart()) - flags_doc[self.MULTIPART_KEY] = msg.is_multipart() - # XXX get lower case for keys? - # XXX get headers doc - content_doc[self.HEADERS_KEY] = headers - # set subject based on message headers and eventually replace by - # subject given as param - if self.SUBJECT_FIELD in headers: - content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - if subject is not None: - content_doc[self.SUBJECT_KEY] = subject - - # XXX could separate body into its own doc - # but should also separate multiparts - # that should be wrapped in MessagePart - content_doc[self.RAW_KEY] = self._stringify(raw) - content_doc[self.SIZE_KEY] = len(raw) - + raw_str = msg.as_string() + chash = self._get_hash(msg) + multi = msg.is_multipart() + + attaches = [] + inner_parts = [] + + if multi: + # XXX should walk down recursively + # in a better way. but fixing this quick + # to have an rc. + # XXX should pick the content-type in txt + body = first(msg.get_payload()).get_payload() + if isinstance(body, list): + # allowing one nesting level for now... + body, rest = body[0].get_payload(), body[1:] + for p in rest: + inner_parts.append(p) + else: + body = msg.get_payload() + logger.debug("adding msg (multipart:%s)" % multi) + + # flags doc --------------------------------------- + fd[self.MBOX_KEY] = self.mbox + fd[self.UID_KEY] = uid + fd[self.CONTENT_HASH_KEY] = chash + fd[self.MULTIPART_KEY] = multi + fd[self.SIZE_KEY] = len(raw_str) + if flags: + fd[self.FLAGS_KEY] = map(self._stringify, flags) + fd[self.SEEN_KEY] = self.SEEN_FLAG in flags + fd[self.RECENT_KEY] = self.RECENT_FLAG in flags + + # headers doc ---------------------------------------- + hd[self.CONTENT_HASH_KEY] = chash + hd[self.HEADERS_KEY] = headers + if not subject and self.SUBJECT_FIELD in headers: + hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + else: + hd[self.SUBJECT_KEY] = subject if not date and self.DATE_FIELD in headers: - content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] + hd[self.DATE_KEY] = headers[self.DATE_FIELD] else: - content_doc[self.DATE_KEY] = date - - logger.debug('enqueuing message for write') - + hd[self.DATE_KEY] = date + if multi: + hd[self.NUM_PARTS_KEY] = len(msg.get_payload()) + + # body doc + bd[self.CONTENT_HASH_KEY] = chash + bd[self.BODY_KEY] = body + # in an ideal world, we would not need to save a copy of the + # raw message. But we'll keep it until we can be sure that + # we can rebuild the original message from the parts. + bd[self.RAW_KEY] = raw_str + + docs = [fd, hd] + + # attachment docs + if multi: + outer_parts = msg.get_payload() + parts = outer_parts + inner_parts + + # skip first part, we already got it in body + to_attach = ((i, m) for i, m in enumerate(parts) if i > 0) + for index, part_msg in to_attach: + att_doc = self._get_empty_doc(self.ATTACHMENT_DOC) + att_doc[self.PART_NUMBER_KEY] = index + att_doc[self.CONTENT_HASH_KEY] = chash + phash = self._get_hash(part_msg) + att_doc[self.PAYLOAD_HASH_KEY] = phash + att_doc[self.RAW_KEY] = part_msg.as_string() + + # keep a pointer to the payload hash in the + # headers doc, under the parts_map + hd[self.PARTS_MAP_KEY][str(index)] = phash + attaches.append(att_doc) + + # Saving ... ------------------------------- + # ok, there we go... + logger.debug('enqueuing message docs for write') ptuple = SoledadWriterPayload + + # first, regular docs: flags and headers + for doc in docs: + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=doc)) + # second, try to create body doc. self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=content_doc)) - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=flags_doc)) + mode=ptuple.BODY_CREATE, payload=bd)) + # and last, but not least, try to create + # attachment docs if not already there. + for at in attaches: + self.soledad_writer.put(ptuple( + mode=ptuple.ATTACHMENT_CREATE, payload=at)) def remove(self, msg): """ @@ -563,8 +1066,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :param msg: a Leapmessage instance :type msg: LeapMessage """ - # XXX remove - #self._soledad.delete_doc(msg) msg.remove() # getters @@ -596,14 +1097,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :rtype: list of SoledadDocument """ if _type not in fields.__dict__.values(): - raise TypeError("Wrong type passed to get_all") + raise TypeError("Wrong type passed to get_all_docs") if sameProxiedObjects(self._soledad, None): logger.warning('Tried to get messages but soledad is None!') return [] all_docs = [doc for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, + fields.TYPE_MBOX_IDX, _type, self.mbox)] # inneficient, but first let's grok it and then @@ -618,8 +1119,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): """ all_uids = (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_FLAGS_VAL, self.mbox)) + fields.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox)) return (u for u in sorted(all_uids)) def count(self): @@ -629,7 +1130,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :rtype: int """ count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, + fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) return count @@ -645,8 +1146,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): """ return (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0')) + fields.TYPE_MBOX_SEEN_IDX, + fields.TYPE_FLAGS_VAL, self.mbox, '0')) def count_unseen(self): """ @@ -656,8 +1157,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :rtype: int """ count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0') + fields.TYPE_MBOX_SEEN_IDX, + fields.TYPE_FLAGS_VAL, self.mbox, '0') return count def get_unseen(self): @@ -681,8 +1182,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): """ return (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1')) + fields.TYPE_MBOX_RECT_IDX, + fields.TYPE_FLAGS_VAL, self.mbox, '1')) def get_recent(self): """ @@ -702,8 +1203,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :rtype: int """ count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1') + fields.TYPE_MBOX_RECT_IDX, + fields.TYPE_FLAGS_VAL, self.mbox, '1') return count def __len__(self): @@ -731,5 +1232,5 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): return u"" % ( self.mbox, self.count()) - # XXX should implement __eq__ also !!! --- use a hash - # of content for that, will be used for dedup. + # XXX should implement __eq__ also !!! + # --- use the content hash for that, will be used for dedup. diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py index 1ae19c0..306dcf0 100644 --- a/src/leap/mail/imap/parser.py +++ b/src/leap/mail/imap/parser.py @@ -19,10 +19,14 @@ Mail parser mixins. """ import cStringIO import StringIO +import hashlib import re +from email.message import Message from email.parser import Parser +from leap.common.check import leap_assert_type + class MailParser(object): """ @@ -34,16 +38,30 @@ class MailParser(object): """ self._parser = Parser() - def _get_parsed_msg(self, raw): + def _get_parsed_msg(self, raw, headersonly=False): """ Return a parsed Message. :param raw: the raw string to parse :type raw: basestring, or StringIO object + + :param headersonly: True for parsing only the headers. + :type headersonly: bool """ - msg = self._get_parser_fun(raw)(raw, True) + msg = self._get_parser_fun(raw)(raw, headersonly=headersonly) return msg + def _get_hash(self, msg): + """ + Returns a hash of the string representation of the raw message, + suitable for indexing the inmutable pieces. + + :param msg: a Message object + :type msg: Message + """ + leap_assert_type(msg, Message) + return hashlib.sha256(msg.as_string()).hexdigest() + def _get_parser_fun(self, o): """ Retunn the proper parser function for an object. @@ -67,6 +85,8 @@ class MailParser(object): :param o: object :type o: object """ + # XXX Maybe we don't need no more, we're using + # msg.as_string() if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): return o.getvalue() else: -- cgit v1.2.3 From a912729c4788d46d648a72126226741b63e0a37c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 2 Jan 2014 17:14:03 -0400 Subject: add documentation to the decorator, fix errorback. * it also fixes the traceback in the errorback, thanks to chiiph, who reads documentation instead of whinning :D * other minor documentation corrections --- src/leap/mail/decorators.py | 68 ++++++++++++++++++++++++++++++++++++------ src/leap/mail/imap/fetch.py | 4 +-- src/leap/mail/imap/messages.py | 5 +++- 3 files changed, 65 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py index 9e49605..024a139 100644 --- a/src/leap/mail/decorators.py +++ b/src/leap/mail/decorators.py @@ -19,13 +19,10 @@ Useful decorators for mail package. """ import logging import os -import sys -import traceback from functools import wraps from twisted.internet.threads import deferToThread -from twisted.python import log logger = logging.getLogger(__name__) @@ -41,27 +38,68 @@ def deferred(f): method wrapper. """ class descript(object): + """ + The class to be used as decorator. + + It takes any method as the passed object. + """ + def __init__(self, f): + """ + Initializes the decorator object. + + :param f: the decorated function + :type f: callable + """ self.f = f def __get__(self, instance, klass): + """ + Descriptor implementation. + + At creation time, the decorated `method` is unbound. + + It will dispatch the make_unbound method if we still do not + have an instance available, and the make_bound method when the + method has already been bound to the instance. + + :param instance: the instance of the class, or None if not exist. + :type instance: instantiated class or None. + """ if instance is None: # Class method was requested return self.make_unbound(klass) return self.make_bound(instance) def _errback(self, failure): - err = failure.value - logger.warning('error in method: %s' % (self.f.__name__)) - logger.exception(err) - log.err(err) + """ + Errorback that logs the exception catched. + + :param failure: a twisted failure + :type failure: Failure + """ + logger.warning('Error in method: %s' % (self.f.__name__)) + logger.exception(failure.getTraceback()) def make_unbound(self, klass): + """ + Return a wrapped function with the unbound call, during the + early access to the decortad method. This gets passed + only the class (not the instance since it does not yet exist). + + :param klass: the class to which the still unbound method belongs + :type klass: type + """ @wraps(self.f) def wrapper(*args, **kwargs): """ - this doc will vanish + We're temporarily wrapping the decorated method, but this + should not be called, since our application should use + the bound-wrapped method after this decorator class has been + used. + + This documentation will vanish at runtime. """ raise TypeError( 'unbound method {}() must be called with {} instance ' @@ -72,11 +110,23 @@ def deferred(f): return wrapper def make_bound(self, instance): + """ + Return a function that wraps the bound method call, + after we are able to access the instance object. + + :param instance: an instance of the class the decorated method, + now bound, belongs to. + :type instance: object + """ @wraps(self.f) def wrapper(*args, **kwargs): """ - This documentation will disapear + Do a proper function wrapper that defers the decorated method + call to a separated thread if the LEAPMAIL_DEBUG + environment variable is set. + + This documentation will vanish at runtime. """ if not os.environ.get('LEAPMAIL_DEBUG'): d = deferToThread(self.f, instance, *args, **kwargs) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index fdf1412..cb200be 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -455,8 +455,8 @@ class LeapIncomingMail(object): :param senderPubkey: The key of the sender of the message. :type senderPubkey: OpenPGPKey - :return: A unitary tuple containing a decrypted message and - a bool indicating wether the signature is valid. + :return: A tuple containing a decrypted message and + a bool indicating whether the signature is valid. :rtype: (Message, bool) """ log.msg('maybe decrypting inline encrypted msg') diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index c69c023..47c40d5 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -695,6 +695,9 @@ class LeapMessage(fields, MailParser, MBoxParser): SoledadWriterPayload = namedtuple( 'SoledadWriterPayload', ['mode', 'payload']) +# TODO we could consider using enum here: +# https://pypi.python.org/pypi/enum + SoledadWriterPayload.CREATE = 1 SoledadWriterPayload.PUT = 2 SoledadWriterPayload.BODY_CREATE = 3 @@ -758,7 +761,7 @@ class SoledadDocWriter(object): Message deduplication. We do a query for the content hashes before writing to our beloved - slcipher backend of Soledad. This means, by now, that: + sqlcipher backend of Soledad. This means, by now, that: 1. We will not store the same attachment twice, only the hash of it. 2. We will not store the same message body twice, only the hash of it. -- cgit v1.2.3 From 5585ff784940dee267576d097076de66797f9188 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 2 Jan 2014 16:08:09 -0400 Subject: fix tests after rewrite --- src/leap/mail/imap/fields.py | 3 + src/leap/mail/imap/mailbox.py | 41 ++++--- src/leap/mail/imap/messages.py | 94 +++++++++++++--- src/leap/mail/imap/tests/test_imap.py | 196 ++++++++++++++++++++++------------ 4 files changed, 227 insertions(+), 107 deletions(-) (limited to 'src') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 40817cd..bc536fe 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -35,6 +35,7 @@ class WithMsgFields(object): UID_KEY = "uid" MBOX_KEY = "mbox" SEEN_KEY = "seen" + DEL_KEY = "deleted" RECENT_KEY = "recent" FLAGS_KEY = "flags" MULTIPART_KEY = "multi" @@ -95,6 +96,7 @@ class WithMsgFields(object): TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' TYPE_C_HASH_IDX = 'by-type-and-contenthash' TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' TYPE_P_HASH_IDX = 'by-type-and-payloadhash' @@ -128,6 +130,7 @@ class WithMsgFields(object): # messages TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(recent)', 'bool(seen)'], } diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 5ea6f55..10087f6 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -390,18 +390,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags, date, uid_next) + d = self._do_add_message(message, flags=flags, date=date, uid=uid_next) d.addCallback(self._notify_new) return d @deferred - def _do_add_message(self, message, flags, date, uid_next): + def _do_add_message(self, message, flags, date, uid): """ Calls to the messageCollection add_msg method (deferred to thread). Invoked from addMessage. """ - self.messages.add_msg(message, flags=flags, date=date, - uid=uid_next) + self.messages.add_msg(message, flags=flags, date=date, uid=uid) def _notify_new(self, *args): """ @@ -436,21 +435,29 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # we should postpone the removal self._soledad.delete_doc(self._get_mbox()) - @deferred + def _close_cb(self, result): + self.closed = True + + def close(self): + """ + Expunge and mark as closed + """ + d = self.expunge() + d.addCallback(self._close_cb) + return d + + def _expunge_cb(self, result): + return result + def expunge(self): """ Remove all messages flagged \\Deleted """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - deleted = [] - for m in self.messages: - if self.DELETED_FLAG in m.getFlags(): - self.messages.remove(m) - # XXX this would ve more efficient if we can just pass - # a sequence of uids. - deleted.append(m.getUID()) - return deleted + d = self.messages.remove_all_deleted() + d.addCallback(self._expunge_cb) + return d @deferred def fetch(self, messages, uid): @@ -603,14 +610,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._signal_unread_to_ui() return result - @deferred - def close(self): - """ - Expunge and mark as closed - """ - self.expunge() - self.closed = True - # IMessageCopier @deferred diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 47c40d5..80411f9 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -20,9 +20,11 @@ LeapMessage and MessageCollection. import copy import logging import StringIO -from collections import namedtuple + +from collections import defaultdict, namedtuple from twisted.mail import imap4 +from twisted.internet import defer from twisted.python import log from u1db import errors as u1db_errors from zope.interface import implements @@ -182,6 +184,7 @@ class MessageAttachment(object): if not self._msg: return {} headers = dict(self._msg.items()) + names = map(lambda s: s.upper(), names) if negate: cond = lambda key: key.upper() not in names @@ -329,6 +332,7 @@ class LeapMessage(fields, MailParser, MBoxParser): doc.content[self.FLAGS_KEY] = flags doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags + doc.content[self.DEL_KEY] = self.DELETED_FLAG in flags self._soledad.put_doc(doc) def addFlags(self, flags): @@ -455,6 +459,7 @@ class LeapMessage(fields, MailParser, MBoxParser): headers = self._get_headers() if not headers: return {'content-type': ''} + names = map(lambda s: s.upper(), names) if negate: cond = lambda key: key.upper() not in names @@ -465,8 +470,8 @@ class LeapMessage(fields, MailParser, MBoxParser): # twisted imap server expects headers to be lowercase head = dict( - map(str, (key, value)) if key.lower() != "content-type" - else map(str, (key.lower(), value)) + (str(key), map(str, value)) if key.lower() != "content-type" + else (str(key.lower(), map(str, value))) for (key, value) in head.items()) # unpack and filter original dict by negate-condition @@ -670,6 +675,9 @@ class LeapMessage(fields, MailParser, MBoxParser): # until we think about a good way of deorphaning. # Maybe a crawler of unreferenced docs. + uid = self._uid + print "removing...", uid + fd = self._get_flags_doc() hd = self._get_headers_doc() #bd = self._get_body_doc() @@ -682,7 +690,11 @@ class LeapMessage(fields, MailParser, MBoxParser): #docs.append(ad) for d in filter(None, docs): - self._soledad.delete_doc(d) + try: + self._soledad.delete_doc(d) + except Exception as exc: + logger.error(exc) + return uid def does_exist(self): """ @@ -849,6 +861,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): fields.SEEN_KEY: False, fields.RECENT_KEY: True, + fields.DEL_KEY: False, fields.FLAGS_KEY: [], fields.MULTIPART_KEY: False, fields.SIZE_KEY: 0 @@ -921,7 +934,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): self.soledad_writer = MessageProducer( SoledadDocWriter(soledad), - period=0.05) + period=0.02) def _get_empty_doc(self, _type=FLAGS_DOC): """ @@ -966,7 +979,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): (self.FLAGS_DOC, self.HEADERS_DOC, self.BODY_DOC)) msg = self._get_parsed_msg(raw) - headers = dict(msg) + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) raw_str = msg.as_string() chash = self._get_hash(msg) multi = msg.is_multipart() @@ -987,7 +1002,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): inner_parts.append(p) else: body = msg.get_payload() - logger.debug("adding msg (multipart:%s)" % multi) + logger.debug("adding msg with uid %s (multipart:%s)" % ( + uid, multi)) # flags doc --------------------------------------- fd[self.MBOX_KEY] = self.mbox @@ -998,26 +1014,33 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): if flags: fd[self.FLAGS_KEY] = map(self._stringify, flags) fd[self.SEEN_KEY] = self.SEEN_FLAG in flags - fd[self.RECENT_KEY] = self.RECENT_FLAG in flags + fd[self.DEL_KEY] = self.DELETED_FLAG in flags + fd[self.RECENT_KEY] = True # set always by default # headers doc ---------------------------------------- hd[self.CONTENT_HASH_KEY] = chash hd[self.HEADERS_KEY] = headers + + print "headers" + import pprint + pprint.pprint(headers) + if not subject and self.SUBJECT_FIELD in headers: - hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD]) else: hd[self.SUBJECT_KEY] = subject if not date and self.DATE_FIELD in headers: - hd[self.DATE_KEY] = headers[self.DATE_FIELD] + hd[self.DATE_KEY] = first(headers[self.DATE_FIELD]) else: hd[self.DATE_KEY] = date if multi: + # XXX fix for multipart nested case hd[self.NUM_PARTS_KEY] = len(msg.get_payload()) # body doc bd[self.CONTENT_HASH_KEY] = chash bd[self.BODY_KEY] = body - # in an ideal world, we would not need to save a copy of the + # XXX in an ideal world, we would not need to save a copy of the # raw message. But we'll keep it until we can be sure that # we can rebuild the original message from the parts. bd[self.RAW_KEY] = raw_str @@ -1062,14 +1085,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): self.soledad_writer.put(ptuple( mode=ptuple.ATTACHMENT_CREATE, payload=at)) - def remove(self, msg): + def _remove_cb(self, result): + return result + + def remove_all_deleted(self): + """ + Removes all messages flagged as deleted. """ - Removes a message. + delete_deferl = [] + for msg in self.get_deleted(): + delete_deferl.append(msg.remove()) + d1 = defer.gatherResults(delete_deferl, consumeErrors=True) + d1.addCallback(self._remove_cb) + return d1 - :param msg: a Leapmessage instance + def remove(self, msg): + """ + Remove a given msg. + :param msg: the message to be removed :type msg: LeapMessage """ - msg.remove() + d = msg.remove() + d.addCallback(self._remove_cb) + return d # getters @@ -1178,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): def recent_iter(self): """ - Get an iterator for the message docs with `recent` flag. + Get an iterator for the message UIDs with `recent` flag. :return: iterator through recent message docs :rtype: iterable @@ -1210,6 +1248,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): fields.TYPE_FLAGS_VAL, self.mbox, '1') return count + # deleted messages + + def deleted_iter(self): + """ + Get an iterator for the message UIDs with `deleted` flag. + + :return: iterator through deleted message docs + :rtype: iterable + """ + return (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + fields.TYPE_MBOX_DEL_IDX, + fields.TYPE_FLAGS_VAL, self.mbox, '1')) + + def get_deleted(self): + """ + Get all messages with the `Deleted` flag. + + :returns: a generator of LeapMessages + :rtype: generator + """ + return (LeapMessage(self._soledad, docid, self.mbox) + for docid in self.deleted_iter()) + def __len__(self): """ Returns the number of messages on this mailbox. diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index ea75854..e1bed8c 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -25,7 +25,7 @@ XXX add authors from the original twisted tests. @license: GPLv3, see included LICENSE file """ # XXX review license of the original tests!!! -from nose.twistedtools import deferred +from email import parser try: from cStringIO import StringIO @@ -36,9 +36,13 @@ import os import types import tempfile import shutil +import time + +from itertools import chain from mock import Mock +from nose.twistedtools import deferred, stop_reactor from twisted.mail import imap4 @@ -58,9 +62,9 @@ import twisted.cred.portal # import u1db from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.server import SoledadMailbox -from leap.mail.imap.server import SoledadBackedAccount -from leap.mail.imap.server import MessageCollection +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.messages import MessageCollection from leap.soledad.client import Soledad from leap.soledad.client import SoledadCrypto @@ -321,6 +325,9 @@ class IMAP4HelperMixin(BaseLeapTest): for mb in self.server.theAccount.mailboxes: self.server.theAccount.delete(mb) + # email parser + self.parser = parser.Parser() + def tearDown(self): """ tearDown method called after each test. @@ -389,6 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Tests for the MessageCollection class """ + count = 0 def setUp(self): """ @@ -396,34 +404,35 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ - self.messages = MessageCollection("testmbox", self._soledad) - for m in self.messages.get_all(): - self.messages.remove(m) + self.messages = MessageCollection("testmbox%s" % (self.count,), + self._soledad) + MessageCollectionTestCase.count += 1 def tearDown(self): """ tearDown method for each test - Delete the message collection """ del self.messages + def wait(self): + time.sleep(2) + def testEmptyMessage(self): """ Test empty message and collection """ - em = self.messages._get_empty_msg() + em = self.messages._get_empty_doc() self.assertEqual( em, { - "date": '', "flags": [], - "headers": {}, "mbox": "inbox", - "raw": "", "recent": True, "seen": False, - "subject": "", - "type": "msg", + "deleted": False, + "multi": False, + "size": 0, + "type": "flags", "uid": 1, }) self.assertEqual(self.messages.count(), 0) @@ -432,23 +441,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Add multiple messages """ + # TODO really profile addition mc = self.messages + print "messages", self.messages self.assertEqual(self.messages.count(), 0) - mc.add_msg('Stuff', subject="test1") - self.assertEqual(self.messages.count(), 1) - mc.add_msg('Stuff', subject="test2") - self.assertEqual(self.messages.count(), 2) - mc.add_msg('Stuff', subject="test3") - self.assertEqual(self.messages.count(), 3) - mc.add_msg('Stuff', subject="test4") + mc.add_msg('Stuff', uid=1, subject="test1") + mc.add_msg('Stuff', uid=2, subject="test2") + mc.add_msg('Stuff', uid=3, subject="test3") + mc.add_msg('Stuff', uid=4, subject="test4") + self.wait() self.assertEqual(self.messages.count(), 4) - mc.add_msg('Stuff', subject="test5") - mc.add_msg('Stuff', subject="test6") - mc.add_msg('Stuff', subject="test7") - mc.add_msg('Stuff', subject="test8") - mc.add_msg('Stuff', subject="test9") - mc.add_msg('Stuff', subject="test10") - self.assertEqual(self.messages.count(), 10) + mc.add_msg('Stuff', uid=5, subject="test5") + mc.add_msg('Stuff', uid=6, subject="test6") + mc.add_msg('Stuff', uid=7, subject="test7") + self.wait() + self.assertEqual(self.messages.count(), 7) + self.wait() def testRecentCount(self): """ @@ -456,45 +464,48 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ mc = self.messages self.assertEqual(self.messages.count_recent(), 0) - mc.add_msg('Stuff', subject="test1", uid=1) + mc.add_msg('Stuff', uid=1, subject="test1") # For the semantics defined in the RFC, we auto-add the # recent flag by default. + self.wait() self.assertEqual(self.messages.count_recent(), 1) - mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',)) + mc.add_msg('Stuff', subject="test2", uid=2, + flags=('\\Deleted',)) + self.wait() self.assertEqual(self.messages.count_recent(), 2) - mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',)) + mc.add_msg('Stuff', subject="test3", uid=3, + flags=('\\Recent',)) + self.wait() self.assertEqual(self.messages.count_recent(), 3) mc.add_msg('Stuff', subject="test4", uid=4, flags=('\\Deleted', '\\Recent')) + self.wait() self.assertEqual(self.messages.count_recent(), 4) - for m in mc: - msg = self.messages.get_msg_by_uid(m.get('uid')) - msg_newflags = msg.removeFlags(('\\Recent',)) - self._soledad.put_doc(msg_newflags) - + for msg in mc: + msg.removeFlags(('\\Recent',)) self.assertEqual(mc.count_recent(), 0) def testFilterByMailbox(self): """ Test that queries filter by selected mailbox """ + def wait(): + time.sleep(1) + mc = self.messages self.assertEqual(self.messages.count(), 0) - mc.add_msg('', subject="test1") - self.assertEqual(self.messages.count(), 1) - mc.add_msg('', subject="test2") - self.assertEqual(self.messages.count(), 2) - mc.add_msg('', subject="test3") + mc.add_msg('', uid=1, subject="test1") + mc.add_msg('', uid=2, subject="test2") + mc.add_msg('', uid=3, subject="test3") + wait() self.assertEqual(self.messages.count(), 3) - - newmsg = mc._get_empty_msg() + newmsg = mc._get_empty_doc() newmsg['mailbox'] = "mailbox/foo" - newmsg['subject'] = "test another mailbox" mc._soledad.create_doc(newmsg) self.assertEqual(mc.count(), 3) self.assertEqual( - len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4) + len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): @@ -1174,16 +1185,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(0.5) + def append(): return self.client.append( 'root/subthing', message, - ['\\SEEN', '\\DELETED'], + ('\\SEEN', '\\DELETED'), 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', ) d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) + d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -1191,17 +1206,31 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestFullAppend(self, ignored, infile): mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing') + time.sleep(0.5) self.assertEqual(1, len(mb.messages)) + msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ['\\SEEN', '\\DELETED'], - mb.messages[1].content['flags']) + ('\\SEEN', '\\DELETED'), + msg.getFlags()) self.assertEqual( 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - mb.messages[1].content['date']) + msg.getInternalDate()) + + parsed = self.parser.parse(open(infile)) + body = parsed.get_payload() + headers = parsed.items() + self.assertEqual( + body, + msg.getBodyFile().read()) + + msg_headers = msg.getHeaders(True, "",) + gotheaders = list(chain( + *[[(k, item) for item in v] for (k, v) in msg_headers.items()])) - self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + self.assertItemsEqual( + headers, gotheaders) @deferred(timeout=None) def testPartialAppend(self): @@ -1209,12 +1238,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - message = open(infile) SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(1) + def append(): message = file(infile) return self.client.sendCommand( @@ -1226,6 +1257,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ) ) d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -1235,15 +1267,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestPartialAppend(self, ignored, infile): mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - + time.sleep(1) self.assertEqual(1, len(mb.messages)) + msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ['\\SEEN', ], - mb.messages[1].content['flags'] + ('\\SEEN', ), + msg.getFlags() ) + #self.assertEqual( + #'Right now', msg.getInternalDate()) + parsed = self.parser.parse(open(infile)) + body = parsed.get_payload() self.assertEqual( - 'Right now', mb.messages[1].content['date']) - self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + body, + msg.getBodyFile().read()) @deferred(timeout=None) def testCheck(self): @@ -1279,14 +1316,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.server.theAccount.addMailbox(name) m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('', subject="Message 1", + m.messages.add_msg('test 1', uid=1, subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) - m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) + m.messages.add_msg('test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + m.messages.add_msg('test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(1) + def select(): return self.client.select(name) @@ -1294,6 +1336,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.close() d = self.connected.addCallback(strip(login)) + d.addCallbacks(strip(wait), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -1302,8 +1345,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestClose(self, ignored, m): self.assertEqual(len(m.messages), 1) + messages = [msg for msg in m.messages] + self.assertFalse(messages[0] is None) self.assertEqual( - m.messages[1].content['subject'], + messages[0]._hdoc.content['subject'], 'Message 2') self.failUnless(m.closed) @@ -1315,17 +1360,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): name = 'mailbox-expunge' SimpleLEAPServer.theAccount.addMailbox(name) m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('', subject="Message 1", + m.messages.add_msg('test 1', uid=1, subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - self.failUnless(m.messages.count() == 1) - m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) - self.failUnless(m.messages.count() == 2) - m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) - self.failUnless(m.messages.count() == 3) + m.messages.add_msg('test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + m.messages.add_msg('test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(2) + def select(): return self.client.select('mailbox-expunge') @@ -1338,6 +1385,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) @@ -1348,12 +1396,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestExpunge(self, ignored, m): # we only left 1 mssage with no deleted flag - self.assertEqual(m.messages.count(), 1) + self.assertEqual(len(m.messages), 1) + messages = [msg for msg in m.messages] self.assertEqual( - m.messages[1].content['subject'], + messages[0]._hdoc.content['subject'], 'Message 2') - self.assertEqual(self.results, [0, 1]) - # XXX fix this thing with the indexes... + # the uids of the deleted messages + self.assertItemsEqual(self.results, [1, 3]) class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): @@ -1363,3 +1412,10 @@ class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): """ # XXX coming soon to your screens! pass + + +def tearDownModule(): + """ + Tear down functions for module level + """ + stop_reactor() -- cgit v1.2.3 From e1946f1653dbbb6fcf61569bc873ab061965664e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 6 Jan 2014 04:44:05 -0400 Subject: tests infrastructure for multipart --- src/leap/mail/imap/mailbox.py | 5 + .../mail/imap/tests/rfc822.multi-signed.message | 238 +++++++++++++++++++++ src/leap/mail/imap/tests/rfc822.multi.message | 96 +++++++++ src/leap/mail/imap/tests/test_imap.py | 78 ++++++- 4 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 src/leap/mail/imap/tests/rfc822.multi-signed.message create mode 100644 src/leap/mail/imap/tests/rfc822.multi.message (limited to 'src') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 10087f6..1d76d4d 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -478,6 +478,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): LeapMessage """ result = [] + + # XXX DEBUG ------------- + print "getting uid", uid + print "in mbox", self.mbox + sequence = True if uid == 0 else False if not messages.last: diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message new file mode 100644 index 0000000..9907c2d --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message new file mode 100644 index 0000000..30f74e5 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) +From: Doug Sauder +To: Joe Blow +Subject: Test message from PINE +Message-ID: +MIME-Version: 1.0 +Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" + + This message is in MIME format. The first part should be readable text, + while the remaining parts are likely unreadable without MIME-aware tools. + Send mail to mime@docserver.cac.washington.edu for more info. + +---1463757054-952513540-958744548=:8452 +Content-Type: TEXT/PLAIN; charset=US-ASCII + +This is a test message from PINE MUA. + + +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="redball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="redball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj +AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv +AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 +GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf +hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl +rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL +ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS +AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP +AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP +AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI +AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR +AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 +AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO +AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 +AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 +LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF +BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp +6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow +8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G +ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn +OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 +a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T +VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 +Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f +lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ +joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms +1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA +JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 +vAAAAABJRU5ErkJggg== +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="blueball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="blueball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI +IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y +Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S +hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM +vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB +Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b +fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo +Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp +LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX +P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M +jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 ++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE +1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 +YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY +mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp +Z3VldDZzO7wAAAAASUVORK5CYII= +---1463757054-952513540-958744548=:8452-- diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index e1bed8c..8c1cf20 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -357,11 +357,11 @@ class IMAP4HelperMixin(BaseLeapTest): # XXX we also should put this in a mailbox! - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") + self._soledad.messages.add_msg('', uid=1, subject="test1") + self._soledad.messages.add_msg('', uid=2, subject="test2") + self._soledad.messages.add_msg('', uid=3, subject="test3") # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") + self._soledad.messages.add_msg('', uid=4, subject="test4") def delete_all_docs(self): """ @@ -1405,10 +1405,78 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertItemsEqual(self.results, [1, 3]) +class StoreAndFetchTestCase(unittest.TestCase, IMAP4HelperMixin): + """ + Several tests to check that the internal storage representation + is able to render the message structures as we expect them. + """ + # TODO get rid of the fucking sleeps with a proper defer + # management. + + def setUp(self): + IMAP4HelperMixin.setUp(self) + MBOX_NAME = "multipart/SIGNED" + self.received_messages = self.received_uid = None + self.result = None + + self.server.state = 'select' + + infile = util.sibpath(__file__, 'rfc822.multi-signed.message') + raw = open(infile).read() + + self.server.theAccount.addMailbox(MBOX_NAME) + mbox = self.server.theAccount.getMailbox(MBOX_NAME) + time.sleep(1) + self.server.mbox = mbox + self.server.mbox.messages.add_msg(raw, uid=1) + time.sleep(1) + + def addListener(self, x): + pass + + def removeListener(self, x): + pass + + def _fetchWork(self, uids): + + def result(R): + self.result = R + + self.connected.addCallback( + lambda _: self.function( + uids, uid=1) # do NOT use seq numbers! + ).addCallback(result).addCallback( + self._cbStopClient).addErrback(self._ebGeneral) + + d = loopback.loopbackTCP(self.server, self.client, noisy=False) + d.addCallback(lambda x: self.assertEqual(self.result, self.expected)) + return d + + @deferred(timeout=None) + def testMultiBody(self): + """ + Test that a multipart signed message is retrieved the same + as we stored it. + """ + time.sleep(1) + self.function = self.client.fetchBody + messages = '1' + + # XXX review. This probably should give everything? + + self.expected = {1: { + 'RFC822.TEXT': 'This is an example of a signed message,\n' + 'with attachments.\n\n\n--=20\n' + 'Nihil sine chao! =E2=88=B4\n', + 'UID': '1'}} + print "test multi: fetch uid", messages + return self._fetchWork(messages) + + class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): """ - Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}. + Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ # XXX coming soon to your screens! pass -- cgit v1.2.3 From a203337d155a6e7186980ef175642adc91d472fe Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 14:23:25 -0400 Subject: move utility to its own --- src/leap/mail/imap/messages.py | 11 +---------- src/leap/mail/utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 src/leap/mail/utils.py (limited to 'src') diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 80411f9..bfe913c 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -33,6 +33,7 @@ from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset +from leap.mail.utils import first from leap.mail.decorators import deferred from leap.mail.imap.index import IndexedDB from leap.mail.imap.fields import fields, WithMsgFields @@ -42,16 +43,6 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer logger = logging.getLogger(__name__) -def first(things): - """ - Return the head of a collection. - """ - try: - return things[0] - except (IndexError, TypeError): - return None - - class MessageBody(object): """ IMessagePart implementor for the main diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py new file mode 100644 index 0000000..2480efc --- /dev/null +++ b/src/leap/mail/utils.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Small utilities. +""" + + +def first(things): + """ + Return the head of a collection. + """ + try: + return things[0] + except (IndexError, TypeError): + return None -- cgit v1.2.3 From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 11:34:08 -0400 Subject: Second stage of the new year's storage rewrite. * documents of only three types: * flags * headers * content * add algorithm for walking the parsed message tree. * treat special cases like a multipart with a single part. * modify add_msg to use the walk routine * modify twisted interfaces to use the new storage schema. * tests for different multipart cases * fix multipart detection typo in the fetch This is a merge proposal for the 0.5.0-rc3. known bugs ---------- Some things are still know not to work well at this point (some cases of multipart messages do not display the bodies). IMAP server also is left in a bad internal state after a logout/login. --- src/leap/mail/decorators.py | 5 + src/leap/mail/imap/fetch.py | 2 +- src/leap/mail/imap/fields.py | 26 +- src/leap/mail/imap/messages.py | 722 +++++++++++---------- src/leap/mail/imap/service/imap.py | 2 + .../mail/imap/tests/rfc822.multi-minimal.message | 16 + src/leap/mail/imap/tests/rfc822.plain.message | 66 ++ src/leap/mail/imap/tests/walktree.py | 117 ++++ src/leap/mail/walk.py | 160 +++++ 9 files changed, 768 insertions(+), 348 deletions(-) create mode 100644 src/leap/mail/imap/tests/rfc822.multi-minimal.message create mode 100644 src/leap/mail/imap/tests/rfc822.plain.message create mode 100644 src/leap/mail/imap/tests/walktree.py create mode 100644 src/leap/mail/walk.py (limited to 'src') diff --git a/src/leap/mail/decorators.py b/src/leap/mail/decorators.py index 024a139..d5eac97 100644 --- a/src/leap/mail/decorators.py +++ b/src/leap/mail/decorators.py @@ -27,6 +27,11 @@ from twisted.internet.threads import deferToThread logger = logging.getLogger(__name__) +# TODO +# Should write a helper to be able to pass a timeout argument. +# See this answer: http://stackoverflow.com/a/19019648/1157664 +# And the notes by glyph and jpcalderone + def deferred(f): """ Decorator, for deferring methods to Threads. diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index cb200be..604a2ea 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -404,7 +404,7 @@ class LeapIncomingMail(object): """ log.msg('decrypting multipart encrypted msg') msg = copy.deepcopy(msg) - self._multipart_sanity_check(msg) + self._msg_multipart_sanity_check(msg) # parse message and get encrypted content pgpencmsg = msg.get_payload()[1] diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index bc536fe..2545adf 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -43,17 +43,17 @@ class WithMsgFields(object): # headers HEADERS_KEY = "headers" - NUM_PARTS_KEY = "numparts" - PARTS_MAP_KEY = "partmap" DATE_KEY = "date" SUBJECT_KEY = "subject" - - # attachment - PART_NUMBER_KEY = "part" - RAW_KEY = "raw" + # XXX DELETE-ME + #NUM_PARTS_KEY = "numparts" # not needed?! + PARTS_MAP_KEY = "part_map" + BODY_KEY = "body" # link to phash of body # content - BODY_KEY = "body" + LINKED_FROM_KEY = "lkf" + RAW_KEY = "raw" + CTYPE_KEY = "ctype" # Mailbox specific keys CLOSED_KEY = "closed" @@ -65,11 +65,13 @@ class WithMsgFields(object): # Document Type, for indexing TYPE_KEY = "type" TYPE_MBOX_VAL = "mbox" - TYPE_MESSAGE_VAL = "msg" TYPE_FLAGS_VAL = "flags" TYPE_HEADERS_VAL = "head" - TYPE_ATTACHMENT_VAL = "attach" - # should add also a headers val + TYPE_CONTENT_VAL = "cnt" + + # XXX DEPRECATE + #TYPE_MESSAGE_VAL = "msg" + #TYPE_ATTACHMENT_VAL = "attach" INBOX_VAL = "inbox" @@ -109,7 +111,6 @@ class WithMsgFields(object): MBOX_VAL = TYPE_MBOX_VAL CHASH_VAL = CONTENT_HASH_KEY PHASH_VAL = PAYLOAD_HASH_KEY - PART_VAL = PART_NUMBER_KEY INDEXES = { # generic @@ -122,8 +123,7 @@ class WithMsgFields(object): # content, headers doc TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - # attachment docs - TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL], + # attachment payload dedup TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index bfe913c..37e4311 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -33,6 +33,7 @@ from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset +from leap.mail import walk from leap.mail.utils import first from leap.mail.decorators import deferred from leap.mail.imap.index import IndexedDB @@ -43,65 +44,58 @@ from leap.mail.messageflow import IMessageConsumer, MessageProducer logger = logging.getLogger(__name__) -class MessageBody(object): - """ - IMessagePart implementor for the main - body of a multipart message. - - Excusatio non petita: see the interface documentation. - """ +# TODO ------------------------------------------------------------ - implements(imap4.IMessagePart) - - def __init__(self, fdoc, bdoc): - self._fdoc = fdoc - self._bdoc = bdoc - - def getSize(self): - return len(self._bdoc.content[fields.BODY_KEY]) +# [ ] Add linked-from info. +# [ ] Delete incoming mail only after successful write! +# [ ] Remove UID from syncable db. Store only those indexes locally. +# [ ] Send patch to twisted for bug in imap4.py:5717 (content-type can be +# none? lower-case?) - def getBodyFile(self): - fd = StringIO.StringIO() - - if self._bdoc: - body = self._bdoc.content[fields.BODY_KEY] - else: - body = "" - charset = self._get_charset(body) - try: - body = body.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - body = body.encode(charset, 'replace') - fd.write(body) - fd.seek(0) - return fd - - @memoized_method - def _get_charset(self, stuff): - return get_email_charset(unicode(stuff)) - - def getHeaders(self, negate, *names): - return {} +def lowerdict(_dict): + """ + Return a dict with the keys in lowercase. - def isMultipart(self): - return False + :param _dict: the dict to convert + :rtype: dict + """ + return dict((key.lower(), value) + for key, value in _dict.items()) - def getSubPart(self, part): - return None +class MessagePart(object): + """ + IMessagePart implementor. + It takes a subpart message and is able to find + the inner parts. -class MessageAttachment(object): + Excusatio non petita: see the interface documentation. + """ implements(imap4.IMessagePart) - def __init__(self, msg): + def __init__(self, soledad, part_map): """ - Initializes the messagepart with a Message instance. - :param msg: a message instance - :type msg: Message + Initializes the MessagePart. + + :param part_map: a dictionary containing the parts map for this + message + :type part_map: dict """ - self._msg = msg + # TODO + # It would be good to pass the uid/mailbox also + # for references while debugging. + + # We have a problem on bulk moves, and is + # that when the fetch on the new mailbox is done + # the parts maybe are not complete. + # So we should be able to fail with empty + # docs until we solve that. The ideal would be + # to gather the results of the deferred operations + # to signal the operation is complete. + #leap_assert(part_map, "part map dict cannot be null") + self._soledad = soledad + self._pmap = part_map def getSize(self): """ @@ -110,9 +104,12 @@ class MessageAttachment(object): :return: size of the message, in octets :rtype: int """ - if not self._msg: + if not self._pmap: return 0 - return len(self._msg.as_string()) + size = self._pmap.get('size', None) + if not size: + logger.error("Message part cannot find size in the partmap") + return size def getBodyFile(self): """ @@ -122,24 +119,91 @@ class MessageAttachment(object): :rtype: StringIO """ fd = StringIO.StringIO() - if self._msg: - body = self._msg.get_payload() + if self._pmap: + multi = self._pmap.get('multi') + if not multi: + phash = self._pmap.get("phash", None) + else: + pmap = self._pmap.get('part_map') + first_part = pmap.get('1', None) + if first_part: + phash = first_part['phash'] + + if not phash: + logger.warning("Could not find phash for this subpart!") + payload = str("") + else: + payload = self._get_payload_from_document(phash) + else: - logger.debug("Empty message!") - body = "" - - # XXX should only do the dance if we're sure it's - # content/text-plain!!! - #charset = self._get_charset(body) - #try: - #body = body.encode(charset) - #except (UnicodeEncodeError, UnicodeDecodeError) as e: - #logger.error("Unicode error {0}".format(e)) - #body = body.encode(charset, 'replace') - fd.write(body) + logger.warning("Message with no part_map!") + payload = str("") + + if payload: + #headers = self.getHeaders(True) + #headers = lowerdict(headers) + #content_type = headers.get('content-type', "") + content_type = self._get_ctype_from_document(phash) + charset_split = content_type.split('charset=') + # XXX fuck all this, use a regex! + if len(charset_split) > 1: + charset = charset_split[1] + if charset: + charset = charset.strip() + else: + charset = None + if not charset: + charset = self._get_charset(payload) + try: + payload = payload.encode(charset) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + payload = payload.encode(charset, 'replace') + + fd.write(payload) fd.seek(0) return fd + # TODO cache the phash retrieval + def _get_payload_from_document(self, phash): + """ + Gets the message payload from the content document. + + :param phash: the payload hash to retrieve by. + :type phash: basestring + """ + cdocs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(phash)) + + cdoc = first(cdocs) + if not cdoc: + logger.warning( + "Could not find the content doc " + "for phash %s" % (phash,)) + payload = cdoc.content.get(fields.RAW_KEY, "") + return payload + + # TODO cache the pahash retrieval + def _get_ctype_from_document(self, phash): + """ + Gets the content-type from the content document. + + :param phash: the payload hash to retrieve by. + :type phash: basestring + """ + cdocs = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(phash)) + + cdoc = first(cdocs) + if not cdoc: + logger.warning( + "Could not find the content doc " + "for phash %s" % (phash,)) + ctype = cdoc.content.get('ctype', "") + return ctype + @memoized_method def _get_charset(self, stuff): # TODO put in a common class with LeapMessage @@ -150,8 +214,6 @@ class MessageAttachment(object): :type stuff: basestring :returns: charset """ - # XXX existential doubt 1. wouldn't be smarter to - # peek into the mail headers? # XXX existential doubt 2. shouldn't we make the scope # of the decorator somewhat more persistent? # ah! yes! and put memory bounds. @@ -172,9 +234,17 @@ class MessageAttachment(object): :return: A mapping of header field names to header field values :rtype: dict """ - if not self._msg: + if not self._pmap: + logger.warning("No pmap in Subpart!") return {} - headers = dict(self._msg.items()) + headers = dict(self._pmap.get("headers", [])) + + # twisted imap server expects *some* headers to be lowercase + # We could use a CaseInsensitiveDict here... + headers = dict( + (str(key), str(value)) if key.lower() != "content-type" + else (str(key.lower()), str(value)) + for (key, value) in headers.items()) names = map(lambda s: s.upper(), names) if negate: @@ -187,13 +257,18 @@ class MessageAttachment(object): map(str, (key, val)) for key, val in headers.items() if cond(key)] - return dict(filter_by_cond) + filtered = dict(filter_by_cond) + return filtered def isMultipart(self): """ Return True if this message is multipart. """ - return self._msg.is_multipart() + if not self._pmap: + logger.warning("Could not get part map!") + return False + multi = self._pmap.get("multi", False) + return multi def getSubPart(self, part): """ @@ -206,10 +281,30 @@ class MessageAttachment(object): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - return self._msg.get_payload() + if not self.isMultipart(): + raise TypeError + sub_pmap = self._pmap.get("part_map", {}) + try: + part_map = sub_pmap[str(part + 1)] + except KeyError: + logger.debug("getSubpart for %s: KeyError" % (part,)) + raise IndexError + + # XXX check for validity + return MessagePart(self._soledad, part_map) class LeapMessage(fields, MailParser, MBoxParser): + """ + The main representation of a message. + + It indexes the messages in one mailbox by a combination + of uid+mailbox name. + """ + + # TODO this has to change. + # Should index primarily by chash, and keep a local-lonly + # UID table. implements(imap4.IMessage) @@ -268,6 +363,8 @@ class LeapMessage(fields, MailParser, MBoxParser): """ An accessor to the body document. """ + if not self._hdoc: + return None if not self.__bdoc: self.__bdoc = self._get_body_doc() return self.__bdoc @@ -320,6 +417,11 @@ class LeapMessage(fields, MailParser, MBoxParser): log.msg('setting flags: %s' % (self._uid)) doc = self._fdoc + if not doc: + logger.warning( + "Could not find FDOC for %s:%s while setting flags!" % + (self._mbox, self._uid)) + return doc.content[self.FLAGS_KEY] = flags doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags @@ -384,16 +486,25 @@ class LeapMessage(fields, MailParser, MBoxParser): fd = StringIO.StringIO() bdoc = self._bdoc if bdoc: - body = self._bdoc.content.get(self.BODY_KEY, "") + body = str(self._bdoc.content.get(self.RAW_KEY, "")) else: - body = "" + logger.warning("No BDOC found for message.") + body = str("") + + # XXX not needed, isn't it? ---- ivan? + #if bdoc: + #content_type = bdoc.content.get('content-type', "") + #charset = content_type.split('charset=')[1] + #if charset: + #charset = charset.strip() + #if not charset: + #charset = self._get_charset(body) + #try: + #body = str(body.encode(charset)) + #except (UnicodeEncodeError, UnicodeDecodeError) as e: + #logger.error("Unicode error {0}".format(e)) + #body = str(body.encode(charset, 'replace')) - charset = self._get_charset(body) - try: - body = body.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - body = body.encode(charset, 'replace') fd.write(body) fd.seek(0) return fd @@ -407,8 +518,7 @@ class LeapMessage(fields, MailParser, MBoxParser): :type stuff: basestring :returns: charset """ - # XXX existential doubt 1. wouldn't be smarter to - # peek into the mail headers? + # TODO get from subpart headers # XXX existential doubt 2. shouldn't we make the scope # of the decorator somewhat more persistent? # ah! yes! and put memory bounds. @@ -447,9 +557,11 @@ class LeapMessage(fields, MailParser, MBoxParser): :return: A mapping of header field names to header field values :rtype: dict """ + # TODO split in smaller methods headers = self._get_headers() if not headers: - return {'content-type': ''} + logger.warning("No headers found") + return {str('content-type'): str('')} names = map(lambda s: s.upper(), names) if negate: @@ -457,16 +569,20 @@ class LeapMessage(fields, MailParser, MBoxParser): else: cond = lambda key: key.upper() in names - head = copy.deepcopy(dict(headers.items())) + if isinstance(headers, list): + headers = dict(headers) - # twisted imap server expects headers to be lowercase - head = dict( - (str(key), map(str, value)) if key.lower() != "content-type" - else (str(key.lower(), map(str, value))) - for (key, value) in head.items()) + # twisted imap server expects *some* headers to be lowercase + # XXX refactor together with MessagePart method + headers = dict( + (str(key), str(value)) if key.lower() != "content-type" + else (str(key.lower()), str(value)) + for (key, value) in headers.items()) # unpack and filter original dict by negate-condition - filter_by_cond = [(key, val) for key, val in head.items() if cond(key)] + filter_by_cond = [(key, val) for key, val + in headers.items() if cond(key)] + return dict(filter_by_cond) def _get_headers(self): @@ -474,7 +590,9 @@ class LeapMessage(fields, MailParser, MBoxParser): Return the headers dict for this message. """ if self._hdoc is not None: - return self._hdoc.content.get(self.HEADERS_KEY, {}) + headers = self._hdoc.content.get(self.HEADERS_KEY, {}) + return headers + else: logger.warning( "No HEADERS doc for msg %s:%s" % ( @@ -486,12 +604,13 @@ class LeapMessage(fields, MailParser, MBoxParser): Return True if this message is multipart. """ if self._fdoc: - return self._fdoc.content.get(self.MULTIPART_KEY, False) + is_multipart = self._fdoc.content.get(self.MULTIPART_KEY, False) + return is_multipart else: logger.warning( "No FLAGS doc for msg %s:%s" % ( - self.mbox, - self.uid)) + self._mbox, + self._uid)) def getSubPart(self, part): """ @@ -504,27 +623,33 @@ class LeapMessage(fields, MailParser, MBoxParser): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - logger.debug("Getting subpart: %s" % part) if not self.isMultipart(): raise TypeError - - if part == 0: - # Let's get the first part, which - # is really the body. - return MessageBody(self._fdoc, self._bdoc) - - attach_doc = self._get_attachment_doc(part) - if not attach_doc: - # so long and thanks for all the fish - logger.debug("...not today") + try: + pmap_dict = self._get_part_from_parts_map(part + 1) + except KeyError: + logger.debug("getSubpart for %s: KeyError" % (part,)) raise IndexError - msg_part = self._get_parsed_msg(attach_doc.content[self.RAW_KEY]) - return MessageAttachment(msg_part) + return MessagePart(self._soledad, pmap_dict) # # accessors # + def _get_part_from_parts_map(self, part): + """ + Get a part map from the headers doc + + :raises: KeyError if key does not exist + :rtype: dict + """ + if not self._hdoc: + logger.warning("Tried to get part but no HDOC found!") + return None + + pmap = self._hdoc.content.get(fields.PARTS_MAP_KEY, {}) + return pmap[str(part)] + def _get_flags_doc(self): """ Return the document that keeps the flags for this @@ -550,63 +675,16 @@ class LeapMessage(fields, MailParser, MBoxParser): Return the document that keeps the body for this message. """ - body_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_MESSAGE_VAL, str(self._chash)) - return first(body_docs) - - def _get_num_parts(self): - """ - Return the number of parts for a multipart message. - """ - if not self.isMultipart(): - raise TypeError( - "Tried to get num parts in a non-multipart message") - if not self._hdoc: - return None - return self._hdoc.content.get(fields.NUM_PARTS_KEY, 2) - - def _get_attachment_doc(self, part): - """ - Return the document that keeps the headers for this - message. - - :param part: the part number for the multipart message. - :type part: int - """ - if not self._hdoc: - return None - try: - phash = self._hdoc.content[self.PARTS_MAP_KEY][str(part)] - except KeyError: - # this is the remnant of a debug session until - # I found that the index is actually a string... - # It should be safe to just raise the KeyError now, - # but leaving it here while the blood is fresh... - logger.warning("We expected a phash in the " - "index %s, but noone found" % (part, )) - logger.debug(self._hdoc.content[self.PARTS_MAP_KEY]) + body_phash = self._hdoc.content.get( + fields.BODY_KEY, None) + if not body_phash: + logger.warning("No body phash for this document!") return None - attach_docs = self._soledad.get_from_index( + body_docs = self._soledad.get_from_index( fields.TYPE_P_HASH_IDX, - fields.TYPE_ATTACHMENT_VAL, str(phash)) - - # The following is true for the fist owner. - # We could use this relationship to flag the "owner" - # and orphan when we delete it. + fields.TYPE_CONTENT_VAL, str(body_phash)) - #attach_docs = self._soledad.get_from_index( - #fields.TYPE_C_HASH_PART_IDX, - #fields.TYPE_ATTACHMENT_VAL, str(self._chash), str(part)) - return first(attach_docs) - - def _get_raw_msg(self): - """ - Return the raw msg. - :rtype: basestring - """ - # TODO deprecate this. - return self._bdoc.content.get(self.RAW_KEY, '') + return first(body_docs) def __getitem__(self, key): """ @@ -658,27 +736,22 @@ class LeapMessage(fields, MailParser, MBoxParser): """ Remove all docs associated with this message. """ - # XXX this would ve more efficient if we can just pass - # a sequence of uids. - # XXX For the moment we are only removing the flags and headers # docs. The rest we leave there polluting your hard disk, # until we think about a good way of deorphaning. # Maybe a crawler of unreferenced docs. + # XXX implement elijah's idea of using a PUT document as a + # token to ensure consistency in the removal. + uid = self._uid - print "removing...", uid fd = self._get_flags_doc() - hd = self._get_headers_doc() + #hd = self._get_headers_doc() #bd = self._get_body_doc() #docs = [fd, hd, bd] - docs = [fd, hd] - - #for pn in range(self._get_num_parts()[1:]): - #ad = self._get_attachment_doc(pn) - #docs.append(ad) + docs = [fd] for d in filter(None, docs): try: @@ -703,8 +776,7 @@ SoledadWriterPayload = namedtuple( SoledadWriterPayload.CREATE = 1 SoledadWriterPayload.PUT = 2 -SoledadWriterPayload.BODY_CREATE = 3 -SoledadWriterPayload.ATTACHMENT_CREATE = 4 +SoledadWriterPayload.CONTENT_CREATE = 3 class SoledadDocWriter(object): @@ -723,6 +795,38 @@ class SoledadDocWriter(object): """ self._soledad = soledad + def _get_call_for_item(self, item): + """ + Return the proper call type for a given item. + + :param item: one of the types defined under the + attributes of SoledadWriterPayload + :type item: int + """ + call = None + payload = item.payload + + if item.mode == SoledadWriterPayload.CREATE: + call = self._soledad.create_doc + elif (item.mode == SoledadWriterPayload.CONTENT_CREATE + and not self._content_does_exist(payload)): + call = self._soledad.create_doc + elif item.mode == SoledadWriterPayload.PUT: + call = self._soledad.put_doc + return call + + def _process(self, queue): + """ + Return the item and the proper call type for the next + item in the queue if any. + + :param queue: the queue from where we'll pick item. + :type queue: Queue + """ + item = queue.get() + call = self._get_call_for_item(item) + return item, call + def consume(self, queue): """ Creates a new document in soledad db. @@ -733,24 +837,10 @@ class SoledadDocWriter(object): """ empty = queue.empty() while not empty: - item = queue.get() - call = None - payload = item.payload - - if item.mode == SoledadWriterPayload.CREATE: - call = self._soledad.create_doc - elif item.mode == SoledadWriterPayload.BODY_CREATE: - if not self._body_does_exist(payload): - call = self._soledad.create_doc - elif item.mode == SoledadWriterPayload.ATTACHMENT_CREATE: - if not self._attachment_does_exist(payload): - call = self._soledad.create_doc - elif item.mode == SoledadWriterPayload.PUT: - call = self._soledad.put_doc - - # XXX delete? + item, call = self._process(queue) if call: + # XXX should handle the delete case # should handle errors try: call(item.payload) @@ -779,33 +869,10 @@ class SoledadDocWriter(object): Stack. """ - def _body_does_exist(self, doc): + def _content_does_exist(self, doc): """ - Check whether we already have a body payload with this hash in our - database. - - :param doc: tentative body document - :type doc: dict - :returns: True if that happens, False otherwise. - """ - if not doc: - return False - chash = doc[fields.CONTENT_HASH_KEY] - body_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_MESSAGE_VAL, str(chash)) - if not body_docs: - return False - if len(body_docs) != 1: - logger.warning("Found more than one copy of chash %s!" - % (chash,)) - logger.debug("Found body doc with that hash! Skipping save!") - return True - - def _attachment_does_exist(self, doc): - """ - Check whether we already have an attachment payload with this hash - in our database. + Check whether we already have a content document for a payload + with this hash in our database. :param doc: tentative body document :type doc: dict @@ -816,7 +883,7 @@ class SoledadDocWriter(object): phash = doc[fields.PAYLOAD_HASH_KEY] attach_docs = self._soledad.get_from_index( fields.TYPE_P_HASH_IDX, - fields.TYPE_ATTACHMENT_VAL, str(phash)) + fields.TYPE_CONTENT_VAL, str(phash)) if not attach_docs: return False @@ -840,15 +907,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): # into a template for the class. FLAGS_DOC = "FLAGS" HEADERS_DOC = "HEADERS" - ATTACHMENT_DOC = "ATTACHMENT" - BODY_DOC = "BODY" + CONTENT_DOC = "CONTENT" templates = { FLAGS_DOC: { fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, + fields.UID_KEY: 1, # XXX moe to a local table fields.MBOX_KEY: fields.INBOX_VAL, + fields.CONTENT_HASH_KEY: "", fields.SEEN_KEY: False, fields.RECENT_KEY: True, @@ -862,35 +929,28 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, fields.CONTENT_HASH_KEY: "", + fields.DATE_KEY: "", + fields.SUBJECT_KEY: "", + fields.HEADERS_KEY: {}, - fields.NUM_PARTS_KEY: 0, fields.PARTS_MAP_KEY: {}, - fields.DATE_KEY: "", - fields.SUBJECT_KEY: "" }, - ATTACHMENT_DOC: { - fields.TYPE_KEY: fields.TYPE_ATTACHMENT_VAL, - fields.PART_NUMBER_KEY: 0, - fields.CONTENT_HASH_KEY: "", + CONTENT_DOC: { + fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, fields.PAYLOAD_HASH_KEY: "", + fields.LINKED_FROM_KEY: [], + fields.CTYPE_KEY: "", # should index by this too - fields.RAW_KEY: "" - }, - - BODY_DOC: { - fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.BODY_KEY: "", - - # this should not be needed, - # but let's keep the raw msg for some time - # until we are sure we can reconstruct - # the original msg from our disection. + # should only get inmutable headers parts + # (for indexing) + fields.HEADERS_KEY: {}, fields.RAW_KEY: "", + fields.PARTS_MAP_KEY: {}, + fields.HEADERS_KEY: {}, + fields.MULTIPART_KEY: False, + }, - } } def __init__(self, mbox=None, soledad=None): @@ -938,128 +998,124 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): raise TypeError("Improper type passed to _get_empty_doc") return copy.deepcopy(self.templates[_type]) - @deferred - def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): + def _do_parse(self, raw): """ - Creates a new message document. + Parse raw message and return it along with + relevant information about its outer level. :param raw: the raw message - :type raw: str - - :param subject: subject of the message. - :type subject: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :param uid: the message uid for this mailbox - :type uid: int + :type raw: StringIO or basestring + :return: msg, chash, size, multi + :rtype: tuple """ - # TODO: split in smaller methods - logger.debug('adding message') - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - - # docs for flags, headers, and body - fd, hd, bd = map( - lambda t: self._get_empty_doc(t), - (self.FLAGS_DOC, self.HEADERS_DOC, self.BODY_DOC)) - msg = self._get_parsed_msg(raw) - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - raw_str = msg.as_string() chash = self._get_hash(msg) + size = len(msg.as_string()) multi = msg.is_multipart() + return msg, chash, size, multi - attaches = [] - inner_parts = [] - - if multi: - # XXX should walk down recursively - # in a better way. but fixing this quick - # to have an rc. - # XXX should pick the content-type in txt - body = first(msg.get_payload()).get_payload() - if isinstance(body, list): - # allowing one nesting level for now... - body, rest = body[0].get_payload(), body[1:] - for p in rest: - inner_parts.append(p) - else: - body = msg.get_payload() - logger.debug("adding msg with uid %s (multipart:%s)" % ( - uid, multi)) + def _populate_flags(self, flags, uid, chash, size, multi): + """ + Return a flags doc. + + XXX Missing DOC ----------- + """ + fd = self._get_empty_doc(self.FLAGS_DOC) - # flags doc --------------------------------------- fd[self.MBOX_KEY] = self.mbox fd[self.UID_KEY] = uid fd[self.CONTENT_HASH_KEY] = chash + fd[self.SIZE_KEY] = size fd[self.MULTIPART_KEY] = multi - fd[self.SIZE_KEY] = len(raw_str) if flags: fd[self.FLAGS_KEY] = map(self._stringify, flags) fd[self.SEEN_KEY] = self.SEEN_FLAG in flags fd[self.DEL_KEY] = self.DELETED_FLAG in flags fd[self.RECENT_KEY] = True # set always by default + return fd - # headers doc ---------------------------------------- + def _populate_headr(self, msg, chash, subject, date): + """ + Return a headers doc. + + XXX Missing DOC ----------- + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + + # "fix" for repeated headers. + for k, v in headers.items(): + newline = "\n%s: " % (k,) + headers[k] = newline.join(v) + + hd = self._get_empty_doc(self.HEADERS_DOC) hd[self.CONTENT_HASH_KEY] = chash hd[self.HEADERS_KEY] = headers - print "headers" - import pprint - pprint.pprint(headers) - if not subject and self.SUBJECT_FIELD in headers: hd[self.SUBJECT_KEY] = first(headers[self.SUBJECT_FIELD]) else: hd[self.SUBJECT_KEY] = subject + if not date and self.DATE_FIELD in headers: hd[self.DATE_KEY] = first(headers[self.DATE_FIELD]) else: hd[self.DATE_KEY] = date - if multi: - # XXX fix for multipart nested case - hd[self.NUM_PARTS_KEY] = len(msg.get_payload()) - - # body doc - bd[self.CONTENT_HASH_KEY] = chash - bd[self.BODY_KEY] = body - # XXX in an ideal world, we would not need to save a copy of the - # raw message. But we'll keep it until we can be sure that - # we can rebuild the original message from the parts. - bd[self.RAW_KEY] = raw_str + return hd + + @deferred + def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): + """ + Creates a new message document. + + :param raw: the raw message + :type raw: str + + :param subject: subject of the message. + :type subject: str + + :param flags: flags + :type flags: list + + :param date: the received date for the message + :type date: str + + :param uid: the message uid for this mailbox + :type uid: int + """ + # TODO signal that we can delete the original message!----- + # when all the processing is done. + + # TODO add the linked-from info ! + + logger.debug('adding message') + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + + # parse + msg, chash, size, multi = self._do_parse(raw) + + fd = self._populate_flags(flags, uid, chash, size, multi) + hd = self._populate_headr(msg, chash, subject, date) + + parts = walk.get_parts(msg) + body_phash_fun = [walk.get_body_phash_simple, + walk.get_body_phash_multi][int(multi)] + body_phash = body_phash_fun(walk.get_payloads(msg)) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + + # add parts map to header doc + # (body, multi, part_map) + for key in parts_map: + hd[key] = parts_map[key] + del parts_map docs = [fd, hd] + cdocs = walk.get_raw_docs(msg, parts) - # attachment docs - if multi: - outer_parts = msg.get_payload() - parts = outer_parts + inner_parts - - # skip first part, we already got it in body - to_attach = ((i, m) for i, m in enumerate(parts) if i > 0) - for index, part_msg in to_attach: - att_doc = self._get_empty_doc(self.ATTACHMENT_DOC) - att_doc[self.PART_NUMBER_KEY] = index - att_doc[self.CONTENT_HASH_KEY] = chash - phash = self._get_hash(part_msg) - att_doc[self.PAYLOAD_HASH_KEY] = phash - att_doc[self.RAW_KEY] = part_msg.as_string() - - # keep a pointer to the payload hash in the - # headers doc, under the parts_map - hd[self.PARTS_MAP_KEY][str(index)] = phash - attaches.append(att_doc) - - # Saving ... ------------------------------- - # ok, there we go... + # Saving logger.debug('enqueuing message docs for write') ptuple = SoledadWriterPayload @@ -1067,14 +1123,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): for doc in docs: self.soledad_writer.put(ptuple( mode=ptuple.CREATE, payload=doc)) - # second, try to create body doc. - self.soledad_writer.put(ptuple( - mode=ptuple.BODY_CREATE, payload=bd)) + # and last, but not least, try to create - # attachment docs if not already there. - for at in attaches: + # content docs if not already there. + for cd in cdocs: self.soledad_writer.put(ptuple( - mode=ptuple.ATTACHMENT_CREATE, payload=at)) + mode=ptuple.CONTENT_CREATE, payload=cd)) def _remove_cb(self, result): return result diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 26e14c3..234996d 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -87,6 +87,8 @@ class LeapIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ + print "RECV: STATE (%s)" % self.state + if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message new file mode 100644 index 0000000..582297c --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message new file mode 100644 index 0000000..fc627c3 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py new file mode 100644 index 0000000..1626f65 --- /dev/null +++ b/src/leap/mail/imap/tests/walktree.py @@ -0,0 +1,117 @@ +#t -*- coding: utf-8 -*- +# walktree.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the walktree module. +""" +import os +from email import parser + +from leap.mail import walk as W + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + +p = parser.Parser() + +# TODO pass an argument of the type of message + +################################################## +# Input from hell + +#msg = p.parse(open('rfc822.multi-signed.message')) +#msg = p.parse(open('rfc822.plain.message')) +msg = p.parse(open('rfc822.multi-minimal.message')) +DO_CHECK = False +################################################# + +parts = W.get_parts(msg) + +if DEBUG: + def trim(item): + item = item[:10] + [trim(part["phash"]) for part in parts if part.get('phash', None)] + +raw_docs = list(W.get_raw_docs(msg, parts)) + +body_phash_fun = [W.get_body_phash_simple, + W.get_body_phash_multi][int(msg.is_multipart())] +body_phash = body_phash_fun(W.get_payloads(msg)) +parts_map = W.walk_msg_tree(parts, body_phash=body_phash) + + +# TODO add missing headers! +expected = { + 'body': '1ddfa80485', + 'multi': True, + 'part_map': { + 1: { + 'headers': {'Content-Disposition': 'inline', + 'Content-Type': 'multipart/mixed; ' + 'boundary="z0eOaCaDLjvTGF2l"'}, + 'multi': True, + 'part_map': {1: {'ctype': 'text/plain', + 'headers': [ + ('Content-Type', + 'text/plain; charset=utf-8'), + ('Content-Disposition', + 'inline'), + ('Content-Transfer-Encoding', + 'quoted-printable')], + 'multi': False, + 'parts': 1, + 'phash': '1ddfa80485', + 'size': 206}, + 2: {'ctype': 'text/plain', + 'headers': [('Content-Type', + 'text/plain; charset=us-ascii'), + ('Content-Disposition', + 'attachment; ' + 'filename="attach.txt"')], + 'multi': False, + 'parts': 1, + 'phash': '7a94e4d769', + 'size': 133}, + 3: {'ctype': 'application/octet-stream', + 'headers': [('Content-Type', + 'application/octet-stream'), + ('Content-Disposition', + 'attachment; filename="hack.ico"'), + ('Content-Transfer-Encoding', + 'base64')], + 'multi': False, + 'parts': 1, + 'phash': 'c42cccebbd', + 'size': 12736}}}, + 2: {'ctype': 'application/pgp-signature', + 'headers': [('Content-Type', 'application/pgp-signature')], + 'multi': False, + 'parts': 1, + 'phash': '8f49fbf749', + 'size': 877}}} + +if DEBUG and DO_CHECK: + # TODO turn this into a proper unittest + assert(parts_map == expected) + print "Structure: OK" + + +import pprint +print +print "RAW DOCS" +pprint.pprint(raw_docs) +print +print "PARTS MAP" +pprint.pprint(parts_map) diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py new file mode 100644 index 0000000..820b8c7 --- /dev/null +++ b/src/leap/mail/walk.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# walk.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Utilities for walking along a message tree. +""" +import hashlib +import os + +from leap.mail.utils import first + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + +if DEBUG: + get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10] +else: + get_hash = lambda s: hashlib.sha256(s).hexdigest() + + +""" +Get interesting message parts +""" +get_parts = lambda msg: [ + {'multi': part.is_multipart(), + 'ctype': part.get_content_type(), + 'size': len(part.as_string()), + 'parts': len(part.get_payload()) + if isinstance(part.get_payload(), list) + else 1, + 'headers': part.items(), + 'phash': get_hash(part.get_payload()) + if not part.is_multipart() else None} + for part in msg.walk()] + +""" +Utility lambda functions for getting the parts vector and the +payloads from the original message. +""" + +get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts) +get_payloads = lambda msg: ((x.get_payload(), + dict(((str.lower(k), v) for k, v in (x.items())))) + for x in msg.walk()) + +get_body_phash_simple = lambda payloads: first( + [get_hash(payload) for payload, headers in payloads + if "text/plain" in headers.get('content-type')]) + +get_body_phash_multi = lambda payloads: (first( + [get_hash(payload) for payload, headers in payloads + if "text/plain" in headers.get('content-type')]) + or get_body_phash_simple(payloads)) + +""" +On getting the raw docs, we get also some of the headers to be able to +index the content. Here we remove any mutable part, as the the filename +in the content disposition. +""" + +get_raw_docs = lambda msg, parts: ( + {"type": "cnt", # type content they'll be + "raw": payload if not DEBUG else payload[:100], + "phash": get_hash(payload), + "content-disposition": first(headers.get( + 'content-disposition', '').split(';')), + "content-type": headers.get( + 'content-type', ''), + "content-transfer-encoding": headers.get( + 'content-transfer-type', '')} + for payload, headers in get_payloads(msg) + if not isinstance(payload, list)) + + +def walk_msg_tree(parts, body_phash=None): + """ + Take a list of interesting items of a message subparts structure, + and return a dict of dicts almost ready to be written to the content + documents that will be stored in Soledad. + + It walks down the subparts in the parsed message tree, and collapses + the leaf docuents into a wrapper document until no multipart submessages + are left. To achieve this, it iteratively calculates a wrapper vector of + all documents in the sequence that have more than one part and have unitary + documents to their right. To collapse a multipart, take as many + unitary documents as parts the submessage contains, and replace the object + in the sequence with the new wrapper document. + + :param parts: A list of dicts containing the interesting properties for + the message structure. Normally this has been generated by + doing a message walk. + :type parts: list of dicts. + :param body_phash: the payload hash of the body part, to be included + in the outer content doc for convenience. + :type body_phash: basestring or None + """ + # parts vector + pv = list(get_parts_vector(parts)) + + if len(parts) == 2: + inner_headers = parts[1].get("headers", None) + + if DEBUG: + print "parts vector: ", pv + print + + # wrappers vector + getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False + for i in range(len(pv) - 1)] + wv = getwv(pv) + + # do until no wrapper document is left + while any(wv): + wind = wv.index(True) # wrapper index + nsub = pv[wind] # number of subparts to pick + slic = parts[wind + 1:wind + 1 + nsub] # slice with subparts + + cwra = { + "multi": True, + "part_map": dict((index + 1, part) # content wrapper + for index, part in enumerate(slic)), + "headers": dict(parts[wind]['headers']) + } + + # remove subparts and substitue wrapper + map(lambda i: parts.remove(i), slic) + parts[wind] = cwra + + # refresh vectors for this iteration + pv = list(get_parts_vector(parts)) + wv = getwv(pv) + + outer = parts[0] + outer.pop('headers') + if not "part_map" in outer: + # we have a multipart with 1 part only, so kind of fix it + # although it would be prettier if I take this special case at + # the beginning of the walk. + pdoc = {"multi": True, + "part_map": {1: outer}} + pdoc["part_map"][1]["multi"] = False + if not pdoc["part_map"][1].get("phash", None): + pdoc["part_map"][1]["phash"] = body_phash + pdoc["part_map"][1]["headers"] = inner_headers + else: + pdoc = outer + pdoc["body"] = body_phash + return pdoc -- cgit v1.2.3 From e4f2914517ea11aeef60aa74be50116e1979f34d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Jan 2014 21:39:27 -0400 Subject: handle all fetches as sequential * this allows quick testing using telnet, and the use of other less sofisticated MUAs. --- src/leap/mail/imap/mailbox.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 1d76d4d..7c01490 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -479,11 +479,13 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ result = [] - # XXX DEBUG ------------- - print "getting uid", uid - print "in mbox", self.mbox + # For the moment our UID is sequential, so we + # can treat them all the same. + # Change this to the flag that twisted expects when we + # switch to content-hash based index + local UID table. - sequence = True if uid == 0 else False + sequence = False + #sequence = True if uid == 0 else False if not messages.last: try: -- cgit v1.2.3 From 28236ecdf46360acd09a53130646e2a082583de6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Jan 2014 22:24:46 -0400 Subject: add a quick message fetching utility for tests --- src/leap/mail/imap/tests/getmail | 282 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100755 src/leap/mail/imap/tests/getmail (limited to 'src') diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail new file mode 100755 index 0000000..17e195c --- /dev/null +++ b/src/leap/mail/imap/tests/getmail @@ -0,0 +1,282 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE in twisted for details. + +# Modifications by LEAP Developers 2014 to fit +# Bitmask configuration settings. + + +""" +Simple IMAP4 client which displays the subjects of all messages in a +particular mailbox. +""" + +import sys + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +class TrivialPrompter(basic.LineReceiver): + from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + insecure-login. + """ + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + """ + result = [e[2] for e in result] + s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, result + ) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.examine(mbox + ).addCallback(cbExamineMbox, proto + ) + + +def cbExamineMbox(result, proto): + """ + Callback invoked when examine command completes. + + Retrieve the subject header of every message in the mailbox. + """ + return proto.fetchSpecific('1:*', + headerType='HEADER.FIELDS', + headerArgs=['SUBJECT'], + ).addCallback(cbFetch, proto, + ) + + +def cbFetch(result, proto): + """ + Display headers. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty mailbox!" + + return proto.prompt("\nWhich message? [1] (Q quits) " + ).addCallback(cbPickMessage, proto) + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + if result == "Q": + print "Bye!" + return proto.logout() + + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbShowmessage, proto) + + +def cbShowmessage(result, proto): + """ + Display message. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty message!" + + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import sys + + if len(sys.argv) != 3: + print "Usage: getmail " + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + if port == '993': + reactor.connectSSL( + hostname, int(port), factory, ssl.ClientContextFactory()) + else: + if not port: + port = 143 + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() -- cgit v1.2.3