diff options
| -rw-r--r-- | src/leap/mail/imap/account.py | 426 | ||||
| -rw-r--r-- | src/leap/mail/imap/fields.py | 127 | ||||
| -rw-r--r-- | src/leap/mail/imap/index.py | 69 | ||||
| -rw-r--r-- | src/leap/mail/imap/mailbox.py | 617 | ||||
| -rw-r--r-- | src/leap/mail/imap/messages.py | 735 | ||||
| -rw-r--r-- | src/leap/mail/imap/parser.py | 93 | ||||
| -rw-r--r-- | src/leap/mail/imap/server.py | 1897 | ||||
| -rw-r--r-- | src/leap/mail/imap/service/imap.py | 2 |
8 files changed, 2068 insertions, 1898 deletions
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py new file mode 100644 index 0000000..fd861e7 --- /dev/null +++ b/src/leap/mail/imap/account.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- +# account.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad Backed Account. +""" +import copy +import time + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.imap.index import IndexedDB +from leap.mail.imap.fields import WithMsgFields +from leap.mail.imap.parser import MBoxParser +from leap.mail.imap.mailbox import SoledadMailbox +from leap.soledad.client import Soledad + + +####################################### +# Soledad Account +####################################### + + +class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): + """ + An implementation of IAccount and INamespacePresenteer + that is backed by Soledad Encrypted Documents. + """ + + implements(imap4.IAccount, imap4.INamespacePresenter) + + _soledad = None + selected = None + + def __init__(self, account_name, soledad=None): + """ + Creates a SoledadAccountIndex that keeps track of the mailboxes + and subscriptions handled by this account. + + :param acct_name: The name of the account (user id). + :type acct_name: str + + :param soledad: a Soledad instance. + :param soledad: Soledad + """ + leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert_type(soledad, Soledad) + + # XXX SHOULD assert too that the name matches the user/uuid with which + # soledad has been initialized. + + self._account_name = self._parse_mailbox_name(account_name) + self._soledad = soledad + + self.initialize_db() + + # every user should have the right to an inbox folder + # at least, so let's make one! + + if not self.mailboxes: + self.addMailbox(self.INBOX_NAME) + + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + return copy.deepcopy(self.EMPTY_MBOX) + + def _get_mailbox_by_name(self, name): + """ + Return an mbox document by name. + + :param name: the name of the mailbox + :type name: str + + :rtype: SoledadDocument + """ + doc = self._soledad.get_from_index( + self.TYPE_MBOX_IDX, self.MBOX_KEY, + self._parse_mailbox_name(name)) + return doc[0] if doc else None + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + """ + return [doc.content[self.MBOX_KEY] + for doc in self._soledad.get_from_index( + self.TYPE_IDX, self.MBOX_KEY)] + + @property + def subscriptions(self): + """ + A list of the current subscriptions for this account. + """ + return [doc.content[self.MBOX_KEY] + for doc in self._soledad.get_from_index( + self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + + def getMailbox(self, name): + """ + Returns a Mailbox with that name, without selecting it. + + :param name: name of the mailbox + :type name: str + + :returns: a a SoledadMailbox instance + :rtype: SoledadMailbox + """ + name = self._parse_mailbox_name(name) + + if name not in self.mailboxes: + raise imap4.MailboxException("No such mailbox") + + return SoledadMailbox(name, soledad=self._soledad) + + ## + ## IAccount + ## + + def addMailbox(self, name, creation_ts=None): + """ + Add a mailbox to the account. + + :param name: the name of the mailbox + :type name: str + + :param creation_ts: an optional creation timestamp to be used as + mailbox id. A timestamp will be used if no + one is provided. + :type creation_ts: int + + :returns: True if successful + :rtype: bool + """ + name = self._parse_mailbox_name(name) + + if name in self.mailboxes: + raise imap4.MailboxCollision, name + + if not creation_ts: + # by default, we pass an int value + # taken from the current time + # we make sure to take enough decimals to get a unique + # mailbox-uidvalidity. + creation_ts = int(time.time() * 10E2) + + mbox = self._get_empty_mailbox() + mbox[self.MBOX_KEY] = name + mbox[self.CREATED_KEY] = creation_ts + + doc = self._soledad.create_doc(mbox) + return bool(doc) + + def create(self, pathspec): + """ + Create a new mailbox from the given hierarchical name. + + :param pathspec: The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. + :type pathspec: str + + :return: A true value if the creation succeeds. + :rtype: bool + + :raise MailboxException: Raised if this mailbox cannot be added. + """ + # TODO raise MailboxException + paths = filter( + None, + self._parse_mailbox_name(pathspec).split('/')) + for accum in range(1, len(paths)): + try: + self.addMailbox('/'.join(paths[:accum])) + except imap4.MailboxCollision: + pass + try: + self.addMailbox('/'.join(paths)) + except imap4.MailboxCollision: + if not pathspec.endswith('/'): + return False + return True + + def select(self, name, readwrite=1): + """ + Selects a mailbox. + + :param name: the mailbox to select + :type name: str + + :param readwrite: 1 for readwrite permissions. + :type readwrite: int + + :rtype: bool + """ + name = self._parse_mailbox_name(name) + + if name not in self.mailboxes: + return None + + self.selected = name + + return SoledadMailbox( + name, rw=readwrite, + soledad=self._soledad) + + def delete(self, name, force=False): + """ + Deletes a mailbox. + + Right now it does not purge the messages, but just removes the mailbox + name from the mailboxes list!!! + + :param name: the mailbox to be deleted + :type name: str + + :param force: if True, it will not check for noselect flag or inferior + names. use with care. + :type force: bool + """ + name = self._parse_mailbox_name(name) + + if not name in self.mailboxes: + raise imap4.MailboxException("No such mailbox") + + mbox = self.getMailbox(name) + + if force is False: + # See if this box is flagged \Noselect + # XXX use mbox.flags instead? + if self.NOSELECT_FLAG in mbox.getFlags(): + # Check for hierarchically inferior mailboxes with this one + # as part of their root. + for others in self.mailboxes: + if others != name and others.startswith(name): + raise imap4.MailboxException, ( + "Hierarchically inferior mailboxes " + "exist and \\Noselect is set") + mbox.destroy() + + # XXX FIXME --- not honoring the inferior names... + + # if there are no hierarchically inferior names, we will + # delete it from our ken. + #if self._inferiorNames(name) > 1: + # ??! -- can this be rite? + #self._index.removeMailbox(name) + + def rename(self, oldname, newname): + """ + Renames a mailbox. + + :param oldname: old name of the mailbox + :type oldname: str + + :param newname: new name of the mailbox + :type newname: str + """ + oldname = self._parse_mailbox_name(oldname) + newname = self._parse_mailbox_name(newname) + + if oldname not in self.mailboxes: + raise imap4.NoSuchMailbox, oldname + + inferiors = self._inferiorNames(oldname) + inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + + for (old, new) in inferiors: + if new in self.mailboxes: + raise imap4.MailboxCollision, new + + for (old, new) in inferiors: + mbox = self._get_mailbox_by_name(old) + mbox.content[self.MBOX_KEY] = new + self._soledad.put_doc(mbox) + + # XXX ---- FIXME!!!! ------------------------------------ + # until here we just renamed the index... + # We have to rename also the occurrence of this + # mailbox on ALL the messages that are contained in it!!! + # ... we maybe could use a reference to the doc_id + # in each msg, instead of the "mbox" field in msgs + # ------------------------------------------------------- + + def _inferiorNames(self, name): + """ + Return hierarchically inferior mailboxes. + + :param name: name of the mailbox + :rtype: list + """ + # XXX use wildcard query instead + inferiors = [] + for infname in self.mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + def isSubscribed(self, name): + """ + Returns True if user is subscribed to this mailbox. + + :param name: the mailbox to be checked. + :type name: str + + :rtype: bool + """ + mbox = self._get_mailbox_by_name(name) + return mbox.content.get('subscribed', False) + + def _set_subscription(self, name, value): + """ + Sets the subscription value for a given mailbox + + :param name: the mailbox + :type name: str + + :param value: the boolean value + :type value: bool + """ + # maybe we should store subscriptions in another + # document... + if not name in self.mailboxes: + self.addMailbox(name) + mbox = self._get_mailbox_by_name(name) + + if mbox: + mbox.content[self.SUBSCRIBED_KEY] = value + self._soledad.put_doc(mbox) + + def subscribe(self, name): + """ + Subscribe to this mailbox + + :param name: name of the mailbox + :type name: str + """ + name = self._parse_mailbox_name(name) + if name not in self.subscriptions: + self._set_subscription(name, True) + + def unsubscribe(self, name): + """ + Unsubscribe from this mailbox + + :param name: name of the mailbox + :type name: str + """ + name = self._parse_mailbox_name(name) + if name not in self.subscriptions: + raise imap4.MailboxException, "Not currently subscribed to " + name + self._set_subscription(name, False) + + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames( + self._parse_mailbox_name(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + ## + ## INamespacePresenter + ## + + def getPersonalNamespaces(self): + return [["", "/"]] + + def getSharedNamespaces(self): + return None + + def getOtherNamespaces(self): + return None + + # extra, for convenience + + def deleteAllMessages(self, iknowhatiamdoing=False): + """ + Deletes all messages from all mailboxes. + Danger! high voltage! + + :param iknowhatiamdoing: confirmation parameter, needs to be True + to proceed. + """ + if iknowhatiamdoing is True: + for mbox in self.mailboxes: + self.delete(mbox, force=True) + + def __repr__(self): + """ + Representation string for this object. + """ + return "<SoledadBackedAccount (%s)>" % self._account_name diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py new file mode 100644 index 0000000..96b937e --- /dev/null +++ b/src/leap/mail/imap/fields.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# fields.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Fields for Mailbox and Message. +""" +from leap.mail.imap.parser import MBoxParser + + +class WithMsgFields(object): + """ + Container class for class-attributes to be shared by + several message-related classes. + """ + # Internal representation of Message + DATE_KEY = "date" + HEADERS_KEY = "headers" + FLAGS_KEY = "flags" + MBOX_KEY = "mbox" + CONTENT_HASH_KEY = "chash" + RAW_KEY = "raw" + SUBJECT_KEY = "subject" + UID_KEY = "uid" + MULTIPART_KEY = "multi" + SIZE_KEY = "size" + + # Mailbox specific keys + CLOSED_KEY = "closed" + CREATED_KEY = "created" + SUBSCRIBED_KEY = "subscribed" + RW_KEY = "rw" + LAST_UID_KEY = "lastuid" + + # Document Type, for indexing + TYPE_KEY = "type" + TYPE_MBOX_VAL = "mbox" + TYPE_MESSAGE_VAL = "msg" + TYPE_FLAGS_VAL = "flags" + TYPE_HEADERS_VAL = "head" + TYPE_ATTACHMENT_VAL = "attach" + # should add also a headers val + + INBOX_VAL = "inbox" + + # Flags for SoledadDocument for indexing. + SEEN_KEY = "seen" + RECENT_KEY = "recent" + + # Flags in Mailbox and Message + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) + + # Fields in mail object + SUBJECT_FIELD = "Subject" + DATE_FIELD = "Date" + + # Index types + # -------------- + + TYPE_IDX = 'by-type' + TYPE_MBOX_IDX = 'by-type-and-mbox' + TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' + TYPE_SUBS_IDX = 'by-type-and-subscribed' + TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' + TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + TYPE_HASH_IDX = 'by-type-and-hash' + + # Tomas created the `recent and seen index`, but the semantic is not too + # correct since the recent flag is volatile. + TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + + KTYPE = TYPE_KEY + MBOX_VAL = TYPE_MBOX_VAL + HASH_VAL = CONTENT_HASH_KEY + + INDEXES = { + # generic + TYPE_IDX: [KTYPE], + TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], + TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], + + # mailboxes + TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], + + # content, headers doc + TYPE_HASH_IDX: [KTYPE, HASH_VAL], + + # messages + TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], + TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, + 'bool(recent)', 'bool(seen)'], + } + + MBOX_KEY = MBOX_VAL + + EMPTY_MBOX = { + TYPE_KEY: MBOX_KEY, + TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, + SUBJECT_KEY: "", + FLAGS_KEY: [], + CLOSED_KEY: False, + SUBSCRIBED_KEY: False, + RW_KEY: 1, + LAST_UID_KEY: 0 + } + +fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py new file mode 100644 index 0000000..2280d86 --- /dev/null +++ b/src/leap/mail/imap/index.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# index.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Index for SoledadBackedAccount, Mailbox and Messages. +""" +import logging + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail.imap.account import SoledadBackedAccount + + +logger = logging.getLogger(__name__) + + +class IndexedDB(object): + """ + Methods dealing with the index. + + This is a MixIn that needs access to the soledad instance, + and also assumes that a INDEXES attribute is accessible to the instance. + + INDEXES must be a dictionary of type: + {'index-name': ['field1', 'field2']} + """ + # TODO we might want to move this to soledad itself, check + + def initialize_db(self): + """ + Initialize the database. + """ + leap_assert(self._soledad, + "Need a soledad attribute accesible in the instance") + leap_assert_type(self.INDEXES, dict) + + # Ask the database for currently existing indexes. + if not self._soledad: + logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") + return + db_indexes = dict() + if self._soledad is not None: + db_indexes = dict(self._soledad.list_indexes()) + for name, expression in SoledadBackedAccount.INDEXES.items(): + if name not in db_indexes: + # The index does not yet exist. + self._soledad.create_index(name, *expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so we + # delete it and add the proper index expression. + self._soledad.delete_index(name) + self._soledad.create_index(name, *expression) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py new file mode 100644 index 0000000..09c06a2 --- /dev/null +++ b/src/leap/mail/imap/mailbox.py @@ -0,0 +1,617 @@ +# *- coding: utf-8 -*- +# mailbox.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad Mailbox. +""" +import logging +from collections import defaultdict + +from twisted.internet import defer +from twisted.python import log + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common import events as leap_events +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.check import leap_assert, leap_assert_type +from leap.mail.decorators import deferred +from leap.mail.imap.fields import WithMsgFields, fields +from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.parser import MBoxParser + +logger = logging.getLogger(__name__) + + +class SoledadMailbox(WithMsgFields, MBoxParser): + """ + A Soledad-backed IMAP mailbox. + + Implements the high-level method needed for the Mailbox interfaces. + The low-level database methods are contained in MessageCollection class, + which we instantiate and make accessible in the `messages` attribute. + """ + implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) + # XXX should finish the implementation of IMailboxListener + # XXX should implement IMessageCopier too + + messages = None + _closed = False + + INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, + WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, + WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, + WithMsgFields.LIST_FLAG) + flags = None + + CMD_MSG = "MESSAGES" + CMD_RECENT = "RECENT" + CMD_UIDNEXT = "UIDNEXT" + CMD_UIDVALIDITY = "UIDVALIDITY" + CMD_UNSEEN = "UNSEEN" + + _listeners = defaultdict(set) + + def __init__(self, mbox, soledad=None, rw=1): + """ + SoledadMailbox constructor. Needs to get passed a name, plus a + Soledad instance. + + :param mbox: the mailbox name + :type mbox: str + + :param soledad: a Soledad instance. + :type soledad: Soledad + + :param rw: read-and-write flags + :type rw: int + """ + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(soledad, "Need a soledad instance to initialize") + + # XXX should move to wrapper + #leap_assert(isinstance(soledad._db, SQLCipherDatabase), + #"soledad._db must be an instance of SQLCipherDatabase") + + self.mbox = self._parse_mailbox_name(mbox) + self.rw = rw + + self._soledad = soledad + + self.messages = MessageCollection( + mbox=mbox, soledad=self._soledad) + + if not self.getFlags(): + self.setFlags(self.INIT_FLAGS) + + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox] + + def addListener(self, listener): + """ + Adds a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + logger.debug('adding mailbox listener: %s' % listener) + self.listeners.add(listener) + + def removeListener(self, listener): + """ + Removes a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + self.listeners.remove(listener) + + def _get_mbox(self): + """ + Returns mailbox document. + + :return: A SoledadDocument containing this mailbox, or None if + the query failed. + :rtype: SoledadDocument or None. + """ + try: + query = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_MBOX_VAL, self.mbox) + if query: + return query.pop() + except Exception as exc: + logger.error("Unhandled error %r" % exc) + + def getFlags(self): + """ + Returns the flags defined for this mailbox. + + :returns: tuple of flags for this mailbox + :rtype: tuple of str + """ + mbox = self._get_mbox() + if not mbox: + return None + flags = mbox.content.get(self.FLAGS_KEY, []) + return map(str, flags) + + def setFlags(self, flags): + """ + Sets flags for this mailbox. + + :param flags: a tuple with the flags + :type flags: tuple of str + """ + leap_assert(isinstance(flags, tuple), + "flags expected to be a tuple") + mbox = self._get_mbox() + if not mbox: + return None + mbox.content[self.FLAGS_KEY] = map(str, flags) + self._soledad.put_doc(mbox) + + # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + + def _get_closed(self): + """ + Return the closed attribute for this mailbox. + + :return: True if the mailbox is closed + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.CLOSED_KEY, False) + + def _set_closed(self, closed): + """ + Set the closed attribute for this mailbox. + + :param closed: the state to be set + :type closed: bool + """ + leap_assert(isinstance(closed, bool), "closed needs to be boolean") + mbox = self._get_mbox() + mbox.content[self.CLOSED_KEY] = closed + self._soledad.put_doc(mbox) + + closed = property( + _get_closed, _set_closed, doc="Closed attribute.") + + def _get_last_uid(self): + """ + Return the last uid for this mailbox. + + :return: the last uid for messages in this mailbox + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.LAST_UID_KEY, 1) + + def _set_last_uid(self, uid): + """ + Sets the last uid for this mailbox. + + :param uid: the uid to be set + :type uid: int + """ + leap_assert(isinstance(uid, int), "uid has to be int") + mbox = self._get_mbox() + key = self.LAST_UID_KEY + + count = self.getMessageCount() + + # XXX safety-catch. If we do get duplicates, + # we want to avoid further duplication. + + if uid >= count: + value = uid + else: + # something is wrong, + # just set the last uid + # beyond the max msg count. + logger.debug("WRONG uid < count. Setting last uid to %s", count) + value = count + + mbox.content[key] = value + self._soledad.put_doc(mbox) + + last_uid = property( + _get_last_uid, _set_last_uid, doc="Last_UID attribute.") + + def getUIDValidity(self): + """ + Return the unique validity identifier for this mailbox. + + :return: unique validity identifier + :rtype: int + """ + mbox = self._get_mbox() + return mbox.content.get(self.CREATED_KEY, 1) + + def getUID(self, message): + """ + Return the UID of a message in the mailbox + + .. note:: this implementation does not make much sense RIGHT NOW, + but in the future will be useful to get absolute UIDs from + message sequence numbers. + + :param message: the message uid + :type message: int + + :rtype: int + """ + msg = self.messages.get_msg_by_uid(message) + return msg.getUID() + + def getUIDNext(self): + """ + Return the likely UID for the next message added to this + mailbox. Currently it returns the higher UID incremented by + one. + + We increment the next uid *each* time this function gets called. + In this way, there will be gaps if the message with the allocated + uid cannot be saved. But that is preferable to having race conditions + if we get to parallel message adding. + + :rtype: int + """ + self.last_uid += 1 + return self.last_uid + + def getMessageCount(self): + """ + Returns the total count of messages in this mailbox. + + :rtype: int + """ + return self.messages.count() + + def getUnseenCount(self): + """ + Returns the number of messages with the 'Unseen' flag. + + :return: count of messages flagged `unseen` + :rtype: int + """ + return self.messages.count_unseen() + + def getRecentCount(self): + """ + Returns the number of messages with the 'Recent' flag. + + :return: count of messages flagged `recent` + :rtype: int + """ + return self.messages.count_recent() + + def isWriteable(self): + """ + Get the read/write status of the mailbox. + + :return: 1 if mailbox is read-writeable, 0 otherwise. + :rtype: int + """ + return self.rw + + def getHierarchicalDelimiter(self): + """ + Returns the character used to delimite hierarchies in mailboxes. + + :rtype: str + """ + return '/' + + def requestStatus(self, names): + """ + Handles a status request by gathering the output of the different + status commands. + + :param names: a list of strings containing the status commands + :type names: iter + """ + r = {} + if self.CMD_MSG in names: + r[self.CMD_MSG] = self.getMessageCount() + if self.CMD_RECENT in names: + r[self.CMD_RECENT] = self.getRecentCount() + if self.CMD_UIDNEXT in names: + r[self.CMD_UIDNEXT] = self.last_uid + 1 + if self.CMD_UIDVALIDITY in names: + r[self.CMD_UIDVALIDITY] = self.getUID() + if self.CMD_UNSEEN in names: + r[self.CMD_UNSEEN] = self.getUnseenCount() + return defer.succeed(r) + + def addMessage(self, message, flags, date=None): + """ + Adds a message to this mailbox. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str + + :return: a deferred that evals to None + """ + # XXX we should treat the message as an IMessage from here + leap_assert_type(message, basestring) + uid_next = self.getUIDNext() + logger.debug('Adding msg with UID :%s' % uid_next) + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + d = self._do_add_messages(message, flags, date, uid_next) + d.addCallback(self._notify_new) + + @deferred + def _do_add_messages(self, message, flags, date, uid_next): + """ + Calls to the messageCollection add_msg method (deferred to thread). + Invoked from addMessage. + """ + self.messages.add_msg(message, flags=flags, date=date, + uid=uid_next) + + def _notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + :param args: ignored. + """ + exists = self.getMessageCount() + recent = self.getRecentCount() + logger.debug("NOTIFY: there are %s messages, %s recent" % ( + exists, + recent)) + + logger.debug("listeners: %s", str(self.listeners)) + for l in self.listeners: + logger.debug('notifying...') + l.newMessages(exists, recent) + + # commands, do not rename methods + + def destroy(self): + """ + Called before this mailbox is permanently deleted. + + Should cleanup resources, and set the \\Noselect flag + on the mailbox. + """ + self.setFlags((self.NOSELECT_FLAG,)) + self.deleteAllDocs() + + # XXX removing the mailbox in situ for now, + # we should postpone the removal + self._soledad.delete_doc(self._get_mbox()) + + def expunge(self): + """ + Remove all messages flagged \\Deleted + """ + if not self.isWriteable(): + raise imap4.ReadOnlyMailbox + delete = [] + deleted = [] + + for m in self.messages.get_all_docs(): + # XXX should operate with LeapMessages instead, + # so we don't expose the implementation. + # (so, iterate for m in self.messages) + if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: + delete.append(m) + for m in delete: + deleted.append(m.content) + self.messages.remove(m) + + # XXX should return the UIDs of the deleted messages + # more generically + return [x for x in range(len(deleted))] + + @deferred + def fetch(self, messages, uid): + """ + Retrieve one or more messages in this mailbox. + + from rfc 3501: The data items to be fetched can be either a single atom + or a parenthesized list. + + :param messages: IDs of the messages to retrieve information about + :type messages: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :rtype: A tuple of two-tuples of message sequence numbers and + LeapMessage + """ + result = [] + sequence = True if uid == 0 else False + + if not messages.last: + try: + iter(messages) + except TypeError: + # looks like we cannot iterate + messages.last = self.last_uid + + # for sequence numbers (uid = 0) + if sequence: + logger.debug("Getting msg by index: INEFFICIENT call!") + raise NotImplementedError + + else: + for msg_id in messages: + msg = self.messages.get_msg_by_uid(msg_id) + if msg: + result.append((msg_id, msg)) + else: + logger.debug("fetch %s, no msg found!!!" % msg_id) + + if self.isWriteable(): + self._unset_recent_flag() + self._signal_unread_to_ui() + + # XXX workaround for hangs in thunderbird + #return tuple(result[:100]) # --- doesn't show all!! + return tuple(result) + + @deferred + def _unset_recent_flag(self): + """ + Unsets `Recent` flag from a tuple of messages. + Called from fetch. + + From RFC, about `Recent`: + + Message is "recently" arrived in this mailbox. This session + is the first session to have been notified about this + message; if the session is read-write, subsequent sessions + will not see \Recent set for this message. This flag can not + be altered by the client. + + If it is not possible to determine whether or not this + session is the first session to be notified about a message, + then that message SHOULD be considered recent. + """ + log.msg('unsetting recent flags...') + for msg in self.messages.get_recent(): + msg.removeFlags((fields.RECENT_FLAG,)) + self._signal_unread_to_ui() + + @deferred + def _signal_unread_to_ui(self): + """ + Sends unread event to ui. + """ + unseen = self.getUnseenCount() + leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + + @deferred + def store(self, messages, flags, mode, uid): + """ + Sets the flags of one or more messages. + + :param messages: The identifiers of the messages to set the flags + :type messages: A MessageSet object with the list of messages requested + + :param flags: The flags to set, unset, or add. + :type flags: sequence of str + + :param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be + added to the specified messages. If mode is 0, all + existing flags should be cleared and these flags should be + added. + :type mode: -1, 0, or 1 + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A dict mapping message sequence numbers to sequences of + str representing the flags set on the message after this + operation has been performed. + :rtype: dict + + :raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + # XXX implement also sequence (uid = 0) + # XXX we should prevent cclient from setting Recent flag. + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) + + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + if not messages.last: + messages.last = self.messages.count() + + result = {} + for msg_id in messages: + log.msg("MSG ID = %s" % msg_id) + msg = self.messages.get_msg_by_uid(msg_id) + if mode == 1: + msg.addFlags(flags) + elif mode == -1: + msg.removeFlags(flags) + elif mode == 0: + msg.setFlags(flags) + result[msg_id] = msg.getFlags() + + self._signal_unread_to_ui() + return result + + @deferred + def close(self): + """ + Expunge and mark as closed + """ + self.expunge() + self.closed = True + + #@deferred + #def copy(self, messageObject): + #""" + #Copy the given message object into this mailbox. + #""" + # XXX should just: + # 1. Get the message._fdoc + # 2. Change the UID to UIDNext for this mailbox + # 3. Add implements IMessageCopier + + # convenience fun + + def deleteAllDocs(self): + """ + Deletes all docs in this mailbox + """ + docs = self.messages.get_all_docs() + for doc in docs: + self.messages._soledad.delete_doc(doc) + + def __repr__(self): + """ + Representation string for this mailbox. + """ + return u"<SoledadMailbox: mbox '%s' (%s)>" % ( + self.mbox, self.messages.count()) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py new file mode 100644 index 0000000..b0d5da2 --- /dev/null +++ b/src/leap/mail/imap/messages.py @@ -0,0 +1,735 @@ +# -*- coding: utf-8 -*- +# messages.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +LeapMessage and MessageCollection. +""" +import copy +import logging +import StringIO +from collections import namedtuple + +from twisted.mail import imap4 +from twisted.python import log +from u1db import errors as u1db_errors +from zope.interface import implements +from zope.proxy import sameProxiedObjects + +from leap.common.check import leap_assert, leap_assert_type +from leap.common.mail import get_email_charset +from leap.mail.decorators import deferred +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.index import IndexedDB +from leap.mail.imap.fields import fields, WithMsgFields +from leap.mail.imap.parser import MailParser, MBoxParser +from leap.mail.messageflow import IMessageConsumer, MessageProducer + +logger = logging.getLogger(__name__) + + +class LeapMessage(fields, MailParser, MBoxParser): + + implements(imap4.IMessage) + + def __init__(self, soledad, uid, mbox): + """ + Initializes a LeapMessage. + + :param soledad: a Soledad instance + :type soledad: Soledad + :param uid: the UID for the message. + :type uid: int or basestring + :param mbox: the mbox this message belongs to + :type mbox: basestring + """ + MailParser.__init__(self) + self._soledad = soledad + self._uid = int(uid) + self._mbox = self._parse_mailbox_name(mbox) + self._chash = None + + self.__cdoc = None + + @property + def _fdoc(self): + """ + An accessor to the flags document. + """ + return self._get_flags_doc() + + @property + def _cdoc(self): + """ + An accessor to the content document. + """ + if not self.__cdoc: + self.__cdoc = self._get_content_doc() + return self.__cdoc + + @property + def _chash(self): + """ + An accessor to the content hash for this message. + """ + if not self._fdoc: + return None + return self._fdoc.content.get(fields.CONTENT_HASH_KEY, None) + + # IMessage implementation + + def getUID(self): + """ + Retrieve the unique identifier associated with this message + + :return: uid for this message + :rtype: int + """ + return self._uid + + def getFlags(self): + """ + Retrieve the flags associated with this message + + :return: The flags, represented as strings + :rtype: tuple + """ + if self._uid is None: + return [] + + flags = [] + flag_doc = self._fdoc + if flag_doc: + flags = flag_doc.content.get(self.FLAGS_KEY, None) + if flags: + flags = map(str, flags) + return tuple(flags) + + # setFlags, addFlags, removeFlags are not in the interface spec + # but we use them with store command. + + def setFlags(self, flags): + """ + Sets the flags for this message + + Returns a SoledadDocument that needs to be updated by the caller. + + :param flags: the flags to update in the message. + :type flags: tuple of str + + :return: a SoledadDocument instance + :rtype: SoledadDocument + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + log.msg('setting flags: %s' % (self._uid)) + + doc = self._fdoc + doc.content[self.FLAGS_KEY] = flags + doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags + doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags + self._soledad.put_doc(doc) + + def addFlags(self, flags): + """ + Adds flags to this message. + + Returns a SoledadDocument that needs to be updated by the caller. + + :param flags: the flags to add to the message. + :type flags: tuple of str + + :return: a SoledadDocument instance + :rtype: SoledadDocument + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + oldflags = self.getFlags() + self.setFlags(tuple(set(flags + oldflags))) + + def removeFlags(self, flags): + """ + Remove flags from this message. + + Returns a SoledadDocument that needs to be updated by the caller. + + :param flags: the flags to be removed from the message. + :type flags: tuple of str + + :return: a SoledadDocument instance + :rtype: SoledadDocument + """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + oldflags = self.getFlags() + self.setFlags(tuple(set(oldflags) - set(flags))) + + def getInternalDate(self): + """ + Retrieve the date internally associated with this message + + :rtype: C{str} + :return: An RFC822-formatted date string. + """ + return str(self._cdoc.content.get(self.DATE_KEY, '')) + + # + # IMessagePart + # + + # XXX we should implement this interface too for the subparts + # so we allow nested parts... + + def getBodyFile(self): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: StringIO + """ + fd = StringIO.StringIO() + + cdoc = self._cdoc + content = cdoc.content.get(self.RAW_KEY, '') + charset = get_email_charset( + unicode(cdoc.content.get(self.RAW_KEY, ''))) + try: + content = content.encode(charset) + except (UnicodeEncodeError, UnicodeDecodeError) as e: + logger.error("Unicode error {0}".format(e)) + content = content.encode(charset, 'replace') + + raw = self._get_raw_msg() + msg = self._get_parsed_msg(raw) + body = msg.get_payload() + fd.write(body) + # XXX SHOULD use a separate BODY FIELD ... + fd.seek(0) + return fd + + def getSize(self): + """ + Return the total size, in octets, of this message. + + :return: size of the message, in octets + :rtype: int + """ + size = self._cdoc.content.get(self.SIZE_KEY, False) + if not size: + # XXX fallback, should remove when all migrated. + size = self.getBodyFile().len + return size + + def _get_headers(self): + """ + Return the headers dict stored in this message document. + """ + # XXX get from the headers doc + return self._cdoc.content.get(self.HEADERS_KEY, {}) + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + headers = self._get_headers() + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + # unpack and filter original dict by negate-condition + filter_by_cond = [ + map(str, (key, val)) for + key, val in headers.items() + if cond(key)] + return dict(filter_by_cond) + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + if self._cdoc: + retval = self._fdoc.content.get(self.MULTIPART_KEY, False) + return retval + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + if not self.isMultipart(): + raise TypeError + + msg = self._get_parsed_msg() + # XXX should wrap IMessagePart + return msg.get_payload()[part] + + # + # accessors + # + + def _get_flags_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + flag_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + flag_doc = flag_docs[0] if flag_docs else None + return flag_doc + + def _get_content_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + cont_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_HASH_IDX, + fields.TYPE_MESSAGE_VAL, self._content_hash, str(self._uid)) + cont_doc = cont_docs[0] if cont_docs else None + return cont_doc + + def _get_raw_msg(self): + """ + Return the raw msg. + :rtype: basestring + """ + return self._cdoc.content.get(self.RAW_KEY, '') + + def __getitem__(self, key): + """ + Return the content of the message document. + + :param key: The key + :type key: str + + :return: The content value indexed by C{key} or None + :rtype: str + """ + return self._cdoc.content.get(key, None) + + def does_exist(self): + """ + Return True if there is actually a message for this + UID and mbox. + """ + return bool(self._fdoc) + + +SoledadWriterPayload = namedtuple( + 'SoledadWriterPayload', ['mode', 'payload']) + +SoledadWriterPayload.CREATE = 1 +SoledadWriterPayload.PUT = 2 + + +class SoledadDocWriter(object): + """ + This writer will create docs serially in the local soledad database. + """ + + implements(IMessageConsumer) + + def __init__(self, soledad): + """ + Initialize the writer. + + :param soledad: the soledad instance + :type soledad: Soledad + """ + self._soledad = soledad + + def consume(self, queue): + """ + Creates a new document in soledad db. + + :param queue: queue to get item from, with content of the document + to be inserted. + :type queue: Queue + """ + empty = queue.empty() + while not empty: + item = queue.get() + if item.mode == SoledadWriterPayload.CREATE: + call = self._soledad.create_doc + elif item.mode == SoledadWriterPayload.PUT: + call = self._soledad.put_doc + + # should handle errors + try: + call(item.payload) + except u1db_errors.RevisionConflict as exc: + logger.error("Error: %r" % (exc,)) + raise exc + + empty = queue.empty() + + +class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): + """ + A collection of messages, surprisingly. + + It is tied to a selected mailbox name that is passed to constructor. + Implements a filter query over the messages contained in a soledad + database. + """ + # XXX this should be able to produce a MessageSet methinks + + EMPTY_MSG = { + fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.SUBJECT_KEY: "", + fields.DATE_KEY: "", + fields.RAW_KEY: "", + + # XXX should separate headers into another doc + fields.HEADERS_KEY: {}, + } + + EMPTY_FLAGS = { + fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.FLAGS_KEY: [], + fields.SEEN_KEY: False, + fields.RECENT_KEY: True, + fields.MULTIPART_KEY: False, + } + + # get from SoledadBackedAccount the needed index-related constants + INDEXES = SoledadBackedAccount.INDEXES + TYPE_IDX = SoledadBackedAccount.TYPE_IDX + + def __init__(self, mbox=None, soledad=None): + """ + Constructor for MessageCollection. + + :param mbox: the name of the mailbox. It is the name + with which we filter the query over the + messages database + :type mbox: str + + :param soledad: Soledad database + :type soledad: Soledad instance + """ + MailParser.__init__(self) + leap_assert(mbox, "Need a mailbox name to initialize") + leap_assert(mbox.strip() != "", "mbox cannot be blank space") + leap_assert(isinstance(mbox, (str, unicode)), + "mbox needs to be a string") + leap_assert(soledad, "Need a soledad instance to initialize") + + # okay, all in order, keep going... + self.mbox = self._parse_mailbox_name(mbox) + self._soledad = soledad + self.initialize_db() + + # I think of someone like nietzsche when reading this + + # this will be the producer that will enqueue the content + # to be processed serially by the consumer (the writer). We just + # need to `put` the new material on its plate. + + self.soledad_writer = MessageProducer( + SoledadDocWriter(soledad), + period=0.05) + + def _get_empty_msg(self): + """ + Returns an empty message. + + :return: a dict containing a default empty message + :rtype: dict + """ + return copy.deepcopy(self.EMPTY_MSG) + + def _get_empty_flags_doc(self): + """ + Returns an empty doc for storing flags. + + :return: + :rtype: + """ + return copy.deepcopy(self.EMPTY_FLAGS) + + @deferred + def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): + """ + Creates a new message document. + + :param raw: the raw message + :type raw: str + + :param subject: subject of the message. + :type subject: str + + :param flags: flags + :type flags: list + + :param date: the received date for the message + :type date: str + + :param uid: the message uid for this mailbox + :type uid: int + """ + # TODO: split in smaller methods + logger.debug('adding message') + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + + content_doc = self._get_empty_msg() + flags_doc = self._get_empty_flags_doc() + + content_doc[self.MBOX_KEY] = self.mbox + flags_doc[self.MBOX_KEY] = self.mbox + # ...should get a sanity check here. + content_doc[self.UID_KEY] = uid + flags_doc[self.UID_KEY] = uid + + if flags: + flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) + flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags + + msg = self._get_parsed_msg(raw) + headers = dict(msg) + + logger.debug("adding. is multipart:%s" % msg.is_multipart()) + flags_doc[self.MULTIPART_KEY] = msg.is_multipart() + # XXX get lower case for keys? + # XXX get headers doc + content_doc[self.HEADERS_KEY] = headers + # set subject based on message headers and eventually replace by + # subject given as param + if self.SUBJECT_FIELD in headers: + content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + if subject is not None: + content_doc[self.SUBJECT_KEY] = subject + + # XXX could separate body into its own doc + # but should also separate multiparts + # that should be wrapped in MessagePart + content_doc[self.RAW_KEY] = self._stringify(raw) + content_doc[self.SIZE_KEY] = len(raw) + + if not date and self.DATE_FIELD in headers: + content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] + else: + content_doc[self.DATE_KEY] = date + + logger.debug('enqueuing message for write') + + ptuple = SoledadWriterPayload + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=content_doc)) + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=flags_doc)) + + def remove(self, msg): + """ + Removes a message. + + :param msg: a Leapmessage instance + :type msg: LeapMessage + """ + # XXX remove + #self._soledad.delete_doc(msg) + msg.remove() + + # getters + + def get_msg_by_uid(self, uid): + """ + Retrieves a LeapMessage by UID. + + :param uid: the message uid to query by + :type uid: int + + :return: A LeapMessage instance matching the query, + or None if not found. + :rtype: LeapMessage + """ + msg = LeapMessage(self._soledad, uid, self.mbox) + if not msg.does_exist(): + return None + return msg + + def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + """ + Get all documents for the selected mailbox of the + passed type. By default, it returns the flag docs. + + If you want acess to the content, use __iter__ instead + + :return: a list of u1db documents + :rtype: list of SoledadDocument + """ + if _type not in fields.__dict__.values(): + raise TypeError("Wrong type passed to get_all") + + if sameProxiedObjects(self._soledad, None): + logger.warning('Tried to get messages but soledad is None!') + return [] + + all_docs = [doc for doc in self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + _type, self.mbox)] + + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad + return sorted(all_docs, key=lambda item: item.content['uid']) + + def all_msg_iter(self): + """ + Return an iterator trhough the UIDs of all messages, sorted in + ascending order. + """ + all_uids = (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + self.TYPE_FLAGS_VAL, self.mbox)) + return (u for u in sorted(all_uids)) + + def count(self): + """ + Return the count of messages for this mailbox. + + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + return count + + # unseen messages + + def unseen_iter(self): + """ + Get an iterator for the message UIDs with no `seen` flag + for this mailbox. + + :return: iterator through unseen message doc UIDs + :rtype: iterable + """ + return (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '0')) + + def count_unseen(self): + """ + Count all messages with the `Unseen` flag. + + :returns: count + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '0') + return count + + def get_unseen(self): + """ + Get all messages with the `Unseen` flag + + :returns: a list of LeapMessages + :rtype: list + """ + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.unseen_iter()] + + # recent messages + + def recent_iter(self): + """ + Get an iterator for the message docs with `recent` flag. + + :return: iterator through recent message docs + :rtype: iterable + """ + return (doc.content[self.UID_KEY] for doc in + self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_RECT_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '1')) + + def get_recent(self): + """ + Get all messages with the `Recent` flag. + + :returns: a list of LeapMessages + :rtype: list + """ + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.recent_iter()] + + def count_recent(self): + """ + Count all messages with the `Recent` flag. + + :returns: count + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_RECT_IDX, + self.TYPE_FLAGS_VAL, self.mbox, '1') + return count + + def __len__(self): + """ + Returns the number of messages on this mailbox. + + :rtype: int + """ + return self.count() + + def __iter__(self): + """ + Returns an iterator over all messages. + + :returns: iterator of dicts with content for all messages. + :rtype: iterable + """ + return (LeapMessage(self._soledad, docuid, self.mbox) + for docuid in self.all_msg_iter()) + + def __repr__(self): + """ + Representation string for this object. + """ + return u"<MessageCollection: mbox '%s' (%s)>" % ( + self.mbox, self.count()) + + # XXX should implement __eq__ also !!! --- use a hash + # of content for that, will be used for dedup. diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py new file mode 100644 index 0000000..1ae19c0 --- /dev/null +++ b/src/leap/mail/imap/parser.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# parser.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Mail parser mixins. +""" +import cStringIO +import StringIO +import re + +from email.parser import Parser + + +class MailParser(object): + """ + Mixin with utility methods to parse raw messages. + """ + def __init__(self): + """ + Initializes the mail parser. + """ + self._parser = Parser() + + def _get_parsed_msg(self, raw): + """ + Return a parsed Message. + + :param raw: the raw string to parse + :type raw: basestring, or StringIO object + """ + msg = self._get_parser_fun(raw)(raw, True) + return msg + + def _get_parser_fun(self, o): + """ + Retunn the proper parser function for an object. + + :param o: object + :type o: object + :param parser: an instance of email.parser.Parser + :type parser: email.parser.Parser + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return self._parser.parse + if isinstance(o, basestring): + return self._parser.parsestr + # fallback + return self._parser.parsestr + + def _stringify(self, o): + """ + Return a string object. + + :param o: object + :type o: object + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return o.getvalue() + else: + return o + + +class MBoxParser(object): + """ + Utility function to parse mailbox names. + """ + INBOX_NAME = "INBOX" + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + def _parse_mailbox_name(self, name): + """ + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py deleted file mode 100644 index 57587a5..0000000 --- a/src/leap/mail/imap/server.py +++ /dev/null @@ -1,1897 +0,0 @@ -# -*- coding: utf-8 -*- -# server.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Soledad-backed IMAP Server. -""" -import copy -import logging -import StringIO -import cStringIO -import time -import re - -from collections import defaultdict, namedtuple -from email.parser import Parser - -from zope.interface import implements -from zope.proxy import sameProxiedObjects - -from twisted.mail import imap4 -from twisted.internet import defer -from twisted.python import log - -from u1db import errors as u1db_errors - -from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.check import leap_assert, leap_assert_type -from leap.common.mail import get_email_charset -from leap.mail.messageflow import IMessageConsumer, MessageProducer -from leap.mail.decorators import deferred -from leap.soledad.client import Soledad - -logger = logging.getLogger(__name__) - - -class MissingIndexError(Exception): - """ - Raises when tried to access a non existent index document. - """ - - -class BadIndexError(Exception): - """ - Raises when index is malformed or has the wrong cardinality. - """ - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # Internal representation of Message - DATE_KEY = "date" - HEADERS_KEY = "headers" - FLAGS_KEY = "flags" - MBOX_KEY = "mbox" - RAW_KEY = "raw" - SUBJECT_KEY = "subject" - UID_KEY = "uid" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" - - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MESSAGE_VAL = "msg" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - # should add also a headers val - - INBOX_VAL = "inbox" - - # Flags for SoledadDocument for indexing. - SEEN_KEY = "seen" - RECENT_KEY = "recent" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - -fields = WithMsgFields # alias for convenience - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - db_indexes = dict() - if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in SoledadBackedAccount.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) - - -class MailParser(object): - """ - Mixin with utility methods to parse raw messages. - """ - def __init__(self): - """ - Initializes the mail parser. - """ - self._parser = Parser() - - def _get_parsed_msg(self, raw): - """ - Return a parsed Message. - - :param raw: the raw string to parse - :type raw: basestring, or StringIO object - """ - msg = self._get_parser_fun(raw)(raw, True) - return msg - - def _get_parser_fun(self, o): - """ - Retunn the proper parser function for an object. - - :param o: object - :type o: object - :param parser: an instance of email.parser.Parser - :type parser: email.parser.Parser - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, basestring): - return self._parser.parsestr - # fallback - return self._parser.parsestr - - def _stringify(self, o): - """ - Return a string object. - - :param o: object - :type o: object - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() - else: - return o - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name - - -####################################### -# Soledad Account -####################################### - - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): - """ - An implementation of IAccount and INamespacePresenteer - that is backed by Soledad Encrypted Documents. - """ - - implements(imap4.IAccount, imap4.INamespacePresenter) - - _soledad = None - selected = None - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - KTYPE = WithMsgFields.TYPE_KEY - MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - WithMsgFields.TYPE_KEY: MBOX_KEY, - WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.CLOSED_KEY: False, - WithMsgFields.SUBSCRIBED_KEY: False, - WithMsgFields.RW_KEY: 1, - WithMsgFields.LAST_UID_KEY: 0 - } - - def __init__(self, account_name, soledad=None): - """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. - - :param acct_name: The name of the account (user id). - :type acct_name: str - - :param soledad: a Soledad instance. - :param soledad: Soledad - """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) - - # XXX SHOULD assert too that the name matches the user/uuid with which - # soledad has been initialized. - - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - - self.initialize_db() - - # every user should have the right to an inbox folder - # at least, so let's make one! - - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - - def _get_mailbox_by_name(self, name): - """ - Return an mbox document by name. - - :param name: the name of the mailbox - :type name: str - - :rtype: SoledadDocument - """ - doc = self._soledad.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) - return doc[0] if doc else None - - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)] - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] - - def getMailbox(self, name): - """ - Returns a Mailbox with that name, without selecting it. - - :param name: name of the mailbox - :type name: str - - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox - """ - name = self._parse_mailbox_name(name) - - if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - return SoledadMailbox(name, soledad=self._soledad) - - ## - ## IAccount - ## - - def addMailbox(self, name, creation_ts=None): - """ - Add a mailbox to the account. - - :param name: the name of the mailbox - :type name: str - - :param creation_ts: an optional creation timestamp to be used as - mailbox id. A timestamp will be used if no - one is provided. - :type creation_ts: int - - :returns: True if successful - :rtype: bool - """ - name = self._parse_mailbox_name(name) - - if name in self.mailboxes: - raise imap4.MailboxCollision, name - - if not creation_ts: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # mailbox-uidvalidity. - creation_ts = int(time.time() * 10E2) - - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - doc = self._soledad.create_doc(mbox) - return bool(doc) - - def create(self, pathspec): - """ - Create a new mailbox from the given hierarchical name. - - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. - :type pathspec: str - - :return: A true value if the creation succeeds. - :rtype: bool - - :raise MailboxException: Raised if this mailbox cannot be added. - """ - # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) - for accum in range(1, len(paths)): - try: - self.addMailbox('/'.join(paths[:accum])) - except imap4.MailboxCollision: - pass - try: - self.addMailbox('/'.join(paths)) - except imap4.MailboxCollision: - if not pathspec.endswith('/'): - return False - return True - - def select(self, name, readwrite=1): - """ - Selects a mailbox. - - :param name: the mailbox to select - :type name: str - - :param readwrite: 1 for readwrite permissions. - :type readwrite: int - - :rtype: bool - """ - name = self._parse_mailbox_name(name) - - if name not in self.mailboxes: - return None - - self.selected = name - - return SoledadMailbox( - name, rw=readwrite, - soledad=self._soledad) - - def delete(self, name, force=False): - """ - Deletes a mailbox. - - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - - :param name: the mailbox to be deleted - :type name: str - - :param force: if True, it will not check for noselect flag or inferior - names. use with care. - :type force: bool - """ - name = self._parse_mailbox_name(name) - - if not name in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - mbox = self.getMailbox(name) - - if force is False: - # See if this box is flagged \Noselect - # XXX use mbox.flags instead? - if self.NOSELECT_FLAG in mbox.getFlags(): - # Check for hierarchically inferior mailboxes with this one - # as part of their root. - for others in self.mailboxes: - if others != name and others.startswith(name): - raise imap4.MailboxException, ( - "Hierarchically inferior mailboxes " - "exist and \\Noselect is set") - mbox.destroy() - - # XXX FIXME --- not honoring the inferior names... - - # if there are no hierarchically inferior names, we will - # delete it from our ken. - #if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - #self._index.removeMailbox(name) - - def rename(self, oldname, newname): - """ - Renames a mailbox. - - :param oldname: old name of the mailbox - :type oldname: str - - :param newname: new name of the mailbox - :type newname: str - """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox, oldname - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] - - for (old, new) in inferiors: - if new in self.mailboxes: - raise imap4.MailboxCollision, new - - for (old, new) in inferiors: - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new - self._soledad.put_doc(mbox) - - # XXX ---- FIXME!!!! ------------------------------------ - # until here we just renamed the index... - # We have to rename also the occurrence of this - # mailbox on ALL the messages that are contained in it!!! - # ... we maybe could use a reference to the doc_id - # in each msg, instead of the "mbox" field in msgs - # ------------------------------------------------------- - - def _inferiorNames(self, name): - """ - Return hierarchically inferior mailboxes. - - :param name: name of the mailbox - :rtype: list - """ - # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors - - def isSubscribed(self, name): - """ - Returns True if user is subscribed to this mailbox. - - :param name: the mailbox to be checked. - :type name: str - - :rtype: bool - """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) - - def _set_subscription(self, name, value): - """ - Sets the subscription value for a given mailbox - - :param name: the mailbox - :type name: str - - :param value: the boolean value - :type value: bool - """ - # maybe we should store subscriptions in another - # document... - if not name in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) - - def subscribe(self, name): - """ - Subscribe to this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) - - def unsubscribe(self, name): - """ - Unsubscribe from this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException, "Not currently subscribed to " + name - self._set_subscription(name, False) - - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - - ## - ## INamespacePresenter - ## - - def getPersonalNamespaces(self): - return [["", "/"]] - - def getSharedNamespaces(self): - return None - - def getOtherNamespaces(self): - return None - - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - - def __repr__(self): - """ - Representation string for this object. - """ - return "<SoledadBackedAccount (%s)>" % self._account_name - -####################################### -# LeapMessage, MessageCollection -# and Mailbox -####################################### - - -class LeapMessage(fields, MailParser, MBoxParser): - - implements(imap4.IMessage) - - def __init__(self, soledad, uid, mbox): - """ - Initializes a LeapMessage. - - :param soledad: a Soledad instance - :type soledad: Soledad - :param uid: the UID for the message. - :type uid: int or basestring - :param mbox: the mbox this message belongs to - :type mbox: basestring - """ - MailParser.__init__(self) - self._soledad = soledad - self._uid = int(uid) - self._mbox = self._parse_mailbox_name(mbox) - - self.__cdoc = None - - @property - def _fdoc(self): - """ - An accessor to the flags docuemnt - """ - return self._get_flags_doc() - - @property - def _cdoc(self): - """ - An accessor to the content docuemnt - """ - if not self.__cdoc: - self.__cdoc = self._get_content_doc() - return self.__cdoc - - def getUID(self): - """ - Retrieve the unique identifier associated with this message - - :return: uid for this message - :rtype: int - """ - return self._uid - - def getFlags(self): - """ - Retrieve the flags associated with this message - - :return: The flags, represented as strings - :rtype: tuple - """ - if self._uid is None: - return [] - - flags = [] - flag_doc = self._fdoc - if flag_doc: - flags = flag_doc.content.get(self.FLAGS_KEY, None) - if flags: - flags = map(str, flags) - return tuple(flags) - - # setFlags, addFlags, removeFlags are not in the interface spec - # but we use them with store command. - - def setFlags(self, flags): - """ - Sets the flags for this message - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to update in the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - log.msg('setting flags: %s' % (self._uid)) - - doc = self._fdoc - doc.content[self.FLAGS_KEY] = flags - doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags - doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags - self._soledad.put_doc(doc) - - def addFlags(self, flags): - """ - Adds flags to this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to add to the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - oldflags = self.getFlags() - self.setFlags(tuple(set(flags + oldflags))) - - def removeFlags(self, flags): - """ - Remove flags from this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to be removed from the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - oldflags = self.getFlags() - self.setFlags(tuple(set(oldflags) - set(flags))) - - def getInternalDate(self): - """ - Retrieve the date internally associated with this message - - :rtype: C{str} - :return: An RFC822-formatted date string. - """ - return str(self._cdoc.content.get(self.DATE_KEY, '')) - - # - # IMessagePart - # - - # XXX we should implement this interface too for the subparts - # so we allow nested parts... - - def getBodyFile(self): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO - """ - fd = StringIO.StringIO() - - cdoc = self._cdoc - content = cdoc.content.get(self.RAW_KEY, '') - charset = get_email_charset( - unicode(cdoc.content.get(self.RAW_KEY, ''))) - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - - raw = self._get_raw_msg() - msg = self._get_parsed_msg(raw) - body = msg.get_payload() - fd.write(body) - # XXX SHOULD use a separate BODY FIELD ... - fd.seek(0) - return fd - - def getSize(self): - """ - Return the total size, in octets, of this message. - - :return: size of the message, in octets - :rtype: int - """ - size = self._cdoc.content.get(self.SIZE_KEY, False) - if not size: - # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len - return size - - def _get_headers(self): - """ - Return the headers dict stored in this message document. - """ - # XXX get from the headers doc - return self._cdoc.content.get(self.HEADERS_KEY, {}) - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - headers = self._get_headers() - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # unpack and filter original dict by negate-condition - filter_by_cond = [ - map(str, (key, val)) for - key, val in headers.items() - if cond(key)] - return dict(filter_by_cond) - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - if self._cdoc: - retval = self._fdoc.content.get(self.MULTIPART_KEY, False) - return retval - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - if not self.isMultipart(): - raise TypeError - - msg = self._get_parsed_msg() - # XXX should wrap IMessagePart - return msg.get_payload()[part] - - # - # accessors - # - - def _get_flags_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - flag_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - flag_doc = flag_docs[0] if flag_docs else None - return flag_doc - - def _get_content_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - cont_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid)) - cont_doc = cont_docs[0] if cont_docs else None - return cont_doc - - def _get_raw_msg(self): - """ - Return the raw msg. - :rtype: basestring - """ - return self._cdoc.content.get(self.RAW_KEY, '') - - def __getitem__(self, key): - """ - Return the content of the message document. - - :param key: The key - :type key: str - - :return: The content value indexed by C{key} or None - :rtype: str - """ - return self._cdoc.content.get(key, None) - - def does_exist(self): - """ - Return True if there is actually a message for this - UID and mbox. - """ - return bool(self._fdoc) - - -SoledadWriterPayload = namedtuple( - 'SoledadWriterPayload', ['mode', 'payload']) - -SoledadWriterPayload.CREATE = 1 -SoledadWriterPayload.PUT = 2 - - -class SoledadDocWriter(object): - """ - This writer will create docs serially in the local soledad database. - """ - - implements(IMessageConsumer) - - def __init__(self, soledad): - """ - Initialize the writer. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - self._soledad = soledad - - def consume(self, queue): - """ - Creates a new document in soledad db. - - :param queue: queue to get item from, with content of the document - to be inserted. - :type queue: Queue - """ - empty = queue.empty() - while not empty: - item = queue.get() - if item.mode == SoledadWriterPayload.CREATE: - call = self._soledad.create_doc - elif item.mode == SoledadWriterPayload.PUT: - call = self._soledad.put_doc - - # should handle errors - try: - call(item.payload) - except u1db_errors.RevisionConflict as exc: - logger.error("Error: %r" % (exc,)) - raise exc - - empty = queue.empty() - - -class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): - """ - A collection of messages, surprisingly. - - It is tied to a selected mailbox name that is passed to constructor. - Implements a filter query over the messages contained in a soledad - database. - """ - # XXX this should be able to produce a MessageSet methinks - - EMPTY_MSG = { - fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.SUBJECT_KEY: "", - fields.DATE_KEY: "", - fields.RAW_KEY: "", - - # XXX should separate headers into another doc - fields.HEADERS_KEY: {}, - } - - EMPTY_FLAGS = { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.FLAGS_KEY: [], - fields.SEEN_KEY: False, - fields.RECENT_KEY: True, - fields.MULTIPART_KEY: False, - } - - # get from SoledadBackedAccount the needed index-related constants - INDEXES = SoledadBackedAccount.INDEXES - TYPE_IDX = SoledadBackedAccount.TYPE_IDX - - def __init__(self, mbox=None, soledad=None): - """ - Constructor for MessageCollection. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database - :type mbox: str - - :param soledad: Soledad database - :type soledad: Soledad instance - """ - MailParser.__init__(self) - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") - - # okay, all in order, keep going... - self.mbox = self._parse_mailbox_name(mbox) - self._soledad = soledad - self.initialize_db() - - # I think of someone like nietzsche when reading this - - # this will be the producer that will enqueue the content - # to be processed serially by the consumer (the writer). We just - # need to `put` the new material on its plate. - - self.soledad_writer = MessageProducer( - SoledadDocWriter(soledad), - period=0.05) - - def _get_empty_msg(self): - """ - Returns an empty message. - - :return: a dict containing a default empty message - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MSG) - - def _get_empty_flags_doc(self): - """ - Returns an empty doc for storing flags. - - :return: - :rtype: - """ - return copy.deepcopy(self.EMPTY_FLAGS) - - @deferred - def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): - """ - Creates a new message document. - - :param raw: the raw message - :type raw: str - - :param subject: subject of the message. - :type subject: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :param uid: the message uid for this mailbox - :type uid: int - """ - # TODO: split in smaller methods - logger.debug('adding message') - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - - content_doc = self._get_empty_msg() - flags_doc = self._get_empty_flags_doc() - - content_doc[self.MBOX_KEY] = self.mbox - flags_doc[self.MBOX_KEY] = self.mbox - # ...should get a sanity check here. - content_doc[self.UID_KEY] = uid - flags_doc[self.UID_KEY] = uid - - if flags: - flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) - flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags - - msg = self._get_parsed_msg(raw) - headers = dict(msg) - - logger.debug("adding. is multipart:%s" % msg.is_multipart()) - flags_doc[self.MULTIPART_KEY] = msg.is_multipart() - # XXX get lower case for keys? - # XXX get headers doc - content_doc[self.HEADERS_KEY] = headers - # set subject based on message headers and eventually replace by - # subject given as param - if self.SUBJECT_FIELD in headers: - content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - if subject is not None: - content_doc[self.SUBJECT_KEY] = subject - - # XXX could separate body into its own doc - # but should also separate multiparts - # that should be wrapped in MessagePart - content_doc[self.RAW_KEY] = self._stringify(raw) - content_doc[self.SIZE_KEY] = len(raw) - - if not date and self.DATE_FIELD in headers: - content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - content_doc[self.DATE_KEY] = date - - logger.debug('enqueuing message for write') - - ptuple = SoledadWriterPayload - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=content_doc)) - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=flags_doc)) - - def remove(self, msg): - """ - Removes a message. - - :param msg: a u1db doc containing the message - :type msg: SoledadDocument - """ - self._soledad.delete_doc(msg) - - # getters - - def get_msg_by_uid(self, uid): - """ - Retrieves a LeapMessage by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - msg = LeapMessage(self._soledad, uid, self.mbox) - if not msg.does_exist(): - return None - return msg - - def get_all(self): - """ - Get all message documents for the selected mailbox. - If you want acess to the content, use __iter__ instead - - :return: a list of u1db documents - :rtype: list of SoledadDocument - """ - # TODO change to get_all_docs and turn this - # into returning messages - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - all_docs = [doc for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox)] - - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - return sorted(all_docs, key=lambda item: item.content['uid']) - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - return count - - # unseen messages - - def unseen_iter(self): - """ - Get an iterator for the message docs with no `seen` flag - - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - return (doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0')) - - def count_unseen(self): - """ - Count all messages with the `Unseen` flag. - - :returns: count - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0') - return count - - def get_unseen(self): - """ - Get all messages with the `Unseen` flag - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox) - for docid in self.unseen_iter()] - - # recent messages - - def recent_iter(self): - """ - Get an iterator for the message docs with `recent` flag. - - :return: iterator through recent message docs - :rtype: iterable - """ - return (doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1')) - - def get_recent(self): - """ - Get all messages with the `Recent` flag. - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox) - for docid in self.recent_iter()] - - def count_recent(self): - """ - Count all messages with the `Recent` flag. - - :returns: count - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1') - return count - - def __len__(self): - """ - Returns the number of messages on this mailbox. - - :rtype: int - """ - return self.count() - - def __iter__(self): - """ - Returns an iterator over all messages. - - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - # XXX return LeapMessage instead?! (change accordingly) - return (m.content for m in self.get_all()) - - def __repr__(self): - """ - Representation string for this object. - """ - return u"<MessageCollection: mbox '%s' (%s)>" % ( - self.mbox, self.count()) - - # XXX should implement __eq__ also !!! --- use a hash - # of content for that, will be used for dedup. - - -class SoledadMailbox(WithMsgFields, MBoxParser): - """ - A Soledad-backed IMAP mailbox. - - Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. - """ - implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) - # XXX should finish the implementation of IMailboxListener - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None - - CMD_MSG = "MESSAGES" - CMD_RECENT = "RECENT" - CMD_UIDNEXT = "UIDNEXT" - CMD_UIDVALIDITY = "UIDVALIDITY" - CMD_UNSEEN = "UNSEEN" - - _listeners = defaultdict(set) - - def __init__(self, mbox, soledad=None, rw=1): - """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. - - :param mbox: the mailbox name - :type mbox: str - - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param rw: read-and-write flags - :type rw: int - """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") - - # XXX should move to wrapper - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - - self.mbox = self._parse_mailbox_name(mbox) - self.rw = rw - - self._soledad = soledad - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad) - - if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - @property - def listeners(self): - """ - Returns listeners for this mbox. - - The server itself is a listener to the mailbox. - so we can notify it (and should!) after changes in flags - and number of messages. - - :rtype: set - """ - return self._listeners[self.mbox] - - def addListener(self, listener): - """ - Adds a listener to the listeners queue. - The server adds itself as a listener when there is a SELECT, - so it can send EXIST commands. - - :param listener: listener to add - :type listener: an object that implements IMailboxListener - """ - logger.debug('adding mailbox listener: %s' % listener) - self.listeners.add(listener) - - def removeListener(self, listener): - """ - Removes a listener from the listeners queue. - - :param listener: listener to remove - :type listener: an object that implements IMailboxListener - """ - self.listeners.remove(listener) - - def _get_mbox(self): - """ - Returns mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - try: - query = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() - except Exception as exc: - logger.error("Unhandled error %r" % exc) - - def getFlags(self): - """ - Returns the flags defined for this mailbox. - - :returns: tuple of flags for this mailbox - :rtype: tuple of str - """ - mbox = self._get_mbox() - if not mbox: - return None - flags = mbox.content.get(self.FLAGS_KEY, []) - return map(str, flags) - - def setFlags(self, flags): - """ - Sets flags for this mailbox. - - :param flags: a tuple with the flags - :type flags: tuple of str - """ - leap_assert(isinstance(flags, tuple), - "flags expected to be a tuple") - mbox = self._get_mbox() - if not mbox: - return None - mbox.content[self.FLAGS_KEY] = map(str, flags) - self._soledad.put_doc(mbox) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. - - def _get_closed(self): - """ - Return the closed attribute for this mailbox. - - :return: True if the mailbox is closed - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.CLOSED_KEY, False) - - def _set_closed(self, closed): - """ - Set the closed attribute for this mailbox. - - :param closed: the state to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - mbox = self._get_mbox() - mbox.content[self.CLOSED_KEY] = closed - self._soledad.put_doc(mbox) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - - :return: the last uid for messages in this mailbox - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.LAST_UID_KEY, 1) - - def _set_last_uid(self, uid): - """ - Sets the last uid for this mailbox. - - :param uid: the uid to be set - :type uid: int - """ - leap_assert(isinstance(uid, int), "uid has to be int") - mbox = self._get_mbox() - key = self.LAST_UID_KEY - - count = self.getMessageCount() - - # XXX safety-catch. If we do get duplicates, - # we want to avoid further duplication. - - if uid >= count: - value = uid - else: - # something is wrong, - # just set the last uid - # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to %s", count) - value = count - - mbox.content[key] = value - self._soledad.put_doc(mbox) - - last_uid = property( - _get_last_uid, _set_last_uid, doc="Last_UID attribute.") - - def getUIDValidity(self): - """ - Return the unique validity identifier for this mailbox. - - :return: unique validity identifier - :rtype: int - """ - mbox = self._get_mbox() - return mbox.content.get(self.CREATED_KEY, 1) - - def getUID(self, message): - """ - Return the UID of a message in the mailbox - - .. note:: this implementation does not make much sense RIGHT NOW, - but in the future will be useful to get absolute UIDs from - message sequence numbers. - - :param message: the message uid - :type message: int - - :rtype: int - """ - msg = self.messages.get_msg_by_uid(message) - return msg.getUID() - - def getUIDNext(self): - """ - Return the likely UID for the next message added to this - mailbox. Currently it returns the higher UID incremented by - one. - - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int - """ - self.last_uid += 1 - return self.last_uid - - def getMessageCount(self): - """ - Returns the total count of messages in this mailbox. - - :rtype: int - """ - return self.messages.count() - - def getUnseenCount(self): - """ - Returns the number of messages with the 'Unseen' flag. - - :return: count of messages flagged `unseen` - :rtype: int - """ - return self.messages.count_unseen() - - def getRecentCount(self): - """ - Returns the number of messages with the 'Recent' flag. - - :return: count of messages flagged `recent` - :rtype: int - """ - return self.messages.count_recent() - - def isWriteable(self): - """ - Get the read/write status of the mailbox. - - :return: 1 if mailbox is read-writeable, 0 otherwise. - :rtype: int - """ - return self.rw - - def getHierarchicalDelimiter(self): - """ - Returns the character used to delimite hierarchies in mailboxes. - - :rtype: str - """ - return '/' - - def requestStatus(self, names): - """ - Handles a status request by gathering the output of the different - status commands. - - :param names: a list of strings containing the status commands - :type names: iter - """ - r = {} - if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() - if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() - if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 - if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUID() - if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) - - def addMessage(self, message, flags, date=None): - """ - Adds a message to this mailbox. - - :param message: the raw message - :type message: str - - :param flags: flag list - :type flags: list of str - - :param date: timestamp - :type date: str - - :return: a deferred that evals to None - """ - # XXX we should treat the message as an IMessage from here - leap_assert_type(message, basestring) - uid_next = self.getUIDNext() - logger.debug('Adding msg with UID :%s' % uid_next) - if flags is None: - flags = tuple() - else: - flags = tuple(str(flag) for flag in flags) - - d = self._do_add_messages(message, flags, date, uid_next) - d.addCallback(self._notify_new) - - @deferred - def _do_add_messages(self, message, flags, date, uid_next): - """ - Calls to the messageCollection add_msg method (deferred to thread). - Invoked from addMessage. - """ - self.messages.add_msg(message, flags=flags, date=date, - uid=uid_next) - - def _notify_new(self, *args): - """ - Notify of new messages to all the listeners. - - :param args: ignored. - """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY: there are %s messages, %s recent" % ( - exists, - recent)) - - logger.debug("listeners: %s", str(self.listeners)) - for l in self.listeners: - logger.debug('notifying...') - l.newMessages(exists, recent) - - # commands, do not rename methods - - def destroy(self): - """ - Called before this mailbox is permanently deleted. - - Should cleanup resources, and set the \\Noselect flag - on the mailbox. - """ - self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - self._soledad.delete_doc(self._get_mbox()) - - def expunge(self): - """ - Remove all messages flagged \\Deleted - """ - if not self.isWriteable(): - raise imap4.ReadOnlyMailbox - delete = [] - deleted = [] - - for m in self.messages.get_all(): - if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: - delete.append(m) - for m in delete: - deleted.append(m.content) - self.messages.remove(m) - - # XXX should return the UIDs of the deleted messages - # more generically - return [x for x in range(len(deleted))] - - @deferred - def fetch(self, messages, uid): - """ - Retrieve one or more messages in this mailbox. - - from rfc 3501: The data items to be fetched can be either a single atom - or a parenthesized list. - - :param messages: IDs of the messages to retrieve information about - :type messages: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ - result = [] - sequence = True if uid == 0 else False - - if not messages.last: - try: - iter(messages) - except TypeError: - # looks like we cannot iterate - messages.last = self.last_uid - - # for sequence numbers (uid = 0) - if sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") - raise NotImplementedError - - else: - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) - else: - logger.debug("fetch %s, no msg found!!!" % msg_id) - - if self.isWriteable(): - self._unset_recent_flag() - self._signal_unread_to_ui() - - # XXX workaround for hangs in thunderbird - #return tuple(result[:100]) # --- doesn't show all!! - return tuple(result) - - @deferred - def _unset_recent_flag(self): - """ - Unsets `Recent` flag from a tuple of messages. - Called from fetch. - - From RFC, about `Recent`: - - Message is "recently" arrived in this mailbox. This session - is the first session to have been notified about this - message; if the session is read-write, subsequent sessions - will not see \Recent set for this message. This flag can not - be altered by the client. - - If it is not possible to determine whether or not this - session is the first session to be notified about a message, - then that message SHOULD be considered recent. - """ - log.msg('unsetting recent flags...') - for msg in self.messages.get_recent(): - msg.removeFlags((fields.RECENT_FLAG,)) - self._signal_unread_to_ui() - - @deferred - def _signal_unread_to_ui(self): - """ - Sends unread event to ui. - """ - unseen = self.getUnseenCount() - leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) - - @deferred - def store(self, messages, flags, mode, uid): - """ - Sets the flags of one or more messages. - - :param messages: The identifiers of the messages to set the flags - :type messages: A MessageSet object with the list of messages requested - - :param flags: The flags to set, unset, or add. - :type flags: sequence of str - - :param mode: If mode is -1, these flags should be removed from the - specified messages. If mode is 1, these flags should be - added to the specified messages. If mode is 0, all - existing flags should be cleared and these flags should be - added. - :type mode: -1, 0, or 1 - - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool - - :return: A dict mapping message sequence numbers to sequences of - str representing the flags set on the message after this - operation has been performed. - :rtype: dict - - :raise ReadOnlyMailbox: Raised if this mailbox is not open for - read-write. - """ - # XXX implement also sequence (uid = 0) - # XXX we should prevent cclient from setting Recent flag. - leap_assert(not isinstance(flags, basestring), - "flags cannot be a string") - flags = tuple(flags) - - if not self.isWriteable(): - log.msg('read only mailbox!') - raise imap4.ReadOnlyMailbox - - if not messages.last: - messages.last = self.messages.count() - - result = {} - for msg_id in messages: - log.msg("MSG ID = %s" % msg_id) - msg = self.messages.get_msg_by_uid(msg_id) - if mode == 1: - msg.addFlags(flags) - elif mode == -1: - msg.removeFlags(flags) - elif mode == 0: - msg.setFlags(flags) - result[msg_id] = msg.getFlags() - - self._signal_unread_to_ui() - return result - - @deferred - def close(self): - """ - Expunge and mark as closed - """ - self.expunge() - self.closed = True - - # convenience fun - - def deleteAllDocs(self): - """ - Deletes all docs in this mailbox - """ - docs = self.messages.get_all() - for doc in docs: - self.messages._soledad.delete_doc(doc) - - def __repr__(self): - """ - Representation string for this mailbox. - """ - return u"<SoledadMailbox: mbox '%s' (%s)>" % ( - self.mbox, self.messages.count()) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8756ddc..26e14c3 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.server import SoledadBackedAccount +from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail from leap.soledad.client import Soledad |