From ea4373132458f906b6270744bcfd3e76b64dbd0a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Nov 2014 15:04:26 +0100 Subject: Serializable Models + Soledad Adaptor --- src/leap/mail/mail.py | 248 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 src/leap/mail/mail.py (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py new file mode 100644 index 0000000..ea9c95e --- /dev/null +++ b/src/leap/mail/mail.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Access to Mail objects: Public LEAP Mail API. +""" +from twisted.internet import defer + +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors.soledad import SoledadMailAdaptor + + +# TODO +# [ ] Probably change the name of this module to "api" or "account", mail is +# too generic (there's also IncomingMail, and OutgoingMail + + +class Message(object): + + def __init__(self, wrapper): + """ + :param wrapper: an instance of an implementor of IMessageWrapper + """ + self._wrapper = wrapper + + def get_wrapper(self): + return self._wrapper + + # imap.IMessage methods + + def get_flags(): + """ + """ + + def get_internal_date(): + """ + """ + + # imap.IMessageParts + + def get_headers(): + """ + """ + + def get_body_file(): + """ + """ + + def get_size(): + """ + """ + + def is_multipart(): + """ + """ + + def get_subpart(part): + """ + """ + + # Custom methods. + + def get_tags(): + """ + """ + + +class MessageCollection(object): + """ + A generic collection of messages. It can be messages sharing the same + mailbox, tag, the result of a given query, or just a bunch of ids for + master documents. + + Since LEAP Mail is primarily oriented to store mail in Soledad, the default + (and, so far, only) implementation of the store is contained in this + Soledad Mail Adaptor. If you need to use a different adaptor, change the + adaptor class attribute in your Account object. + + Store is a reference to a particular instance of the message store (soledad + instance or proxy, for instance). + """ + + # TODO look at IMessageSet methods + + # Account should provide an adaptor instance when creating this collection. + adaptor = None + store = None + + def get_message_by_doc_id(self, doc_id): + # ... get from soledad etc + # ... but that should be part of adaptor/store too... :/ + fdoc, hdoc = None + return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + + # TODO review if this is the best place for: + + def create_docs(): + pass + + def udpate_flags(): + # 1. update the flags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + def update_tags(): + # 1. update the tags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + # TODO add delete methods here? + + +class Account(object): + """ + Account is the top level abstraction to access collections of messages + associated with a LEAP Mail Account. + + It primarily handles creation and access of Mailboxes, which will be the + basic collection handled by traditional MUAs, but it can also handle other + types of Collections (tag based, for instance). + + leap.mail.imap.SoledadBackedAccount partially proxies methods in this + class. + """ + + # Adaptor is passed to the returned MessageCollections, so if you want to + # use a different adaptor this is the place to change it, by subclassing + # the Account class. + + adaptor_class = SoledadMailAdaptor + store = None + mailboxes = None + + def __init__(self, store): + self.store = store + self.adaptor = self.adaptor_class() + + self.__mailboxes = set([]) + self._initialized = False + self._deferred_initialization = defer.Deferred() + + self._initialize_storage() + + def _initialize_storage(self): + + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.add_mailbox(INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.adaptor.initialize_store(self.store) + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + # XXX this could use adaptor.store_ready instead...?? + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + :rtype: set + """ + return sorted(self.__mailboxes) + + def _load_mailboxes(self): + + def update_mailboxes(mbox_names): + self.__mailboxes.update(mbox_names) + + d = self.adaptor.get_all_mboxes(self.store) + d.addCallback(update_mailboxes) + return d + + # + # Public API Starts + # + + # XXX params for IMAP only??? + def list_mailboxes(self, ref, wildcard): + self.adaptor.get_all_mboxes(self.store) + + def add_mailbox(self, name, mbox=None): + pass + + def create_mailbox(self, pathspec): + pass + + def delete_mailbox(self, name): + pass + + def rename_mailbox(self, oldname, newname): + pass + + # FIXME yet to be decided if it belongs here... + + def get_collection_by_mailbox(self, name): + """ + :rtype: MessageCollection + """ + # imap select will use this, passing the collection to SoledadMailbox + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_docs(self, docs): + """ + :rtype: MessageCollection + """ + # get a collection of docs by a list of doc_id + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_tag(self, tag): + """ + :rtype: MessageCollection + """ + # is this a good idea? + pass -- cgit v1.2.3 From c8dbbb6bad0463ba84ca9186693e21e9c99161a0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 26 Dec 2014 18:25:36 -0400 Subject: MessageCollections + MailboxIndexer --- src/leap/mail/mail.py | 296 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 230 insertions(+), 66 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ea9c95e..ca07f67 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -27,8 +28,17 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +def _get_mdoc_id(mbox, chash): + """ + Get the doc_id for the metamsg document. + """ + return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) + class Message(object): + """ + Represents a single message, and gives access to all its attributes. + """ def __init__(self, wrapper): """ @@ -37,45 +47,56 @@ class Message(object): self._wrapper = wrapper def get_wrapper(self): + """ + Get the wrapper for this message. + """ return self._wrapper # imap.IMessage methods - def get_flags(): + def get_flags(self): """ """ + return tuple(self._wrapper.fdoc.flags) - def get_internal_date(): + def get_internal_date(self): """ """ + return self._wrapper.fdoc.date # imap.IMessageParts - def get_headers(): + def get_headers(self): """ """ + # XXX process here? from imap.messages + return self._wrapper.hdoc.headers - def get_body_file(): + def get_body_file(self): """ """ - def get_size(): + def get_size(self): """ """ + return self._wrapper.fdoc.size - def is_multipart(): + def is_multipart(self): """ """ + return self._wrapper.fdoc.multi - def get_subpart(part): + def get_subpart(self, part): """ """ + # XXX ??? return MessagePart? # Custom methods. - def get_tags(): + def get_tags(self): """ """ + return tuple(self._wrapper.fdoc.tags) class MessageCollection(object): @@ -85,43 +106,174 @@ class MessageCollection(object): master documents. Since LEAP Mail is primarily oriented to store mail in Soledad, the default - (and, so far, only) implementation of the store is contained in this - Soledad Mail Adaptor. If you need to use a different adaptor, change the + (and, so far, only) implementation of the store is contained in the + Soledad Mail Adaptor, which is passed to every collection on creation by + the root Account object. If you need to use a different adaptor, change the adaptor class attribute in your Account object. Store is a reference to a particular instance of the message store (soledad instance or proxy, for instance). """ - # TODO look at IMessageSet methods + # TODO + # [ ] look at IMessageSet methods + # [ ] make constructor with a per-instance deferredLock to use on + # creation/deletion? + # [ ] instead of a mailbox, we could pass an arbitrary container with + # pointers to different doc_ids (type: foo) + # [ ] To guarantee synchronicity of the documents sent together during a + # sync, we could get hold of a deferredLock that inhibits + # synchronization while we are updating (think more about this!) # Account should provide an adaptor instance when creating this collection. adaptor = None store = None + messageklass = Message - def get_message_by_doc_id(self, doc_id): - # ... get from soledad etc - # ... but that should be part of adaptor/store too... :/ - fdoc, hdoc = None - return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + """ + """ + self.adaptor = adaptor + self.store = store - # TODO review if this is the best place for: + # TODO I have to think about what to do when there is no mbox passed to + # the initialization. We could still get the MetaMsg by index, instead + # of by doc_id. See get_message_by_content_hash + self.mbox_indexer = mbox_indexer + self.mbox_wrapper = mbox_wrapper - def create_docs(): - pass + def is_mailbox_collection(self): + """ + Return True if this collection represents a Mailbox. + :rtype: bool + """ + return bool(self.mbox_wrapper) + + # Get messages + + def get_message_by_content_hash(self, chash, get_cdocs=False): + """ + Retrieve a message by its content hash. + :rtype: Deferred + """ + + if not self.is_mailbox_collection(): + # instead of getting the metamsg by chash, query by (meta) index + # or use the internal collection of pointers-to-docs. + raise NotImplementedError() + + metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + metamsg_id, get_cdocs=get_cdocs) + + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): + """ + Retrieve a message by its Unique Identifier. + + If this is a Mailbox collection, that is the message UID, unique for a + given mailbox, or a relative sequence number depending on the absolute + flag. For now, only absolute identifiers are supported. + :rtype: Deferred + """ + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_msg_from_mdoc_id(doc_id): + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + doc_id, get_cdocs=get_cdocs) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d.addCallback(get_msg_from_mdoc_id) + return d + + def count(self): + """ + Count the messages in this collection. + :rtype: int + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.mbox_indexer.count(self.mbox_wrapper.mbox) + + # Manipulate messages + + def add_msg(self, raw_msg): + """ + Add a message to this collection. + """ + msg = self.adaptor.get_msg_from_string(Message, raw_msg) + wrapper = msg.get_wrapper() + + if self.is_mailbox_collection(): + mbox = self.mbox_wrapper.mbox + wrapper.set_mbox(mbox) + + def insert_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.insert_doc( + self.mbox_wrapper.mbox, doc_id) + + d = wrapper.create(self.store) + d.addCallback(insert_mdoc_id) + return d + + def copy_msg(self, msg, newmailbox): + """ + Copy the message to another collection. (it only makes sense for + mailbox collections) + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def insert_copied_mdoc_id(wrapper): + return self.mbox_indexer.insert_doc( + newmailbox, wrapper.mdoc.doc_id) - def udpate_flags(): + wrapper = msg.get_wrapper() + d = wrapper.copy(self.store, newmailbox) + d.addCallback(insert_copied_mdoc_id) + return d + + def delete_msg(self, msg): + """ + Delete this message. + """ + wrapper = msg.get_wrapper() + + def delete_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.delete_doc_by_hash( + self.mbox_wrapper.mbox, doc_id) + d = wrapper.delete(self.store) + d.addCallback(delete_mdoc_id) + return d + + # TODO should add a delete-by-uid to collection? + + def udpate_flags(self, msg, flags, mode): + """ + Update flags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the flags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) + # 2. update the special flags in the wrapper (seen, etc) + # 3. call adaptor.update_msg(store) pass - def update_tags(): + def update_tags(self, msg, tags, mode): + """ + Update tags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the tags in the message wrapper --- stored where??? # 2. call adaptor.update_msg(store) pass - # TODO add delete methods here? - class Account(object): """ @@ -147,8 +299,8 @@ class Account(object): def __init__(self, store): self.store = store self.adaptor = self.adaptor_class() + self.mbox_indexer = MailboxIndexer(self.store) - self.__mailboxes = set([]) self._initialized = False self._deferred_initialization = defer.Deferred() @@ -156,23 +308,16 @@ class Account(object): def _initialize_storage(self): - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: + def add_mailbox_if_none(mboxes): + if not mboxes: self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True self._deferred_initialization.callback(None) - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - d = self.adaptor.initialize_store(self.store) - d.addCallback(load_mbox_cache) + d.addCallback(self.list_all_mailbox_names) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) @@ -185,64 +330,83 @@ class Account(object): self._deferred_initialization.addCallback(cb) return self._deferred_initialization - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) + # + # Public API Starts + # - def _load_mailboxes(self): + def list_all_mailbox_names(self): + def filter_names(mboxes): + return [m.name for m in mboxes] - def update_mailboxes(mbox_names): - self.__mailboxes.update(mbox_names) + d = self.get_all_mailboxes() + d.addCallback(filter_names) + return d + def get_all_mailboxes(self): d = self.adaptor.get_all_mboxes(self.store) - d.addCallback(update_mailboxes) return d - # - # Public API Starts - # + def add_mailbox(self, name): - # XXX params for IMAP only??? - def list_mailboxes(self, ref, wildcard): - self.adaptor.get_all_mboxes(self.store) - - def add_mailbox(self, name, mbox=None): - pass + def create_uid_table_cb(res): + d = self.mbox_uid.create_table(name) + d.addCallback(lambda _: res) + return d - def create_mailbox(self, pathspec): - pass + d = self.adaptor.__class__.get_or_create(name) + d.addCallback(create_uid_table_cb) + return d def delete_mailbox(self, name): - pass + def delete_uid_table_cb(res): + d = self.mbox_uid.delete_table(name) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.delete_mbox(self.store) + d.addCallback(delete_uid_table_cb) + return d def rename_mailbox(self, oldname, newname): - pass + def _rename_mbox(wrapper): + wrapper.mbox = newname + return wrapper.update() - # FIXME yet to be decided if it belongs here... + def rename_uid_table_cb(res): + d = self.mbox_uid.rename_table(oldname, newname) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.__class__.get_or_create(oldname) + d.addCallback(_rename_mbox) + d.addCallback(rename_uid_table_cb) + return d def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection """ # imap select will use this, passing the collection to SoledadMailbox - # XXX pass adaptor to MessageCollection - pass + def get_collection_for_mailbox(mbox_wrapper): + return MessageCollection( + self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + + mboxwrapper_klass = self.adaptor.mboxwrapper_klass + d = mboxwrapper_klass.get_or_create(name) + d.addCallback(get_collection_for_mailbox) + return d def get_collection_by_docs(self, docs): """ :rtype: MessageCollection """ # get a collection of docs by a list of doc_id - # XXX pass adaptor to MessageCollection - pass + # get.docs(...) --> it should be a generator. does it behave in the + # threadpool? + raise NotImplementedError() def get_collection_by_tag(self, tag): """ :rtype: MessageCollection """ - # is this a good idea? - pass + raise NotImplementedError() -- cgit v1.2.3 From 30387b83756ba078d04f4af701e3f595e30d9c50 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 1 Jan 2015 18:21:44 -0400 Subject: cleanup imap implementation --- src/leap/mail/mail.py | 93 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ca07f67..482b64d 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -61,6 +62,18 @@ class Message(object): def get_internal_date(self): """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str """ return self._wrapper.fdoc.date @@ -99,6 +112,15 @@ class Message(object): return tuple(self._wrapper.fdoc.tags) +class Flagsmode(object): + """ + Modes for setting the flags/tags. + """ + APPEND = 1 + REMOVE = -1 + SET = 0 + + class MessageCollection(object): """ A generic collection of messages. It can be messages sharing the same @@ -132,6 +154,7 @@ class MessageCollection(object): def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ + Constructore for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -149,6 +172,20 @@ class MessageCollection(object): """ return bool(self.mbox_wrapper) + @property + def mbox_name(self): + wrapper = getattr(self, "mbox_wrapper", None) + if not wrapper: + return None + return wrapper.mbox + + def get_mbox_attr(self, attr): + return getattr(self.mbox_wrapper, attr) + + def set_mbox_attr(self, attr, value): + setattr(self.mbox_wrapper, attr, value) + return self.mbox_wrapper.update(self.store) + # Get messages def get_message_by_content_hash(self, chash, get_cdocs=False): @@ -162,7 +199,7 @@ class MessageCollection(object): # or use the internal collection of pointers-to-docs. raise NotImplementedError() - metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + metamsg_id = _get_mdoc_id(self.mbox_name, chash) return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, @@ -181,25 +218,37 @@ class MessageCollection(object): raise NotImplementedError("Does not support relative ids yet") def get_msg_from_mdoc_id(doc_id): + # XXX pass UID? return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, doc_id, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) d.addCallback(get_msg_from_mdoc_id) return d def count(self): """ Count the messages in this collection. - :rtype: int + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_wrapper.mbox) + return self.mbox_indexer.count(self.mbox_name) + + def get_uid_next(self): + """ + Get the next integer beyond the highest UID count for this mailbox. + + :return: a Deferred that will fire with the integer for the next uid. + :rtype: Deferred + """ + return self.mbox_indexer.get_uid_next(self.mbox_name) # Manipulate messages + # TODO pass flags, date too... def add_msg(self, raw_msg): """ Add a message to this collection. @@ -208,14 +257,14 @@ class MessageCollection(object): wrapper = msg.get_wrapper() if self.is_mailbox_collection(): - mbox = self.mbox_wrapper.mbox + mbox = self.mbox_name wrapper.set_mbox(mbox) def insert_mdoc_id(_): # XXX does this work? doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( - self.mbox_wrapper.mbox, doc_id) + self.mbox_name, doc_id) d = wrapper.create(self.store) d.addCallback(insert_mdoc_id) @@ -248,31 +297,45 @@ class MessageCollection(object): # XXX does this work? doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( - self.mbox_wrapper.mbox, doc_id) + self.mbox_name, doc_id) d = wrapper.delete(self.store) d.addCallback(delete_mdoc_id) return d # TODO should add a delete-by-uid to collection? + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + def udpate_flags(self, msg, flags, mode): """ Update flags for a given message. """ wrapper = msg.get_wrapper() - # 1. update the flags in the message wrapper --- stored where??? - # 2. update the special flags in the wrapper (seen, etc) - # 3. call adaptor.update_msg(store) - pass + current = wrapper.fdoc.flags + newflags = self._update_flags_or_tags(current, flags, mode) + wrapper.fdoc.flags = newflags + + wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags + wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags + + return self.adaptor.update_msg(self.store, msg) def update_tags(self, msg, tags, mode): """ Update tags for a given message. """ wrapper = msg.get_wrapper() - # 1. update the tags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) - pass + current = wrapper.fdoc.tags + newtags = self._update_flags_or_tags(current, tags, mode) + wrapper.fdoc.tags = newtags + return self.adaptor.update_msg(self.store, msg) class Account(object): @@ -382,6 +445,8 @@ class Account(object): d.addCallback(rename_uid_table_cb) return d + # Get Collections + def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection -- cgit v1.2.3 From 60eecafcc8a516985f900ef81bc7941a5234930a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 4 Jan 2015 03:37:18 -0400 Subject: tests for mail.mail module: Message --- src/leap/mail/mail.py | 165 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 147 insertions(+), 18 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 482b64d..55e50f7 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,15 +17,24 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import logging +import StringIO + from twisted.internet import defer +from leap.common.check import leap_assert_type +from leap.common.mail import get_email_charset + +from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.utils import empty, find_charset + +logger = logging.getLogger(name=__name__) -# TODO +# TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail @@ -36,16 +45,92 @@ def _get_mdoc_id(mbox, chash): return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) +def _write_and_rewind(payload): + fd = StringIO.StringIO() + fd.write(payload) + fd.seek(0) + return fd + + +class MessagePart(object): + + # TODO pass cdocs in init + + def __init__(self, part_map, cdocs={}): + self._pmap = part_map + self._cdocs = cdocs + + def get_size(self): + return self._pmap['size'] + + def get_body_file(self): + pmap = self._pmap + multi = pmap.get('multi') + if not multi: + phash = pmap.get("phash") + else: + pmap_ = pmap.get('part_map') + first_part = pmap_.get('1', None) + if not empty(first_part): + phash = first_part['phash'] + else: + phash = "" + + payload = self._get_payload(phash) + + if payload: + # FIXME + # content_type = self._get_ctype_from_document(phash) + # charset = find_charset(content_type) + charset = None + if charset is None: + charset = get_email_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + + return _write_and_rewind(payload) + + def get_headers(self): + return self._pmap.get("headers", []) + + def is_multipart(self): + multi = self._pmap.get("multi", False) + return multi + + def get_subpart(self, part): + if not self.is_multipart(): + raise TypeError + + sub_pmap = self._pmap.get("part_map", {}) + try: + part_map = sub_pmap[str(part + 1)] + except KeyError: + logger.debug("getSubpart for %s: KeyError" % (part,)) + raise IndexError + return MessagePart(self._soledad, part_map) + + def _get_payload(self, phash): + return self._cdocs.get(phash, "") + + class Message(object): """ Represents a single message, and gives access to all its attributes. """ - def __init__(self, wrapper): + def __init__(self, wrapper, uid=None): """ :param wrapper: an instance of an implementor of IMessageWrapper + :param uid: + :type uid: int """ self._wrapper = wrapper + self._uid = uid def get_wrapper(self): """ @@ -53,10 +138,18 @@ class Message(object): """ return self._wrapper + def get_uid(self): + """ + Get the (optional) UID. + """ + return self._uid + # imap.IMessage methods def get_flags(self): """ + Get flags for this message. + :rtype: tuple """ return tuple(self._wrapper.fdoc.flags) @@ -81,28 +174,50 @@ class Message(object): def get_headers(self): """ + Get the raw headers document. """ - # XXX process here? from imap.messages - return self._wrapper.hdoc.headers + return [tuple(item) for item in self._wrapper.hdoc.headers] - def get_body_file(self): + def get_body_file(self, store): """ """ + def write_and_rewind_if_found(cdoc): + if not cdoc: + return None + return _write_and_rewind(cdoc.raw) + + d = defer.maybeDeferred(self._wrapper.get_body, store) + d.addCallback(write_and_rewind_if_found) + return d def get_size(self): """ + Size, in octets. """ return self._wrapper.fdoc.size def is_multipart(self): """ + Return True if this message is multipart. """ return self._wrapper.fdoc.multi def get_subpart(self, part): """ - """ - # XXX ??? return MessagePart? + :param part: The number of the part to retrieve, indexed from 0. + :type part: int + :rtype: MessagePart + """ + if not self.is_multipart(): + raise TypeError + part_index = part + 1 + try: + subpart_dict = self._wrapper.get_subpart_dict( + part_index) + except KeyError: + raise TypeError + # XXX pass cdocs + return MessagePart(subpart_dict) # Custom methods. @@ -137,7 +252,7 @@ class MessageCollection(object): instance or proxy, for instance). """ - # TODO + # TODO LIST # [ ] look at IMessageSet methods # [ ] make constructor with a per-instance deferredLock to use on # creation/deletion? @@ -159,7 +274,7 @@ class MessageCollection(object): self.adaptor = adaptor self.store = store - # TODO I have to think about what to do when there is no mbox passed to + # XXX I have to think about what to do when there is no mbox passed to # the initialization. We could still get the MetaMsg by index, instead # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer @@ -218,10 +333,11 @@ class MessageCollection(object): raise NotImplementedError("Does not support relative ids yet") def get_msg_from_mdoc_id(doc_id): - # XXX pass UID? + if doc_id is None: + return None return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, - doc_id, get_cdocs=get_cdocs) + doc_id, uid=uid, get_cdocs=get_cdocs) d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) d.addCallback(get_msg_from_mdoc_id) @@ -248,26 +364,37 @@ class MessageCollection(object): # Manipulate messages - # TODO pass flags, date too... - def add_msg(self, raw_msg): + def add_msg(self, raw_msg, flags=None, tags=None, date=None): """ Add a message to this collection. """ + if not flags: + flags = tuple() + if not tags: + tags = tuple() + leap_assert_type(flags, tuple) + leap_assert_type(date, str) + msg = self.adaptor.get_msg_from_string(Message, raw_msg) wrapper = msg.get_wrapper() - if self.is_mailbox_collection(): + if not self.is_mailbox_collection(): + raise NotImplementedError() + + else: mbox = self.mbox_name + wrapper.set_flags(flags) + wrapper.set_tags(tags) + wrapper.set_date(date) wrapper.set_mbox(mbox) - def insert_mdoc_id(_): - # XXX does this work? + def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( self.mbox_name, doc_id) d = wrapper.create(self.store) - d.addCallback(insert_mdoc_id) + d.addCallback(insert_mdoc_id, wrapper) return d def copy_msg(self, msg, newmailbox): @@ -338,6 +465,8 @@ class MessageCollection(object): return self.adaptor.update_msg(self.store, msg) +# TODO -------------------- split into account object? + class Account(object): """ Account is the top level abstraction to access collections of messages -- cgit v1.2.3 From 751ace7534280187ecc83efb32e0cf9e18ac5bb2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 01:31:26 -0400 Subject: tests for mail.mail module: MessageCollection --- src/leap/mail/mail.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 55e50f7..671642a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -420,13 +420,12 @@ class MessageCollection(object): """ wrapper = msg.get_wrapper() - def delete_mdoc_id(_): - # XXX does this work? + def delete_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( self.mbox_name, doc_id) d = wrapper.delete(self.store) - d.addCallback(delete_mdoc_id) + d.addCallback(delete_mdoc_id, wrapper) return d # TODO should add a delete-by-uid to collection? -- cgit v1.2.3 From 1b457bbe0eefa12d3e75b58247b53cc62aecc356 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 02:19:02 -0400 Subject: tests for mail.mail module: Account --- src/leap/mail/mail.py | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 671642a..0c9b7a3 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -37,6 +37,9 @@ logger = logging.getLogger(name=__name__) # TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +# [ ] Change the doc_ids scheme for part-docs: use mailbox UID validity +# identifier, instead of name! (renames are broken!) +# [ ] Profile add_msg. def _get_mdoc_id(mbox, chash): """ @@ -360,7 +363,7 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_uid_next(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_name) # Manipulate messages @@ -464,8 +467,6 @@ class MessageCollection(object): return self.adaptor.update_msg(self.store, msg) -# TODO -------------------- split into account object? - class Account(object): """ Account is the top level abstraction to access collections of messages @@ -485,7 +486,6 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - mailboxes = None def __init__(self, store): self.store = store @@ -508,17 +508,17 @@ class Account(object): self._deferred_initialization.callback(None) d = self.adaptor.initialize_store(self.store) - d.addCallback(self.list_all_mailbox_names) + d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) - def callWhenReady(self, cb): - # XXX this could use adaptor.store_ready instead...?? + def callWhenReady(self, cb, *args, **kw): + # use adaptor.store_ready instead? if self._initialized: - cb(self) + cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb) + self._deferred_initialization.addCallback(cb, *args, **kw) return self._deferred_initialization # @@ -527,7 +527,7 @@ class Account(object): def list_all_mailbox_names(self): def filter_names(mboxes): - return [m.name for m in mboxes] + return [m.mbox for m in mboxes] d = self.get_all_mailboxes() d.addCallback(filter_names) @@ -540,35 +540,44 @@ class Account(object): def add_mailbox(self, name): def create_uid_table_cb(res): - d = self.mbox_uid.create_table(name) + d = self.mbox_indexer.create_table(name) d.addCallback(lambda _: res) return d - d = self.adaptor.__class__.get_or_create(name) + d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(create_uid_table_cb) return d def delete_mailbox(self, name): def delete_uid_table_cb(res): - d = self.mbox_uid.delete_table(name) + d = self.mbox_indexer.delete_table(name) d.addCallback(lambda _: res) return d - d = self.adaptor.delete_mbox(self.store) + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback( + lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) d.addCallback(delete_uid_table_cb) return d def rename_mailbox(self, oldname, newname): + # TODO incomplete/wrong!!! + # Should rename also ALL of the document ids that are pointing + # to the old mailbox!!! + + # TODO part-docs identifiers should have the UID_validity of the + # mailbox embedded, instead of the name! (so they can survive a rename) + def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update() + return wrapper.update(self.store) def rename_uid_table_cb(res): - d = self.mbox_uid.rename_table(oldname, newname) + d = self.mbox_indexer.rename_table(oldname, newname) d.addCallback(lambda _: res) return d - d = self.adaptor.__class__.get_or_create(oldname) + d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) d.addCallback(rename_uid_table_cb) return d @@ -585,7 +594,8 @@ class Account(object): self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) mboxwrapper_klass = self.adaptor.mboxwrapper_klass - d = mboxwrapper_klass.get_or_create(name) + #d = mboxwrapper_klass.get_or_create(name) + d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d -- cgit v1.2.3 From c1fc9b52d8b577814e921d128357afdbd9278662 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 12 Jan 2015 20:47:29 -0400 Subject: Use mailbox uuids The previous implementation is naive, since it imposes a burden when renaming mailboxes. We're using uuids in the local uid tables instead, which is more cryptic but way more efficient. * receive mbox uuid instead of name * use mailbox uuid in identifiers --- src/leap/mail/mail.py | 57 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 17 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 0c9b7a3..b2caa33 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,6 +17,7 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import uuid import logging import StringIO @@ -283,6 +284,20 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + # TODO need to initialize count here because imap server does not + # expect a defered for the count. caller should return the deferred for + # prime_count (ie, initialize) when returning the collection + # TODO should increment and decrement when adding/deleting. + # TODO recent count should also be static. + + if not count: + count = 0 + self._count = count + + #def initialize(self): + #d = self.prime_count() + #return d + def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. @@ -297,6 +312,13 @@ class MessageCollection(object): return None return wrapper.mbox + @property + def mbox_uuid(self): + wrapper = getattr(self, "mbox_wrapper", None) + if not wrapper: + return None + return wrapper.mbox_uuid + def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -385,16 +407,16 @@ class MessageCollection(object): raise NotImplementedError() else: - mbox = self.mbox_name + mbox_id = self.mbox_uuid wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox(mbox) + wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( - self.mbox_name, doc_id) + self.mbox_uuid, doc_id) d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) @@ -410,7 +432,7 @@ class MessageCollection(object): def insert_copied_mdoc_id(wrapper): return self.mbox_indexer.insert_doc( - newmailbox, wrapper.mdoc.doc_id) + newmailbox_uuid, wrapper.mdoc.doc_id) wrapper = msg.get_wrapper() d = wrapper.copy(self.store, newmailbox) @@ -539,25 +561,32 @@ class Account(object): def add_mailbox(self, name): - def create_uid_table_cb(res): - d = self.mbox_indexer.create_table(name) - d.addCallback(lambda _: res) + def create_uuid(wrapper): + if not wrapper.uuid: + wrapper.uuid = uuid.uuid4() + return wrapper.update(self.store) + + def create_uid_table_cb(wrapper): + d = self.mbox_indexer.create_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(create_uuid) d.addCallback(create_uid_table_cb) return d def delete_mailbox(self, name): - def delete_uid_table_cb(res): - d = self.mbox_indexer.delete_table(name) - d.addCallback(lambda _: res) + + def delete_uid_table_cb(wrapper): + d = self.mbox_indexer.delete_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(delete_uid_table_cb) d.addCallback( lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) - d.addCallback(delete_uid_table_cb) return d def rename_mailbox(self, oldname, newname): @@ -572,14 +601,8 @@ class Account(object): wrapper.mbox = newname return wrapper.update(self.store) - def rename_uid_table_cb(res): - d = self.mbox_indexer.rename_table(oldname, newname) - d.addCallback(lambda _: res) - return d - d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) - d.addCallback(rename_uid_table_cb) return d # Get Collections -- cgit v1.2.3 From 68500fb15dbb7531eeb397ccee2c160d71284d97 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 7 Jan 2015 12:12:24 -0400 Subject: Complete IMAP implementation, update tests --- src/leap/mail/mail.py | 114 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 21 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b2caa33..8137f97 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -172,7 +172,7 @@ class Message(object): :return: An RFC822-formatted date string. :rtype: str """ - return self._wrapper.fdoc.date + return self._wrapper.hdoc.date # imap.IMessageParts @@ -271,9 +271,10 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, + count=None): """ - Constructore for a MessageCollection. + Constructor for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -317,7 +318,7 @@ class MessageCollection(object): wrapper = getattr(self, "mbox_wrapper", None) if not wrapper: return None - return wrapper.mbox_uuid + return wrapper.uuid def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -364,10 +365,18 @@ class MessageCollection(object): self.messageklass, self.store, doc_id, uid=uid, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) d.addCallback(get_msg_from_mdoc_id) return d + # TODO deprecate ??? --- + def _prime_count(self): + def update_count(count): + self._count = count + d = self.mbox_indexer.count(self.mbox_name) + d.addCallback(update_count) + return d + def count(self): """ Count the messages in this collection. @@ -376,7 +385,17 @@ class MessageCollection(object): """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_name) + + d = self.mbox_indexer.count(self.mbox_uuid) + return d + + def count_recent(self): + # FIXME HACK + return 0 + + def count_unseen(self): + # FIXME hack + return 0 def get_uid_next(self): """ @@ -385,7 +404,13 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_next_uid(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_uuid) + + def all_uid_iter(self): + """ + Iterator through all the uids for this collection. + """ + return self.mbox_indexer.all_uid_iter(self.mbox_uuid) # Manipulate messages @@ -397,6 +422,7 @@ class MessageCollection(object): flags = tuple() if not tags: tags = tuple() + leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -408,10 +434,10 @@ class MessageCollection(object): else: mbox_id = self.mbox_uuid + wrapper.set_mbox_uuid(mbox_id) wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id @@ -420,6 +446,8 @@ class MessageCollection(object): d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) + d.addErrback(lambda f: f.printTraceback()) + return d def copy_msg(self, msg, newmailbox): @@ -453,8 +481,47 @@ class MessageCollection(object): d.addCallback(delete_mdoc_id, wrapper) return d + def delete_all_flagged(self): + """ + Delete all messages flagged as \\Deleted. + Used from IMAPMailbox.expunge() + """ + def get_uid_list(hashes): + d = [] + for h in hashes: + d.append(self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, h)) + return defer.gatherResults(d), hashes + + def delete_uid_entries((uids, hashes)): + d = [] + for h in hashes: + d.append(self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, h)) + return defer.gatherResults(d).addCallback( + lambda _: uids) + + mdocs_deleted = self.adaptor.del_all_flagged_messages( + self.store, self.mbox_uuid) + mdocs_deleted.addCallback(get_uid_list) + mdocs_deleted.addCallback(delete_uid_entries) + return mdocs_deleted + # TODO should add a delete-by-uid to collection? + def delete_all_docs(self): + def del_all_uid(uid_list): + deferreds = [] + for uid in uid_list: + d = self.get_message_by_uid(uid) + d.addCallback(lambda msg: msg.delete()) + deferreds.append(d) + return defer.gatherResults(deferreds) + + d = self.all_uid_iter() + d.addCallback(del_all_uid) + return d + def _update_flags_or_tags(self, old, new, mode): if mode == Flagsmode.APPEND: final = list((set(tuple(old) + new))) @@ -509,45 +576,49 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - def __init__(self, store): + def __init__(self, store, ready_cb=None): self.store = store self.adaptor = self.adaptor_class() self.mbox_indexer = MailboxIndexer(self.store) + self.deferred_initialization = defer.Deferred() self._initialized = False - self._deferred_initialization = defer.Deferred() + self._ready_cb = ready_cb - self._initialize_storage() + self._init_d = self._initialize_storage() def _initialize_storage(self): def add_mailbox_if_none(mboxes): if not mboxes: - self.add_mailbox(INBOX_NAME) + return self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True - self._deferred_initialization.callback(None) + self.deferred_initialization.callback(None) + if self._ready_cb is not None: + self._ready_cb() d = self.adaptor.initialize_store(self.store) d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) + return d def callWhenReady(self, cb, *args, **kw): - # use adaptor.store_ready instead? if self._initialized: cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb, *args, **kw) - return self._deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts # def list_all_mailbox_names(self): + def filter_names(mboxes): return [m.mbox for m in mboxes] @@ -563,8 +634,11 @@ class Account(object): def create_uuid(wrapper): if not wrapper.uuid: - wrapper.uuid = uuid.uuid4() - return wrapper.update(self.store) + wrapper.uuid = str(uuid.uuid4()) + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + return wrapper def create_uid_table_cb(wrapper): d = self.mbox_indexer.create_table(wrapper.uuid) @@ -599,7 +673,7 @@ class Account(object): def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update(self.store) + return wrapper, wrapper.update(self.store) d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) @@ -616,8 +690,6 @@ class Account(object): return MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - mboxwrapper_klass = self.adaptor.mboxwrapper_klass - #d = mboxwrapper_klass.get_or_create(name) d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d -- cgit v1.2.3 From c13e17114cfb58acc4911ed98c367faf47717ec0 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 14 Jan 2015 13:50:02 -0600 Subject: Refactor fetch into leap.mail.incoming IService --- src/leap/mail/mail.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8137f97..cb37d25 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -414,15 +414,10 @@ class MessageCollection(object): # Manipulate messages - def add_msg(self, raw_msg, flags=None, tags=None, date=None): + def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date=""): """ Add a message to this collection. """ - if not flags: - flags = tuple() - if not tags: - tags = tuple() - leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -582,7 +577,6 @@ class Account(object): self.mbox_indexer = MailboxIndexer(self.store) self.deferred_initialization = defer.Deferred() - self._initialized = False self._ready_cb = ready_cb self._init_d = self._initialize_storage() @@ -594,7 +588,6 @@ class Account(object): return self.add_mailbox(INBOX_NAME) def finish_initialization(result): - self._initialized = True self.deferred_initialization.callback(None) if self._ready_cb is not None: self._ready_cb() @@ -606,12 +599,8 @@ class Account(object): return d def callWhenReady(self, cb, *args, **kw): - if self._initialized: - cb(self, *args, **kw) - return defer.succeed(None) - else: - self.deferred_initialization.addCallback(cb, *args, **kw) - return self.deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts -- cgit v1.2.3 From 22d2bf496de91690bbced08fbb2e24e6f5040779 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 15 Jan 2015 15:50:20 -0400 Subject: update mail/imap tests --- src/leap/mail/mail.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index cb37d25..8629d0e 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -285,6 +285,8 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + # TODO --- review this count shit. I think it's better to patch server + # to accept deferreds. # TODO need to initialize count here because imap server does not # expect a defered for the count. caller should return the deferred for # prime_count (ie, initialize) when returning the collection @@ -295,10 +297,6 @@ class MessageCollection(object): count = 0 self._count = count - #def initialize(self): - #d = self.prime_count() - #return d - def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. -- cgit v1.2.3 From 8c65f09a16e4e00452dffa7d72771d9fac21c9c0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 13:48:21 -0400 Subject: imap: complete FETCH implementation --- src/leap/mail/mail.py | 49 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8629d0e..976df5a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -155,7 +155,7 @@ class Message(object): Get flags for this message. :rtype: tuple """ - return tuple(self._wrapper.fdoc.flags) + return self._wrapper.fdoc.get_flags() def get_internal_date(self): """ @@ -184,6 +184,7 @@ class Message(object): def get_body_file(self, store): """ + Get a file descriptor with the body content. """ def write_and_rewind_if_found(cdoc): if not cdoc: @@ -367,14 +368,36 @@ class MessageCollection(object): d.addCallback(get_msg_from_mdoc_id) return d - # TODO deprecate ??? --- - def _prime_count(self): - def update_count(count): - self._count = count - d = self.mbox_indexer.count(self.mbox_name) - d.addCallback(update_count) + def get_flags_by_uid(self, uid, absolute=True): + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_flags_from_mdoc_id(doc_id): + if doc_id is None: # XXX needed? or bug? + return None + return self.adaptor.get_flags_from_mdoc_id( + self.store, doc_id) + + def wrap_in_tuple(flags): + return (uid, flags) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) + d.addCallback(get_flags_from_mdoc_id) + d.addCallback(wrap_in_tuple) return d + # TODO ------------------------------ FIXME FIXME FIXME implement this! + def set_flags(self, *args, **kw): + pass + + # TODO deprecate ??? --- + #def _prime_count(self): + #def update_count(count): + #self._count = count + #d = self.mbox_indexer.count(self.mbox_name) + #d.addCallback(update_count) + #return d + def count(self): """ Count the messages in this collection. @@ -389,11 +412,13 @@ class MessageCollection(object): def count_recent(self): # FIXME HACK - return 0 + # TODO ------------------------ implement this + return 3 def count_unseen(self): # FIXME hack - return 0 + # TODO ------------------------ implement this + return 3 def get_uid_next(self): """ @@ -404,6 +429,12 @@ class MessageCollection(object): """ return self.mbox_indexer.get_next_uid(self.mbox_uuid) + def get_last_uid(self): + """ + Get the last UID for this mailbox. + """ + return self.mbox_indexer.get_last_uid(self.mbox_uuid) + def all_uid_iter(self): """ Iterator through all the uids for this collection. -- cgit v1.2.3 From d327d8416475493864b1e0922b51812be566e028 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 21 Jan 2015 00:50:27 -0400 Subject: imap: implement setting of message flags --- src/leap/mail/mail.py | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 976df5a..9b7a562 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -272,8 +272,7 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, - count=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ Constructor for a MessageCollection. """ @@ -288,15 +287,6 @@ class MessageCollection(object): # TODO --- review this count shit. I think it's better to patch server # to accept deferreds. - # TODO need to initialize count here because imap server does not - # expect a defered for the count. caller should return the deferred for - # prime_count (ie, initialize) when returning the collection - # TODO should increment and decrement when adding/deleting. - # TODO recent count should also be static. - - if not count: - count = 0 - self._count = count def is_mailbox_collection(self): """ @@ -386,18 +376,6 @@ class MessageCollection(object): d.addCallback(wrap_in_tuple) return d - # TODO ------------------------------ FIXME FIXME FIXME implement this! - def set_flags(self, *args, **kw): - pass - - # TODO deprecate ??? --- - #def _prime_count(self): - #def update_count(count): - #self._count = count - #d = self.mbox_indexer.count(self.mbox_name) - #d.addCallback(update_count) - #return d - def count(self): """ Count the messages in this collection. @@ -479,6 +457,7 @@ class MessageCollection(object): Copy the message to another collection. (it only makes sense for mailbox collections) """ + # TODO currently broken ------------------FIXME- if not self.is_mailbox_collection(): raise NotImplementedError() @@ -555,19 +534,21 @@ class MessageCollection(object): final = new return final - def udpate_flags(self, msg, flags, mode): + def update_flags(self, msg, flags, mode): """ Update flags for a given message. """ wrapper = msg.get_wrapper() current = wrapper.fdoc.flags - newflags = self._update_flags_or_tags(current, flags, mode) + newflags = map(str, self._update_flags_or_tags(current, flags, mode)) wrapper.fdoc.flags = newflags wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags - return self.adaptor.update_msg(self.store, msg) + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(lambda _: newflags) + return d def update_tags(self, msg, tags, mode): """ @@ -576,8 +557,11 @@ class MessageCollection(object): wrapper = msg.get_wrapper() current = wrapper.fdoc.tags newtags = self._update_flags_or_tags(current, tags, mode) + wrapper.fdoc.tags = newtags - return self.adaptor.update_msg(self.store, msg) + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(newtags) + return d class Account(object): -- cgit v1.2.3 From aab87b503184619b5637d6b326ec7717e34dfb99 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:20:37 -0400 Subject: lots of little fixes after meskio's review mostly having to do with poor, missing or outdated documentation, naming of confusing things and reordering of code blocks for improved readability. --- src/leap/mail/mail.py | 70 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 25 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 9b7a562..59fd57c 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -58,9 +58,16 @@ def _write_and_rewind(payload): class MessagePart(object): - # TODO pass cdocs in init - def __init__(self, part_map, cdocs={}): + """ + :param part_map: a dictionary mapping the subparts for + this MessagePart (1-indexed). + :type part_map: dict + :param cdoc: optional, a dict of content documents + """ + # TODO document the expected keys in the part_map dict. + # TODO add abstraction layer between the cdocs and this class. Only + # adaptor should know about the format of the cdocs. self._pmap = part_map self._cdocs = cdocs @@ -266,6 +273,10 @@ class MessageCollection(object): # [ ] To guarantee synchronicity of the documents sent together during a # sync, we could get hold of a deferredLock that inhibits # synchronization while we are updating (think more about this!) + # [ ] review the serveral count_ methods. I think it's better to patch + # server to accept deferreds. + # [ ] Use inheritance for the mailbox-collection instead of handling the + # special cases everywhere? # Account should provide an adaptor instance when creating this collection. adaptor = None @@ -285,9 +296,6 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper - # TODO --- review this count shit. I think it's better to patch server - # to accept deferreds. - def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. @@ -297,22 +305,26 @@ class MessageCollection(object): @property def mbox_name(self): - wrapper = getattr(self, "mbox_wrapper", None) - if not wrapper: + # TODO raise instead? + if self.mbox_wrapper is None: return None - return wrapper.mbox + return self.mbox_wrapper.mbox @property def mbox_uuid(self): - wrapper = getattr(self, "mbox_wrapper", None) - if not wrapper: + # TODO raise instead? + if self.mbox_wrapper is None: return None - return wrapper.uuid + return self.mbox_wrapper.uuid def get_mbox_attr(self, attr): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") return getattr(self.mbox_wrapper, attr) def set_mbox_attr(self, attr, value): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") setattr(self.mbox_wrapper, attr, value) return self.mbox_wrapper.update(self.store) @@ -323,10 +335,10 @@ class MessageCollection(object): Retrieve a message by its content hash. :rtype: Deferred """ - if not self.is_mailbox_collection(): - # instead of getting the metamsg by chash, query by (meta) index - # or use the internal collection of pointers-to-docs. + # TODO instead of getting the metamsg by chash, in this case we + # should query by (meta) index or use the internal collection of + # pointers-to-docs. raise NotImplementedError() metamsg_id = _get_mdoc_id(self.mbox_name, chash) @@ -462,6 +474,9 @@ class MessageCollection(object): raise NotImplementedError() def insert_copied_mdoc_id(wrapper): + # TODO this needs to be implemented before the copy + # interface works. + newmailbox_uuid = get_mbox_uuid_from_msg_wrapper(wrapper) return self.mbox_indexer.insert_doc( newmailbox_uuid, wrapper.mdoc.doc_id) @@ -525,15 +540,6 @@ class MessageCollection(object): d.addCallback(del_all_uid) return d - def _update_flags_or_tags(self, old, new, mode): - if mode == Flagsmode.APPEND: - final = list((set(tuple(old) + new))) - elif mode == Flagsmode.REMOVE: - final = list(set(old).difference(set(new))) - elif mode == Flagsmode.SET: - final = new - return final - def update_flags(self, msg, flags, mode): """ Update flags for a given message. @@ -563,6 +569,15 @@ class MessageCollection(object): d.addCallback(newtags) return d + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + class Account(object): """ @@ -573,7 +588,7 @@ class Account(object): basic collection handled by traditional MUAs, but it can also handle other types of Collections (tag based, for instance). - leap.mail.imap.SoledadBackedAccount partially proxies methods in this + leap.mail.imap.IMAPAccount partially proxies methods in this class. """ @@ -582,7 +597,6 @@ class Account(object): # the Account class. adaptor_class = SoledadMailAdaptor - store = None def __init__(self, store, ready_cb=None): self.store = store @@ -612,6 +626,12 @@ class Account(object): return d def callWhenReady(self, cb, *args, **kw): + """ + Execute the callback when the initialization of the Account is ready. + Note that the callback will receive a first meaningless parameter. + """ + # TODO this should ignore the first parameter explicitely + # lambda _: cb(*args, **kw) self.deferred_initialization.addCallback(cb, *args, **kw) return self.deferred_initialization -- cgit v1.2.3 From 98def315e5f48df6eec713dbe175df8bdfe406dd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 03:16:32 -0400 Subject: re-add support for basic multipart messages --- src/leap/mail/mail.py | 102 +++++++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 42 deletions(-) (limited to 'src/leap/mail/mail.py') diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 59fd57c..aa499c0 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -22,6 +22,7 @@ import logging import StringIO from twisted.internet import defer +from twisted.python import log from leap.common.check import leap_assert_type from leap.common.mail import get_email_charset @@ -30,7 +31,7 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.utils import empty, find_charset +from leap.mail.utils import empty # find_charset logger = logging.getLogger(name=__name__) @@ -57,61 +58,57 @@ def _write_and_rewind(payload): class MessagePart(object): + # TODO This class should be better abstracted from the data model. + # TODO support arbitrarily nested multiparts (right now we only support + # the trivial case) - def __init__(self, part_map, cdocs={}): + def __init__(self, part_map, index=1, cdocs={}): """ :param part_map: a dictionary mapping the subparts for this MessagePart (1-indexed). :type part_map: dict - :param cdoc: optional, a dict of content documents + + The format for the part_map is as follows: + + {u'1': {u'ctype': u'text/plain', + u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], + [u'Content-Transfer-Encoding', u'8bit']], + u'multi': False, + u'parts': 1, + u'phash': u'02D82B29F6BB0C8612D1C', + u'size': 132}} + + :param index: which index in the content-doc is this subpart + representing. + :param cdocs: optional, a reference to the top-level dict of wrappers + for content-docs (1-indexed). """ - # TODO document the expected keys in the part_map dict. - # TODO add abstraction layer between the cdocs and this class. Only - # adaptor should know about the format of the cdocs. + # TODO: Pass only the cdoc wrapper for this part. self._pmap = part_map + self._index = index self._cdocs = cdocs def get_size(self): return self._pmap['size'] def get_body_file(self): + payload = "" pmap = self._pmap multi = pmap.get('multi') if not multi: - phash = pmap.get("phash") + payload = self._get_payload(self._index) else: - pmap_ = pmap.get('part_map') - first_part = pmap_.get('1', None) - if not empty(first_part): - phash = first_part['phash'] - else: - phash = "" - - payload = self._get_payload(phash) - + # XXX uh, multi also... should recurse" + raise NotImplementedError if payload: - # FIXME - # content_type = self._get_ctype_from_document(phash) - # charset = find_charset(content_type) - charset = None - if charset is None: - charset = get_email_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - + payload = self._format_payload(payload) return _write_and_rewind(payload) def get_headers(self): return self._pmap.get("headers", []) def is_multipart(self): - multi = self._pmap.get("multi", False) - return multi + return self._pmap.get("multi", False) def get_subpart(self, part): if not self.is_multipart(): @@ -123,10 +120,30 @@ class MessagePart(object): except KeyError: logger.debug("getSubpart for %s: KeyError" % (part,)) raise IndexError - return MessagePart(self._soledad, part_map) - - def _get_payload(self, phash): - return self._cdocs.get(phash, "") + return MessagePart(part_map, cdocs={1: self._cdocs.get(1, {})}) + + def _get_payload(self, index): + cdoc_wrapper = self._cdocs.get(index, None) + if cdoc_wrapper: + return cdoc_wrapper.raw + return "" + + def _format_payload(self, payload): + # FIXME ----------------------------------------------- + # Test against unicode payloads... + # content_type = self._get_ctype_from_document(phash) + # charset = find_charset(content_type) + charset = None + if charset is None: + charset = get_email_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + return payload class Message(object): @@ -224,17 +241,18 @@ class Message(object): raise TypeError part_index = part + 1 try: - subpart_dict = self._wrapper.get_subpart_dict( - part_index) + subpart_dict = self._wrapper.get_subpart_dict(part_index) except KeyError: - raise TypeError - # XXX pass cdocs - return MessagePart(subpart_dict) + raise IndexError + + return MessagePart( + subpart_dict, index=part_index, cdocs=self._wrapper.cdocs) # Custom methods. def get_tags(self): """ + Get the tags for this message. """ return tuple(self._wrapper.fdoc.tags) @@ -290,7 +308,7 @@ class MessageCollection(object): self.adaptor = adaptor self.store = store - # XXX I have to think about what to do when there is no mbox passed to + # XXX think about what to do when there is no mbox passed to # the initialization. We could still get the MetaMsg by index, instead # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer -- cgit v1.2.3