summaryrefslogtreecommitdiff
path: root/src/leap/mail/imap/messages.py
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2015-01-01 18:21:44 -0400
committerKali Kaneko <kali@leap.se>2015-02-11 14:05:43 -0400
commitfa60b76ce9cdf6684945c6bc724f10818104166b (patch)
tree8c11ca5d65aaa54c3a3ab2fd5b1db59110a0621f /src/leap/mail/imap/messages.py
parent4fa3d27225f44d9598d24d9e59a97baaa9bbb90e (diff)
cleanup imap implementation
Diffstat (limited to 'src/leap/mail/imap/messages.py')
-rw-r--r--src/leap/mail/imap/messages.py1007
1 files changed, 181 insertions, 826 deletions
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index d47c8eb..7e0f973 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# messages.py
-# Copyright (C) 2013, 2014 LEAP
+# imap/messages.py
+# Copyright (C) 2013-2015 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,85 +15,41 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-LeapMessage and MessageCollection.
+IMAPMessage and IMAPMessageCollection.
"""
-import copy
import logging
-import threading
-import StringIO
-
-from collections import defaultdict
-from functools import partial
-
+# import StringIO
from twisted.mail import imap4
-from twisted.internet import reactor
from zope.interface import implements
-from zope.proxy import sameProxiedObjects
from leap.common.check import leap_assert, leap_assert_type
from leap.common.decorators import memoized_method
from leap.common.mail import get_email_charset
-from leap.mail.adaptors import soledad_indexes as indexes
-from leap.mail.constants import INBOX_NAME
-from leap.mail.utils import find_charset, empty
-from leap.mail.imap.index import IndexedDB
-from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
-from leap.mail.imap.parser import MBoxParser
-
-logger = logging.getLogger(__name__)
-
-# TODO ------------------------------------------------------------
-# [ ] Add ref to incoming message during add_msg
-# [ ] Add linked-from info.
-# * Need a new type of documents: linkage info.
-# * HDOCS are linked from FDOCs (ref to chash)
-# * CDOCS are linked from HDOCS (ref to chash)
+from leap.mail.utils import find_charset
-# [ ] Delete incoming mail only after successful write!
-# [ ] Remove UID from syncable db. Store only those indexes locally.
+from leap.mail.imap.messageparts import MessagePart
+# from leap.mail.imap.messagepargs import MessagePartDoc
+logger = logging.getLogger(__name__)
-def try_unique_query(curried):
- """
- Try to execute a query that is expected to have a
- single outcome, and log a warning if more than one document found.
-
- :param curried: a curried function
- :type curried: callable
- """
- # XXX FIXME ---------- convert to deferreds
- leap_assert(callable(curried), "A callable is expected")
- try:
- query = curried()
- if query:
- if len(query) > 1:
- # TODO we could take action, like trigger a background
- # process to kill dupes.
- name = getattr(curried, 'expected', 'doc')
- logger.warning(
- "More than one %s found for this mbox, "
- "we got a duplicate!!" % (name,))
- return query.pop()
- else:
- return None
- except Exception as exc:
- logger.exception("Unhandled error %r" % exc)
-
+# TODO ------------------------------------------------------------
-# FIXME remove-me
-#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+# [ ] Add ref to incoming message during add_msg.
+# [ ] Delete incoming mail only after successful write.
-class IMAPMessage(fields, MBoxParser):
+class IMAPMessage(object):
"""
The main representation of a message.
"""
implements(imap4.IMessage)
- def __init__(self, soledad, uid, mbox):
+ # TODO ---- see what should we pass here instead
+ # where's UID added to the message?
+ # def __init__(self, soledad, uid, mbox):
+ def __init__(self, message, collection):
"""
Initializes a LeapMessage.
@@ -103,81 +59,14 @@ class IMAPMessage(fields, MBoxParser):
:type uid: int or basestring
:param mbox: the mbox this message belongs to
:type mbox: str or unicode
- :param collection: a reference to the parent collection object
- :type collection: MessageCollection
- :param container: a IMessageContainer implementor instance
- :type container: IMessageContainer
"""
- self._soledad = soledad
- self._uid = int(uid) if uid is not None else None
- self._mbox = self._parse_mailbox_name(mbox)
-
- self.__chash = None
- self.__bdoc = None
-
- # TODO collection and container are deprecated.
-
- # TODO move to adaptor
-
- #@property
- #def fdoc(self):
- #"""
- #An accessor to the flags document.
- #"""
- #if all(map(bool, (self._uid, self._mbox))):
- #fdoc = None
- #if self._container is not None:
- #fdoc = self._container.fdoc
- #if not fdoc:
- #fdoc = self._get_flags_doc()
- #if fdoc:
- #fdoc_content = fdoc.content
- #self.__chash = fdoc_content.get(
- #fields.CONTENT_HASH_KEY, None)
- #return fdoc
-#
- #@property
- #def hdoc(self):
- #"""
- #An accessor to the headers document.
- #"""
- #container = self._container
- #if container is not None:
- #hdoc = self._container.hdoc
- #if hdoc and not empty(hdoc.content):
- #return hdoc
- #hdoc = self._get_headers_doc()
-#
- #if container and not empty(hdoc.content):
- # mem-cache it
- #hdoc_content = hdoc.content
- #chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
- #hdocs = {chash: hdoc_content}
- #container.memstore.load_header_docs(hdocs)
- #return hdoc
-#
- #@property
- #def chash(self):
- #"""
- #An accessor to the content hash for this message.
- #"""
- #if not self.fdoc:
- #return None
- #if not self.__chash and self.fdoc:
- #self.__chash = self.fdoc.content.get(
- #fields.CONTENT_HASH_KEY, None)
- #return self.__chash
-
- #@property
- #def bdoc(self):
- #"""
- #An accessor to the body document.
- #"""
- #if not self.hdoc:
- #return None
- #if not self.__bdoc:
- #self.__bdoc = self._get_body_doc()
- #return self.__bdoc
+ #self._uid = int(uid) if uid is not None else None
+ #self._mbox = normalize_mailbox(mbox)
+
+ self.message = message
+
+ # TODO maybe not needed, see setFlags below
+ self.collection = collection
# IMessage implementation
@@ -188,12 +77,7 @@ class IMAPMessage(fields, MBoxParser):
:return: uid for this message
:rtype: int
"""
- # TODO ----> return lookup in local sqlcipher table.
- return self._uid
-
- # --------------------------------------------------------------
- # TODO -- from here on, all the methods should be proxied to the
- # instance of leap.mail.mail.Message
+ return self.message.get_uid()
def getFlags(self):
"""
@@ -202,24 +86,14 @@ class IMAPMessage(fields, MBoxParser):
:return: The flags, represented as strings
:rtype: tuple
"""
- uid = self._uid
-
- flags = set([])
- fdoc = self.fdoc
- if fdoc:
- flags = set(fdoc.content.get(self.FLAGS_KEY, None))
+ return self.message.get_flags()
- msgcol = self._collection
+ # setFlags not in the interface spec but we use it with store command.
- # We treat the recent flag specially: gotten from
- # a mailbox-level document.
- if msgcol and uid in msgcol.recent_flags:
- flags.add(fields.RECENT_FLAG)
- if flags:
- flags = map(str, flags)
- return tuple(flags)
+ # XXX if we can move it to a collection method, we don't need to pass
+ # collection to the IMAPMessage
- # setFlags not in the interface spec but we use it with store command.
+ # lookup method? IMAPMailbox?
def setFlags(self, flags, mode):
"""
@@ -231,32 +105,11 @@ class IMAPMessage(fields, MBoxParser):
:type mode: int
"""
leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
- mbox, uid = self._mbox, self._uid
-
- APPEND = 1
- REMOVE = -1
- SET = 0
-
- doc = self.fdoc
- if not doc:
- logger.warning(
- "Could not find FDOC for %r:%s while setting flags!" %
- (mbox, uid))
- return
- current = doc.content[self.FLAGS_KEY]
- if mode == APPEND:
- newflags = tuple(set(tuple(current) + flags))
- elif mode == REMOVE:
- newflags = tuple(set(current).difference(set(flags)))
- elif mode == SET:
- newflags = flags
- new_fdoc = {
- self.FLAGS_KEY: newflags,
- self.SEEN_KEY: self.SEEN_FLAG in newflags,
- self.DEL_KEY: self.DELETED_FLAG in newflags}
- self._collection.memstore.update_flags(mbox, uid, new_fdoc)
-
- return map(str, newflags)
+ # XXX
+ # return new flags
+ # map to str
+ #self.message.set_flags(flags, mode)
+ self.collection.update_flags(self.message, flags, mode)
def getInternalDate(self):
"""
@@ -273,8 +126,7 @@ class IMAPMessage(fields, MBoxParser):
:return: An RFC822-formatted date string.
:rtype: str
"""
- date = self.hdoc.content.get(fields.DATE_KEY, '')
- return date
+ return self.message.get_internal_date()
#
# IMessagePart
@@ -290,42 +142,40 @@ class IMAPMessage(fields, MBoxParser):
:return: file-like object opened for reading
:rtype: StringIO
"""
- def write_fd(body):
- fd.write(body)
- fd.seek(0)
- return fd
-
+ #def write_fd(body):
+ #fd.write(body)
+ #fd.seek(0)
+ #return fd
+#
# TODO refactor with getBodyFile in MessagePart
-
- fd = StringIO.StringIO()
-
- if self.bdoc is not None:
- bdoc_content = self.bdoc.content
- if empty(bdoc_content):
- logger.warning("No BDOC content found for message!!!")
- return write_fd("")
-
- body = bdoc_content.get(self.RAW_KEY, "")
- content_type = bdoc_content.get('content-type', "")
- charset = find_charset(content_type)
- if charset is None:
- charset = self._get_charset(body)
- try:
- if isinstance(body, unicode):
- body = body.encode(charset)
- except UnicodeError as exc:
- logger.error(
- "Unicode error, using 'replace'. {0!r}".format(exc))
- logger.debug("Attempted to encode with: %s" % charset)
- body = body.encode(charset, 'replace')
- finally:
- return write_fd(body)
-
- # We are still returning funky characters from here.
- else:
- logger.warning("No BDOC found for message.")
- return write_fd("")
-
+#
+ #fd = StringIO.StringIO()
+#
+ #if self.bdoc is not None:
+ #bdoc_content = self.bdoc.content
+ #if empty(bdoc_content):
+ #logger.warning("No BDOC content found for message!!!")
+ #return write_fd("")
+#
+ #body = bdoc_content.get(self.RAW_KEY, "")
+ #content_type = bdoc_content.get('content-type', "")
+ #charset = find_charset(content_type)
+ #if charset is None:
+ #charset = self._get_charset(body)
+ #try:
+ #if isinstance(body, unicode):
+ #body = body.encode(charset)
+ #except UnicodeError as exc:
+ #logger.error(
+ #"Unicode error, using 'replace'. {0!r}".format(exc))
+ #logger.debug("Attempted to encode with: %s" % charset)
+ #body = body.encode(charset, 'replace')
+ #finally:
+ #return write_fd(body)
+
+ return self.message.get_body_file()
+
+ # TODO move to mail.mail
@memoized_method
def _get_charset(self, stuff):
"""
@@ -337,7 +187,7 @@ class IMAPMessage(fields, MBoxParser):
"""
# XXX shouldn't we make the scope
# of the decorator somewhat more persistent?
- # ah! yes! and put memory bounds.
+ # and put memory bounds.
return get_email_charset(stuff)
def getSize(self):
@@ -347,17 +197,11 @@ class IMAPMessage(fields, MBoxParser):
:return: size of the message, in octets
:rtype: int
"""
- size = None
- if self.fdoc is not None:
- fdoc_content = self.fdoc.content
- size = fdoc_content.get(self.SIZE_KEY, False)
- else:
- logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
- self._uid))
- #if not size:
- # XXX fallback, should remove when all migrated.
- #size = self.getBodyFile().len
- return size
+ #size = None
+ #fdoc_content = self.fdoc.content
+ #size = fdoc_content.get(self.SIZE_KEY, False)
+ #return size
+ return self.message.get_size()
def getHeaders(self, negate, *names):
"""
@@ -374,10 +218,10 @@ class IMAPMessage(fields, MBoxParser):
:return: A mapping of header field names to header field values
:rtype: dict
"""
- # TODO split in smaller methods
+ # TODO split in smaller methods -- format_headers()?
# XXX refactor together with MessagePart method
- headers = self._get_headers()
+ headers = self.message.get_headers()
# XXX keep this in the imap imessage implementation,
# because the server impl. expects content-type to be present.
@@ -417,34 +261,15 @@ class IMAPMessage(fields, MBoxParser):
headers2[key] = value
return headers2
- def _get_headers(self):
- """
- Return the headers dict for this message.
- """
- if self.hdoc is not None:
- hdoc_content = self.hdoc.content
- headers = hdoc_content.get(self.HEADERS_KEY, {})
- return headers
-
- else:
- logger.warning(
- "No HEADERS doc for msg %s:%s" % (
- self._mbox,
- self._uid))
-
def isMultipart(self):
"""
Return True if this message is multipart.
"""
- if self.fdoc:
- fdoc_content = self.fdoc.content
- is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
- return is_multipart
- else:
- logger.warning(
- "No FLAGS doc for msg %s:%s" % (
- self._mbox,
- self._uid))
+ #fdoc_content = self.fdoc.content
+ #is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
+ #return is_multipart
+
+ return self.message.fdoc.is_multi
def getSubPart(self, part):
"""
@@ -463,12 +288,16 @@ class IMAPMessage(fields, MBoxParser):
pmap_dict = self._get_part_from_parts_map(part + 1)
except KeyError:
raise IndexError
+
+ # TODO move access to adaptor ----
return MessagePart(self._soledad, pmap_dict)
#
# accessors
#
+ # FIXME
+ # -- move to wrapper/adaptor
def _get_part_from_parts_map(self, part):
"""
Get a part map from the headers doc
@@ -476,100 +305,44 @@ class IMAPMessage(fields, MBoxParser):
:raises: KeyError if key does not exist
:rtype: dict
"""
- if not self.hdoc:
- logger.warning("Tried to get part but no HDOC found!")
- return None
-
- hdoc_content = self.hdoc.content
- pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
+ raise NotImplementedError()
+ #hdoc_content = self.hdoc.content
+ #pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
+#
# remember, lads, soledad is using strings in its keys,
# not integers!
- return pmap[str(part)]
+ #return pmap[str(part)]
- # XXX moved to memory store
- # move the rest too. ------------------------------------------
- def _get_flags_doc(self):
- """
- Return the document that keeps the flags for this
- message.
- """
- def get_first_if_any(docs):
- result = first(docs)
- return result if result else {}
-
- d = self._soledad.get_from_index(
- fields.TYPE_MBOX_UID_IDX,
- fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
- d.addCallback(get_first_if_any)
- return d
-
- # TODO move to soledadstore instead of accessing soledad directly
- def _get_headers_doc(self):
- """
- Return the document that keeps the headers for this
- message.
- """
- d = self._soledad.get_from_index(
- fields.TYPE_C_HASH_IDX,
- fields.TYPE_HEADERS_VAL, str(self.chash))
- d.addCallback(lambda docs: first(docs))
- return d
-
- # TODO move to soledadstore instead of accessing soledad directly
+ # TODO move to wrapper/adaptor
def _get_body_doc(self):
"""
Return the document that keeps the body for this
message.
"""
- # XXX FIXME --- this might need a maybedeferred
- # on the receiving side...
- hdoc_content = self.hdoc.content
- body_phash = hdoc_content.get(
- fields.BODY_KEY, None)
- if not body_phash:
- logger.warning("No body phash for this document!")
- return None
-
- # XXX get from memstore too...
- # if memstore: memstore.get_phrash
- # memstore should keep a dict with weakrefs to the
- # phash doc...
-
- if self._container is not None:
- bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
- if not empty(bdoc) and not empty(bdoc.content):
- return bdoc
-
+ # FIXME
+ # -- just get the body and retrieve the cdoc P-<phash>
+ #hdoc_content = self.hdoc.content
+ #body_phash = hdoc_content.get(
+ #fields.BODY_KEY, None)
+ #if not body_phash:
+ #logger.warning("No body phash for this document!")
+ #return None
+#
+ #if self._container is not None:
+ #bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
+ #if not empty(bdoc) and not empty(bdoc.content):
+ #return bdoc
+#
# no memstore, or no body doc found there
- d = self._soledad.get_from_index(
- fields.TYPE_P_HASH_IDX,
- fields.TYPE_CONTENT_VAL, str(body_phash))
- d.addCallback(lambda docs: first(docs))
- return d
-
- def __getitem__(self, key):
- """
- Return an item from the content of the flags document,
- for convenience.
-
- :param key: The key
- :type key: str
-
- :return: The content value indexed by C{key} or None
- :rtype: str
- """
- return self.fdoc.content.get(key, None)
-
- def does_exist(self):
- """
- Return True if there is actually a flags document for this
- UID and mbox.
- """
- return not empty(self.fdoc)
+ #d = self._soledad.get_from_index(
+ #fields.TYPE_P_HASH_IDX,
+ #fields.TYPE_CONTENT_VAL, str(body_phash))
+ #d.addCallback(lambda docs: first(docs))
+ #return d
-class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
+class IMAPMessageCollection(object):
"""
A collection of messages, surprisingly.
@@ -578,9 +351,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
database.
"""
- # XXX this should be able to produce a MessageSet methinks
- # could validate these kinds of objects turning them
- # into a template for the class.
+ messageklass = IMAPMessage
+
+ # TODO
+ # [ ] Add RECENT flags docs to mailbox-doc attributes (list-of-uids)
+ # [ ] move Query for all the headers documents to Collection
+
+ # TODO this should be able to produce a MessageSet methinks
+ # TODO --- reimplement, review and prune documentation below.
+
FLAGS_DOC = "FLAGS"
HEADERS_DOC = "HEADERS"
CONTENT_DOC = "CONTENT"
@@ -604,145 +383,40 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
"""
HDOCS_SET_DOC = "HDOCS_SET"
- templates = {
-
- # Mailbox Level
-
- RECENT_DOC: {
- "type": indexes.RECENT,
- "mbox": INBOX_NAME,
- fields.RECENTFLAGS_KEY: [],
- },
-
- HDOCS_SET_DOC: {
- "type": indexes.HDOCS_SET,
- "mbox": INBOX_NAME,
- fields.HDOCS_SET_KEY: [],
- }
-
-
- }
-
- # Different locks for wrapping both the u1db document getting/setting
- # and the property getting/settting in an atomic operation.
-
- # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks
- _rdoc_lock = defaultdict(lambda: threading.Lock())
- _rdoc_write_lock = defaultdict(lambda: threading.Lock())
- _rdoc_read_lock = defaultdict(lambda: threading.Lock())
- _rdoc_property_lock = defaultdict(lambda: threading.Lock())
-
- _initialized = {}
-
- def __init__(self, mbox=None, soledad=None, memstore=None):
- """
- Constructor for MessageCollection.
-
- On initialization, we ensure that we have a document for
- storing the recent flags. The nature of this flag make us wanting
- to store the set of the UIDs with this flag at the level of the
- MessageCollection for each mailbox, instead of treating them
- as a property of each message.
-
- We are passed an instance of MemoryStore, the same for the
- SoledadBackedAccount, that we use as a read cache and a buffer
- for writes.
-
- :param mbox: the name of the mailbox. It is the name
- with which we filter the query over the
- messages database.
- :type mbox: str
- :param soledad: Soledad database
- :type soledad: Soledad instance
- :param memstore: a MemoryStore instance
- :type memstore: MemoryStore
+ def __init__(self, collection):
"""
- leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(mbox.strip() != "", "mbox cannot be blank space")
- leap_assert(isinstance(mbox, (str, unicode)),
- "mbox needs to be a string")
- leap_assert(soledad, "Need a soledad instance to initialize")
+ Constructor for IMAPMessageCollection.
- # okay, all in order, keep going...
-
- self.mbox = self._parse_mailbox_name(mbox)
-
- # XXX get a SoledadStore passed instead
- self._soledad = soledad
- self.memstore = memstore
-
- self.__rflags = None
-
- if not self._initialized.get(mbox, False):
- try:
- self.initialize_db()
- # ensure that we have a recent-flags doc
- self._get_or_create_rdoc()
- except Exception:
- logger.debug("Error initializing %r" % (mbox,))
- else:
- self._initialized[mbox] = True
-
- def _get_empty_doc(self, _type=FLAGS_DOC):
- """
- Returns an empty doc for storing different message parts.
- Defaults to returning a template for a flags document.
- :return: a dict with the template
- :rtype: dict
- """
- if _type not in self.templates.keys():
- raise TypeError("Improper type passed to _get_empty_doc")
- return copy.deepcopy(self.templates[_type])
-
- def _get_or_create_rdoc(self):
- """
- Try to retrieve the recent-flags doc for this MessageCollection,
- and create one if not found.
+ :param collection: an instance of a MessageCollection
+ :type collection: MessageCollection
"""
- # XXX should move this to memstore too
- with self._rdoc_write_lock[self.mbox]:
- rdoc = self._get_recent_doc_from_soledad()
- if rdoc is None:
- rdoc = self._get_empty_doc(self.RECENT_DOC)
- if self.mbox != fields.INBOX_VAL:
- rdoc[fields.MBOX_KEY] = self.mbox
- self._soledad.create_doc(rdoc)
-
- # --------------------------------------------------------------------
+ leap_assert(
+ collection.is_mailbox_collection(),
+ "Need a mailbox name to initialize")
+ mbox_name = collection.mbox_name
+ leap_assert(mbox_name.strip() != "", "mbox cannot be blank space")
+ leap_assert(isinstance(mbox_name, (str, unicode)),
+ "mbox needs to be a string")
+ self.collection = collection
- # -----------------------------------------------------------------------
+ # XXX this has to be done in IMAPAccount
+ # (Where the collection must be instantiated and passed to us)
+ # self.mbox = normalize_mailbox(mbox)
- def _fdoc_already_exists(self, chash):
+ @property
+ def mbox_name(self):
"""
- Check whether we can find a flags doc for this mailbox with the
- given content-hash. It enforces that we can only have the same maessage
- listed once for a a given mailbox.
-
- :param chash: the content-hash to check about.
- :type chash: basestring
- :return: False, if it does not exist, or UID.
+ Return the string that identifies this mailbox.
"""
- exist = False
- exist = self.memstore.get_fdoc_from_chash(chash, self.mbox)
+ return self.collection.mbox_name
- if not exist:
- exist = self._get_fdoc_from_chash(chash)
- if exist and exist.content is not None:
- return exist.content.get(fields.UID_KEY, "unknown-uid")
- else:
- return False
-
- def add_msg(self, raw, subject=None, flags=None, date=None,
- notify_on_disk=False):
+ def add_msg(self, raw, flags=None, date=None):
"""
Creates a new message document.
:param raw: the raw message
:type raw: str
- :param subject: subject of the message.
- :type subject: str
-
:param flags: flags
:type flags: list
@@ -756,212 +430,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
if flags is None:
flags = tuple()
leap_assert_type(flags, tuple)
+ return self.collection.add_msg(raw, flags, date)
- # TODO ---- proxy to MessageCollection addMessage
-
- #observer = defer.Deferred()
- #d = self._do_parse(raw)
- #d.addCallback(lambda result: reactor.callInThread(
- #self._do_add_msg, result, flags, subject, date,
- #notify_on_disk, observer))
- #return observer
-
- # TODO ---------------------------------------------------
- # move this to leap.mail.adaptors.soledad
-
- def _do_add_msg(self, parse_result, flags, subject,
- date, notify_on_disk, observer):
- """
- """
- msg, parts, chash, size, multi = parse_result
-
- # XXX move to SoledadAdaptor write operation ... ???
- # check for uniqueness --------------------------------
- # Watch out! We're reserving a UID right after this!
- existing_uid = self._fdoc_already_exists(chash)
- if existing_uid:
- msg = self.get_msg_by_uid(existing_uid)
- reactor.callFromThread(observer.callback, existing_uid)
- msg.setFlags((fields.DELETED_FLAG,), -1)
- return
-
- # TODO move UID autoincrement to MessageCollection.addMessage(mailbox)
- # TODO S2 -- get FUCKING UID from autoincremental table
- #uid = self.memstore.increment_last_soledad_uid(self.mbox)
- #self.set_recent_flag(uid)
-
-
- # ------------------------------------------------------------
-
- #
- # getters: specific queries
- #
-
- # recent flags
-
- def _get_recent_flags(self):
- """
- An accessor for the recent-flags set for this mailbox.
+ def get_msg_by_uid(self, uid, absolute=True):
"""
- # XXX check if we should remove this
- if self.__rflags is not None:
- return self.__rflags
-
- if self.memstore is not None:
- with self._rdoc_lock[self.mbox]:
- rflags = self.memstore.get_recent_flags(self.mbox)
- if not rflags:
- # not loaded in the memory store yet.
- # let's fetch them from soledad...
- rdoc = self._get_recent_doc_from_soledad()
- if rdoc is None:
- return set([])
- rflags = set(rdoc.content.get(
- fields.RECENTFLAGS_KEY, []))
- # ...and cache them now.
- self.memstore.load_recent_flags(
- self.mbox,
- {'doc_id': rdoc.doc_id, 'set': rflags})
- return rflags
-
- def _set_recent_flags(self, value):
- """
- Setter for the recent-flags set for this mailbox.
- """
- if self.memstore is not None:
- self.memstore.set_recent_flags(self.mbox, value)
-
- recent_flags = property(
- _get_recent_flags, _set_recent_flags,
- doc="Set of UIDs with the recent flag for this mailbox.")
-
- def _get_recent_doc_from_soledad(self):
- """
- Get recent-flags document from Soledad for this mailbox.
- :rtype: SoledadDocument or None
- """
- # FIXME ----- use deferreds.
- curried = partial(
- self._soledad.get_from_index,
- fields.TYPE_MBOX_IDX,
- fields.TYPE_RECENT_VAL, self.mbox)
- curried.expected = "rdoc"
- with self._rdoc_read_lock[self.mbox]:
- return try_unique_query(curried)
-
- # Property-set modification (protected by a different
- # lock to give atomicity to the read/write operation)
-
- def unset_recent_flags(self, uids):
- """
- Unset Recent flag for a sequence of uids.
-
- :param uids: the uids to unset
- :type uid: sequence
- """
- # FIXME ----- use deferreds.
- with self._rdoc_property_lock[self.mbox]:
- self.recent_flags.difference_update(
- set(uids))
-
- # Individual flags operations
-
- def unset_recent_flag(self, uid):
- """
- Unset Recent flag for a given uid.
-
- :param uid: the uid to unset
- :type uid: int
- """
- # FIXME ----- use deferreds.
- with self._rdoc_property_lock[self.mbox]:
- self.recent_flags.difference_update(
- set([uid]))
-
- def set_recent_flag(self, uid):
- """
- Set Recent flag for a given uid.
+ Retrieves a IMAPMessage by UID.
+ This is used primarity in the Mailbox fetch and store methods.
- :param uid: the uid to set
+ :param uid: the message uid to query by
:type uid: int
- """
- # FIXME ----- use deferreds.
- with self._rdoc_property_lock[self.mbox]:
- self.recent_flags = self.recent_flags.union(
- set([uid]))
-
- # individual doc getters, message layer.
- def _get_fdoc_from_chash(self, chash):
+ :rtype: IMAPMessage
"""
- Return a flags document for this mailbox with a given chash.
+ def make_imap_msg(msg):
+ kls = self.messageklass
+ # TODO --- remove ref to collection
+ return kls(msg, self.collection)
- :return: A SoledadDocument containing the Flags Document, or None if
- the query failed.
- :rtype: SoledadDocument or None.
- """
- # USED from:
- # [ ] duplicated fdoc detection
- # [ ] _get_uid_from_msgidCb
-
- # FIXME ----- use deferreds.
- curried = partial(
- self._soledad.get_from_index,
- fields.TYPE_MBOX_C_HASH_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox, chash)
- curried.expected = "fdoc"
- fdoc = try_unique_query(curried)
- if fdoc is not None:
- return fdoc
- else:
- # probably this should be the other way round,
- # ie, try fist on memstore...
- cf = self.memstore._chash_fdoc_store
- fdoc = cf[chash][self.mbox]
- # hey, I just needed to wrap fdoc thing into
- # a "content" attribute, look a better way...
- if not empty(fdoc):
- return MessagePartDoc(
- new=None, dirty=None, part=None,
- store=None, doc_id=None,
- content=fdoc)
-
- def _get_uid_from_msgidCb(self, msgid):
- hdoc = None
- curried = partial(
- self._soledad.get_from_index,
- fields.TYPE_MSGID_IDX,
- fields.TYPE_HEADERS_VAL, msgid)
- curried.expected = "hdoc"
- hdoc = try_unique_query(curried)
-
- # XXX this is only a quick hack to avoid regression
- # on the "multiple copies of the draft" issue, but
- # this is currently broken since it's not efficient to
- # look for this. Should lookup better.
- # FIXME!
-
- if hdoc is not None:
- hdoc_dict = hdoc.content
+ d = self.collection.get_msg_by_uid(uid, absolute=absolute)
+ d.addCalback(make_imap_msg)
+ return d
- else:
- hdocstore = self.memstore._hdoc_store
- match = [x for _, x in hdocstore.items() if x['msgid'] == msgid]
- hdoc_dict = first(match)
-
- if hdoc_dict is None:
- logger.warning("Could not find hdoc for msgid %s"
- % (msgid,))
- return None
- msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY)
-
- fdoc = self._get_fdoc_from_chash(msg_chash)
- if not fdoc:
- logger.warning("Could not find fdoc for msgid %s"
- % (msgid,))
- return None
- return fdoc.content.get(fields.UID_KEY, None)
+ # TODO -- move this to collection too
+ # Used for the Search (Drafts) queries?
def _get_uid_from_msgid(self, msgid):
"""
Return a UID for a given message-id.
@@ -972,15 +464,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:return: A UID, or None
"""
- # We need to wait a little bit, cause in some of the cases
- # the query is received right after we've saved the document,
- # and we cannot find it otherwise. This seems to be enough.
-
- # XXX do a deferLater instead ??
- # XXX is this working?
return self._get_uid_from_msgidCb(msgid)
- def set_flags(self, mbox, messages, flags, mode, observer):
+ # TODO handle deferreds
+ def set_flags(self, messages, flags, mode):
"""
Set flags for a sequence of messages.
@@ -1000,142 +487,27 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
getmsg = self.get_msg_by_uid
def set_flags(uid, flags, mode):
- msg = getmsg(uid, mem_only=True, flags_only=True)
+ msg = getmsg(uid)
if msg is not None:
+ # XXX IMAPMessage needs access to the collection
+ # to be able to set flags. Better if we make use
+ # of collection... here.
return uid, msg.setFlags(flags, mode)
setted_flags = [set_flags(uid, flags, mode) for uid in messages]
result = dict(filter(None, setted_flags))
+ # XXX return gatherResults or something
+ return result
- # TODO -- remove
- reactor.callFromThread(observer.callback, result)
-
- # getters: generic for a mailbox
-
- def get_msg_by_uid(self, uid, mem_only=False, flags_only=False):
- """
- Retrieves a LeapMessage by UID.
- This is used primarity in the Mailbox fetch and store methods.
-
- :param uid: the message uid to query by
- :type uid: int
- :param mem_only: a flag that indicates whether this Message should
- pass a reference to soledad to retrieve missing pieces
- or not.
- :type mem_only: bool
- :param flags_only: whether the message should carry only a reference
- to the flags document.
- :type flags_only: bool
-
- :return: A LeapMessage instance matching the query,
- or None if not found.
- :rtype: LeapMessage
- """
- msg_container = self.memstore.get_message(
- self.mbox, uid, flags_only=flags_only)
-
- if msg_container is not None:
- if mem_only:
- msg = IMAPMessage(None, uid, self.mbox, collection=self,
- container=msg_container)
- else:
- # We pass a reference to soledad just to be able to retrieve
- # missing parts that cannot be found in the container, like
- # the content docs after a copy.
- msg = IMAPMessage(self._soledad, uid, self.mbox,
- collection=self, container=msg_container)
- else:
- msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self)
-
- if not msg.does_exist():
- return None
- return msg
-
- # FIXME --- used where ? ---------------------------------------------
- #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
- #"""
- #Get all documents for the selected mailbox of the
- #passed type. By default, it returns the flag docs.
-#
- #If you want acess to the content, use __iter__ instead
-#
- #:return: a Deferred, that will fire with a list of u1db documents
- #:rtype: Deferred (promise of list of SoledadDocument)
- #"""
- #if _type not in fields.__dict__.values():
- #raise TypeError("Wrong type passed to get_all_docs")
-#
- # FIXME ----- either raise or return a deferred wrapper.
- #if sameProxiedObjects(self._soledad, None):
- #logger.warning('Tried to get messages but soledad is None!')
- #return []
-#
- #def get_sorted_docs(docs):
- #all_docs = [doc for doc in docs]
- # inneficient, but first let's grok it and then
- # let's worry about efficiency.
- # XXX FIXINDEX -- should implement order by in soledad
- # FIXME ----------------------------------------------
- #return sorted(all_docs, key=lambda item: item.content['uid'])
-#
- #d = self._soledad.get_from_index(
- #fields.TYPE_MBOX_IDX, _type, self.mbox)
- #d.addCallback(get_sorted_docs)
- #return d
-
- def all_soledad_uid_iter(self):
- """
- Return an iterator through the UIDs of all messages, sorted in
- ascending order.
- """
- # XXX FIXME ------ sorted???
-
- def get_uids(docs):
- return set([
- doc.content[self.UID_KEY] for doc in docs if not empty(doc)])
-
- d = self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox)
- d.addCallback(get_uids)
- return d
-
- def all_uid_iter(self):
- """
- Return an iterator through the UIDs of all messages, from memory.
+ def count(self):
"""
- mem_uids = self.memstore.get_uids(self.mbox)
- soledad_known_uids = self.memstore.get_soledad_known_uids(
- self.mbox)
- combined = tuple(set(mem_uids).union(soledad_known_uids))
- return combined
+ Return the count of messages for this mailbox.
- def get_all_soledad_flag_docs(self):
+ :rtype: int
"""
- Return a dict with the content of all the flag documents
- in soledad store for the given mbox.
+ return self.collection.count()
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: dict
- """
- # XXX we really could return a reduced version with
- # just {'uid': (flags-tuple,) since the prefetch is
- # only oriented to get the flag tuples.
-
- def get_content(docs):
- all_docs = [(
- doc.content[self.UID_KEY],
- dict(doc.content))
- for doc in docs
- if not empty(doc.content)]
- all_flags = dict(all_docs)
- return all_flags
-
- d = self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox)
- d.addCallback(get_content)
- return d
+ # headers query
def all_headers(self):
"""
@@ -1144,15 +516,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:rtype: dict
"""
- return self.memstore.all_headers(self.mbox)
-
- def count(self):
- """
- Return the count of messages for this mailbox.
-
- :rtype: int
- """
- return self.memstore.count(self.mbox)
+ # Use self.collection.mbox_indexer
+ # and derive all the doc_ids for the hdocs
+ raise NotImplementedError()
# unseen messages
@@ -1164,7 +530,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:return: iterator through unseen message doc UIDs
:rtype: iterable
"""
- return self.memstore.unseen_iter(self.mbox)
+ raise NotImplementedError()
def count_unseen(self):
"""
@@ -1182,13 +548,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:returns: a list of LeapMessages
:rtype: list
"""
- return [IMAPMessage(self._soledad, docid, self.mbox, collection=self)
- for docid in self.unseen_iter()]
+ raise NotImplementedError()
+ #return [self.messageklass(self._soledad, doc_id, self.mbox)
+ #for doc_id in self.unseen_iter()]
# recent messages
- # XXX take it from memstore
- # XXX Used somewhere?
def count_recent(self):
"""
Count all messages with the `Recent` flag.
@@ -1199,32 +564,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:returns: count
:rtype: int
"""
- return len(self.recent_flags)
+ raise NotImplementedError()
+
+ # magic
def __len__(self):
"""
Returns the number of messages on this mailbox.
-
:rtype: int
"""
return self.count()
- def __iter__(self):
- """
- Returns an iterator over all messages.
-
- :returns: iterator of dicts with content for all messages.
- :rtype: iterable
- """
- return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self)
- for docuid in self.all_uid_iter())
-
def __repr__(self):
"""
Representation string for this object.
"""
- return u"<MessageCollection: mbox '%s' (%s)>" % (
- self.mbox, self.count())
+ return u"<IMAPMessageCollection: mbox '%s' (%s)>" % (
+ self.mbox_name, self.count())
- # XXX should implement __eq__ also !!!
- # use chash...
+ # TODO implement __iter__ ?