diff options
Diffstat (limited to 'src/leap/mail/imap')
| -rw-r--r-- | src/leap/mail/imap/account.py | 223 | ||||
| -rw-r--r-- | src/leap/mail/imap/fields.py | 132 | ||||
| -rw-r--r-- | src/leap/mail/imap/index.py | 90 | ||||
| -rw-r--r-- | src/leap/mail/imap/interfaces.py | 2 | ||||
| -rw-r--r-- | src/leap/mail/imap/mailbox.py | 76 | ||||
| -rw-r--r-- | src/leap/mail/imap/messages.py | 484 | ||||
| -rw-r--r-- | src/leap/mail/imap/parser.py | 45 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/test_imap.py | 2 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/utils.py | 15 |
9 files changed, 301 insertions, 768 deletions
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index fe466cb..7dfbbd1 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -28,10 +28,10 @@ from twisted.python import log from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.mail.imap.index import IndexedDB + +from leap.mail.mail import Account from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.parser import MBoxParser -from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -39,7 +39,6 @@ logger = logging.getLogger(__name__) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") @@ -47,96 +46,43 @@ if PROFILE_CMD: ####################################### -# Soledad Account +# Soledad IMAP Account ####################################### +# TODO remove MsgFields too -# TODO change name to LeapIMAPAccount, since we're using -# the memstore. -# IndexedDB should also not be here anymore. - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): +class IMAPAccount(WithMsgFields): """ - An implementation of IAccount and INamespacePresenteer + An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. """ implements(imap4.IAccount, imap4.INamespacePresenter) - _soledad = None selected = None closed = False - _initialized = False - def __init__(self, account_name, soledad, memstore=None): + def __init__(self, user_id, store): """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. + Keeps track of the mailboxes and subscriptions handled by this account. - :param acct_name: The name of the account (user id). - :type acct_name: str + :param account: The name of the account (user id). + :type account: str - :param soledad: a Soledad instance. - :type soledad: Soledad - :param memstore: a MemoryStore instance. - :type memstore: MemoryStore + :param store: a Soledad instance. + :type store: Soledad """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) + # XXX assert a generic store interface instead, so that we + # can plug the memory store wrapper seamlessly. + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) # XXX SHOULD assert too that the name matches the user/uuid with which # soledad has been initialized. + self.user_id = user_id + self.account = Account(store) - # XXX ??? why is this parsing mailbox name??? it's account... - # userid? homogenize. - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - self._memstore = memstore - - self.__mailboxes = set([]) - - self._deferred_initialization = defer.Deferred() - self._initialize_storage() - - def _initialize_storage(self): - - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def finish_initialization(result): - self._initialized = True - self._deferred_initialization.callback(None) - - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - - d = self.initialize_db() - - d.addCallback(load_mbox_cache) - d.addCallback(add_mailbox_if_none) - d.addCallback(finish_initialization) - - def callWhenReady(self, cb): - if self._initialized: - cb(self) - return defer.succeed(None) - else: - self._deferred_initialization.addCallback(cb) - return self._deferred_initialization - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - + # XXX should hide this in the adaptor... def _get_mailbox_by_name(self, name): """ Return an mbox document by name. @@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ - # XXX use soledadstore instead ...; def get_first_if_any(docs): return docs[0] if docs else None - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) + normalize_mailbox(name)) d.addCallback(get_first_if_any) return d - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) - - def _load_mailboxes(self): - def update_mailboxes(db_indexes): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] for doc in db_indexes]) - d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) - d.addCallback(update_mailboxes) - return d - + # XXX move to Account? + # XXX needed? def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) - if name not in self.mailboxes: + if name not in self.account.mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) - return SoledadMailbox(name, self._soledad, - memstore=self._memstore) + # XXX Does mailbox really need reference to soledad? + return SoledadMailbox(name, self._store) # # IAccount # + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + # XXX move to mailbox module + return copy.deepcopy(mailbox.EMPTY_MBOX) + + # TODO use mail.Account.add_mailbox def addMailbox(self, name, creation_ts=None): """ Add a mailbox to the account. @@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a Deferred that will contain the document if successful. :rtype: bool """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) leap_assert(name, "Need a mailbox name to create a mailbox") @@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d.addCallback(lambda _: result) return d - d = self._soledad.create_doc(mbox) + d = self._store.create_doc(mbox) d.addCallback(load_mbox_cache) return d + # TODO use mail.Account.create_mailbox? + # Watch out, imap specific exceptions raised here. def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. @@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) + paths = filter(None, normalize_mailbox(pathspec).split('/')) subs = [] sep = '/' @@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d1.addCallback(load_mbox_cache) return d1 + # TODO use mail.Account.get_collection_by_mailbox def select(self, name, readwrite=1): """ Selects a mailbox. @@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadMailbox """ - if PROFILE_CMD: - start = time.time() - - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: logger.warning("No such mailbox!") return None self.selected = name - sm = SoledadMailbox( - name, self._soledad, self._memstore, readwrite) - if PROFILE_CMD: - _debugProfiling(None, "SELECT", start) + sm = SoledadMailbox(name, self._store, readwrite) return sm + # TODO use mail.Account.delete_mailbox def delete(self, name, force=False): """ Deletes a mailbox. @@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type force: bool :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # ??! -- can this be rite? # self._index.removeMailbox(name) + # TODO use mail.Account.rename_mailbox def rename(self, oldname, newname): """ Renames a mailbox. @@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param newname: new name of the mailbox :type newname: str """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) if oldname not in self.mailboxes: raise imap4.NoSuchMailbox(repr(oldname)) @@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO use mail.Account.list_mailboxes + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames(normalize_mailbox(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + # TODO ------------------ can we preserve the attr? # maybe add to memory store. def isSubscribed(self, name): @@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: Deferred (will fire with bool) """ + # TODO use Flags class subscribed = self.SUBSCRIBED_KEY def is_subscribed(mbox): @@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def get_docs_content(docs): return [doc.content[self.MBOX_KEY] for doc in docs] - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') d.addCallback(get_docs_content) return d @@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def update_subscribed_value(mbox): mbox.content[subscribed] = value - return self._soledad.put_doc(mbox) + return self._store.put_doc(mbox) # maybe we should store subscriptions in another # document... @@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_subscribe(subscriptions): if name not in subscriptions: @@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_unsubscribe(subscriptions): if name not in subscriptions: @@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def getSubscriptions(self): return self._get_subscriptions() - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - # # INamespacePresenter # @@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): """ Representation string for this object. """ - return "<SoledadBackedAccount (%s)>" % self._account_name + return "<IMAPAccount (%s)>" % self.user_id diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 4576939..a751c6d 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -17,7 +17,9 @@ """ Fields for Mailbox and Message. """ -from leap.mail.imap.parser import MBoxParser + +# TODO deprecate !!! (move all to constants maybe?) +# Flags -> foo class WithMsgFields(object): @@ -25,55 +27,12 @@ class WithMsgFields(object): Container class for class-attributes to be shared by several message-related classes. """ - # indexing - CONTENT_HASH_KEY = "chash" - PAYLOAD_HASH_KEY = "phash" - - # Internal representation of Message - - # flags doc - UID_KEY = "uid" - MBOX_KEY = "mbox" - SEEN_KEY = "seen" - DEL_KEY = "deleted" - RECENT_KEY = "recent" - FLAGS_KEY = "flags" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # headers - HEADERS_KEY = "headers" - DATE_KEY = "date" - SUBJECT_KEY = "subject" - PARTS_MAP_KEY = "part_map" - BODY_KEY = "body" # link to phash of body - MSGID_KEY = "msgid" - - # content - LINKED_FROM_KEY = "lkf" # XXX not implemented yet! - RAW_KEY = "raw" - CTYPE_KEY = "ctype" - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" + CREATED_KEY = "created" # used??? + RECENTFLAGS_KEY = "rct" HDOCS_SET_KEY = "hdocset" - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - TYPE_HEADERS_VAL = "head" - TYPE_CONTENT_VAL = "cnt" - TYPE_RECENT_VAL = "rct" - TYPE_HDOCS_SET_VAL = "hdocset" - - INBOX_VAL = "inbox" - # Flags in Mailbox and Message SEEN_FLAG = "\\Seen" RECENT_FLAG = "\\Recent" @@ -88,86 +47,5 @@ class WithMsgFields(object): SUBJECT_FIELD = "Subject" DATE_FIELD = "Date" - # Index types - # -------------- - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MSGID_IDX = 'by-type-and-message-id' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' - TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' - TYPE_C_HASH_IDX = 'by-type-and-contenthash' - TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' - TYPE_P_HASH_IDX = 'by-type-and-payloadhash' - - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - # Soledad index for incoming mail, without decrypting errors. - JUST_MAIL_IDX = "just-mail" - # XXX the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX = "just-mail-compat" - - INCOMING_KEY = "incoming" - ERROR_DECRYPTING_KEY = "errdecr" - - KTYPE = TYPE_KEY - MBOX_VAL = TYPE_MBOX_VAL - CHASH_VAL = CONTENT_HASH_KEY - PHASH_VAL = PAYLOAD_HASH_KEY - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # fdocs uniqueness - TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], - - # headers doc - search by msgid. - TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], - - # content, headers doc - TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - - # attachment payload dedup - TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - - # incoming queue - JUST_MAIL_IDX: [INCOMING_KEY, - "bool(%s)" % (ERROR_DECRYPTING_KEY,)], - - # the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - TYPE_KEY: MBOX_KEY, - TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - SUBJECT_KEY: "", - FLAGS_KEY: [], - CLOSED_KEY: False, - SUBSCRIBED_KEY: False, - RW_KEY: 1, - LAST_UID_KEY: 0 - } fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py deleted file mode 100644 index ea35fff..0000000 --- a/src/leap/mail/imap/index.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# index.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Index for SoledadBackedAccount, Mailbox and Messages. -""" -import logging - -from twisted.internet import defer - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail.imap.fields import fields - - -logger = logging.getLogger(__name__) - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - _index_creation_deferreds = [] - index_ready = False - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - self._index_creation_deferreds = [] - - def _on_indexes_created(ignored): - self.index_ready = True - - def _create_index(name, expression): - d = self._soledad.create_index(name, *expression) - self._index_creation_deferreds.append(d) - - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - _create_index(name, expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = self._soledad.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - - all_created = defer.gatherResults(self._index_creation_deferreds) - all_created.addCallback(_on_indexes_created) - return all_created - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - if self._soledad is not None: - d = self._soledad.list_indexes() - d.addCallback(_create_indexes) - return d diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py index c906278..f8f25fa 100644 --- a/src/leap/mail/imap/interfaces.py +++ b/src/leap/mail/imap/interfaces.py @@ -20,6 +20,7 @@ Interfaces for the IMAP module. from zope.interface import Interface, Attribute +# TODO remove ---------------- class IMessageContainer(Interface): """ I am a container around the different documents that a message @@ -38,6 +39,7 @@ class IMessageContainer(Interface): """ +# TODO remove -------------------- class IMessageStore(Interface): """ I represent a generic storage for LEAP Messages. diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3c1769a..ea54d33 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ Soledad Mailbox. """ import copy +import re import threading import logging import StringIO @@ -27,6 +28,7 @@ import os from collections import defaultdict from twisted.internet import defer +from twisted.internet import reactor from twisted.internet.task import deferLater from twisted.python import log @@ -36,15 +38,18 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import INBOX_NAME from leap.mail.decorators import deferred_to_thread from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) +# TODO +# [ ] Restore profile_cmd instrumentation + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. @@ -71,7 +76,9 @@ if PROFILE_CMD: d.addErrback(lambda f: log.msg(f.getTraceback())) -class SoledadMailbox(WithMsgFields, MBoxParser): +# TODO Rename to Mailbox +# TODO Remove WithMsgFields +class SoledadMailbox(WithMsgFields): """ A Soledad-backed IMAP mailbox. @@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): _last_uid_primed = {} _known_uids_primed = {} - def __init__(self, mbox, soledad, memstore, rw=1): + # TODO pass the collection to the constructor + # TODO pass the mbox_doc too + def __init__(self, mbox, store, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. @@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param mbox: the mailbox name :type mbox: str - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + :param store: + :type store: Soledad :param rw: read-and-write flag for this mailbox :type rw: int """ leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert(store, "Need a store instance to initialize") - from twisted.internet import reactor - self.reactor = reactor - - self.mbox = self._parse_mailbox_name(mbox) + self.mbox = normalize_mailbox(mbox) self.rw = rw - self._soledad = soledad - self._memstore = memstore - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad, memstore=self._memstore) + self.store = store + self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None # XXX careful with this get/set (it would be @@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._memstore.get_mbox_doc(self.mbox) - # XXX the memstore->soledadstore method in memstore is not complete def getFlags(self): """ Returns the flags defined for this mailbox. @@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = self.INIT_FLAGS return map(str, flags) - # XXX the memstore->soledadstore method in memstore is not complete def setFlags(self, flags): """ Sets flags for this mailbox. @@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date, notify_on_disk=notify_on_disk) - if PROFILE_CMD: - do_profile_cmd(d, "APPEND") + #if PROFILE_CMD: + #do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # to work fine for now. def notifyCallback(x): - self.reactor.callLater(0, self.notify_new) + reactor.callLater(0, self.notify_new) return x d.addCallback(notifyCallback) @@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: deferred """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH") + + # XXX do not need no thread... + reactor.callInThread(self._do_fetch, messages_asked, uid, d) d.addCallback(self.cb_signal_unread_to_ui) return d @@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self.__cb_signal_unread_to_ui) return result - @deferred_to_thread def _get_unseen_deferred(self): return self.getUnseenCount() @@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: C{list} or C{Deferred} """ # TODO see if we can raise w/o interrupting flow - #:raise IllegalQueryError: Raised when query is not valid. + # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', # '52D44F11.9060107@dev.bitmask.net'] @@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(createCopy) d.addErrback(lambda f: log.msg(f.getTraceback())) - @deferred_to_thread + #@deferred_to_thread def _get_msg_copy(self, message): """ Get a copy of the fdoc for this message, and check whether @@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return u"<SoledadMailbox: mbox '%s' (%s)>" % ( self.mbox, self.messages.count()) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index c761091..d47c8eb 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # messages.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,30 +19,25 @@ LeapMessage and MessageCollection. """ import copy import logging -import re import threading import StringIO from collections import defaultdict -from email import message_from_string from functools import partial -from pycryptopp.hash import sha256 from twisted.mail import imap4 -from twisted.internet import defer, reactor +from twisted.internet import reactor from zope.interface import implements from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset -from leap.mail import walk -from leap.mail.utils import first, find_charset, lowerdict, empty -from leap.mail.utils import stringify_parts_map -from leap.mail.decorators import deferred_to_thread +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.utils import find_charset, empty from leap.mail.imap.index import IndexedDB from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.memorystore import MessageWrapper from leap.mail.imap.messageparts import MessagePart, MessagePartDoc from leap.mail.imap.parser import MBoxParser @@ -59,9 +54,6 @@ logger = logging.getLogger(__name__) # [ ] Delete incoming mail only after successful write! # [ ] Remove UID from syncable db. Store only those indexes locally. -MSGID_PATTERN = r"""<([\w@.]+)>""" -MSGID_RE = re.compile(MSGID_PATTERN) - def try_unique_query(curried): """ @@ -90,28 +82,18 @@ def try_unique_query(curried): logger.exception("Unhandled error %r" % exc) -""" -A dictionary that keeps one lock per mbox and uid. -""" -# XXX too much overhead? -fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# FIXME remove-me +#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) -class LeapMessage(fields, MBoxParser): +class IMAPMessage(fields, MBoxParser): """ The main representation of a message. - - It indexes the messages in one mailbox by a combination - of uid+mailbox name. """ - # TODO this has to change. - # Should index primarily by chash, and keep a local-only - # UID table. - implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox, collection=None, container=None): + def __init__(self, soledad, uid, mbox): """ Initializes a LeapMessage. @@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser): self._soledad = soledad self._uid = int(uid) if uid is not None else None self._mbox = self._parse_mailbox_name(mbox) - self._collection = collection - self._container = container self.__chash = None self.__bdoc = None - # XXX make these properties public - - # XXX FIXME ------ the documents can be - # deferreds too.... niice. - - @property - def fdoc(self): - """ - An accessor to the flags document. - """ - if all(map(bool, (self._uid, self._mbox))): - fdoc = None - if self._container is not None: - fdoc = self._container.fdoc - if not fdoc: - fdoc = self._get_flags_doc() - if fdoc: - fdoc_content = fdoc.content - self.__chash = fdoc_content.get( - fields.CONTENT_HASH_KEY, None) - return fdoc - - @property - def hdoc(self): - """ - An accessor to the headers document. - """ - container = self._container - if container is not None: - hdoc = self._container.hdoc - if hdoc and not empty(hdoc.content): - return hdoc - hdoc = self._get_headers_doc() - - if container and not empty(hdoc.content): + # TODO collection and container are deprecated. + + # TODO move to adaptor + + #@property + #def fdoc(self): + #""" + #An accessor to the flags document. + #""" + #if all(map(bool, (self._uid, self._mbox))): + #fdoc = None + #if self._container is not None: + #fdoc = self._container.fdoc + #if not fdoc: + #fdoc = self._get_flags_doc() + #if fdoc: + #fdoc_content = fdoc.content + #self.__chash = fdoc_content.get( + #fields.CONTENT_HASH_KEY, None) + #return fdoc +# + #@property + #def hdoc(self): + #""" + #An accessor to the headers document. + #""" + #container = self._container + #if container is not None: + #hdoc = self._container.hdoc + #if hdoc and not empty(hdoc.content): + #return hdoc + #hdoc = self._get_headers_doc() +# + #if container and not empty(hdoc.content): # mem-cache it - hdoc_content = hdoc.content - chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - hdocs = {chash: hdoc_content} - container.memstore.load_header_docs(hdocs) - return hdoc - - @property - def chash(self): - """ - An accessor to the content hash for this message. - """ - if not self.fdoc: - return None - if not self.__chash and self.fdoc: - self.__chash = self.fdoc.content.get( - fields.CONTENT_HASH_KEY, None) - return self.__chash - - @property - def bdoc(self): - """ - An accessor to the body document. - """ - if not self.hdoc: - return None - if not self.__bdoc: - self.__bdoc = self._get_body_doc() - return self.__bdoc + #hdoc_content = hdoc.content + #chash = hdoc_content.get(fields.CONTENT_HASH_KEY) + #hdocs = {chash: hdoc_content} + #container.memstore.load_header_docs(hdocs) + #return hdoc +# + #@property + #def chash(self): + #""" + #An accessor to the content hash for this message. + #""" + #if not self.fdoc: + #return None + #if not self.__chash and self.fdoc: + #self.__chash = self.fdoc.content.get( + #fields.CONTENT_HASH_KEY, None) + #return self.__chash + + #@property + #def bdoc(self): + #""" + #An accessor to the body document. + #""" + #if not self.hdoc: + #return None + #if not self.__bdoc: + #self.__bdoc = self._get_body_doc() + #return self.__bdoc # IMessage implementation @@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ + # TODO ----> return lookup in local sqlcipher table. return self._uid + # -------------------------------------------------------------- + # TODO -- from here on, all the methods should be proxied to the + # instance of leap.mail.mail.Message + def getFlags(self): """ Retrieve the flags associated with this Message. @@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser): REMOVE = -1 SET = 0 - with fdoc_locks[mbox][uid]: - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) + doc = self.fdoc + if not doc: + logger.warning( + "Could not find FDOC for %r:%s while setting flags!" % + (mbox, uid)) + return + current = doc.content[self.FLAGS_KEY] + if mode == APPEND: + newflags = tuple(set(tuple(current) + flags)) + elif mode == REMOVE: + newflags = tuple(set(current).difference(set(flags))) + elif mode == SET: + newflags = flags + new_fdoc = { + self.FLAGS_KEY: newflags, + self.SEEN_KEY: self.SEEN_FLAG in newflags, + self.DEL_KEY: self.DELETED_FLAG in newflags} + self._collection.memstore.update_flags(mbox, uid, new_fdoc) return map(str, newflags) @@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser): else: logger.warning("No FLAGS doc for %s:%s" % (self._mbox, self._uid)) - if not size: + #if not size: # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len + #size = self.getBodyFile().len return size def getHeaders(self, negate, *names): @@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser): # XXX refactor together with MessagePart method headers = self._get_headers() + + # XXX keep this in the imap imessage implementation, + # because the server impl. expects content-type to be present. if not headers: logger.warning("No headers found") return {str('content-type'): str('')} @@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): (the u1db index) for all the headers documents for a given mailbox. We use it to prefetch massively all the headers for a mailbox. This is the second massive query, after fetching all the FLAGS, that - a MUA will do in a case where we do not have local disk cache. + a typical IMAP MUA will do in a case where we do not have local disk cache. """ HDOCS_SET_DOC = "HDOCS_SET" templates = { - # Message Level - - FLAGS_DOC: { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, # XXX moe to a local table - fields.MBOX_KEY: fields.INBOX_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.SEEN_KEY: False, - fields.DEL_KEY: False, - fields.FLAGS_KEY: [], - fields.MULTIPART_KEY: False, - fields.SIZE_KEY: 0 - }, - - HEADERS_DOC: { - fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.DATE_KEY: "", - fields.SUBJECT_KEY: "", - - fields.HEADERS_KEY: {}, - fields.PARTS_MAP_KEY: {}, - }, - - CONTENT_DOC: { - fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, - fields.PAYLOAD_HASH_KEY: "", - fields.LINKED_FROM_KEY: [], - fields.CTYPE_KEY: "", # should index by this too - - # should only get inmutable headers parts - # (for indexing) - fields.HEADERS_KEY: {}, - fields.RAW_KEY: "", - fields.PARTS_MAP_KEY: {}, - fields.HEADERS_KEY: {}, - fields.MULTIPART_KEY: False, - }, - # Mailbox Level RECENT_DOC: { - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.RECENT, + "mbox": INBOX_NAME, fields.RECENTFLAGS_KEY: [], }, HDOCS_SET_DOC: { - fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.HDOCS_SET, + "mbox": INBOX_NAME, fields.HDOCS_SET_KEY: [], } @@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # Different locks for wrapping both the u1db document getting/setting # and the property getting/settting in an atomic operation. - # TODO we would abstract this to a SoledadProperty class - + # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks _rdoc_lock = defaultdict(lambda: threading.Lock()) _rdoc_write_lock = defaultdict(lambda: threading.Lock()) _rdoc_read_lock = defaultdict(lambda: threading.Lock()) @@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): rdoc[fields.MBOX_KEY] = self.mbox self._soledad.create_doc(rdoc) - @deferred_to_thread - def _do_parse(self, raw): - """ - Parse raw message and return it along with - relevant information about its outer level. - - This is done in a separate thread, and the callback is passed - to `_do_add_msg` method. + # -------------------------------------------------------------------- - :param raw: the raw message - :type raw: StringIO or basestring - :return: msg, parts, chash, size, multi - :rtype: tuple - """ - msg = message_from_string(raw) - parts = walk.get_parts(msg) - size = len(raw) - chash = sha256.SHA256(raw).hexdigest() - multi = msg.is_multipart() - return msg, parts, chash, size, multi - - def _populate_flags(self, flags, uid, chash, size, multi): - """ - Return a flags doc. - - XXX Missing DOC ----------- - """ - fd = self._get_empty_doc(self.FLAGS_DOC) - - fd[self.MBOX_KEY] = self.mbox - fd[self.UID_KEY] = uid - fd[self.CONTENT_HASH_KEY] = chash - fd[self.SIZE_KEY] = size - fd[self.MULTIPART_KEY] = multi - if flags: - fd[self.FLAGS_KEY] = flags - fd[self.SEEN_KEY] = self.SEEN_FLAG in flags - fd[self.DEL_KEY] = self.DELETED_FLAG in flags - fd[self.RECENT_KEY] = True # set always by default - return fd - - def _populate_headr(self, msg, chash, subject, date): - """ - Return a headers doc. - - XXX Missing DOC ----------- - """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) - - lower_headers = lowerdict(headers) - msgid = first(MSGID_RE.findall( - lower_headers.get('message-id', ''))) - - hd = self._get_empty_doc(self.HEADERS_DOC) - hd[self.CONTENT_HASH_KEY] = chash - hd[self.HEADERS_KEY] = headers - hd[self.MSGID_KEY] = msgid - - if not subject and self.SUBJECT_FIELD in headers: - hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - else: - hd[self.SUBJECT_KEY] = subject - - if not date and self.DATE_FIELD in headers: - hd[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - hd[self.DATE_KEY] = date - return hd + # ----------------------------------------------------------------------- def _fdoc_already_exists(self, chash): """ @@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) - # TODO return soledad deferred instead - observer = defer.Deferred() - d = self._do_parse(raw) - d.addCallback(lambda result: reactor.callInThread( - self._do_add_msg, result, flags, subject, date, - notify_on_disk, observer)) - return observer + # TODO ---- proxy to MessageCollection addMessage + + #observer = defer.Deferred() + #d = self._do_parse(raw) + #d.addCallback(lambda result: reactor.callInThread( + #self._do_add_msg, result, flags, subject, date, + #notify_on_disk, observer)) + #return observer + + # TODO --------------------------------------------------- + # move this to leap.mail.adaptors.soledad - # Called in thread def _do_add_msg(self, parse_result, flags, subject, date, notify_on_disk, observer): """ - Helper that creates a new message document. - Here lives the magic of the leap mail. Well, in soledad, really. - - See `add_msg` docstring for parameter info. - - :param parse_result: a tuple with the results of `self._do_parse` - :type parse_result: tuple - :param observer: a deferred that will be fired with the message - uid when the adding succeed. - :type observer: deferred """ - # TODO signal that we can delete the original message!----- - # when all the processing is done. - - # TODO add the linked-from info ! - # TODO add reference to the original message - msg, parts, chash, size, multi = parse_result + # XXX move to SoledadAdaptor write operation ... ??? # check for uniqueness -------------------------------- # Watch out! We're reserving a UID right after this! existing_uid = self._fdoc_already_exists(chash) if existing_uid: msg = self.get_msg_by_uid(existing_uid) - - # We can say the observer that we're done - # TODO return soledad deferred instead reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return + # TODO move UID autoincrement to MessageCollection.addMessage(mailbox) # TODO S2 -- get FUCKING UID from autoincremental table - uid = self.memstore.increment_last_soledad_uid(self.mbox) - - # We can say the observer that we're done at this point, but - # before that we should make sure it has no serious consequences - # if we're issued, for instance, a fetch command right after... - # reactor.callFromThread(observer.callback, uid) - # if we did the notify, we need to invalidate the deferred - # so not to try to fire it twice. - # observer = None - - fd = self._populate_flags(flags, uid, chash, size, multi) - hd = self._populate_headr(msg, chash, subject, date) - - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) - parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) - - # add parts map to header doc - # (body, multi, part_map) - for key in parts_map: - hd[key] = parts_map[key] - del parts_map + #uid = self.memstore.increment_last_soledad_uid(self.mbox) + #self.set_recent_flag(uid) - hd = stringify_parts_map(hd) - # The MessageContainer expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) - - self.set_recent_flag(uid) - msg_container = MessageWrapper(fd, hd, cdocs) - - # TODO S1 -- just pass this to memstore and return that deferred. - self.memstore.create_message( - self.mbox, uid, msg_container, - observer=observer, notify_on_disk=notify_on_disk) + # ------------------------------------------------------------ # # getters: specific queries @@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # USED from: + # [ ] duplicated fdoc detection + # [ ] _get_uid_from_msgidCb + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, @@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): if msg_container is not None: if mem_only: - msg = LeapMessage(None, uid, self.mbox, collection=self, + msg = IMAPMessage(None, uid, self.mbox, collection=self, container=msg_container) else: # We pass a reference to soledad just to be able to retrieve # missing parts that cannot be found in the container, like # the content docs after a copy. - msg = LeapMessage(self._soledad, uid, self.mbox, + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self, container=msg_container) else: - msg = LeapMessage(self._soledad, uid, self.mbox, collection=self) + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self) if not msg.does_exist(): return None return msg - def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - """ - Get all documents for the selected mailbox of the - passed type. By default, it returns the flag docs. - - If you want acess to the content, use __iter__ instead - - :return: a Deferred, that will fire with a list of u1db documents - :rtype: Deferred (promise of list of SoledadDocument) - """ - if _type not in fields.__dict__.values(): - raise TypeError("Wrong type passed to get_all_docs") - + # FIXME --- used where ? --------------------------------------------- + #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + #""" + #Get all documents for the selected mailbox of the + #passed type. By default, it returns the flag docs. +# + #If you want acess to the content, use __iter__ instead +# + #:return: a Deferred, that will fire with a list of u1db documents + #:rtype: Deferred (promise of list of SoledadDocument) + #""" + #if _type not in fields.__dict__.values(): + #raise TypeError("Wrong type passed to get_all_docs") +# # FIXME ----- either raise or return a deferred wrapper. - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - def get_sorted_docs(docs): - all_docs = [doc for doc in docs] + #if sameProxiedObjects(self._soledad, None): + #logger.warning('Tried to get messages but soledad is None!') + #return [] +# + #def get_sorted_docs(docs): + #all_docs = [doc for doc in docs] # inneficient, but first let's grok it and then # let's worry about efficiency. # XXX FIXINDEX -- should implement order by in soledad # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, _type, self.mbox) - d.addCallback(get_sorted_docs) - return d + #return sorted(all_docs, key=lambda item: item.content['uid']) +# + #d = self._soledad.get_from_index( + #fields.TYPE_MBOX_IDX, _type, self.mbox) + #d.addCallback(get_sorted_docs) + #return d def all_soledad_uid_iter(self): """ @@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(self._soledad, docid, self.mbox, collection=self) + return [IMAPMessage(self._soledad, docid, self.mbox, collection=self) for docid in self.unseen_iter()] # recent messages @@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: iterator of dicts with content for all messages. :rtype: iterable """ - return (LeapMessage(self._soledad, docuid, self.mbox, collection=self) + return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self) for docuid in self.all_uid_iter()) def __repr__(self): diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py deleted file mode 100644 index 4a801b0..0000000 --- a/src/leap/mail/imap/parser.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# parser.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Mail parser mixin. -""" -import re - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - Return a normalized representation of the mailbox C{name}. - - This method ensures that an eventual initial 'inbox' part of a - mailbox name is made uppercase. - - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dd4294c..5af499f 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): MessageCollection interface in this particular TestCase """ super(MessageCollectionTestCase, self).setUp() + + # TODO deprecate memstore memstore = MemoryStore() self.messages = MessageCollection("testmbox%s" % (self.count,), self._soledad, memstore=memstore) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 9a3868c..920eeb0 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() +# XXX move to common helper def initialize_soledad(email, gnupg_home, tempdir): """ Initializes soledad by hand @@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest): """ Setup method for each test. - Initializes and run a LEAP IMAP4 Server, - but passing the same Soledad instance (it's costly to initialize), - so we have to be sure to restore state across tests. + Initializes and run a LEAP IMAP4 Server. """ self.old_path = os.environ['PATH'] self.old_home = os.environ['HOME'] @@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest): def tearDown(self): """ tearDown method called after each test. - - Deletes all documents in the Index, and deletes - instances of server and client. """ try: self._soledad.close() + except Exception: + print "ERROR WHILE CLOSING SOLEDAD" + finally: os.environ["PATH"] = self.old_path os.environ["HOME"] = self.old_home # safety check assert 'leap_tests-' in self.tempdir shutil.rmtree(self.tempdir) - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" def populateMessages(self): """ @@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest): def loopback(self): return loopback.loopbackAsync(self.server, self.client) - - |