diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/leap/mail/adaptors/soledad.py | 183 | ||||
| -rw-r--r-- | src/leap/mail/adaptors/soledad_indexes.py | 15 | ||||
| -rw-r--r-- | src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 7 | ||||
| -rw-r--r-- | src/leap/mail/imap/account.py | 174 | ||||
| -rw-r--r-- | src/leap/mail/imap/mailbox.py | 116 | ||||
| -rw-r--r-- | src/leap/mail/imap/messages.py | 13 | ||||
| -rw-r--r-- | src/leap/mail/imap/server.py | 30 | ||||
| -rw-r--r-- | src/leap/mail/imap/service/imap.py | 98 | ||||
| -rwxr-xr-x | src/leap/mail/imap/tests/stress_tests_imap.zsh (renamed from src/leap/mail/imap/tests/leap_tests_imap.zsh) | 0 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/test_imap.py | 539 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/utils.py | 212 | ||||
| -rw-r--r-- | src/leap/mail/mail.py | 114 | ||||
| -rw-r--r-- | src/leap/mail/mailbox_indexer.py | 70 | ||||
| -rw-r--r-- | src/leap/mail/tests/common.py | 17 | ||||
| -rw-r--r-- | src/leap/mail/tests/test_mailbox_indexer.py | 41 |
15 files changed, 920 insertions, 709 deletions
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index c5cfce0..d99f677 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -23,7 +23,6 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer -from twisted.python import util from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -56,6 +55,14 @@ class DuplicatedDocumentError(Exception): pass +def cleanup_deferred_locks(): + """ + Need to use this from within trial to cleanup the reactor before + each run. + """ + SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock) + + class SoledadDocumentWrapper(models.DocumentWrapper): """ A Wrapper object that can be manipulated, passed around, and serialized in @@ -526,7 +533,7 @@ class MessageWrapper(object): This method should only be used before the Documents for the MessageWrapper have been created, will raise otherwise. """ - mbox_uuid = mbox.uuid.replace('-', '_') + mbox_uuid = mbox_uuid.replace('-', '_') self.mdoc.set_mbox_uuid(mbox_uuid) self.fdoc.set_mbox_uuid(mbox_uuid) @@ -536,6 +543,9 @@ class MessageWrapper(object): flags = tuple() leap_assert_type(flags, tuple) self.fdoc.flags = list(flags) + self.fdoc.deleted = "\\Deleted" in flags + self.fdoc.seen = "\\Seen" in flags + self.fdoc.recent = "\\Recent" in flags def set_tags(self, tags): # TODO serialize the get + update @@ -593,26 +603,30 @@ class MailboxWrapper(SoledadDocumentWrapper): # Soledad Adaptor # -# TODO make this an interface? class SoledadIndexMixin(object): """ - this will need a class attribute `indexes`, that is a dictionary containing + This will need a class attribute `indexes`, that is a dictionary containing the index definitions for the underlying u1db store underlying soledad. It needs to be in the following format: {'index-name': ['field1', 'field2']} + + You can also add a class attribute `wait_for_indexes` to any class + inheriting from this Mixin, that should be a list of strings representing + the methods that need to wait until the indexes have been initialized + before being able to work properly. """ + # TODO move this mixin to soledad itself + # so that each application can pass a set of indexes for their data model. + # TODO could have a wrapper class for indexes, supporting introspection # and __getattr__ - indexes = {} - store_ready = False - _index_creation_deferreds = [] + # TODO make this an interface? - # TODO we might want to move this logic to soledad itself - # so that each application can pass a set of indexes for their data model. - # TODO check also the decorator used in keymanager for waiting for indexes - # to be ready. + indexes = {} + wait_for_indexes = [] + store_ready = False def initialize_store(self, store): """ @@ -626,47 +640,81 @@ class SoledadIndexMixin(object): # TODO I think we *should* get another deferredLock in here, but # global to the soledad namespace, to protect from several points # initializing soledad indexes at the same time. + self._wait_for_indexes() - leap_assert(store, "Need a store") - leap_assert_type(self.indexes, dict) + d = self._init_indexes(store) + d.addCallback(self._restore_waiting_methods) + return d - self._index_creation_deferreds = [] + def _init_indexes(self, store): + """ + Initialize the database indexes. + """ + leap_assert(store, "Cannot init indexes with null soledad") + leap_assert_type(self.indexes, dict) def _create_index(name, expression): return store.create_index(name, *expression) - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - + def init_idexes(indexes): + deferreds = [] + db_indexes = dict(indexes) + # Loop through the indexes we expect to find. for name, expression in self.indexes.items(): if name not in db_indexes: # The index does not yet exist. d = _create_index(name, expression) - self._index_creation_deferreds.append(d) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = store.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - self._index_creation_deferreds.append(d1) - - all_created = defer.gatherResults( - self._index_creation_deferreds, consumeErrors=True) - all_created.addCallback(_on_indexes_created) - return all_created - - def _on_indexes_created(ignored): + deferreds.append(d) + elif expression != db_indexes[name]: + # The index exists but the definition is not what expected, + # so we delete it and add the proper index expression. + d = store.delete_index(name) + d.addCallback( + lambda _: _create_index(name, *expression)) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def store_ready(whatever): self.store_ready = True + return whatever - # Ask the database for currently existing indexes, and create them - # if not found. - d = store.list_indexes() - d.addCallback(_create_indexes) - return d + self.deferred_indexes = store.list_indexes() + self.deferred_indexes.addCallback(init_idexes) + self.deferred_indexes.addCallback(store_ready) + return self.deferred_indexes + + def _wait_for_indexes(self): + """ + Make the marked methods to wait for the indexes to be ready. + Heavily based on + http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/ + + :param methods: methods that need to wait for the indexes to be ready + :type methods: tuple(str) + """ + leap_assert_type(self.wait_for_indexes, list) + methods = self.wait_for_indexes + + self.waiting = [] + self.stored = {} + + def makeWrapper(method): + def wrapper(*args, **kw): + d = defer.Deferred() + d.addCallback(lambda _: self.stored[method](*args, **kw)) + self.waiting.append(d) + return d + return wrapper + + for method in methods: + self.stored[method] = getattr(self, method) + setattr(self, method, makeWrapper(method)) + + def _restore_waiting_methods(self, _): + for method in self.stored: + setattr(self, method, self.stored[method]) + for d in self.waiting: + d.callback(None) class SoledadMailAdaptor(SoledadIndexMixin): @@ -675,8 +723,18 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] + mboxwrapper_klass = MailboxWrapper + def __init__(self): + SoledadIndexMixin.__init__(self) + + mboxwrapper_klass = MailboxWrapper + + def __init__(self): + SoledadIndexMixin.__init__(self) + # Message handling def get_msg_from_string(self, MessageClass, raw_msg): @@ -762,10 +820,10 @@ class SoledadMailAdaptor(SoledadIndexMixin): chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox=mbox, chash=chash) + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) def _get_hdoc_id_from_mdoc_id(): - return constants.HDOCID.format(mbox=mbox, chash=chash) + return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) d_docs = [] fdoc_id = _get_fdoc_id_from_mdoc_id() @@ -816,6 +874,47 @@ class SoledadMailAdaptor(SoledadIndexMixin): wrapper = msg.get_wrapper() return wrapper.update(store) + # batch deletion + + def del_all_flagged_messages(self, store, mbox_uuid): + """ + Delete all messages flagged as deleted. + """ + def err(f): + print "ERROR GETTING FROM INDEX" + f.printTraceback() + + def delete_fdoc_and_mdoc_flagged(fdocs): + # low level here, not using the wrappers... + # get meta doc ids from the flag doc ids + fdoc_ids = [doc.doc_id for doc in fdocs] + mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) + + def delete_all_docs(mdocs, fdocs): + mdocs = list(mdocs) + doc_ids = [m.doc_id for m in mdocs] + _d = [] + docs = mdocs + fdocs + for doc in docs: + _d.append(store.delete_doc(doc)) + d = defer.gatherResults(_d) + # return the mdocs ids only + d.addCallback(lambda _: doc_ids) + return d + + d = store.get_docs(mdoc_ids) + d.addCallback(delete_all_docs, fdocs) + d.addErrback(err) + return d + + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + deleted_index = indexes.TYPE_MBOX_DEL_IDX + + d = store.get_from_index(deleted_index, type_, uuid, "1") + d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) + return d + # Mailbox handling def get_or_create_mbox(self, store, name): diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py index f3e990d..856dfb4 100644 --- a/src/leap/mail/adaptors/soledad_indexes.py +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -25,6 +25,7 @@ Soledad Indexes for Mail Documents. TYPE = "type" MBOX = "mbox" +MBOX_UUID = "mbox_uuid" FLAGS = "flags" HEADERS = "head" CONTENT = "cnt" @@ -46,7 +47,7 @@ UID = "uid" TYPE_IDX = 'by-type' TYPE_MBOX_IDX = 'by-type-and-mbox' -#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' +TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MSGID_IDX = 'by-type-and-message-id' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' @@ -62,9 +63,6 @@ TYPE_P_HASH_IDX = 'by-type-and-payloadhash' JUST_MAIL_IDX = "just-mail" JUST_MAIL_COMPAT_IDX = "just-mail-compat" -# Tomas created the `recent and seen index`, but the semantic is not too -# correct since the recent flag is volatile --- XXX review and delete. -#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' # TODO # it would be nice to measure the cost of indexing @@ -77,6 +75,7 @@ MAIL_INDEXES = { # generic TYPE_IDX: [TYPE], TYPE_MBOX_IDX: [TYPE, MBOX], + TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID], # XXX deprecate 0.4.0 # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], @@ -97,11 +96,9 @@ MAIL_INDEXES = { TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], # messages - TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'], - #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX, - #'bool(recent)', 'bool(seen)'], + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], # incoming queue JUST_MAIL_IDX: [INCOMING_KEY, diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 7bdeef5..3dc79fe 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -276,8 +276,9 @@ HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): - def __init__(self, wrapper): + def __init__(self, wrapper, uid): self.wrapper = wrapper + self.uid = uid def get_wrapper(self): return self.wrapper @@ -322,9 +323,9 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): self.assertTrue(msg.wrapper.cdocs is not None) self.assertEquals(len(msg.wrapper.cdocs), 1) self.assertEquals(msg.wrapper.fdoc.chash, chash) - self.assertEquals(msg.wrapper.fdoc.size, 3834) + self.assertEquals(msg.wrapper.fdoc.size, 3837) self.assertEquals(msg.wrapper.hdoc.chash, chash) - self.assertEqual(msg.wrapper.hdoc.headers['subject'], + self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'], subject) self.assertEqual(msg.wrapper.hdoc.subject, subject) self.assertEqual(msg.wrapper.cdocs[1].phash, phash) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0baf078..dfc0d62 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -63,7 +63,7 @@ class IMAPAccount(object): selected = None closed = False - def __init__(self, user_id, store): + def __init__(self, user_id, store, d=None): """ Keeps track of the mailboxes and subscriptions handled by this account. @@ -72,21 +72,31 @@ class IMAPAccount(object): :param store: a Soledad instance. :type store: Soledad + + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred """ leap_assert(store, "Need a store instance to initialize") leap_assert_type(store, Soledad) # TODO assert too that the name matches the user/uuid with which - # soledad has been initialized. + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. + self.user_id = user_id - self.account = Account(store) + self.account = Account(store, ready_cb=lambda: d.callback(self)) def _return_mailbox_from_collection(self, collection, readwrite=1): if collection is None: return None - return IMAPMailbox(collection, rw=readwrite) + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + + def callWhenReady(self, cb, *args, **kw): + d = self.account.callWhenReady(cb, *args, **kw) + return d - # XXX Where's this used from? -- self.delete... def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -102,11 +112,12 @@ class IMAPAccount(object): def check_it_exists(mailboxes): if name not in mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) + return True d = self.account.list_all_mailbox_names() d.addCallback(check_it_exists) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) - d.addCallbacK(self._return_mailbox_from_collection) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) return d # @@ -130,12 +141,9 @@ class IMAPAccount(object): """ name = normalize_mailbox(name) + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... leap_assert(name, "Need a mailbox name to create a mailbox") - - def check_it_does_not_exist(mailboxes): - if name in mailboxes: - raise imap4.MailboxCollision(repr(name)) - if creation_ts is None: # by default, we pass an int value # taken from the current time @@ -143,14 +151,20 @@ class IMAPAccount(object): # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes + def set_mbox_creation_ts(collection): - d = collection.set_mbox_attr("created") + d = collection.set_mbox_attr("created", creation_ts) d.addCallback(lambda _: collection) return d d = self.account.list_all_mailbox_names() d.addCallback(check_it_does_not_exist) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallback(lambda _: self.account.add_mailbox(name)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) d.addCallback(set_mbox_creation_ts) d.addCallback(self._return_mailbox_from_collection) return d @@ -172,39 +186,40 @@ class IMAPAccount(object): :raise MailboxException: Raised if this mailbox cannot be added. """ - # TODO raise MailboxException paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] sep = '/' + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + for accum in range(1, len(paths)): - try: - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - subs.append(d) - # XXX should this be handled by the deferred? - except imap4.MailboxCollision: - pass - try: - df = self.addMailbox(sep.join(paths)) - except imap4.MailboxCollision: + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): - df = defer.succeed(False) + return defer.succeed(False) else: - df = defer.succeed(True) - finally: - subs.append(df) + return defer.succeed(True) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) def all_good(result): return all(result) if subs: - d1 = defer.gatherResults(subs, consumeErrors=True) + d1 = defer.gatherResults(subs) d1.addCallback(all_good) + return d1 else: - d1 = defer.succeed(False) - return d1 + return defer.succeed(False) def select(self, name, readwrite=1): """ @@ -216,7 +231,7 @@ class IMAPAccount(object): :param readwrite: 1 for readwrite permissions. :type readwrite: int - :rtype: SoledadMailbox + :rtype: IMAPMailbox """ name = normalize_mailbox(name) @@ -245,9 +260,6 @@ class IMAPAccount(object): """ Deletes a mailbox. - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - :param name: the mailbox to be deleted :type name: str @@ -258,10 +270,10 @@ class IMAPAccount(object): :rtype: Deferred """ name = normalize_mailbox(name) - _mboxes = [] + _mboxes = None def check_it_exists(mailboxes): - # FIXME works? -- pass variable ref to outer scope + global _mboxes _mboxes = mailboxes if name not in mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -274,6 +286,7 @@ class IMAPAccount(object): return mbox.destroy() def check_can_be_deleted(mbox): + global _mboxes # See if this box is flagged \Noselect mbox_flags = mbox.getFlags() if MessageFlags.NOSELECT_FLAG in mbox_flags: @@ -317,29 +330,27 @@ class IMAPAccount(object): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) - # FIXME check that scope works (test) - _mboxes = [] - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox(repr(oldname)) - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + def rename_inferiors(inferiors_result): + inferiors, mailboxes = inferiors_result + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] - for (old, new) in inferiors: - if new in _mboxes: - raise imap4.MailboxCollision(repr(new)) + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) - rename_deferreds = [] + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) - for (old, new) in inferiors: - d = self.account.rename_mailbox(old, new) - rename_deferreds.append(d) + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 - d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - return d1 + d = self._inferiorNames(oldname) + d.addCallback(rename_inferiors) + return d - # FIXME use deferreds (list_all_mailbox_names, etc) def _inferiorNames(self, name): """ Return hierarchically inferior mailboxes. @@ -348,13 +359,17 @@ class IMAPAccount(object): :rtype: list """ # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d - # TODO use mail.Account.list_mailboxes def listMailboxes(self, ref, wildcard): """ List the mailboxes. @@ -371,11 +386,21 @@ class IMAPAccount(object): :param wildcard: mailbox name with possible wildcards :type wildcard: str """ - # XXX use wildcard in index query - # TODO get deferreds wildcard = imap4.wildcardToRegexp(wildcard, '/') - ref = self._inferiorNames(normalize_mailbox(ref)) - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) + + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d # # The rest of the methods are specific for leap.mail.imap.account.Account @@ -393,7 +418,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def get_subscribed(mbox): - return mbox.get_mbox_attr("subscribed") + return mbox.collection.get_mbox_attr("subscribed") d = self.getMailbox(name) d.addCallback(get_subscribed) @@ -410,7 +435,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_subscribed(mbox): - return mbox.set_mbox_attr("subscribed", True) + return mbox.collection.set_mbox_attr("subscribed", True) d = self.getMailbox(name) d.addCallback(set_subscribed) @@ -427,16 +452,19 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_unsubscribed(mbox): - return mbox.set_mbox_attr("subscribed", False) + return mbox.collection.set_mbox_attr("subscribed", False) d = self.getMailbox(name) d.addCallback(set_unsubscribed) return d - # TODO -- get__all_mboxes, return tuple - # with ... name? and subscribed bool... def getSubscriptions(self): - raise NotImplementedError() + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] + + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d # # INamespacePresenter diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index f2cbf75..e1eb6bf 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -21,9 +21,11 @@ import re import logging import StringIO import cStringIO +import time import os from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -36,6 +38,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage logger = logging.getLogger(__name__) @@ -88,7 +91,7 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - imap4.ICloseableMailbox, + #imap4.ICloseableMailbox, imap4.ISearchableMailbox, imap4.IMessageCopier) @@ -105,8 +108,7 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. + SoledadMailbox constructor. :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -115,6 +117,7 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw + self.closed = False self._uidvalidity = None self.collection = collection @@ -139,6 +142,11 @@ class IMAPMailbox(object): """ return self._listeners[self.mbox_name] + def get_imap_message(self, message): + msg = IMAPMessage(message) + msg.store = self.collection.store + return msg + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. @@ -308,17 +316,24 @@ class IMAPMailbox(object): :type names: iter """ r = {} + maybe = defer.maybeDeferred if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() + r[self.CMD_MSG] = maybe(self.getMessageCount) if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() + r[self.CMD_RECENT] = maybe(self.getRecentCount) if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getUIDNext() + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d def addMessage(self, message, flags, date=None): """ @@ -341,11 +356,15 @@ class IMAPMailbox(object): # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) + if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) + if date is None: + date = formatdate(time.time()) + # if PROFILE_CMD: # do_profile_cmd(d, "APPEND") @@ -419,10 +438,11 @@ class IMAPMailbox(object): self.setFlags((MessageFlags.NOSELECT_FLAG,)) def remove_mbox(_): - # FIXME collection does not have a delete_mbox method, - # it's in account. - # XXX should take care of deleting the uid table too. - return self.collection.delete_mbox(self.mbox_name) + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d d = self.deleteAllDocs() d.addCallback(remove_mbox) @@ -431,13 +451,14 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - def close(self): - """ - Expunge and mark as closed - """ - d = self.expunge() - d.addCallback(self._close_cb) - return d + # TODO server already calls expunge for closing + #def close(self): + #""" + #Expunge and mark as closed + #""" + #d = self.expunge() + #d.addCallback(self._close_cb) + #return d def expunge(self): """ @@ -445,10 +466,7 @@ class IMAPMailbox(object): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - d = defer.Deferred() - # FIXME actually broken. - # Iterate through index, and do a expunge. - return d + return self.collection.delete_all_flagged() # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): @@ -465,12 +483,12 @@ class IMAPMailbox(object): except TypeError: # looks like we cannot iterate try: + # XXX fixme, does not exist messages_asked.last = self.last_uid except ValueError: pass return messages_asked - # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -480,10 +498,16 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: set """ - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - return seq_messg + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(sequence): + set_asked = set(messages_asked) + set_exist = set(sequence) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d def fetch(self, messages_asked, uid): """ @@ -509,26 +533,41 @@ class IMAPMailbox(object): sequence = False # sequence = True if uid == 0 else False + getmsg = self.collection.get_message_by_uid messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - getmsg = self.collection.get_msg_by_uid + d_sequence = self._filter_msg_seq(messages_asked) + + def get_imap_messages_for_sequence(sequence): + def _zip_msgid(messages): + return zip( + list(sequence), + map(self.get_imap_message, messages)) + + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in sequence: + d_msg.append(getmsg(msgid)) + + d = defer.gatherResults(d_msg) + d.addCallback(_zip_msgid) + return d # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError + else: - got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) - result = ((msgid, msg) for msgid, msg in got_msg - if msg is not None) - reactor.callLater(0, self.unset_recent_flags, seq_messg) + d_sequence.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - - return result + return d_sequence def fetch_flags(self, messages_asked, uid): """ @@ -755,6 +794,7 @@ class IMAPMailbox(object): # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on @@ -821,7 +861,7 @@ class IMAPMailbox(object): Representation string for this mailbox. """ return u"<IMAPMailbox: mbox '%s' (%s)>" % ( - self.mbox_name, self.messages.count()) + self.mbox_name, self.collection.count()) _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 883da35..9b00162 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -18,18 +18,12 @@ IMAPMessage and IMAPMessageCollection. """ import logging -# import StringIO from twisted.mail import imap4 from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset - from leap.mail.utils import find_charset -from leap.mail.imap.messageparts import MessagePart -# from leap.mail.imap.messagepargs import MessagePartDoc logger = logging.getLogger(__name__) @@ -116,13 +110,17 @@ class IMAPMessage(object): # IMessagePart # - def getBodyFile(self): + def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading :rtype: StringIO """ + if store is None: + store = self.store + return self.message.get_body_file(store) + # TODO refactor with getBodyFile in MessagePart #body = bdoc_content.get(self.RAW_KEY, "") @@ -141,7 +139,6 @@ class IMAPMessage(object): #finally: #return write_fd(body) - return self.message.get_body_file() def getSize(self): """ diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index cf0ba74..b4f320a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Leap IMAP4 Server Implementation. +LEAP IMAP4 Server Implementation. """ from copy import copy @@ -36,9 +36,9 @@ from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile -class LeapIMAPServer(imap4.IMAP4Server): +class LEAPIMAPServer(imap4.IMAP4Server): """ - An IMAP4 Server with mailboxes backed by soledad + An IMAP4 Server with a LEAP Storage Backend. """ def __init__(self, *args, **kwargs): # pop extraneous arguments @@ -51,7 +51,6 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid - self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -146,12 +145,15 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ - self.reactor.callFromThread(self.mbox.notify_new) + reactor.callFromThread(self.mbox.notify_new) def _cbSelectWork(self, mbox, cmdName, tag): """ - Callback for selectWork, patched to avoid conformance errors due to - incomplete UIDVALIDITY line. + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count """ if mbox is None: self.sendNegativeResponse(tag, 'No such mailbox') @@ -161,9 +163,9 @@ class LeapIMAPServer(imap4.IMAP4Server): return flags = mbox.getFlags() + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') - self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding @@ -353,12 +355,9 @@ class LeapIMAPServer(imap4.IMAP4Server): self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) # -------------------- end isSubscribed patch ----------- - # TODO ---- - # subscribe method had also to be changed to accomodate - # deferred - # Revert to regular methods as soon as we implement non-deferred memory - # cache. + # TODO subscribe method had also to be changed to accomodate deferred def do_SUBSCRIBE(self, tag, name): + print "DOING SUBSCRIBE" name = self._parseMbox(name) def _subscribeCb(_): @@ -421,7 +420,8 @@ class LeapIMAPServer(imap4.IMAP4Server): def _renameEb(failure): m = failure.value - print "rename failure!" + print "SERVER rename failure!" + print m if failure.check(TypeError): self.sendBadResponse(tag, 'Invalid command syntax') elif failure.check(imap4.MailboxException): @@ -479,7 +479,7 @@ class LeapIMAPServer(imap4.IMAP4Server): if failure.check(imap4.MailboxException): self.sendNegativeResponse(tag, str(m)) else: - print "other error" + print "SERVER: other error" log.err() self.sendBadResponse( tag, diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 10ba32a..5d88a79 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # imap.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,15 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Imap service initialization +IMAP service initialization """ import logging import os -import time -from twisted.internet import defer, threads -from twisted.internet.protocol import ServerFactory +from twisted.internet import reactor from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory from twisted.mail import imap4 from twisted.python import log @@ -32,42 +31,14 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.account import IMAPAccount from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.server import LEAPIMAPServer from leap.soledad.client import Soledad -# The default port in which imap service will run -IMAP_PORT = 1984 - -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -###################################################### -# Temporary workaround for RecursionLimit when using -# qt4reactor. Do remove when we move to poll or select -# reactor, which do not show those problems. See #4974 -import resource -import sys - -try: - sys.setrecursionlimit(10**7) -except Exception: - print "Error setting recursion limit" -try: - # Increase max stack size from 8MB to 256MB - resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) -except Exception: - print "Error setting stack size" - -###################################################### - DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole @@ -81,6 +52,13 @@ if DO_PROFILE: pr = cProfile.Profile() pr.enable() +# The default port in which imap service will run +IMAP_PORT = 1984 + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + class IMAPAuthRealm(object): """ @@ -114,12 +92,8 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad - self._memstore = MemoryStore( - permanent_store=SoledadStore(soledad)) - theAccount = SoledadBackedAccount( - uuid, soledad=soledad, - memstore=self._memstore) + theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount # XXX how to pass the store along? @@ -131,7 +105,8 @@ class LeapIMAPFactory(ServerFactory): :param addr: remote ip address :type addr: str """ - imapProtocol = LeapIMAPServer( + # XXX addr not used??! + imapProtocol = LEAPIMAPServer( uuid=self._uuid, userid=self._userid, soledad=self._soledad) @@ -139,41 +114,18 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol - def doStop(self, cv=None): + def doStop(self): """ Stops imap service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition - :return: a Deferred that stops and flushes the in memory store data to - disk in another thread. - :rtype: Deferred """ + # TODO should wait for all the pending deferreds, + # the twisted way! if DO_PROFILE: log.msg("Stopping PROFILING") pr.disable() pr.dump_stats(PROFILE_DAT) - ServerFactory.doStop(self) - - if cv is not None: - def _stop_imap_cb(): - logger.debug('Stopping in memory store.') - self._memstore.stop_and_flush() - while not self._memstore.producer.is_queue_empty(): - logger.debug('Waiting for queue to be empty.') - # TODO use a gatherResults over the new/dirty - # deferred list, - # as in memorystore's expunge() method. - time.sleep(1) - # notify that service has stopped - logger.debug('Notifying that service has stopped.') - cv.acquire() - cv.notify() - cv.release() - - return threads.deferToThread(_stop_imap_cb) + return ServerFactory.doStop(self) def run_service(*args, **kwargs): @@ -185,11 +137,6 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ - from twisted.internet import reactor - # it looks like qtreactor does not honor this, - # but other reactors should. - reactor.suggestThreadPoolSize(20) - leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -201,13 +148,14 @@ def run_service(*args, **kwargs): leap_check(userid is not None, "need an user id") offline = kwargs.get('offline', False) - uuid = soledad._get_uuid() + uuid = soledad.uuid factory = LeapIMAPFactory(uuid, userid, soledad) try: tport = reactor.listenTCP(port, factory, interface="localhost") if not offline: + # FIXME --- update after meskio's work fetcher = LeapIncomingMail( keymanager, soledad, @@ -236,6 +184,8 @@ def run_service(*args, **kwargs): interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature return fetcher, tport, factory # not ok, signal error. diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/stress_tests_imap.zsh index 544faca..544faca 100755 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ b/src/leap/mail/imap/tests/stress_tests_imap.zsh diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dbb823f..d7fcdce 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -17,7 +17,7 @@ """ Test case for leap.email.imap.server TestCases taken from twisted tests and modified to make them work -against SoledadBackedAccount. +against our implementation of the IMAPAccount. @authors: Kali Kaneko, <kali@leap.se> XXX add authors from the original twisted tests. @@ -32,19 +32,13 @@ import types from twisted.mail import imap4 from twisted.internet import defer -from twisted.trial import unittest from twisted.python import util from twisted.python import failure from twisted import cred - -# import u1db - -from leap.mail.imap.mailbox import SoledadMailbox -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import IMAPMessageCollection from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -81,7 +75,7 @@ class TestRealm: # TestCases # -class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): +class MessageCollectionTestCase(IMAP4HelperMixin): """ Tests for the MessageCollection class """ @@ -95,10 +89,9 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ super(MessageCollectionTestCase, self).setUp() - # TODO deprecate memstore - memstore = MemoryStore() - self.messages = MessageCollection("testmbox%s" % (self.count,), - self._soledad, memstore=memstore) + # FIXME --- update initialization + self.messages = IMAPMessageCollection( + "testmbox%s" % (self.count,), self._soledad) MessageCollectionTestCase.count += 1 def tearDown(self): @@ -207,23 +200,18 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): add_1().addCallback(lambda ignored: self.assertEqual( mc.count(), 3)) - # XXX this has to be redone to fit memstore ------------# - #newmsg = mc._get_empty_doc() - #newmsg['mailbox'] = "mailbox/foo" - #mc._soledad.create_doc(newmsg) - #self.assertEqual(mc.count(), 3) - #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + +# DEBUG --- +#from twisted.internet.base import DelayedCall +#DelayedCall.debug = True -class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): - # TODO this currently will use a memory-only store. - # create a different one for testing soledad sync. +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ - Tests for the generic behavior of the LeapIMAP4Server + Tests for the generic behavior of the LEAPIMAP4Server which, right now, it's just implemented in this test file as - LeapIMAPServer. We will move the implementation, together with + LEAPIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -243,6 +231,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') + acc = self.server.theAccount def cb(): self.result.append(1) @@ -267,24 +256,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1 = self.connected.addCallback(strip(login)) d1.addCallback(strip(create)) d2 = self.loopback() - d = defer.gatherResults([d1, d2]) + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestCreate, succeed, fail) - def _cbTestCreate(self, ignored, succeed, fail): + def _cbTestCreate(self, mailboxes, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = LeapIMAPServer.theAccount.mailboxes - answers = ([u'INBOX', u'testbox', u'test/box', u'test', u'test/box/box', 'foobox']) - self.assertEqual(sorted(mboxes), sorted([a for a in answers])) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - acc = LeapIMAPServer.theAccount - d0 = lambda: acc.addMailbox('test-delete/me') + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -292,15 +280,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def delete(): return self.client.delete('test-delete/me') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback( - lambda _: self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, ['INBOX'])) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) return d def testIllegalInboxDelete(self): @@ -359,29 +349,34 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - acc = LeapIMAPServer.theAccount - d_del0 = lambda: acc.addMailbox('delete') - d_del1 = lambda: acc.addMailbox('delete/me') - - def set_noselect_flag(): - mbox = acc.getMailbox('delete') - mbox.setFlags((r'\Noselect',)) + acc = self.server.theAccount def login(): return self.client.login(TEST_USER, TEST_PASSWD) - def delete(): + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): return self.client.delete('delete') def deleteFailed(failure): self.failure = failure self.failure = None + d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d_del0)) - d1.addCallback(strip(d_del1)) - d1.addCallback(strip(set_noselect_flag)) - d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -393,11 +388,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly def testRename(self): """ Test whether we can rename a mailbox """ - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -405,16 +404,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, - ['INBOX', 'newname'])) + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) return d def testIllegalInboxRename(self): @@ -448,7 +447,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - acc = LeapIMAPServer.theAccount + acc = LEAPIMAPServer.theAccount dc1 = lambda: acc.create('oldmbox/m1') dc2 = lambda: acc.create('oldmbox/m2') @@ -468,7 +467,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, ignored): - mboxes = LeapIMAPServer.theAccount.mailboxes + mboxes = LEAPIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] self.assertEqual(sorted(mboxes), sorted([s for s in expected])) @@ -476,6 +475,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can mark a mailbox as subscribed to """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -483,9 +487,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.subscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return self.server.theAccount.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -500,7 +505,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can unsubscribe from a set of mailboxes """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) dc1 = lambda: acc.subscribe('this/mbox') dc2 = lambda: acc.subscribe('that/mbox') @@ -512,9 +522,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.unsubscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return acc.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallback(strip(dc1)) d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) @@ -531,10 +542,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to select a mailbox """ - acc = self.server.theAccount - d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + mbox_name = "TESTMAILBOXSELECT" self.selectedArgs = None + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -542,30 +557,26 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def selected(args): self.selectedArgs = args self._cbStopClient(None) - d = self.client.select('TESTMAILBOX-SELECT') + d = self.client.select(mbox_name) d.addCallback(selected) return d - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(select)) - d1.addErrback(self._ebGeneral) + #d1.addErrback(self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d def _cbTestSelect(self, ignored): - mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. + self.assertTrue(self.selectedArgs is not None) self.assertEqual(self.selectedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': True @@ -583,13 +594,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): caps.update(c) self.server.transport.loseConnection() return self.client.getCapabilities().addCallback(gotCaps) - d1 = self.connected.addCallback( + + d1 = self.connected + d1.addCallback( strip(getCaps)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, 'IDLE': None} - - return d.addCallback(lambda _: self.assertEqual(expected, caps)) + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d def testCapabilityWithAuth(self): caps = {} @@ -610,7 +624,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'IDLE': None, 'LITERAL+': None, 'AUTH': ['CRAM-MD5']} - return d.addCallback(lambda _: self.assertEqual(expCap, caps)) + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d # # authentication @@ -658,7 +673,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') def testFailedLogin(self): @@ -696,7 +710,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # @@ -743,11 +756,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): for details. """ # TODO implement the IMAP4ClientExamineTests testcase. - - self.server.theAccount.addMailbox('test-mailbox-e', - creation_ts=42) + mbox_name = "test_mailbox_e" + acc = self.server.theAccount self.examinedArgs = None + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -755,11 +770,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def examined(args): self.examinedArgs = args self._cbStopClient(None) - d = self.client.examine('test-mailbox-e') + d = self.client.examine(mbox_name) d.addCallback(examined) return d - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(examine)) d1.addErrback(self._ebGeneral) d2 = self.loopback() @@ -767,27 +783,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('test-mailbox-e') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. self.assertEqual(self.examinedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) def _listSetup(self, f, f2=None): - acc = LeapIMAPServer.theAccount - dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) - dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) - dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) + acc = self.server.theAccount + + dc1 = lambda: acc.addMailbox('root_subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root_another_thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non_root_subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -816,12 +824,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ def list(): return self.client.list('root', '%') + d = self._listSetup(list) d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), - (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") ]) )) return d @@ -830,28 +839,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def subs_mailbox(): # why not client.subscribe instead? - return acc.subscribe('root/subthing') + return acc.subscribe('root_subthing') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) + [(IMAPMailbox.init_flags, "/", "root_subthing")]) return d def testStatus(self): """ Test Status command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def add_mailbox(): - return acc.addMailbox('root/subthings') + return acc.addMailbox('root_subthings') # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -861,7 +870,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def status(): return self.client.status( - 'root/subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') def statused(result): self.statused = result @@ -927,7 +936,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) acc = self.server.theAccount - mailbox_name = "root_subthing" + mailbox_name = "root/subthing" def add_mailbox(): return acc.addMailbox(mailbox_name) @@ -948,42 +957,62 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - def print_mb(mb): - print "MB ----", mb - return mb - d.addCallback(print_mb) - d.addCallback(lambda mb: mb.collection.get_message_by_uid(1)) + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback(self._cbTestFullAppend, infile) - def _cbTestFullAppend(self, msg, infile): - # TODO --- move to deferreds - self.assertEqual( - set(('\\Recent', '\\SEEN', '\\DELETED')), - set(msg.getFlags())) + def _cbTestFullAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = dict(parsed.items()) - self.assertEqual( - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - msg.getInternalDate()) + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) - parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - headers = dict(parsed.items()) - self.assertEqual( - body, - msg.getBodyFile().read()) - gotheaders = msg.getHeaders(True) + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(expected_headers, headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) - self.assertItemsEqual( - headers, gotheaders) + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d def testPartialAppend(self): """ Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -998,34 +1027,46 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): (), self.client._IMAP4Client__cbContinueAppend, message ) ) - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback( self._cbTestPartialAppend, infile) - def _cbTestPartialAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - self.assertEqual(1, len(mb.messages)) - msg = mb.messages.get_msg_by_uid(1) - self.assertEqual( - set(('\\SEEN', '\\Recent')), - set(msg.getFlags()) - ) + def _cbTestPartialAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - self.assertEqual( - body, - msg.getBodyFile().read()) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d def testCheck(self): """ Test check command """ - LeapIMAPServer.theAccount.addMailbox('root/subthing') + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1036,98 +1077,106 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def check(): return self.client.check() - d = self.connected.addCallback(strip(login)) + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(check), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) - return self.loopback() - - # Okay, that was fun - - def testClose(self): - """ - Test closing the mailbox. We expect to get deleted all messages flagged - as such. - """ - acc = self.server.theAccount - name = 'mailbox-close' - - d0 = lambda: acc.addMailbox(name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select(name) - - def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) - - def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) - return d - - def close(): - return self.client.close() - - d = self.connected.addCallback(strip(login)) - d.addCallback(strip(d0)) - d.addCallbacks(strip(select), self._ebGeneral) - d.addCallback(strip(get_mailbox)) - d.addCallbacks(strip(add_messages), self._ebGeneral) - d.addCallbacks(strip(close), self._ebGeneral) - d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - - def _cbTestClose(self, ignored): - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - self.assertTrue(msg is not None) - - self.assertEqual( - dict(msg.hdoc.content)['subject'], - 'Message 2') - self.failUnless(self.mailbox.closed) + return defer.gatherResults([d, d2]) + + # Okay, that was much fun indeed + + # skipping close test: we just need expunge for now. + #def testClose(self): + #""" + #Test closing the mailbox. We expect to get deleted all messages flagged + #as such. + #""" + #acc = self.server.theAccount + #mailbox_name = 'mailbox-close' +# + #def add_mailbox(): + #return acc.addMailbox(mailbox_name) +# + #def login(): + #return self.client.login(TEST_USER, TEST_PASSWD) +# + #def select(): + #return self.client.select(mailbox_name) +# + #def get_mailbox(): + #def _save_mbox(mailbox): + #self.mailbox = mailbox + #d = self.server.theAccount.getMailbox(mailbox_name) + #d.addCallback(_save_mbox) + #return d +# + #def add_messages(): + #d1 = self.mailbox.addMessage( + #'test 1', flags=('\\Deleted', 'AnotherFlag')) + #d2 = self.mailbox.addMessage( + #'test 2', flags=('AnotherFlag',)) + #d3 = self.mailbox.addMessage( + #'test 3', flags=('\\Deleted',)) + #d = defer.gatherResults([d1, d2, d3]) + #return d +# + #def close(): + #return self.client.close() +# + #d = self.connected.addCallback(strip(add_mailbox)) + #d.addCallback(strip(login)) + #d.addCallbacks(strip(select), self._ebGeneral) + #d.addCallback(strip(get_mailbox)) + #d.addCallbacks(strip(add_messages), self._ebGeneral) + #d.addCallbacks(strip(close), self._ebGeneral) + #d.addCallbacks(self._cbStopClient, self._ebGeneral) + #d2 = self.loopback() + #d1 = defer.gatherResults([d, d2]) + #d1.addCallback(lambda _: self.mailbox.getMessageCount()) + #d1.addCallback(self._cbTestClose) + #return d1 +# + #def _cbTestClose(self, count): + # TODO is this correct? count should not take into account those + # flagged as deleted??? + #self.assertEqual(count, 1) + # TODO --- assert flags are those of the message #2 + #self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ acc = self.server.theAccount - name = 'mailbox-expunge' + mailbox_name = 'mailboxexpunge' - d0 = lambda: acc.addMailbox(name) + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def select(): - return self.client.select('mailbox-expunge') + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag')) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',))) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',))) return d def expunge(): @@ -1138,49 +1187,53 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = results self.results = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) - d1.addCallbacks(strip(select), self._ebGeneral) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.mailbox.getMessageCount()) return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored): + def _cbTestExpunge(self, count): # we only left 1 mssage with no deleted flag - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - - msg = list(self.mailbox.messages)[0] - self.assertTrue(msg is not None) - - self.assertEqual( - msg.hdoc.content['subject'], - 'Message 2') - + self.assertEqual(count, 1) # the uids of the deleted messages self.assertItemsEqual(self.results, [1, 3]) -class AccountTestCase(IMAP4HelperMixin, unittest.TestCase): +class AccountTestCase(IMAP4HelperMixin): """ Test the Account. """ def _create_empty_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('') + return self.server.theAccount.addMailbox('') def _create_one_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('one') + return self.server.theAccount.addMailbox('one') def test_illegalMailboxCreate(self): - self.assertRaises(AssertionError, self._create_empty_mailbox) + # FIXME --- account.addMailbox needs to raise a failure, + # not the direct exception. + self.stashed = None + + def stash(result): + self.stashed = result + + d = self._create_empty_mailbox() + d.addBoth(stash) + d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, + failure.Failure))) + return d + #self.assertRaises(AssertionError, self._create_empty_mailbox) -class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): """ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 920eeb0..5708787 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -1,30 +1,44 @@ -import os -import tempfile -import shutil - +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Common utilities for testing Soledad IMAP Server. +""" from email import parser from mock import Mock from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback +from twisted.python import log -from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.soledad.client import Soledad +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.tests.common import SoledadTestMixin TEST_USER = "testuser@leap.se" TEST_PASSWD = "1234" + # # Simple IMAP4 Client for testing # - class SimpleClient(imap4.IMAP4Client): - """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -51,160 +65,57 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() -# XXX move to common helper -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - - _soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return _soledad - - -# XXX this is not properly a mixin, since helper already inherits from -# uniittest.Testcase -class IMAP4HelperMixin(BaseLeapTest): +class IMAP4HelperMixin(SoledadTestMixin): """ MixIn containing several utilities to be shared across different TestCases """ - serverCTX = None clientCTX = None - # setUpClass cannot be a classmethod in trial, see: - # https://twistedmatrix.com/trac/ticket/1870 - def setUp(self): - """ - Setup method for each test. - - Initializes and run a LEAP IMAP4 Server. - """ - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - UUID = 'deadbeef', - USERID = TEST_USER - memstore = MemoryStore() - ########### + soledad_adaptor.cleanup_deferred_locks() - d_server_ready = defer.Deferred() + UUID = 'deadbeef', + USERID = TEST_USER - self.server = LeapIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - soledad=self._soledad) + def setup_server(account): + self.server = LEAPIMAPServer( + uuid=UUID, userid=USERID, + contextFactory=self.serverCTX, + soledad=self._soledad) + self.server.theAccount = account - self.client = SimpleClient( - d_server_ready, contextFactory=self.clientCTX) + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready - theAccount = SoledadBackedAccount( - USERID, - soledad=self._soledad, - memstore=memstore) - d_account_ready = theAccount.callWhenReady(lambda r: None) - LeapIMAPServer.theAccount = theAccount + def setup_account(_): + self.parser = parser.Parser() - self.connected = defer.gatherResults( - [d_server_ready, d_account_ready]) + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + self._soledad.sync = Mock() - # XXX FIXME -------------------------------------------- - # XXX this needs to be done differently, - # have to be hooked on initialization callback instead. - # in case we get something from previous tests... - #for mb in self.server.theAccount.mailboxes: - #self.server.theAccount.delete(mb) + d = defer.Deferred() + self.acc = IMAPAccount(USERID, self._soledad, d=d) + return d - # email parser - self.parser = parser.Parser() + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d def tearDown(self): - """ - tearDown method called after each test. - """ - try: - self._soledad.close() - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - def populateMessages(self): - """ - Populates soledad instance with several simple messages - """ - # XXX we should encapsulate this thru SoledadBackedAccount - # instead. - - # XXX we also should put this in a mailbox! - - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") - # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") - - def delete_all_docs(self): - """ - Deletes all the docs in the testing instance of the - SoledadBackedAccount. - """ - self.server.theAccount.deleteAllMessages( - iknowhatiamdoing=True) + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected def _cbStopClient(self, ignore): self.client.transport.loseConnection() @@ -212,11 +123,8 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - # can we do something similar? - # I guess this was ok with trial, but not in noseland... - # log.err(failure, "Problem with %r" % (self.function,)) - raise failure.value - # failure.trap(Exception) + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) def loopback(self): return loopback.loopbackAsync(self.server, self.client) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b2caa33..8137f97 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -172,7 +172,7 @@ class Message(object): :return: An RFC822-formatted date string. :rtype: str """ - return self._wrapper.fdoc.date + return self._wrapper.hdoc.date # imap.IMessageParts @@ -271,9 +271,10 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, + count=None): """ - Constructore for a MessageCollection. + Constructor for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -317,7 +318,7 @@ class MessageCollection(object): wrapper = getattr(self, "mbox_wrapper", None) if not wrapper: return None - return wrapper.mbox_uuid + return wrapper.uuid def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -364,10 +365,18 @@ class MessageCollection(object): self.messageklass, self.store, doc_id, uid=uid, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) d.addCallback(get_msg_from_mdoc_id) return d + # TODO deprecate ??? --- + def _prime_count(self): + def update_count(count): + self._count = count + d = self.mbox_indexer.count(self.mbox_name) + d.addCallback(update_count) + return d + def count(self): """ Count the messages in this collection. @@ -376,7 +385,17 @@ class MessageCollection(object): """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_name) + + d = self.mbox_indexer.count(self.mbox_uuid) + return d + + def count_recent(self): + # FIXME HACK + return 0 + + def count_unseen(self): + # FIXME hack + return 0 def get_uid_next(self): """ @@ -385,7 +404,13 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_next_uid(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_uuid) + + def all_uid_iter(self): + """ + Iterator through all the uids for this collection. + """ + return self.mbox_indexer.all_uid_iter(self.mbox_uuid) # Manipulate messages @@ -397,6 +422,7 @@ class MessageCollection(object): flags = tuple() if not tags: tags = tuple() + leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -408,10 +434,10 @@ class MessageCollection(object): else: mbox_id = self.mbox_uuid + wrapper.set_mbox_uuid(mbox_id) wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id @@ -420,6 +446,8 @@ class MessageCollection(object): d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) + d.addErrback(lambda f: f.printTraceback()) + return d def copy_msg(self, msg, newmailbox): @@ -453,8 +481,47 @@ class MessageCollection(object): d.addCallback(delete_mdoc_id, wrapper) return d + def delete_all_flagged(self): + """ + Delete all messages flagged as \\Deleted. + Used from IMAPMailbox.expunge() + """ + def get_uid_list(hashes): + d = [] + for h in hashes: + d.append(self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, h)) + return defer.gatherResults(d), hashes + + def delete_uid_entries((uids, hashes)): + d = [] + for h in hashes: + d.append(self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, h)) + return defer.gatherResults(d).addCallback( + lambda _: uids) + + mdocs_deleted = self.adaptor.del_all_flagged_messages( + self.store, self.mbox_uuid) + mdocs_deleted.addCallback(get_uid_list) + mdocs_deleted.addCallback(delete_uid_entries) + return mdocs_deleted + # TODO should add a delete-by-uid to collection? + def delete_all_docs(self): + def del_all_uid(uid_list): + deferreds = [] + for uid in uid_list: + d = self.get_message_by_uid(uid) + d.addCallback(lambda msg: msg.delete()) + deferreds.append(d) + return defer.gatherResults(deferreds) + + d = self.all_uid_iter() + d.addCallback(del_all_uid) + return d + def _update_flags_or_tags(self, old, new, mode): if mode == Flagsmode.APPEND: final = list((set(tuple(old) + new))) @@ -509,45 +576,49 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - def __init__(self, store): + def __init__(self, store, ready_cb=None): self.store = store self.adaptor = self.adaptor_class() self.mbox_indexer = MailboxIndexer(self.store) + self.deferred_initialization = defer.Deferred() self._initialized = False - self._deferred_initialization = defer.Deferred() + self._ready_cb = ready_cb - self._initialize_storage() + self._init_d = self._initialize_storage() def _initialize_storage(self): def add_mailbox_if_none(mboxes): if not mboxes: - self.add_mailbox(INBOX_NAME) + return self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True - self._deferred_initialization.callback(None) + self.deferred_initialization.callback(None) + if self._ready_cb is not None: + self._ready_cb() d = self.adaptor.initialize_store(self.store) d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) + return d def callWhenReady(self, cb, *args, **kw): - # use adaptor.store_ready instead? if self._initialized: cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb, *args, **kw) - return self._deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts # def list_all_mailbox_names(self): + def filter_names(mboxes): return [m.mbox for m in mboxes] @@ -563,8 +634,11 @@ class Account(object): def create_uuid(wrapper): if not wrapper.uuid: - wrapper.uuid = uuid.uuid4() - return wrapper.update(self.store) + wrapper.uuid = str(uuid.uuid4()) + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + return wrapper def create_uid_table_cb(wrapper): d = self.mbox_indexer.create_table(wrapper.uuid) @@ -599,7 +673,7 @@ class Account(object): def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update(self.store) + return wrapper, wrapper.update(self.store) d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) @@ -616,8 +690,6 @@ class Account(object): return MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - mboxwrapper_klass = self.adaptor.mboxwrapper_klass - #d = mboxwrapper_klass.get_or_create(name) d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 6155a7a..22e57d4 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -114,6 +114,24 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=sanitize(oldmailbox), new=sanitize(newmailbox))) + return self._query(sql) + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -134,7 +152,7 @@ class MailboxIndexer(object): assert doc_id mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -148,9 +166,11 @@ class MailboxIndexer(object): sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) + d.addErrback(lambda f: f.printTraceback()) return d def delete_doc_by_uid(self, mailbox_id, uid): @@ -199,13 +219,14 @@ class MailboxIndexer(object): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox uuid + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) @@ -218,7 +239,24 @@ class MailboxIndexer(object): d.addCallback(get_hash) return d - def get_doc_ids_from_uids(self, mailbox, uids): + def get_uid_from_doc_id(self, mailbox_id, doc_id): + check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') + + def get_uid(result): + return _maybe_first_query_item(result) + + sql = ("SELECT uid from {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_id))) + values = (doc_id,) + d = self._query(sql, values) + d.addCallback(get_uid) + return d + + + + def get_doc_ids_from_uids(self, mailbox_id, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() @@ -227,8 +265,8 @@ class MailboxIndexer(object): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox: the mailbox name - :type mailbox: str + :param mailbox_id: the mailbox uuid + :type mailbox_id: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ @@ -241,6 +279,7 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) + d.addErrback(lambda _: 0) return d def get_next_uid(self, mailbox_id): @@ -252,7 +291,7 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. @@ -273,3 +312,22 @@ class MailboxIndexer(object): d = self._query(sql) d.addCallback(increment) return d + + def all_uid_iter(self, mailbox_id): + """ + Get a sequence of all the uids in this mailbox. + + :param mailbox_id: the mailbox uuid + :type mailbox_id: str + """ + check_good_uuid(mailbox_id) + + sql = ("SELECT uid from {preffix}{name} ").format( + preffix=self.table_preffix, name=sanitize(mailbox_id)) + + def get_results(result): + return [x[0] for x in result] + + d = self._query(sql) + d.addCallback(get_results) + return d diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py index fefa7ee..a411b2d 100644 --- a/src/leap/mail/tests/common.py +++ b/src/leap/mail/tests/common.py @@ -21,6 +21,9 @@ import os import shutil import tempfile +from twisted.internet import defer +from twisted.trial import unittest + from leap.common.testing.basetest import BaseLeapTest from leap.soledad.client import Soledad @@ -60,7 +63,7 @@ def _initialize_soledad(email, gnupg_home, tempdir): return soledad -class SoledadTestMixin(BaseLeapTest): +class SoledadTestMixin(unittest.TestCase, BaseLeapTest): """ It is **VERY** important that this base is added *AFTER* unittest.TestCase """ @@ -68,15 +71,7 @@ class SoledadTestMixin(BaseLeapTest): def setUp(self): self.results = [] - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir + self.setUpEnv() # Soledad: config info self.gnupg_home = "%s/gnupg" % self.tempdir @@ -88,6 +83,8 @@ class SoledadTestMixin(BaseLeapTest): self.gnupg_home, self.tempdir) + return defer.succeed(True) + def tearDown(self): """ tearDown method called after each test. diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py index 2edf1d8..b82fd2d 100644 --- a/src/leap/mail/tests/test_mailbox_indexer.py +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -84,18 +84,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): d.addCallback(assert_table_deleted) return d - #def test_rename_table(self): - #def assert_table_renamed(tables): - #self.assertEqual( - #tables, ["leapmail_uid_foomailbox"]) -# - #m_uid = self.get_mbox_uid() - #d = m_uid.create_table('inbox') - #d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) - #d.addCallback(self.list_mail_tables_cb) - #d.addCallback(assert_table_renamed) - #return d - def test_insert_doc(self): m_uid = self.get_mbox_uid() @@ -168,7 +156,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_get_doc_id_from_uid(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) @@ -183,7 +170,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_count(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) h2 = fmt_hash(mbox_id, hash_test1) @@ -216,7 +202,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_get_next_uid(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) h2 = fmt_hash(mbox_id, hash_test1) @@ -237,3 +222,29 @@ class MailboxIndexerTestCase(SoledadTestMixin): d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) d.addCallback(partial(assert_next_uid, expected=6)) return d + + def test_all_uid_iter(self): + + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4)) + + def assert_all_uid(result, expected=[2, 3, 5]): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id)) + d.addCallback(partial(assert_all_uid)) + return d |