diff options
Diffstat (limited to 'src/leap/mail/imap')
| -rw-r--r-- | src/leap/mail/imap/account.py | 174 | ||||
| -rw-r--r-- | src/leap/mail/imap/mailbox.py | 116 | ||||
| -rw-r--r-- | src/leap/mail/imap/messages.py | 13 | ||||
| -rw-r--r-- | src/leap/mail/imap/server.py | 30 | ||||
| -rw-r--r-- | src/leap/mail/imap/service/imap.py | 98 | ||||
| -rwxr-xr-x | src/leap/mail/imap/tests/stress_tests_imap.zsh (renamed from src/leap/mail/imap/tests/leap_tests_imap.zsh) | 0 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/test_imap.py | 539 | ||||
| -rw-r--r-- | src/leap/mail/imap/tests/utils.py | 212 |
8 files changed, 579 insertions, 603 deletions
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0baf078..dfc0d62 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -63,7 +63,7 @@ class IMAPAccount(object): selected = None closed = False - def __init__(self, user_id, store): + def __init__(self, user_id, store, d=None): """ Keeps track of the mailboxes and subscriptions handled by this account. @@ -72,21 +72,31 @@ class IMAPAccount(object): :param store: a Soledad instance. :type store: Soledad + + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred """ leap_assert(store, "Need a store instance to initialize") leap_assert_type(store, Soledad) # TODO assert too that the name matches the user/uuid with which - # soledad has been initialized. + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. + self.user_id = user_id - self.account = Account(store) + self.account = Account(store, ready_cb=lambda: d.callback(self)) def _return_mailbox_from_collection(self, collection, readwrite=1): if collection is None: return None - return IMAPMailbox(collection, rw=readwrite) + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + + def callWhenReady(self, cb, *args, **kw): + d = self.account.callWhenReady(cb, *args, **kw) + return d - # XXX Where's this used from? -- self.delete... def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -102,11 +112,12 @@ class IMAPAccount(object): def check_it_exists(mailboxes): if name not in mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) + return True d = self.account.list_all_mailbox_names() d.addCallback(check_it_exists) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) - d.addCallbacK(self._return_mailbox_from_collection) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) return d # @@ -130,12 +141,9 @@ class IMAPAccount(object): """ name = normalize_mailbox(name) + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... leap_assert(name, "Need a mailbox name to create a mailbox") - - def check_it_does_not_exist(mailboxes): - if name in mailboxes: - raise imap4.MailboxCollision(repr(name)) - if creation_ts is None: # by default, we pass an int value # taken from the current time @@ -143,14 +151,20 @@ class IMAPAccount(object): # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes + def set_mbox_creation_ts(collection): - d = collection.set_mbox_attr("created") + d = collection.set_mbox_attr("created", creation_ts) d.addCallback(lambda _: collection) return d d = self.account.list_all_mailbox_names() d.addCallback(check_it_does_not_exist) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallback(lambda _: self.account.add_mailbox(name)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) d.addCallback(set_mbox_creation_ts) d.addCallback(self._return_mailbox_from_collection) return d @@ -172,39 +186,40 @@ class IMAPAccount(object): :raise MailboxException: Raised if this mailbox cannot be added. """ - # TODO raise MailboxException paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] sep = '/' + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + for accum in range(1, len(paths)): - try: - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - subs.append(d) - # XXX should this be handled by the deferred? - except imap4.MailboxCollision: - pass - try: - df = self.addMailbox(sep.join(paths)) - except imap4.MailboxCollision: + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): - df = defer.succeed(False) + return defer.succeed(False) else: - df = defer.succeed(True) - finally: - subs.append(df) + return defer.succeed(True) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) def all_good(result): return all(result) if subs: - d1 = defer.gatherResults(subs, consumeErrors=True) + d1 = defer.gatherResults(subs) d1.addCallback(all_good) + return d1 else: - d1 = defer.succeed(False) - return d1 + return defer.succeed(False) def select(self, name, readwrite=1): """ @@ -216,7 +231,7 @@ class IMAPAccount(object): :param readwrite: 1 for readwrite permissions. :type readwrite: int - :rtype: SoledadMailbox + :rtype: IMAPMailbox """ name = normalize_mailbox(name) @@ -245,9 +260,6 @@ class IMAPAccount(object): """ Deletes a mailbox. - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - :param name: the mailbox to be deleted :type name: str @@ -258,10 +270,10 @@ class IMAPAccount(object): :rtype: Deferred """ name = normalize_mailbox(name) - _mboxes = [] + _mboxes = None def check_it_exists(mailboxes): - # FIXME works? -- pass variable ref to outer scope + global _mboxes _mboxes = mailboxes if name not in mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -274,6 +286,7 @@ class IMAPAccount(object): return mbox.destroy() def check_can_be_deleted(mbox): + global _mboxes # See if this box is flagged \Noselect mbox_flags = mbox.getFlags() if MessageFlags.NOSELECT_FLAG in mbox_flags: @@ -317,29 +330,27 @@ class IMAPAccount(object): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) - # FIXME check that scope works (test) - _mboxes = [] - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox(repr(oldname)) - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + def rename_inferiors(inferiors_result): + inferiors, mailboxes = inferiors_result + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] - for (old, new) in inferiors: - if new in _mboxes: - raise imap4.MailboxCollision(repr(new)) + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) - rename_deferreds = [] + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) - for (old, new) in inferiors: - d = self.account.rename_mailbox(old, new) - rename_deferreds.append(d) + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 - d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - return d1 + d = self._inferiorNames(oldname) + d.addCallback(rename_inferiors) + return d - # FIXME use deferreds (list_all_mailbox_names, etc) def _inferiorNames(self, name): """ Return hierarchically inferior mailboxes. @@ -348,13 +359,17 @@ class IMAPAccount(object): :rtype: list """ # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d - # TODO use mail.Account.list_mailboxes def listMailboxes(self, ref, wildcard): """ List the mailboxes. @@ -371,11 +386,21 @@ class IMAPAccount(object): :param wildcard: mailbox name with possible wildcards :type wildcard: str """ - # XXX use wildcard in index query - # TODO get deferreds wildcard = imap4.wildcardToRegexp(wildcard, '/') - ref = self._inferiorNames(normalize_mailbox(ref)) - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) + + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d # # The rest of the methods are specific for leap.mail.imap.account.Account @@ -393,7 +418,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def get_subscribed(mbox): - return mbox.get_mbox_attr("subscribed") + return mbox.collection.get_mbox_attr("subscribed") d = self.getMailbox(name) d.addCallback(get_subscribed) @@ -410,7 +435,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_subscribed(mbox): - return mbox.set_mbox_attr("subscribed", True) + return mbox.collection.set_mbox_attr("subscribed", True) d = self.getMailbox(name) d.addCallback(set_subscribed) @@ -427,16 +452,19 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_unsubscribed(mbox): - return mbox.set_mbox_attr("subscribed", False) + return mbox.collection.set_mbox_attr("subscribed", False) d = self.getMailbox(name) d.addCallback(set_unsubscribed) return d - # TODO -- get__all_mboxes, return tuple - # with ... name? and subscribed bool... def getSubscriptions(self): - raise NotImplementedError() + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] + + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d # # INamespacePresenter diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index f2cbf75..e1eb6bf 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -21,9 +21,11 @@ import re import logging import StringIO import cStringIO +import time import os from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -36,6 +38,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage logger = logging.getLogger(__name__) @@ -88,7 +91,7 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - imap4.ICloseableMailbox, + #imap4.ICloseableMailbox, imap4.ISearchableMailbox, imap4.IMessageCopier) @@ -105,8 +108,7 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. + SoledadMailbox constructor. :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -115,6 +117,7 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw + self.closed = False self._uidvalidity = None self.collection = collection @@ -139,6 +142,11 @@ class IMAPMailbox(object): """ return self._listeners[self.mbox_name] + def get_imap_message(self, message): + msg = IMAPMessage(message) + msg.store = self.collection.store + return msg + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. @@ -308,17 +316,24 @@ class IMAPMailbox(object): :type names: iter """ r = {} + maybe = defer.maybeDeferred if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() + r[self.CMD_MSG] = maybe(self.getMessageCount) if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() + r[self.CMD_RECENT] = maybe(self.getRecentCount) if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getUIDNext() + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d def addMessage(self, message, flags, date=None): """ @@ -341,11 +356,15 @@ class IMAPMailbox(object): # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) + if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) + if date is None: + date = formatdate(time.time()) + # if PROFILE_CMD: # do_profile_cmd(d, "APPEND") @@ -419,10 +438,11 @@ class IMAPMailbox(object): self.setFlags((MessageFlags.NOSELECT_FLAG,)) def remove_mbox(_): - # FIXME collection does not have a delete_mbox method, - # it's in account. - # XXX should take care of deleting the uid table too. - return self.collection.delete_mbox(self.mbox_name) + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d d = self.deleteAllDocs() d.addCallback(remove_mbox) @@ -431,13 +451,14 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - def close(self): - """ - Expunge and mark as closed - """ - d = self.expunge() - d.addCallback(self._close_cb) - return d + # TODO server already calls expunge for closing + #def close(self): + #""" + #Expunge and mark as closed + #""" + #d = self.expunge() + #d.addCallback(self._close_cb) + #return d def expunge(self): """ @@ -445,10 +466,7 @@ class IMAPMailbox(object): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - d = defer.Deferred() - # FIXME actually broken. - # Iterate through index, and do a expunge. - return d + return self.collection.delete_all_flagged() # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): @@ -465,12 +483,12 @@ class IMAPMailbox(object): except TypeError: # looks like we cannot iterate try: + # XXX fixme, does not exist messages_asked.last = self.last_uid except ValueError: pass return messages_asked - # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -480,10 +498,16 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: set """ - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - return seq_messg + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(sequence): + set_asked = set(messages_asked) + set_exist = set(sequence) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d def fetch(self, messages_asked, uid): """ @@ -509,26 +533,41 @@ class IMAPMailbox(object): sequence = False # sequence = True if uid == 0 else False + getmsg = self.collection.get_message_by_uid messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - getmsg = self.collection.get_msg_by_uid + d_sequence = self._filter_msg_seq(messages_asked) + + def get_imap_messages_for_sequence(sequence): + def _zip_msgid(messages): + return zip( + list(sequence), + map(self.get_imap_message, messages)) + + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in sequence: + d_msg.append(getmsg(msgid)) + + d = defer.gatherResults(d_msg) + d.addCallback(_zip_msgid) + return d # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError + else: - got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) - result = ((msgid, msg) for msgid, msg in got_msg - if msg is not None) - reactor.callLater(0, self.unset_recent_flags, seq_messg) + d_sequence.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - - return result + return d_sequence def fetch_flags(self, messages_asked, uid): """ @@ -755,6 +794,7 @@ class IMAPMailbox(object): # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on @@ -821,7 +861,7 @@ class IMAPMailbox(object): Representation string for this mailbox. """ return u"<IMAPMailbox: mbox '%s' (%s)>" % ( - self.mbox_name, self.messages.count()) + self.mbox_name, self.collection.count()) _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 883da35..9b00162 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -18,18 +18,12 @@ IMAPMessage and IMAPMessageCollection. """ import logging -# import StringIO from twisted.mail import imap4 from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset - from leap.mail.utils import find_charset -from leap.mail.imap.messageparts import MessagePart -# from leap.mail.imap.messagepargs import MessagePartDoc logger = logging.getLogger(__name__) @@ -116,13 +110,17 @@ class IMAPMessage(object): # IMessagePart # - def getBodyFile(self): + def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading :rtype: StringIO """ + if store is None: + store = self.store + return self.message.get_body_file(store) + # TODO refactor with getBodyFile in MessagePart #body = bdoc_content.get(self.RAW_KEY, "") @@ -141,7 +139,6 @@ class IMAPMessage(object): #finally: #return write_fd(body) - return self.message.get_body_file() def getSize(self): """ diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index cf0ba74..b4f320a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Leap IMAP4 Server Implementation. +LEAP IMAP4 Server Implementation. """ from copy import copy @@ -36,9 +36,9 @@ from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile -class LeapIMAPServer(imap4.IMAP4Server): +class LEAPIMAPServer(imap4.IMAP4Server): """ - An IMAP4 Server with mailboxes backed by soledad + An IMAP4 Server with a LEAP Storage Backend. """ def __init__(self, *args, **kwargs): # pop extraneous arguments @@ -51,7 +51,6 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid - self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -146,12 +145,15 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ - self.reactor.callFromThread(self.mbox.notify_new) + reactor.callFromThread(self.mbox.notify_new) def _cbSelectWork(self, mbox, cmdName, tag): """ - Callback for selectWork, patched to avoid conformance errors due to - incomplete UIDVALIDITY line. + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count """ if mbox is None: self.sendNegativeResponse(tag, 'No such mailbox') @@ -161,9 +163,9 @@ class LeapIMAPServer(imap4.IMAP4Server): return flags = mbox.getFlags() + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') - self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding @@ -353,12 +355,9 @@ class LeapIMAPServer(imap4.IMAP4Server): self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) # -------------------- end isSubscribed patch ----------- - # TODO ---- - # subscribe method had also to be changed to accomodate - # deferred - # Revert to regular methods as soon as we implement non-deferred memory - # cache. + # TODO subscribe method had also to be changed to accomodate deferred def do_SUBSCRIBE(self, tag, name): + print "DOING SUBSCRIBE" name = self._parseMbox(name) def _subscribeCb(_): @@ -421,7 +420,8 @@ class LeapIMAPServer(imap4.IMAP4Server): def _renameEb(failure): m = failure.value - print "rename failure!" + print "SERVER rename failure!" + print m if failure.check(TypeError): self.sendBadResponse(tag, 'Invalid command syntax') elif failure.check(imap4.MailboxException): @@ -479,7 +479,7 @@ class LeapIMAPServer(imap4.IMAP4Server): if failure.check(imap4.MailboxException): self.sendNegativeResponse(tag, str(m)) else: - print "other error" + print "SERVER: other error" log.err() self.sendBadResponse( tag, diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 10ba32a..5d88a79 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # imap.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,15 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Imap service initialization +IMAP service initialization """ import logging import os -import time -from twisted.internet import defer, threads -from twisted.internet.protocol import ServerFactory +from twisted.internet import reactor from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory from twisted.mail import imap4 from twisted.python import log @@ -32,42 +31,14 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.account import IMAPAccount from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.server import LEAPIMAPServer from leap.soledad.client import Soledad -# The default port in which imap service will run -IMAP_PORT = 1984 - -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -###################################################### -# Temporary workaround for RecursionLimit when using -# qt4reactor. Do remove when we move to poll or select -# reactor, which do not show those problems. See #4974 -import resource -import sys - -try: - sys.setrecursionlimit(10**7) -except Exception: - print "Error setting recursion limit" -try: - # Increase max stack size from 8MB to 256MB - resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) -except Exception: - print "Error setting stack size" - -###################################################### - DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole @@ -81,6 +52,13 @@ if DO_PROFILE: pr = cProfile.Profile() pr.enable() +# The default port in which imap service will run +IMAP_PORT = 1984 + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + class IMAPAuthRealm(object): """ @@ -114,12 +92,8 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad - self._memstore = MemoryStore( - permanent_store=SoledadStore(soledad)) - theAccount = SoledadBackedAccount( - uuid, soledad=soledad, - memstore=self._memstore) + theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount # XXX how to pass the store along? @@ -131,7 +105,8 @@ class LeapIMAPFactory(ServerFactory): :param addr: remote ip address :type addr: str """ - imapProtocol = LeapIMAPServer( + # XXX addr not used??! + imapProtocol = LEAPIMAPServer( uuid=self._uuid, userid=self._userid, soledad=self._soledad) @@ -139,41 +114,18 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol - def doStop(self, cv=None): + def doStop(self): """ Stops imap service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition - :return: a Deferred that stops and flushes the in memory store data to - disk in another thread. - :rtype: Deferred """ + # TODO should wait for all the pending deferreds, + # the twisted way! if DO_PROFILE: log.msg("Stopping PROFILING") pr.disable() pr.dump_stats(PROFILE_DAT) - ServerFactory.doStop(self) - - if cv is not None: - def _stop_imap_cb(): - logger.debug('Stopping in memory store.') - self._memstore.stop_and_flush() - while not self._memstore.producer.is_queue_empty(): - logger.debug('Waiting for queue to be empty.') - # TODO use a gatherResults over the new/dirty - # deferred list, - # as in memorystore's expunge() method. - time.sleep(1) - # notify that service has stopped - logger.debug('Notifying that service has stopped.') - cv.acquire() - cv.notify() - cv.release() - - return threads.deferToThread(_stop_imap_cb) + return ServerFactory.doStop(self) def run_service(*args, **kwargs): @@ -185,11 +137,6 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ - from twisted.internet import reactor - # it looks like qtreactor does not honor this, - # but other reactors should. - reactor.suggestThreadPoolSize(20) - leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -201,13 +148,14 @@ def run_service(*args, **kwargs): leap_check(userid is not None, "need an user id") offline = kwargs.get('offline', False) - uuid = soledad._get_uuid() + uuid = soledad.uuid factory = LeapIMAPFactory(uuid, userid, soledad) try: tport = reactor.listenTCP(port, factory, interface="localhost") if not offline: + # FIXME --- update after meskio's work fetcher = LeapIncomingMail( keymanager, soledad, @@ -236,6 +184,8 @@ def run_service(*args, **kwargs): interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature return fetcher, tport, factory # not ok, signal error. diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/stress_tests_imap.zsh index 544faca..544faca 100755 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ b/src/leap/mail/imap/tests/stress_tests_imap.zsh diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dbb823f..d7fcdce 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -17,7 +17,7 @@ """ Test case for leap.email.imap.server TestCases taken from twisted tests and modified to make them work -against SoledadBackedAccount. +against our implementation of the IMAPAccount. @authors: Kali Kaneko, <kali@leap.se> XXX add authors from the original twisted tests. @@ -32,19 +32,13 @@ import types from twisted.mail import imap4 from twisted.internet import defer -from twisted.trial import unittest from twisted.python import util from twisted.python import failure from twisted import cred - -# import u1db - -from leap.mail.imap.mailbox import SoledadMailbox -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import IMAPMessageCollection from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -81,7 +75,7 @@ class TestRealm: # TestCases # -class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): +class MessageCollectionTestCase(IMAP4HelperMixin): """ Tests for the MessageCollection class """ @@ -95,10 +89,9 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ super(MessageCollectionTestCase, self).setUp() - # TODO deprecate memstore - memstore = MemoryStore() - self.messages = MessageCollection("testmbox%s" % (self.count,), - self._soledad, memstore=memstore) + # FIXME --- update initialization + self.messages = IMAPMessageCollection( + "testmbox%s" % (self.count,), self._soledad) MessageCollectionTestCase.count += 1 def tearDown(self): @@ -207,23 +200,18 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): add_1().addCallback(lambda ignored: self.assertEqual( mc.count(), 3)) - # XXX this has to be redone to fit memstore ------------# - #newmsg = mc._get_empty_doc() - #newmsg['mailbox'] = "mailbox/foo" - #mc._soledad.create_doc(newmsg) - #self.assertEqual(mc.count(), 3) - #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + +# DEBUG --- +#from twisted.internet.base import DelayedCall +#DelayedCall.debug = True -class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): - # TODO this currently will use a memory-only store. - # create a different one for testing soledad sync. +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ - Tests for the generic behavior of the LeapIMAP4Server + Tests for the generic behavior of the LEAPIMAP4Server which, right now, it's just implemented in this test file as - LeapIMAPServer. We will move the implementation, together with + LEAPIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -243,6 +231,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') + acc = self.server.theAccount def cb(): self.result.append(1) @@ -267,24 +256,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1 = self.connected.addCallback(strip(login)) d1.addCallback(strip(create)) d2 = self.loopback() - d = defer.gatherResults([d1, d2]) + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestCreate, succeed, fail) - def _cbTestCreate(self, ignored, succeed, fail): + def _cbTestCreate(self, mailboxes, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = LeapIMAPServer.theAccount.mailboxes - answers = ([u'INBOX', u'testbox', u'test/box', u'test', u'test/box/box', 'foobox']) - self.assertEqual(sorted(mboxes), sorted([a for a in answers])) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - acc = LeapIMAPServer.theAccount - d0 = lambda: acc.addMailbox('test-delete/me') + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -292,15 +280,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def delete(): return self.client.delete('test-delete/me') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback( - lambda _: self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, ['INBOX'])) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) return d def testIllegalInboxDelete(self): @@ -359,29 +349,34 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - acc = LeapIMAPServer.theAccount - d_del0 = lambda: acc.addMailbox('delete') - d_del1 = lambda: acc.addMailbox('delete/me') - - def set_noselect_flag(): - mbox = acc.getMailbox('delete') - mbox.setFlags((r'\Noselect',)) + acc = self.server.theAccount def login(): return self.client.login(TEST_USER, TEST_PASSWD) - def delete(): + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): return self.client.delete('delete') def deleteFailed(failure): self.failure = failure self.failure = None + d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d_del0)) - d1.addCallback(strip(d_del1)) - d1.addCallback(strip(set_noselect_flag)) - d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -393,11 +388,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly def testRename(self): """ Test whether we can rename a mailbox """ - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -405,16 +404,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, - ['INBOX', 'newname'])) + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) return d def testIllegalInboxRename(self): @@ -448,7 +447,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - acc = LeapIMAPServer.theAccount + acc = LEAPIMAPServer.theAccount dc1 = lambda: acc.create('oldmbox/m1') dc2 = lambda: acc.create('oldmbox/m2') @@ -468,7 +467,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, ignored): - mboxes = LeapIMAPServer.theAccount.mailboxes + mboxes = LEAPIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] self.assertEqual(sorted(mboxes), sorted([s for s in expected])) @@ -476,6 +475,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can mark a mailbox as subscribed to """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -483,9 +487,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.subscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return self.server.theAccount.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -500,7 +505,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can unsubscribe from a set of mailboxes """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) dc1 = lambda: acc.subscribe('this/mbox') dc2 = lambda: acc.subscribe('that/mbox') @@ -512,9 +522,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.unsubscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return acc.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallback(strip(dc1)) d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) @@ -531,10 +542,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to select a mailbox """ - acc = self.server.theAccount - d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + mbox_name = "TESTMAILBOXSELECT" self.selectedArgs = None + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -542,30 +557,26 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def selected(args): self.selectedArgs = args self._cbStopClient(None) - d = self.client.select('TESTMAILBOX-SELECT') + d = self.client.select(mbox_name) d.addCallback(selected) return d - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(select)) - d1.addErrback(self._ebGeneral) + #d1.addErrback(self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d def _cbTestSelect(self, ignored): - mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. + self.assertTrue(self.selectedArgs is not None) self.assertEqual(self.selectedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': True @@ -583,13 +594,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): caps.update(c) self.server.transport.loseConnection() return self.client.getCapabilities().addCallback(gotCaps) - d1 = self.connected.addCallback( + + d1 = self.connected + d1.addCallback( strip(getCaps)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, 'IDLE': None} - - return d.addCallback(lambda _: self.assertEqual(expected, caps)) + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d def testCapabilityWithAuth(self): caps = {} @@ -610,7 +624,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'IDLE': None, 'LITERAL+': None, 'AUTH': ['CRAM-MD5']} - return d.addCallback(lambda _: self.assertEqual(expCap, caps)) + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d # # authentication @@ -658,7 +673,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') def testFailedLogin(self): @@ -696,7 +710,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # @@ -743,11 +756,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): for details. """ # TODO implement the IMAP4ClientExamineTests testcase. - - self.server.theAccount.addMailbox('test-mailbox-e', - creation_ts=42) + mbox_name = "test_mailbox_e" + acc = self.server.theAccount self.examinedArgs = None + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -755,11 +770,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def examined(args): self.examinedArgs = args self._cbStopClient(None) - d = self.client.examine('test-mailbox-e') + d = self.client.examine(mbox_name) d.addCallback(examined) return d - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(examine)) d1.addErrback(self._ebGeneral) d2 = self.loopback() @@ -767,27 +783,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('test-mailbox-e') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. self.assertEqual(self.examinedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) def _listSetup(self, f, f2=None): - acc = LeapIMAPServer.theAccount - dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) - dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) - dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) + acc = self.server.theAccount + + dc1 = lambda: acc.addMailbox('root_subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root_another_thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non_root_subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -816,12 +824,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ def list(): return self.client.list('root', '%') + d = self._listSetup(list) d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), - (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") ]) )) return d @@ -830,28 +839,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def subs_mailbox(): # why not client.subscribe instead? - return acc.subscribe('root/subthing') + return acc.subscribe('root_subthing') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) + [(IMAPMailbox.init_flags, "/", "root_subthing")]) return d def testStatus(self): """ Test Status command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def add_mailbox(): - return acc.addMailbox('root/subthings') + return acc.addMailbox('root_subthings') # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -861,7 +870,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def status(): return self.client.status( - 'root/subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') def statused(result): self.statused = result @@ -927,7 +936,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) acc = self.server.theAccount - mailbox_name = "root_subthing" + mailbox_name = "root/subthing" def add_mailbox(): return acc.addMailbox(mailbox_name) @@ -948,42 +957,62 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - def print_mb(mb): - print "MB ----", mb - return mb - d.addCallback(print_mb) - d.addCallback(lambda mb: mb.collection.get_message_by_uid(1)) + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback(self._cbTestFullAppend, infile) - def _cbTestFullAppend(self, msg, infile): - # TODO --- move to deferreds - self.assertEqual( - set(('\\Recent', '\\SEEN', '\\DELETED')), - set(msg.getFlags())) + def _cbTestFullAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = dict(parsed.items()) - self.assertEqual( - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - msg.getInternalDate()) + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) - parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - headers = dict(parsed.items()) - self.assertEqual( - body, - msg.getBodyFile().read()) - gotheaders = msg.getHeaders(True) + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(expected_headers, headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) - self.assertItemsEqual( - headers, gotheaders) + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d def testPartialAppend(self): """ Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -998,34 +1027,46 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): (), self.client._IMAP4Client__cbContinueAppend, message ) ) - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback( self._cbTestPartialAppend, infile) - def _cbTestPartialAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - self.assertEqual(1, len(mb.messages)) - msg = mb.messages.get_msg_by_uid(1) - self.assertEqual( - set(('\\SEEN', '\\Recent')), - set(msg.getFlags()) - ) + def _cbTestPartialAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - self.assertEqual( - body, - msg.getBodyFile().read()) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d def testCheck(self): """ Test check command """ - LeapIMAPServer.theAccount.addMailbox('root/subthing') + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1036,98 +1077,106 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def check(): return self.client.check() - d = self.connected.addCallback(strip(login)) + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(check), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) - return self.loopback() - - # Okay, that was fun - - def testClose(self): - """ - Test closing the mailbox. We expect to get deleted all messages flagged - as such. - """ - acc = self.server.theAccount - name = 'mailbox-close' - - d0 = lambda: acc.addMailbox(name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select(name) - - def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) - - def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) - return d - - def close(): - return self.client.close() - - d = self.connected.addCallback(strip(login)) - d.addCallback(strip(d0)) - d.addCallbacks(strip(select), self._ebGeneral) - d.addCallback(strip(get_mailbox)) - d.addCallbacks(strip(add_messages), self._ebGeneral) - d.addCallbacks(strip(close), self._ebGeneral) - d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - - def _cbTestClose(self, ignored): - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - self.assertTrue(msg is not None) - - self.assertEqual( - dict(msg.hdoc.content)['subject'], - 'Message 2') - self.failUnless(self.mailbox.closed) + return defer.gatherResults([d, d2]) + + # Okay, that was much fun indeed + + # skipping close test: we just need expunge for now. + #def testClose(self): + #""" + #Test closing the mailbox. We expect to get deleted all messages flagged + #as such. + #""" + #acc = self.server.theAccount + #mailbox_name = 'mailbox-close' +# + #def add_mailbox(): + #return acc.addMailbox(mailbox_name) +# + #def login(): + #return self.client.login(TEST_USER, TEST_PASSWD) +# + #def select(): + #return self.client.select(mailbox_name) +# + #def get_mailbox(): + #def _save_mbox(mailbox): + #self.mailbox = mailbox + #d = self.server.theAccount.getMailbox(mailbox_name) + #d.addCallback(_save_mbox) + #return d +# + #def add_messages(): + #d1 = self.mailbox.addMessage( + #'test 1', flags=('\\Deleted', 'AnotherFlag')) + #d2 = self.mailbox.addMessage( + #'test 2', flags=('AnotherFlag',)) + #d3 = self.mailbox.addMessage( + #'test 3', flags=('\\Deleted',)) + #d = defer.gatherResults([d1, d2, d3]) + #return d +# + #def close(): + #return self.client.close() +# + #d = self.connected.addCallback(strip(add_mailbox)) + #d.addCallback(strip(login)) + #d.addCallbacks(strip(select), self._ebGeneral) + #d.addCallback(strip(get_mailbox)) + #d.addCallbacks(strip(add_messages), self._ebGeneral) + #d.addCallbacks(strip(close), self._ebGeneral) + #d.addCallbacks(self._cbStopClient, self._ebGeneral) + #d2 = self.loopback() + #d1 = defer.gatherResults([d, d2]) + #d1.addCallback(lambda _: self.mailbox.getMessageCount()) + #d1.addCallback(self._cbTestClose) + #return d1 +# + #def _cbTestClose(self, count): + # TODO is this correct? count should not take into account those + # flagged as deleted??? + #self.assertEqual(count, 1) + # TODO --- assert flags are those of the message #2 + #self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ acc = self.server.theAccount - name = 'mailbox-expunge' + mailbox_name = 'mailboxexpunge' - d0 = lambda: acc.addMailbox(name) + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def select(): - return self.client.select('mailbox-expunge') + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag')) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',))) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',))) return d def expunge(): @@ -1138,49 +1187,53 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = results self.results = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) - d1.addCallbacks(strip(select), self._ebGeneral) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.mailbox.getMessageCount()) return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored): + def _cbTestExpunge(self, count): # we only left 1 mssage with no deleted flag - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - - msg = list(self.mailbox.messages)[0] - self.assertTrue(msg is not None) - - self.assertEqual( - msg.hdoc.content['subject'], - 'Message 2') - + self.assertEqual(count, 1) # the uids of the deleted messages self.assertItemsEqual(self.results, [1, 3]) -class AccountTestCase(IMAP4HelperMixin, unittest.TestCase): +class AccountTestCase(IMAP4HelperMixin): """ Test the Account. """ def _create_empty_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('') + return self.server.theAccount.addMailbox('') def _create_one_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('one') + return self.server.theAccount.addMailbox('one') def test_illegalMailboxCreate(self): - self.assertRaises(AssertionError, self._create_empty_mailbox) + # FIXME --- account.addMailbox needs to raise a failure, + # not the direct exception. + self.stashed = None + + def stash(result): + self.stashed = result + + d = self._create_empty_mailbox() + d.addBoth(stash) + d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, + failure.Failure))) + return d + #self.assertRaises(AssertionError, self._create_empty_mailbox) -class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): """ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 920eeb0..5708787 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -1,30 +1,44 @@ -import os -import tempfile -import shutil - +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Common utilities for testing Soledad IMAP Server. +""" from email import parser from mock import Mock from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback +from twisted.python import log -from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.soledad.client import Soledad +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.tests.common import SoledadTestMixin TEST_USER = "testuser@leap.se" TEST_PASSWD = "1234" + # # Simple IMAP4 Client for testing # - class SimpleClient(imap4.IMAP4Client): - """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -51,160 +65,57 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() -# XXX move to common helper -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - - _soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return _soledad - - -# XXX this is not properly a mixin, since helper already inherits from -# uniittest.Testcase -class IMAP4HelperMixin(BaseLeapTest): +class IMAP4HelperMixin(SoledadTestMixin): """ MixIn containing several utilities to be shared across different TestCases """ - serverCTX = None clientCTX = None - # setUpClass cannot be a classmethod in trial, see: - # https://twistedmatrix.com/trac/ticket/1870 - def setUp(self): - """ - Setup method for each test. - - Initializes and run a LEAP IMAP4 Server. - """ - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - UUID = 'deadbeef', - USERID = TEST_USER - memstore = MemoryStore() - ########### + soledad_adaptor.cleanup_deferred_locks() - d_server_ready = defer.Deferred() + UUID = 'deadbeef', + USERID = TEST_USER - self.server = LeapIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - soledad=self._soledad) + def setup_server(account): + self.server = LEAPIMAPServer( + uuid=UUID, userid=USERID, + contextFactory=self.serverCTX, + soledad=self._soledad) + self.server.theAccount = account - self.client = SimpleClient( - d_server_ready, contextFactory=self.clientCTX) + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready - theAccount = SoledadBackedAccount( - USERID, - soledad=self._soledad, - memstore=memstore) - d_account_ready = theAccount.callWhenReady(lambda r: None) - LeapIMAPServer.theAccount = theAccount + def setup_account(_): + self.parser = parser.Parser() - self.connected = defer.gatherResults( - [d_server_ready, d_account_ready]) + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + self._soledad.sync = Mock() - # XXX FIXME -------------------------------------------- - # XXX this needs to be done differently, - # have to be hooked on initialization callback instead. - # in case we get something from previous tests... - #for mb in self.server.theAccount.mailboxes: - #self.server.theAccount.delete(mb) + d = defer.Deferred() + self.acc = IMAPAccount(USERID, self._soledad, d=d) + return d - # email parser - self.parser = parser.Parser() + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d def tearDown(self): - """ - tearDown method called after each test. - """ - try: - self._soledad.close() - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - def populateMessages(self): - """ - Populates soledad instance with several simple messages - """ - # XXX we should encapsulate this thru SoledadBackedAccount - # instead. - - # XXX we also should put this in a mailbox! - - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") - # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") - - def delete_all_docs(self): - """ - Deletes all the docs in the testing instance of the - SoledadBackedAccount. - """ - self.server.theAccount.deleteAllMessages( - iknowhatiamdoing=True) + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected def _cbStopClient(self, ignore): self.client.transport.loseConnection() @@ -212,11 +123,8 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - # can we do something similar? - # I guess this was ok with trial, but not in noseland... - # log.err(failure, "Problem with %r" % (self.function,)) - raise failure.value - # failure.trap(Exception) + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) def loopback(self): return loopback.loopbackAsync(self.server, self.client) |