summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2014-10-16 14:51:53 +0200
committerKali Kaneko <kali@leap.se>2015-02-11 14:05:42 -0400
commit80ed7b9b85686b5d10f359114ca703dba4a5820b (patch)
treef041b51fa949c3b44e9a53a6e3c51b77fa8a6986
parentaa619124b9293b12d732e275c7f57b11dcb01f60 (diff)
adapt to soledad 0.7 async API
-rw-r--r--src/leap/mail/imap/account.py253
-rw-r--r--src/leap/mail/imap/index.py51
-rw-r--r--src/leap/mail/imap/mailbox.py57
-rw-r--r--src/leap/mail/imap/memorystore.py39
-rw-r--r--src/leap/mail/imap/messages.py136
-rw-r--r--src/leap/mail/imap/server.py204
-rw-r--r--src/leap/mail/imap/soledadstore.py11
-rw-r--r--src/leap/mail/imap/tests/test_imap.py185
-rw-r--r--src/leap/mail/imap/tests/utils.py25
9 files changed, 698 insertions, 263 deletions
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 70ed13b..fe466cb 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -22,6 +22,7 @@ import logging
import os
import time
+from twisted.internet import defer
from twisted.mail import imap4
from twisted.python import log
from zope.interface import implements
@@ -65,6 +66,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
_soledad = None
selected = None
closed = False
+ _initialized = False
def __init__(self, account_name, soledad, memstore=None):
"""
@@ -93,14 +95,39 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
self.__mailboxes = set([])
- self.initialize_db()
+ self._deferred_initialization = defer.Deferred()
+ self._initialize_storage()
- # every user should have the right to an inbox folder
- # at least, so let's make one!
- self._load_mailboxes()
+ def _initialize_storage(self):
- if not self.mailboxes:
- self.addMailbox(self.INBOX_NAME)
+ def add_mailbox_if_none(result):
+ # every user should have the right to an inbox folder
+ # at least, so let's make one!
+ if not self.mailboxes:
+ self.addMailbox(self.INBOX_NAME)
+
+ def finish_initialization(result):
+ self._initialized = True
+ self._deferred_initialization.callback(None)
+
+ def load_mbox_cache(result):
+ d = self._load_mailboxes()
+ d.addCallback(lambda _: result)
+ return d
+
+ d = self.initialize_db()
+
+ d.addCallback(load_mbox_cache)
+ d.addCallback(add_mailbox_if_none)
+ d.addCallback(finish_initialization)
+
+ def callWhenReady(self, cb):
+ if self._initialized:
+ cb(self)
+ return defer.succeed(None)
+ else:
+ self._deferred_initialization.addCallback(cb)
+ return self._deferred_initialization
def _get_empty_mailbox(self):
"""
@@ -120,10 +147,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: SoledadDocument
"""
# XXX use soledadstore instead ...;
- doc = self._soledad.get_from_index(
+ def get_first_if_any(docs):
+ return docs[0] if docs else None
+
+ d = self._soledad.get_from_index(
self.TYPE_MBOX_IDX, self.MBOX_KEY,
self._parse_mailbox_name(name))
- return doc[0] if doc else None
+ d.addCallback(get_first_if_any)
+ return d
@property
def mailboxes(self):
@@ -134,19 +165,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
return sorted(self.__mailboxes)
def _load_mailboxes(self):
- self.__mailboxes.update(
- [doc.content[self.MBOX_KEY]
- for doc in self._soledad.get_from_index(
- self.TYPE_IDX, self.MBOX_KEY)])
-
- @property
- def subscriptions(self):
- """
- A list of the current subscriptions for this account.
- """
- return [doc.content[self.MBOX_KEY]
- for doc in self._soledad.get_from_index(
- self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
+ def update_mailboxes(db_indexes):
+ self.__mailboxes.update(
+ [doc.content[self.MBOX_KEY] for doc in db_indexes])
+ d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY)
+ d.addCallback(update_mailboxes)
+ return d
def getMailbox(self, name):
"""
@@ -182,7 +206,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
one is provided.
:type creation_ts: int
- :returns: True if successful
+ :returns: a Deferred that will contain the document if successful.
:rtype: bool
"""
name = self._parse_mailbox_name(name)
@@ -203,21 +227,29 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
mbox[self.MBOX_KEY] = name
mbox[self.CREATED_KEY] = creation_ts
- doc = self._soledad.create_doc(mbox)
- self._load_mailboxes()
- return bool(doc)
+ def load_mbox_cache(result):
+ d = self._load_mailboxes()
+ d.addCallback(lambda _: result)
+ return d
+
+ d = self._soledad.create_doc(mbox)
+ d.addCallback(load_mbox_cache)
+ return d
def create(self, pathspec):
"""
Create a new mailbox from the given hierarchical name.
- :param pathspec: The full hierarchical name of a new mailbox to create.
- If any of the inferior hierarchical names to this one
- do not exist, they are created as well.
+ :param pathspec:
+ The full hierarchical name of a new mailbox to create.
+ If any of the inferior hierarchical names to this one
+ do not exist, they are created as well.
:type pathspec: str
- :return: A true value if the creation succeeds.
- :rtype: bool
+ :return:
+ A deferred that will fire with a true value if the creation
+ succeeds.
+ :rtype: Deferred
:raise MailboxException: Raised if this mailbox cannot be added.
"""
@@ -225,18 +257,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
paths = filter(
None,
self._parse_mailbox_name(pathspec).split('/'))
+
+ subs = []
+ sep = '/'
+
for accum in range(1, len(paths)):
try:
- self.addMailbox('/'.join(paths[:accum]))
+ partial = sep.join(paths[:accum])
+ d = self.addMailbox(partial)
+ subs.append(d)
except imap4.MailboxCollision:
pass
try:
- self.addMailbox('/'.join(paths))
+ df = self.addMailbox(sep.join(paths))
except imap4.MailboxCollision:
if not pathspec.endswith('/'):
- return False
- self._load_mailboxes()
- return True
+ df = defer.succeed(False)
+ else:
+ df = defer.succeed(True)
+ finally:
+ subs.append(df)
+
+ def all_good(result):
+ return all(result)
+
+ def load_mbox_cache(result):
+ d = self._load_mailboxes()
+ d.addCallback(lambda _: result)
+ return d
+
+ if subs:
+ d1 = defer.gatherResults(subs, consumeErrors=True)
+ d1.addCallback(load_mbox_cache)
+ d1.addCallback(all_good)
+ else:
+ d1 = defer.succeed(False)
+ d1.addCallback(load_mbox_cache)
+ return d1
def select(self, name, readwrite=1):
"""
@@ -275,17 +332,20 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param name: the mailbox to be deleted
:type name: str
- :param force: if True, it will not check for noselect flag or inferior
- names. use with care.
+ :param force:
+ if True, it will not check for noselect flag or inferior
+ names. use with care.
:type force: bool
+ :rtype: Deferred
"""
name = self._parse_mailbox_name(name)
if name not in self.mailboxes:
- raise imap4.MailboxException("No such mailbox: %r" % name)
+ err = imap4.MailboxException("No such mailbox: %r" % name)
+ return defer.fail(err)
mbox = self.getMailbox(name)
- if force is False:
+ if not force:
# See if this box is flagged \Noselect
# XXX use mbox.flags instead?
mbox_flags = mbox.getFlags()
@@ -294,11 +354,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
# as part of their root.
for others in self.mailboxes:
if others != name and others.startswith(name):
- raise imap4.MailboxException, (
+ err = imap4.MailboxException(
"Hierarchically inferior mailboxes "
"exist and \\Noselect is set")
+ return defer.fail(err)
self.__mailboxes.discard(name)
- mbox.destroy()
+ return mbox.destroy()
# XXX FIXME --- not honoring the inferior names...
@@ -331,14 +392,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
if new in self.mailboxes:
raise imap4.MailboxCollision(repr(new))
+ rename_deferreds = []
+
+ def load_mbox_cache(result):
+ d = self._load_mailboxes()
+ d.addCallback(lambda _: result)
+ return d
+
+ def update_mbox_doc_name(mbox, oldname, newname, update_deferred):
+ mbox.content[self.MBOX_KEY] = newname
+ d = self._soledad.put_doc(mbox)
+ d.addCallback(lambda r: update_deferred.callback(True))
+
for (old, new) in inferiors:
- self._memstore.rename_fdocs_mailbox(old, new)
- mbox = self._get_mailbox_by_name(old)
- mbox.content[self.MBOX_KEY] = new
self.__mailboxes.discard(old)
- self._soledad.put_doc(mbox)
+ self._memstore.rename_fdocs_mailbox(old, new)
+
+ d0 = defer.Deferred()
+ d = self._get_mailbox_by_name(old)
+ d.addCallback(update_mbox_doc_name, old, new, d0)
+ rename_deferreds.append(d0)
- self._load_mailboxes()
+ d1 = defer.gatherResults(rename_deferreds, consumeErrors=True)
+ d1.addCallback(load_mbox_cache)
+ return d1
def _inferiorNames(self, name):
"""
@@ -354,6 +431,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
inferiors.append(infname)
return inferiors
+ # TODO ------------------ can we preserve the attr?
+ # maybe add to memory store.
def isSubscribed(self, name):
"""
Returns True if user is subscribed to this mailbox.
@@ -361,10 +440,35 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param name: the mailbox to be checked.
:type name: str
- :rtype: bool
+ :rtype: Deferred (will fire with bool)
+ """
+ subscribed = self.SUBSCRIBED_KEY
+
+ def is_subscribed(mbox):
+ subs_bool = bool(mbox.content.get(subscribed, False))
+ return subs_bool
+
+ d = self._get_mailbox_by_name(name)
+ d.addCallback(is_subscribed)
+ return d
+
+ # TODO ------------------ can we preserve the property?
+ # maybe add to memory store.
+
+ def _get_subscriptions(self):
"""
- mbox = self._get_mailbox_by_name(name)
- return mbox.content.get('subscribed', False)
+ Return a list of the current subscriptions for this account.
+
+ :returns: A deferred that will fire with the subscriptions.
+ :rtype: Deferred
+ """
+ def get_docs_content(docs):
+ return [doc.content[self.MBOX_KEY] for doc in docs]
+
+ d = self._soledad.get_from_index(
+ self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')
+ d.addCallback(get_docs_content)
+ return d
def _set_subscription(self, name, value):
"""
@@ -376,26 +480,42 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param value: the boolean value
:type value: bool
"""
+ # XXX Note that this kind of operation has
+ # no guarantees of atomicity. We should not be accessing mbox
+ # documents concurrently.
+
+ subscribed = self.SUBSCRIBED_KEY
+
+ def update_subscribed_value(mbox):
+ mbox.content[subscribed] = value
+ return self._soledad.put_doc(mbox)
+
# maybe we should store subscriptions in another
# document...
if name not in self.mailboxes:
- self.addMailbox(name)
- mbox = self._get_mailbox_by_name(name)
-
- if mbox:
- mbox.content[self.SUBSCRIBED_KEY] = value
- self._soledad.put_doc(mbox)
+ d = self.addMailbox(name)
+ d.addCallback(lambda v: self._get_mailbox_by_name(name))
+ else:
+ d = self._get_mailbox_by_name(name)
+ d.addCallback(update_subscribed_value)
+ return d
def subscribe(self, name):
"""
- Subscribe to this mailbox
+ Subscribe to this mailbox if not already subscribed.
:param name: name of the mailbox
:type name: str
+ :rtype: Deferred
"""
name = self._parse_mailbox_name(name)
- if name not in self.subscriptions:
- self._set_subscription(name, True)
+
+ def check_and_subscribe(subscriptions):
+ if name not in subscriptions:
+ return self._set_subscription(name, True)
+ d = self._get_subscriptions()
+ d.addCallback(check_and_subscribe)
+ return d
def unsubscribe(self, name):
"""
@@ -403,12 +523,21 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param name: name of the mailbox
:type name: str
+ :rtype: Deferred
"""
name = self._parse_mailbox_name(name)
- if name not in self.subscriptions:
- raise imap4.MailboxException(
- "Not currently subscribed to %r" % name)
- self._set_subscription(name, False)
+
+ def check_and_unsubscribe(subscriptions):
+ if name not in subscriptions:
+ raise imap4.MailboxException(
+ "Not currently subscribed to %r" % name)
+ return self._set_subscription(name, False)
+ d = self._get_subscriptions()
+ d.addCallback(check_and_unsubscribe)
+ return d
+
+ def getSubscriptions(self):
+ return self._get_subscriptions()
def listMailboxes(self, ref, wildcard):
"""
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
index 5f0919a..ea35fff 100644
--- a/src/leap/mail/imap/index.py
+++ b/src/leap/mail/imap/index.py
@@ -19,6 +19,8 @@ Index for SoledadBackedAccount, Mailbox and Messages.
"""
import logging
+from twisted.internet import defer
+
from leap.common.check import leap_assert, leap_assert_type
from leap.mail.imap.fields import fields
@@ -39,6 +41,9 @@ class IndexedDB(object):
"""
# TODO we might want to move this to soledad itself, check
+ _index_creation_deferreds = []
+ index_ready = False
+
def initialize_db(self):
"""
Initialize the database.
@@ -46,24 +51,40 @@ class IndexedDB(object):
leap_assert(self._soledad,
"Need a soledad attribute accesible in the instance")
leap_assert_type(self.INDEXES, dict)
+ self._index_creation_deferreds = []
+
+ def _on_indexes_created(ignored):
+ self.index_ready = True
+
+ def _create_index(name, expression):
+ d = self._soledad.create_index(name, *expression)
+ self._index_creation_deferreds.append(d)
+
+ def _create_indexes(db_indexes):
+ db_indexes = dict(db_indexes)
+ for name, expression in fields.INDEXES.items():
+ if name not in db_indexes:
+ # The index does not yet exist.
+ _create_index(name, expression)
+ continue
+
+ if expression == db_indexes[name]:
+ # The index exists and is up to date.
+ continue
+ # The index exists but the definition is not what expected, so
+ # we delete it and add the proper index expression.
+ d1 = self._soledad.delete_index(name)
+ d1.addCallback(lambda _: _create_index(name, expression))
+
+ all_created = defer.gatherResults(self._index_creation_deferreds)
+ all_created.addCallback(_on_indexes_created)
+ return all_created
# Ask the database for currently existing indexes.
if not self._soledad:
logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
return
- db_indexes = dict()
if self._soledad is not None:
- db_indexes = dict(self._soledad.list_indexes())
- for name, expression in fields.INDEXES.items():
- if name not in db_indexes:
- # The index does not yet exist.
- self._soledad.create_index(name, *expression)
- continue
-
- if expression == db_indexes[name]:
- # The index exists and is up to date.
- continue
- # The index exists but the definition is not what expected, so we
- # delete it and add the proper index expression.
- self._soledad.delete_index(name)
- self._soledad.create_index(name, *expression)
+ d = self._soledad.list_indexes()
+ d.addCallback(_create_indexes)
+ return d
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 34cf535..3c1769a 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -303,21 +303,32 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
We do this to be able to filter the requests efficiently.
"""
primed = self._known_uids_primed.get(self.mbox, False)
- if not primed:
- known_uids = self.messages.all_soledad_uid_iter()
+ # XXX handle the maybeDeferred
+
+ def set_primed(known_uids):
self._memstore.set_known_uids(self.mbox, known_uids)
self._known_uids_primed[self.mbox] = True
+ if not primed:
+ d = self.messages.all_soledad_uid_iter()
+ d.addCallback(set_primed)
+ return d
+
def prime_flag_docs_to_memstore(self):
"""
Prime memstore with all the flags documents.
"""
primed = self._fdoc_primed.get(self.mbox, False)
- if not primed:
- all_flag_docs = self.messages.get_all_soledad_flag_docs()
- self._memstore.load_flag_docs(self.mbox, all_flag_docs)
+
+ def set_flag_docs(flag_docs):
+ self._memstore.load_flag_docs(self.mbox, flag_docs)
self._fdoc_primed[self.mbox] = True
+ if not primed:
+ d = self.messages.get_all_soledad_flag_docs()
+ d.addCallback(set_flag_docs)
+ return d
+
def getUIDValidity(self):
"""
Return the unique validity identifier for this mailbox.
@@ -522,21 +533,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
Should cleanup resources, and set the \\Noselect flag
on the mailbox.
+
"""
# XXX this will overwrite all the existing flags!
# should better simply addFlag
self.setFlags((self.NOSELECT_FLAG,))
- self.deleteAllDocs()
# XXX removing the mailbox in situ for now,
# we should postpone the removal
- # XXX move to memory store??
- mbox_doc = self._get_mbox_doc()
- if mbox_doc is None:
- # memory-only store!
- return
- self._soledad.delete_doc(self._get_mbox_doc())
+ def remove_mbox_doc(ignored):
+ # XXX move to memory store??
+
+ def _remove_mbox_doc(doc):
+ if doc is None:
+ # memory-only store!
+ return defer.succeed(True)
+ return self._soledad.delete_doc(doc)
+
+ doc = self._get_mbox_doc()
+ return _remove_mbox_doc(doc)
+
+ d = self.deleteAllDocs()
+ d.addCallback(remove_mbox_doc)
+ return d
def _close_cb(self, result):
self.closed = True
@@ -1006,9 +1026,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
Delete all docs in this mailbox
"""
- docs = self.messages.get_all_docs()
- for doc in docs:
- self.messages._soledad.delete_doc(doc)
+ def del_all_docs(docs):
+ todelete = []
+ for doc in docs:
+ d = self.messages._soledad.delete_doc(doc)
+ todelete.append(d)
+ return defer.gatherResults(todelete)
+
+ d = self.messages.get_all_docs()
+ d.addCallback(del_all_docs)
+ return d
def unset_recent_flags(self, uid_seq):
"""
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index e075394..eda5b96 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+
# memorystore.py
# Copyright (C) 2014 LEAP
#
@@ -112,8 +112,6 @@ class MemoryStore(object):
:param write_period: the interval to dump messages to disk, in seconds.
:type write_period: int
"""
- self.reactor = reactor
-
self._permanent_store = permanent_store
self._write_period = write_period
@@ -241,6 +239,7 @@ class MemoryStore(object):
self.producer = None
self._write_loop = None
+ # TODO -- remove
def _start_write_loop(self):
"""
Start loop for writing to disk database.
@@ -250,6 +249,7 @@ class MemoryStore(object):
if not self._write_loop.running:
self._write_loop.start(self._write_period, now=True)
+ # TODO -- remove
def _stop_write_loop(self):
"""
Stop loop for writing to disk database.
@@ -278,17 +278,18 @@ class MemoryStore(object):
:type uid: int
:param message: a message to be added
:type message: MessageWrapper
- :param observer: the deferred that will fire with the
- UID of the message. If notify_on_disk is True,
- this will happen when the message is written to
- Soledad. Otherwise it will fire as soon as we've
- added the message to the memory store.
+ :param observer:
+ the deferred that will fire with the UID of the message. If
+ notify_on_disk is True, this will happen when the message is
+ written to Soledad. Otherwise it will fire as soon as we've added
+ the message to the memory store.
:type observer: Deferred
- :param notify_on_disk: whether the `observer` deferred should
- wait until the message is written to disk to
- be fired.
+ :param notify_on_disk:
+ whether the `observer` deferred should wait until the message is
+ written to disk to be fired.
:type notify_on_disk: bool
"""
+ # TODO -- return a deferred
log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid))
key = mbox, uid
@@ -306,7 +307,7 @@ class MemoryStore(object):
else:
# Caller does not care, just fired and forgot, so we pass
# a defer that will inmediately have its callback triggered.
- self.reactor.callFromThread(observer.callback, uid)
+ reactor.callFromThread(observer.callback, uid)
def put_message(self, mbox, uid, message, notify_on_disk=True):
"""
@@ -442,6 +443,7 @@ class MemoryStore(object):
:return: MessageWrapper or None
"""
+ # TODO -- return deferred
if dirtystate == DirtyState.dirty:
flags_only = True
@@ -467,6 +469,7 @@ class MemoryStore(object):
chash = fdoc.get(fields.CONTENT_HASH_KEY)
hdoc = self._hdoc_store[chash]
if empty(hdoc):
+ # XXX this will be a deferred
hdoc = self._permanent_store.get_headers_doc(chash)
if empty(hdoc):
return None
@@ -531,7 +534,8 @@ class MemoryStore(object):
# IMessageStoreWriter
- @deferred_to_thread
+ # TODO -- I think we don't need this anymore.
+ # instead, we can have
def write_messages(self, store):
"""
Write the message documents in this MemoryStore to a different store.
@@ -657,7 +661,7 @@ class MemoryStore(object):
with self._last_uid_lock:
self._last_uid[mbox] += 1
value = self._last_uid[mbox]
- self.reactor.callInThread(self.write_last_uid, mbox, value)
+ reactor.callInThread(self.write_last_uid, mbox, value)
return value
def write_last_uid(self, mbox, value):
@@ -1077,6 +1081,7 @@ class MemoryStore(object):
return None
return self._rflags_store[mbox]['set']
+ # XXX -- remove
def all_rdocs_iter(self):
"""
Return an iterator through all in-memory recent flag dicts, wrapped
@@ -1125,6 +1130,7 @@ class MemoryStore(object):
self.remove_message(mbox, uid)
return mem_deleted
+ # TODO -- remove
def stop_and_flush(self):
"""
Stop the write loop and trigger a write to the producer.
@@ -1180,6 +1186,7 @@ class MemoryStore(object):
:type observer: Deferred
"""
mem_deleted = self.remove_all_deleted(mbox)
+ # TODO return a DeferredList
observer.callback(mem_deleted)
def _delete_from_soledad_and_memory(self, result, mbox, observer):
@@ -1313,8 +1320,8 @@ class MemoryStore(object):
:rtype: bool
"""
# FIXME this should return a deferred !!!
- # XXX ----- can fire when all new + dirty deferreds
- # are done (gatherResults)
+ # TODO this should be moved to soledadStore instead
+ # (all pending deferreds)
return getattr(self, self.WRITING_FLAG)
@property
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index e8d64d1..c761091 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -71,6 +71,7 @@ def try_unique_query(curried):
:param curried: a curried function
:type curried: callable
"""
+ # XXX FIXME ---------- convert to deferreds
leap_assert(callable(curried), "A callable is expected")
try:
query = curried()
@@ -134,10 +135,11 @@ class LeapMessage(fields, MBoxParser):
self.__chash = None
self.__bdoc = None
- self.reactor = reactor
-
# XXX make these properties public
+ # XXX FIXME ------ the documents can be
+ # deferreds too.... niice.
+
@property
def fdoc(self):
"""
@@ -506,18 +508,15 @@ class LeapMessage(fields, MBoxParser):
Return the document that keeps the flags for this
message.
"""
- result = {}
- try:
- flag_docs = self._soledad.get_from_index(
- fields.TYPE_MBOX_UID_IDX,
- fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
- result = first(flag_docs)
- except Exception as exc:
- # ugh! Something's broken down there!
- logger.warning("ERROR while getting flags for UID: %s" % self._uid)
- logger.exception(exc)
- finally:
- return result
+ def get_first_if_any(docs):
+ result = first(docs)
+ return result if result else {}
+
+ d = self._soledad.get_from_index(
+ fields.TYPE_MBOX_UID_IDX,
+ fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
+ d.addCallback(get_first_if_any)
+ return d
# TODO move to soledadstore instead of accessing soledad directly
def _get_headers_doc(self):
@@ -525,10 +524,11 @@ class LeapMessage(fields, MBoxParser):
Return the document that keeps the headers for this
message.
"""
- head_docs = self._soledad.get_from_index(
+ d = self._soledad.get_from_index(
fields.TYPE_C_HASH_IDX,
fields.TYPE_HEADERS_VAL, str(self.chash))
- return first(head_docs)
+ d.addCallback(lambda docs: first(docs))
+ return d
# TODO move to soledadstore instead of accessing soledad directly
def _get_body_doc(self):
@@ -536,6 +536,8 @@ class LeapMessage(fields, MBoxParser):
Return the document that keeps the body for this
message.
"""
+ # XXX FIXME --- this might need a maybedeferred
+ # on the receiving side...
hdoc_content = self.hdoc.content
body_phash = hdoc_content.get(
fields.BODY_KEY, None)
@@ -554,13 +556,11 @@ class LeapMessage(fields, MBoxParser):
return bdoc
# no memstore, or no body doc found there
- if self._soledad:
- body_docs = self._soledad.get_from_index(
- fields.TYPE_P_HASH_IDX,
- fields.TYPE_CONTENT_VAL, str(body_phash))
- return first(body_docs)
- else:
- logger.error("No phash in container, and no soledad found!")
+ d = self._soledad.get_from_index(
+ fields.TYPE_P_HASH_IDX,
+ fields.TYPE_CONTENT_VAL, str(body_phash))
+ d.addCallback(lambda docs: first(docs))
+ return d
def __getitem__(self, key):
"""
@@ -739,8 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
else:
self._initialized[mbox] = True
- self.reactor = reactor
-
def _get_empty_doc(self, _type=FLAGS_DOC):
"""
Returns an empty doc for storing different message parts.
@@ -887,9 +885,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
flags = tuple()
leap_assert_type(flags, tuple)
+ # TODO return soledad deferred instead
observer = defer.Deferred()
d = self._do_parse(raw)
- d.addCallback(lambda result: self.reactor.callInThread(
+ d.addCallback(lambda result: reactor.callInThread(
self._do_add_msg, result, flags, subject, date,
notify_on_disk, observer))
return observer
@@ -924,17 +923,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
msg = self.get_msg_by_uid(existing_uid)
# We can say the observer that we're done
- self.reactor.callFromThread(observer.callback, existing_uid)
+ # TODO return soledad deferred instead
+ reactor.callFromThread(observer.callback, existing_uid)
msg.setFlags((fields.DELETED_FLAG,), -1)
return
- # XXX get FUCKING UID from autoincremental table
+ # TODO S2 -- get FUCKING UID from autoincremental table
uid = self.memstore.increment_last_soledad_uid(self.mbox)
# We can say the observer that we're done at this point, but
# before that we should make sure it has no serious consequences
# if we're issued, for instance, a fetch command right after...
- # self.reactor.callFromThread(observer.callback, uid)
+ # reactor.callFromThread(observer.callback, uid)
# if we did the notify, we need to invalidate the deferred
# so not to try to fire it twice.
# observer = None
@@ -960,6 +960,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
self.set_recent_flag(uid)
msg_container = MessageWrapper(fd, hd, cdocs)
+
+ # TODO S1 -- just pass this to memstore and return that deferred.
self.memstore.create_message(
self.mbox, uid, msg_container,
observer=observer, notify_on_disk=notify_on_disk)
@@ -1011,6 +1013,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
Get recent-flags document from Soledad for this mailbox.
:rtype: SoledadDocument or None
"""
+ # FIXME ----- use deferreds.
curried = partial(
self._soledad.get_from_index,
fields.TYPE_MBOX_IDX,
@@ -1029,6 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:param uids: the uids to unset
:type uid: sequence
"""
+ # FIXME ----- use deferreds.
with self._rdoc_property_lock[self.mbox]:
self.recent_flags.difference_update(
set(uids))
@@ -1042,11 +1046,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:param uid: the uid to unset
:type uid: int
"""
+ # FIXME ----- use deferreds.
with self._rdoc_property_lock[self.mbox]:
self.recent_flags.difference_update(
set([uid]))
- @deferred_to_thread
def set_recent_flag(self, uid):
"""
Set Recent flag for a given uid.
@@ -1054,6 +1058,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:param uid: the uid to set
:type uid: int
"""
+ # FIXME ----- use deferreds.
with self._rdoc_property_lock[self.mbox]:
self.recent_flags = self.recent_flags.union(
set([uid]))
@@ -1068,6 +1073,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
the query failed.
:rtype: SoledadDocument or None.
"""
+ # FIXME ----- use deferreds.
curried = partial(
self._soledad.get_from_index,
fields.TYPE_MBOX_C_HASH_IDX,
@@ -1125,7 +1131,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
return None
return fdoc.content.get(fields.UID_KEY, None)
- @deferred_to_thread
def _get_uid_from_msgid(self, msgid):
"""
Return a UID for a given message-id.
@@ -1144,7 +1149,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
# XXX is this working?
return self._get_uid_from_msgidCb(msgid)
- @deferred_to_thread
def set_flags(self, mbox, messages, flags, mode, observer):
"""
Set flags for a sequence of messages.
@@ -1162,7 +1166,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
done.
:type observer: deferred
"""
- reactor = self.reactor
getmsg = self.get_msg_by_uid
def set_flags(uid, flags, mode):
@@ -1173,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
setted_flags = [set_flags(uid, flags, mode) for uid in messages]
result = dict(filter(None, setted_flags))
+ # TODO -- remove
reactor.callFromThread(observer.callback, result)
# getters: generic for a mailbox
@@ -1223,37 +1227,45 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
If you want acess to the content, use __iter__ instead
- :return: a list of u1db documents
- :rtype: list of SoledadDocument
+ :return: a Deferred, that will fire with a list of u1db documents
+ :rtype: Deferred (promise of list of SoledadDocument)
"""
if _type not in fields.__dict__.values():
raise TypeError("Wrong type passed to get_all_docs")
+ # FIXME ----- either raise or return a deferred wrapper.
if sameProxiedObjects(self._soledad, None):
logger.warning('Tried to get messages but soledad is None!')
return []
- all_docs = [doc for doc in self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- _type, self.mbox)]
+ def get_sorted_docs(docs):
+ all_docs = [doc for doc in docs]
+ # inneficient, but first let's grok it and then
+ # let's worry about efficiency.
+ # XXX FIXINDEX -- should implement order by in soledad
+ # FIXME ----------------------------------------------
+ return sorted(all_docs, key=lambda item: item.content['uid'])
- # inneficient, but first let's grok it and then
- # let's worry about efficiency.
- # XXX FIXINDEX -- should implement order by in soledad
- # FIXME ----------------------------------------------
- return sorted(all_docs, key=lambda item: item.content['uid'])
+ d = self._soledad.get_from_index(
+ fields.TYPE_MBOX_IDX, _type, self.mbox)
+ d.addCallback(get_sorted_docs)
+ return d
def all_soledad_uid_iter(self):
"""
Return an iterator through the UIDs of all messages, sorted in
ascending order.
"""
- db_uids = set([doc.content[self.UID_KEY] for doc in
- self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox)
- if not empty(doc)])
- return db_uids
+ # XXX FIXME ------ sorted???
+
+ def get_uids(docs):
+ return set([
+ doc.content[self.UID_KEY] for doc in docs if not empty(doc)])
+
+ d = self._soledad.get_from_index(
+ fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox)
+ d.addCallback(get_uids)
+ return d
def all_uid_iter(self):
"""
@@ -1277,16 +1289,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
# XXX we really could return a reduced version with
# just {'uid': (flags-tuple,) since the prefetch is
# only oriented to get the flag tuples.
- all_docs = [(
- doc.content[self.UID_KEY],
- dict(doc.content))
- for doc in
- self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox)
- if not empty(doc.content)]
- all_flags = dict(all_docs)
- return all_flags
+
+ def get_content(docs):
+ all_docs = [(
+ doc.content[self.UID_KEY],
+ dict(doc.content))
+ for doc in docs
+ if not empty(doc.content)]
+ all_flags = dict(all_docs)
+ return all_flags
+
+ d = self._soledad.get_from_index(
+ fields.TYPE_MBOX_IDX,
+ fields.TYPE_FLAGS_VAL, self.mbox)
+ d.addCallback(get_content)
+ return d
def all_headers(self):
"""
@@ -1339,6 +1356,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
# recent messages
# XXX take it from memstore
+ # XXX Used somewhere?
def count_recent(self):
"""
Count all messages with the `Recent` flag.
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index fe56ea6..cf0ba74 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation.
from copy import copy
from twisted import cred
+from twisted.internet import reactor
from twisted.internet.defer import maybeDeferred
from twisted.mail import imap4
from twisted.python import log
@@ -50,6 +51,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
leap_assert(uuid, "need a user in the initialization")
self._userid = userid
+ self.reactor = reactor
# initialize imap server!
imap4.IMAP4Server.__init__(self, *args, **kwargs)
@@ -59,9 +61,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
# populate the test account properly (and only once
# per session)
- from twisted.internet import reactor
- self.reactor = reactor
-
def lineReceived(self, line):
"""
Attempt to parse a single line from the server.
@@ -311,21 +310,203 @@ class LeapIMAPServer(imap4.IMAP4Server):
return self._fileLiteral(size, literalPlus)
#############################
+ # --------------------------------- isSubscribed patch
+ # TODO -- send patch upstream.
+ # There is a bug in twisted implementation:
+ # in cbListWork, it's assumed that account.isSubscribed IS a callable,
+ # although in the interface documentation it's stated that it can be
+ # a deferred.
+
+ def _listWork(self, tag, ref, mbox, sub, cmdName):
+ mbox = self._parseMbox(mbox)
+ mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox)
+ mailboxes.addCallback(self._cbSubscribed)
+ mailboxes.addCallback(
+ self._cbListWork, tag, sub, cmdName,
+ ).addErrback(self._ebListWork, tag)
+
+ def _cbSubscribed(self, mailboxes):
+ subscribed = [
+ maybeDeferred(self.account.isSubscribed, name)
+ for (name, box) in mailboxes]
+
+ def get_mailboxes_and_subs(result):
+ subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes]
+ return mailboxes, subscribed
+
+ d = defer.gatherResults(subscribed)
+ d.addCallback(get_mailboxes_and_subs)
+ return d
+
+ def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName):
+ mailboxes, subscribed = mailboxes_subscribed
+
+ for (name, box) in mailboxes:
+ if not sub or name in subscribed:
+ flags = box.getFlags()
+ delim = box.getHierarchicalDelimiter()
+ resp = (imap4.DontQuoteMe(cmdName),
+ map(imap4.DontQuoteMe, flags),
+ delim, name.encode('imap4-utf-7'))
+ self.sendUntaggedResponse(
+ imap4.collapseNestedLists(resp))
+ self.sendPositiveResponse(tag, '%s completed' % (cmdName,))
+ # -------------------- end isSubscribed patch -----------
+
+ # TODO ----
+ # subscribe method had also to be changed to accomodate
+ # deferred
+ # Revert to regular methods as soon as we implement non-deferred memory
+ # cache.
+ def do_SUBSCRIBE(self, tag, name):
+ name = self._parseMbox(name)
+
+ def _subscribeCb(_):
+ self.sendPositiveResponse(tag, 'Subscribed')
+
+ def _subscribeEb(failure):
+ m = failure.value
+ log.err()
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while subscribing to mailbox")
+
+ d = self.account.subscribe(name)
+ d.addCallbacks(_subscribeCb, _subscribeEb)
+ return d
+
+ auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
+ select_SUBSCRIBE = auth_SUBSCRIBE
+
+ def do_UNSUBSCRIBE(self, tag, name):
+ # unsubscribe method had also to be changed to accomodate
+ # deferred
+ name = self._parseMbox(name)
+
+ def _unsubscribeCb(_):
+ self.sendPositiveResponse(tag, 'Unsubscribed')
+
+ def _unsubscribeEb(failure):
+ m = failure.value
+ log.err()
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while unsubscribing "
+ "from mailbox")
+
+ d = self.account.unsubscribe(name)
+ d.addCallbacks(_unsubscribeCb, _unsubscribeEb)
+ return d
+
+ auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
+ select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+ def do_RENAME(self, tag, oldname, newname):
+ oldname, newname = [self._parseMbox(n) for n in oldname, newname]
+ if oldname.lower() == 'inbox' or newname.lower() == 'inbox':
+ self.sendNegativeResponse(
+ tag,
+ 'You cannot rename the inbox, or '
+ 'rename another mailbox to inbox.')
+ return
+
+ def _renameCb(_):
+ self.sendPositiveResponse(tag, 'Mailbox renamed')
+
+ def _renameEb(failure):
+ m = failure.value
+ print "rename failure!"
+ if failure.check(TypeError):
+ self.sendBadResponse(tag, 'Invalid command syntax')
+ elif failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ log.err()
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while "
+ "renaming mailbox")
+
+ d = self.account.rename(oldname, newname)
+ d.addCallbacks(_renameCb, _renameEb)
+ return d
+
+ auth_RENAME = (do_RENAME, arg_astring, arg_astring)
+ select_RENAME = auth_RENAME
+
+ def do_CREATE(self, tag, name):
+ name = self._parseMbox(name)
+
+ def _createCb(result):
+ if result:
+ self.sendPositiveResponse(tag, 'Mailbox created')
+ else:
+ self.sendNegativeResponse(tag, 'Mailbox not created')
+
+ def _createEb(failure):
+ c = failure.value
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(c))
+ else:
+ log.err()
+ self.sendBadResponse(
+ tag, "Server error encountered while creating mailbox")
+
+ d = self.account.create(name)
+ d.addCallbacks(_createCb, _createEb)
+ return d
+
+ auth_CREATE = (do_CREATE, arg_astring)
+ select_CREATE = auth_CREATE
+
+ def do_DELETE(self, tag, name):
+ name = self._parseMbox(name)
+ if name.lower() == 'inbox':
+ self.sendNegativeResponse(tag, 'You cannot delete the inbox')
+ return
+
+ def _deleteCb(result):
+ self.sendPositiveResponse(tag, 'Mailbox deleted')
+
+ def _deleteEb(failure):
+ m = failure.value
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ print "other error"
+ log.err()
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while deleting mailbox")
+
+ d = self.account.delete(name)
+ d.addCallbacks(_deleteCb, _deleteEb)
+ return d
+
+ auth_DELETE = (do_DELETE, arg_astring)
+ select_DELETE = auth_DELETE
+
# Need to override the command table after patching
# arg_astring and arg_literal
+ # do_DELETE = imap4.IMAP4Server.do_DELETE
+ # do_CREATE = imap4.IMAP4Server.do_CREATE
+ # do_RENAME = imap4.IMAP4Server.do_RENAME
+ # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
+ # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
do_LOGIN = imap4.IMAP4Server.do_LOGIN
- do_CREATE = imap4.IMAP4Server.do_CREATE
- do_DELETE = imap4.IMAP4Server.do_DELETE
- do_RENAME = imap4.IMAP4Server.do_RENAME
- do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
- do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
do_STATUS = imap4.IMAP4Server.do_STATUS
do_APPEND = imap4.IMAP4Server.do_APPEND
do_COPY = imap4.IMAP4Server.do_COPY
_selectWork = imap4.IMAP4Server._selectWork
- _listWork = imap4.IMAP4Server._listWork
+
arg_plist = imap4.IMAP4Server.arg_plist
arg_seqset = imap4.IMAP4Server.arg_seqset
opt_plist = imap4.IMAP4Server.opt_plist
@@ -342,8 +523,8 @@ class LeapIMAPServer(imap4.IMAP4Server):
auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE')
select_EXAMINE = auth_EXAMINE
- auth_DELETE = (do_DELETE, arg_astring)
- select_DELETE = auth_DELETE
+ # auth_DELETE = (do_DELETE, arg_astring)
+ # select_DELETE = auth_DELETE
auth_RENAME = (do_RENAME, arg_astring, arg_astring)
select_RENAME = auth_RENAME
@@ -369,7 +550,6 @@ class LeapIMAPServer(imap4.IMAP4Server):
select_COPY = (do_COPY, arg_seqset, arg_astring)
-
#############################################################
# END of Twisted imap4 patch to support LITERAL+ extension
#############################################################
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index f3de8eb..fc8ea55 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -40,11 +40,6 @@ from leap.mail.utils import first, empty, accumulator_queue
logger = logging.getLogger(__name__)
-# TODO
-# [ ] Implement a retry queue?
-# [ ] Consider journaling of operations.
-
-
class ContentDedup(object):
"""
Message deduplication.
@@ -132,6 +127,7 @@ A lock per document.
# http://stackoverflow.com/a/2437645/1157664
# Setting this to twice the number of threads in the threadpool
# should be safe.
+
put_locks = defaultdict(lambda: threading.Lock())
mbox_doc_locks = defaultdict(lambda: threading.Lock())
@@ -429,7 +425,6 @@ class SoledadStore(ContentDedup):
continue
if item.part == MessagePartType.fdoc:
- #logger.debug("PUT dirty fdoc")
yield item, call
# XXX also for linkage-doc !!!
@@ -479,7 +474,7 @@ class SoledadStore(ContentDedup):
return query.pop()
else:
logger.error("Could not find mbox document for %r" %
- (mbox,))
+ (mbox,))
except Exception as exc:
logger.exception("Unhandled error %r" % exc)
@@ -552,8 +547,10 @@ class SoledadStore(ContentDedup):
:type uid: int
:rtype: SoledadDocument or None
"""
+ # TODO -- inlineCallbacks
result = None
try:
+ # TODO -- yield
flag_docs = self._soledad.get_from_index(
fields.TYPE_MBOX_UID_IDX,
fields.TYPE_FLAGS_VAL, mbox, str(uid))
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 7837aaa..dd4294c 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -68,7 +68,6 @@ def sortNest(l):
class TestRealm:
-
"""
A minimal auth realm for testing purposes only
"""
@@ -83,7 +82,6 @@ class TestRealm:
#
class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
-
"""
Tests for the MessageCollection class
"""
@@ -254,14 +252,18 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return self.client.login(TEST_USER, TEST_PASSWD)
def create():
+ create_deferreds = []
for name in succeed + fail:
d = self.client.create(name)
d.addCallback(strip(cb)).addErrback(eb)
- d.addCallbacks(self._cbStopClient, self._ebGeneral)
+ create_deferreds.append(d)
+ dd = defer.gatherResults(create_deferreds)
+ dd.addCallbacks(self._cbStopClient, self._ebGeneral)
+ return dd
self.result = []
- d1 = self.connected.addCallback(strip(login)).addCallback(
- strip(create))
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(create))
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
return d.addCallback(self._cbTestCreate, succeed, fail)
@@ -269,24 +271,27 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def _cbTestCreate(self, ignored, succeed, fail):
self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail))
- mboxes = list(LeapIMAPServer.theAccount.mailboxes)
- answers = ([u'INBOX', u'foobox', 'test', u'test/box',
- u'test/box/box', 'testbox'])
- self.assertEqual(mboxes, [a for a in answers])
+ mboxes = LeapIMAPServer.theAccount.mailboxes
+
+ answers = ([u'INBOX', u'testbox', u'test/box', u'test',
+ u'test/box/box', 'foobox'])
+ self.assertEqual(sorted(mboxes), sorted([a for a in answers]))
def testDelete(self):
"""
Test whether we can delete mailboxes
"""
- LeapIMAPServer.theAccount.addMailbox('delete/me')
+ acc = LeapIMAPServer.theAccount
+ d0 = lambda: acc.addMailbox('test-delete/me')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
def delete():
- return self.client.delete('delete/me')
+ return self.client.delete('test-delete/me')
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(d0))
d1.addCallbacks(strip(delete), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
@@ -352,11 +357,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
Try deleting a mailbox with sub-folders, and \NoSelect flag set.
An exception is expected.
"""
- LeapIMAPServer.theAccount.addMailbox('delete')
- to_delete = LeapIMAPServer.theAccount.getMailbox('delete')
- to_delete.setFlags((r'\Noselect',))
- to_delete.getFlags()
- LeapIMAPServer.theAccount.addMailbox('delete/me')
+ acc = LeapIMAPServer.theAccount
+ d_del0 = lambda: acc.addMailbox('delete')
+ d_del1 = lambda: acc.addMailbox('delete/me')
+
+ def set_noselect_flag():
+ mbox = acc.getMailbox('delete')
+ mbox.setFlags((r'\Noselect',))
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -369,6 +376,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.failure = None
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(d_del0))
+ d1.addCallback(strip(d_del1))
+ d1.addCallback(strip(set_noselect_flag))
d1.addCallback(strip(delete)).addErrback(deleteFailed)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
@@ -385,7 +395,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Test whether we can rename a mailbox
"""
- LeapIMAPServer.theAccount.addMailbox('oldmbox')
+ d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -394,6 +404,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return self.client.rename('oldmbox', 'newname')
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(d0))
d1.addCallbacks(strip(rename), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
@@ -435,8 +446,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Try to rename hierarchical mailboxes
"""
- LeapIMAPServer.theAccount.create('oldmbox/m1')
- LeapIMAPServer.theAccount.create('oldmbox/m2')
+ acc = LeapIMAPServer.theAccount
+ dc1 = lambda: acc.create('oldmbox/m1')
+ dc2 = lambda: acc.create('oldmbox/m2')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -445,6 +457,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return self.client.rename('oldmbox', 'newname')
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
d1.addCallbacks(strip(rename), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
@@ -454,7 +468,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def _cbTestHierarchicalRename(self, ignored):
mboxes = LeapIMAPServer.theAccount.mailboxes
expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2']
- self.assertEqual(mboxes, [s for s in expected])
+ self.assertEqual(sorted(mboxes), sorted([s for s in expected]))
def testSubscribe(self):
"""
@@ -466,23 +480,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def subscribe():
return self.client.subscribe('this/mbox')
+ def get_subscriptions(ignored):
+ return LeapIMAPServer.theAccount.getSubscriptions()
+
d1 = self.connected.addCallback(strip(login))
d1.addCallbacks(strip(subscribe), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- d.addCallback(lambda _:
- self.assertEqual(
- LeapIMAPServer.theAccount.subscriptions,
- ['this/mbox']))
+ d.addCallback(get_subscriptions)
+ d.addCallback(lambda subscriptions:
+ self.assertEqual(subscriptions,
+ ['this/mbox']))
return d
def testUnsubscribe(self):
"""
Test whether we can unsubscribe from a set of mailboxes
"""
- LeapIMAPServer.theAccount.subscribe('this/mbox')
- LeapIMAPServer.theAccount.subscribe('that/mbox')
+ acc = LeapIMAPServer.theAccount
+
+ dc1 = lambda: acc.subscribe('this/mbox')
+ dc2 = lambda: acc.subscribe('that/mbox')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -490,22 +509,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def unsubscribe():
return self.client.unsubscribe('this/mbox')
+ def get_subscriptions(ignored):
+ return LeapIMAPServer.theAccount.getSubscriptions()
+
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
d1.addCallbacks(strip(unsubscribe), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- d.addCallback(lambda _:
- self.assertEqual(
- LeapIMAPServer.theAccount.subscriptions,
- ['that/mbox']))
+ d.addCallback(get_subscriptions)
+ d.addCallback(lambda subscriptions:
+ self.assertEqual(subscriptions,
+ ['that/mbox']))
return d
def testSelect(self):
"""
Try to select a mailbox
"""
- self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42)
+ acc = self.server.theAccount
+ d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42)
self.selectedArgs = None
def login():
@@ -520,6 +545,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return d
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(d0))
d1.addCallback(strip(select))
d1.addErrback(self._ebGeneral)
@@ -754,13 +780,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
'\\Deleted', '\\Draft', '\\Recent', 'List'),
'READ-WRITE': False})
- def _listSetup(self, f):
- LeapIMAPServer.theAccount.addMailbox('root/subthingl',
- creation_ts=42)
- LeapIMAPServer.theAccount.addMailbox('root/another-thing',
- creation_ts=42)
- LeapIMAPServer.theAccount.addMailbox('non-root/subthing',
- creation_ts=42)
+ def _listSetup(self, f, f2=None):
+ acc = LeapIMAPServer.theAccount
+
+ dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42)
+ dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42)
+ dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42)
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -770,6 +795,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.listed = None
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
+ d1.addCallback(strip(dc3))
+
+ if f2 is not None:
+ d1.addCallback(f2)
+
d1.addCallbacks(strip(f), self._ebGeneral)
d1.addCallbacks(listed, self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -786,7 +818,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
d.addCallback(lambda listed: self.assertEqual(
sortNest(listed),
sortNest([
- (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"),
+ (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"),
(SoledadMailbox.INIT_FLAGS, "/", "root/another-thing")
])
))
@@ -796,20 +828,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Test LSub command
"""
- LeapIMAPServer.theAccount.subscribe('root/subthingl2')
+ acc = LeapIMAPServer.theAccount
+
+ def subs_mailbox():
+ # why not client.subscribe instead?
+ return acc.subscribe('root/subthing')
def lsub():
return self.client.lsub('root', '%')
- d = self._listSetup(lsub)
+
+ d = self._listSetup(lsub, strip(subs_mailbox))
d.addCallback(self.assertEqual,
- [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")])
+ [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")])
return d
def testStatus(self):
"""
Test Status command
"""
- LeapIMAPServer.theAccount.addMailbox('root/subthings')
+ acc = LeapIMAPServer.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('root/subthings')
+
# XXX FIXME ---- should populate this a little bit,
# with unseen etc...
@@ -824,7 +865,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.statused = result
self.statused = None
- d1 = self.connected.addCallback(strip(login))
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(status), self._ebGeneral)
d1.addCallbacks(statused, self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -930,7 +973,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
Test partially appending a message to the mailbox
"""
infile = util.sibpath(__file__, 'rfc822.message')
- LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
+ d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -946,6 +989,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
)
)
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(d0))
d1.addCallbacks(strip(append), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
@@ -995,10 +1039,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
Test closing the mailbox. We expect to get deleted all messages flagged
as such.
"""
+ acc = self.server.theAccount
name = 'mailbox-close'
- self.server.theAccount.addMailbox(name)
- m = LeapIMAPServer.theAccount.getMailbox(name)
+ d0 = lambda: acc.addMailbox(name)
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -1006,14 +1050,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def select():
return self.client.select(name)
+ def get_mailbox():
+ self.mailbox = LeapIMAPServer.theAccount.getMailbox(name)
+
def add_messages():
- d1 = m.messages.add_msg(
+ d1 = self.mailbox.messages.add_msg(
'test 1', subject="Message 1",
flags=('\\Deleted', 'AnotherFlag'))
- d2 = m.messages.add_msg(
+ d2 = self.mailbox.messages.add_msg(
'test 2', subject="Message 2",
flags=('AnotherFlag',))
- d3 = m.messages.add_msg(
+ d3 = self.mailbox.messages.add_msg(
'test 3', subject="Message 3",
flags=('\\Deleted',))
d = defer.gatherResults([d1, d2, d3])
@@ -1023,30 +1070,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return self.client.close()
d = self.connected.addCallback(strip(login))
+ d.addCallback(strip(d0))
d.addCallbacks(strip(select), self._ebGeneral)
+ d.addCallback(strip(get_mailbox))
d.addCallbacks(strip(add_messages), self._ebGeneral)
d.addCallbacks(strip(close), self._ebGeneral)
d.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
- return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m)
+ return defer.gatherResults([d, d2]).addCallback(self._cbTestClose)
- def _cbTestClose(self, ignored, m):
- self.assertEqual(len(m.messages), 1)
- msg = m.messages.get_msg_by_uid(2)
+ def _cbTestClose(self, ignored):
+ self.assertEqual(len(self.mailbox.messages), 1)
+ msg = self.mailbox.messages.get_msg_by_uid(2)
self.assertTrue(msg is not None)
self.assertEqual(
dict(msg.hdoc.content)['subject'],
'Message 2')
- self.failUnless(m.closed)
+ self.failUnless(self.mailbox.closed)
def testExpunge(self):
"""
Test expunge command
"""
+ acc = self.server.theAccount
name = 'mailbox-expunge'
- self.server.theAccount.addMailbox(name)
- m = LeapIMAPServer.theAccount.getMailbox(name)
+
+ d0 = lambda: acc.addMailbox(name)
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -1054,14 +1104,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def select():
return self.client.select('mailbox-expunge')
+ def get_mailbox():
+ self.mailbox = LeapIMAPServer.theAccount.getMailbox(name)
+
def add_messages():
- d1 = m.messages.add_msg(
+ d1 = self.mailbox.messages.add_msg(
'test 1', subject="Message 1",
flags=('\\Deleted', 'AnotherFlag'))
- d2 = m.messages.add_msg(
+ d2 = self.mailbox.messages.add_msg(
'test 2', subject="Message 2",
flags=('AnotherFlag',))
- d3 = m.messages.add_msg(
+ d3 = self.mailbox.messages.add_msg(
'test 3', subject="Message 3",
flags=('\\Deleted',))
d = defer.gatherResults([d1, d2, d3])
@@ -1076,21 +1129,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.results = None
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(d0))
d1.addCallbacks(strip(select), self._ebGeneral)
+ d1.addCallback(strip(get_mailbox))
d1.addCallbacks(strip(add_messages), self._ebGeneral)
d1.addCallbacks(strip(expunge), self._ebGeneral)
d1.addCallbacks(expunged, self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- return d.addCallback(self._cbTestExpunge, m)
+ return d.addCallback(self._cbTestExpunge)
- def _cbTestExpunge(self, ignored, m):
+ def _cbTestExpunge(self, ignored):
# we only left 1 mssage with no deleted flag
- self.assertEqual(len(m.messages), 1)
- msg = m.messages.get_msg_by_uid(2)
+ self.assertEqual(len(self.mailbox.messages), 1)
+ msg = self.mailbox.messages.get_msg_by_uid(2)
- msg = list(m.messages)[0]
+ msg = list(self.mailbox.messages)[0]
self.assertTrue(msg is not None)
self.assertEqual(
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index 5339acf..9a3868c 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -139,31 +139,32 @@ class IMAP4HelperMixin(BaseLeapTest):
###########
- d = defer.Deferred()
+ d_server_ready = defer.Deferred()
+
self.server = LeapIMAPServer(
uuid=UUID, userid=USERID,
contextFactory=self.serverCTX,
- # XXX do we really need this??
soledad=self._soledad)
- self.client = SimpleClient(d, contextFactory=self.clientCTX)
- self.connected = d
-
- # XXX REVIEW-ME.
- # We're adding theAccount here to server
- # but it was also passed to initialization
- # as it was passed to realm.
- # I THINK we ONLY need to do it at one place now.
+ self.client = SimpleClient(
+ d_server_ready, contextFactory=self.clientCTX)
theAccount = SoledadBackedAccount(
USERID,
soledad=self._soledad,
memstore=memstore)
+ d_account_ready = theAccount.callWhenReady(lambda r: None)
LeapIMAPServer.theAccount = theAccount
+ self.connected = defer.gatherResults(
+ [d_server_ready, d_account_ready])
+
+ # XXX FIXME --------------------------------------------
+ # XXX this needs to be done differently,
+ # have to be hooked on initialization callback instead.
# in case we get something from previous tests...
- for mb in self.server.theAccount.mailboxes:
- self.server.theAccount.delete(mb)
+ #for mb in self.server.theAccount.mailboxes:
+ #self.server.theAccount.delete(mb)
# email parser
self.parser = parser.Parser()