From d073e11d21e0efa496793651869168e8c68b12da Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 30 Jan 2014 18:35:03 -0400
Subject: prime-uids
We pre-fetch the uids from soledad on mailbox initialization
---
mail/src/leap/mail/imap/mailbox.py | 13 +++++++-
mail/src/leap/mail/imap/memorystore.py | 30 +++++++++++++++++++
mail/src/leap/mail/imap/messages.py | 53 +++++++++++++++++++--------------
mail/src/leap/mail/imap/soledadstore.py | 3 +-
4 files changed, 75 insertions(+), 24 deletions(-)
diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py
index 6c8d78db..802ebf33 100644
--- a/mail/src/leap/mail/imap/mailbox.py
+++ b/mail/src/leap/mail/imap/mailbox.py
@@ -126,6 +126,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
self.setFlags(self.INIT_FLAGS)
if self._memstore:
+ self.prime_known_uids_to_memstore()
self.prime_last_uid_to_memstore()
@property
@@ -263,10 +264,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
Prime memstore with last_uid value
"""
set_exist = set(self.messages.all_uid_iter())
- last = max(set_exist) + 1 if set_exist else 1
+ last = max(set_exist) if set_exist else 0
logger.info("Priming Soledad last_uid to %s" % (last,))
self._memstore.set_last_soledad_uid(self.mbox, last)
+ def prime_known_uids_to_memstore(self):
+ """
+ Prime memstore with the set of all known uids.
+
+ We do this to be able to filter the requests efficiently.
+ """
+ known_uids = self.messages.all_soledad_uid_iter()
+ self._memstore.set_known_uids(self.mbox, known_uids)
+
def getUIDValidity(self):
"""
Return the unique validity identifier for this mailbox.
@@ -525,6 +535,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
return seq_messg
@deferred
+ #@profile
def fetch(self, messages_asked, uid):
"""
Retrieve one or more messages in this mailbox.
diff --git a/mail/src/leap/mail/imap/memorystore.py b/mail/src/leap/mail/imap/memorystore.py
index fac66adb..217ad8e6 100644
--- a/mail/src/leap/mail/imap/memorystore.py
+++ b/mail/src/leap/mail/imap/memorystore.py
@@ -149,6 +149,14 @@ class MemoryStore(object):
"""
self._last_uid = {}
+ """
+ known-uids keeps a count of the uids that soledad knows for a given
+ mailbox
+
+ {'mbox-a': set([1,2,3])}
+ """
+ self._known_uids = defaultdict(set)
+
# New and dirty flags, to set MessageWrapper State.
self._new = set([])
self._new_deferreds = {}
@@ -447,10 +455,20 @@ class MemoryStore(object):
:param mbox: the mailbox
:type mbox: str or unicode
+ :rtype: list
"""
all_keys = self._msg_store.keys()
return [uid for m, uid in all_keys if m == mbox]
+ def get_soledad_known_uids(self, mbox):
+ """
+ Get all uids that soledad knows about, from the memory cache.
+ :param mbox: the mailbox
+ :type mbox: str or unicode
+ :rtype: list
+ """
+ return self._known_uids.get(mbox, [])
+
# last_uid
def get_last_uid(self, mbox):
@@ -496,6 +514,18 @@ class MemoryStore(object):
if not self._last_uid.get(mbox, None):
self._last_uid[mbox] = value
+ def set_known_uids(self, mbox, value):
+ """
+ Set the value fo the known-uids set for this mbox.
+
+ :param mbox: the mailbox
+ :type mbox: str or unicode
+ :param value: a sequence of integers to be added to the set.
+ :type value: tuple
+ """
+ current = self._known_uids[mbox]
+ self._known_uids[mbox] = current.union(set(value))
+
def increment_last_soledad_uid(self, mbox):
"""
Increment by one the soledad integer cache for the last_uid for
diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py
index 356145f0..0e5c74a6 100644
--- a/mail/src/leap/mail/imap/messages.py
+++ b/mail/src/leap/mail/imap/messages.py
@@ -219,6 +219,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
# setFlags not in the interface spec but we use it with store command.
+ #@profile
def setFlags(self, flags, mode):
"""
Sets the flags for this message
@@ -934,6 +935,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
# recent flags
+ #@profile
def _get_recent_flags(self):
"""
An accessor for the recent-flags set for this mailbox.
@@ -957,13 +959,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
{'doc_id': rdoc.doc_id, 'set': rflags})
return rflags
- else:
+ #else:
# fallback for cases without memory store
- with self._rdoc_lock:
- rdoc = self._get_recent_doc()
- self.__rflags = set(rdoc.content.get(
- fields.RECENTFLAGS_KEY, []))
- return self.__rflags
+ #with self._rdoc_lock:
+ #rdoc = self._get_recent_doc()
+ #self.__rflags = set(rdoc.content.get(
+ #fields.RECENTFLAGS_KEY, []))
+ #return self.__rflags
def _set_recent_flags(self, value):
"""
@@ -972,21 +974,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
if self.memstore is not None:
self.memstore.set_recent_flags(self.mbox, value)
- else:
+ #else:
# fallback for cases without memory store
- with self._rdoc_lock:
- rdoc = self._get_recent_doc()
- newv = set(value)
- self.__rflags = newv
- rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
+ #with self._rdoc_lock:
+ #rdoc = self._get_recent_doc()
+ #newv = set(value)
+ #self.__rflags = newv
+ #rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
# XXX should deferLater 0 it?
- self._soledad.put_doc(rdoc)
+ #self._soledad.put_doc(rdoc)
recent_flags = property(
_get_recent_flags, _set_recent_flags,
doc="Set of UIDs with the recent flag for this mailbox.")
# XXX change naming, indicate soledad query.
+ #@profile
def _get_recent_doc(self):
"""
Get recent-flags document from Soledad for this mailbox.
@@ -1027,6 +1030,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
self.recent_flags.difference_update(
set([uid]))
+ @deferred
def set_recent_flag(self, uid):
"""
Set Recent flag for a given uid.
@@ -1095,6 +1099,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
# XXX is this working?
return self._get_uid_from_msgidCb(msgid)
+ #@profile
def set_flags(self, mbox, messages, flags, mode):
"""
Set flags for a sequence of messages.
@@ -1183,25 +1188,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
# FIXME ----------------------------------------------
return sorted(all_docs, key=lambda item: item.content['uid'])
- def all_uid_iter(self):
+ #@profile
+ def all_soledad_uid_iter(self):
"""
Return an iterator through the UIDs of all messages, sorted in
ascending order.
"""
- # XXX we should get this from the uid table, local-only
- # XXX FIXME -------------
- # This should be cached in the memstoretoo
db_uids = set([doc.content[self.UID_KEY] for doc in
self._soledad.get_from_index(
fields.TYPE_MBOX_IDX,
fields.TYPE_FLAGS_VAL, self.mbox)])
+ return db_uids
+
+ #@profile
+ def all_uid_iter(self):
+ """
+ Return an iterator through the UIDs of all messages, from memory.
+ """
if self.memstore is not None:
mem_uids = self.memstore.get_uids(self.mbox)
- uids = db_uids.union(set(mem_uids))
- else:
- uids = db_uids
-
- return (u for u in sorted(uids))
+ soledad_known_uids = self.memstore.get_soledad_known_uids(
+ self.mbox)
+ combined = tuple(set(mem_uids).union(soledad_known_uids))
+ return combined
# XXX MOVE to memstore
def all_flags(self):
diff --git a/mail/src/leap/mail/imap/soledadstore.py b/mail/src/leap/mail/imap/soledadstore.py
index ae5c583e..ff5e03b6 100644
--- a/mail/src/leap/mail/imap/soledadstore.py
+++ b/mail/src/leap/mail/imap/soledadstore.py
@@ -192,7 +192,8 @@ class SoledadStore(ContentDedup):
# IMessageConsumer
- @deferred
+ # It's not thread-safe to defer this to a different thread
+
def consume(self, queue):
"""
Creates a new document in soledad db.
--
cgit v1.2.3