prime-uids

We pre-fetch the uids from soledad on mailbox initialization

4 files changed, 75 insertions, 24 deletions

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 6c8d78d..802ebf3 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -126,6 +126,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             self.setFlags(self.INIT_FLAGS)
 
         if self._memstore:
+            self.prime_known_uids_to_memstore()
             self.prime_last_uid_to_memstore()
 
     @property
@@ -263,10 +264,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         Prime memstore with last_uid value
         """
         set_exist = set(self.messages.all_uid_iter())
-        last = max(set_exist) + 1 if set_exist else 1
+        last = max(set_exist) if set_exist else 0
         logger.info("Priming Soledad last_uid to %s" % (last,))
         self._memstore.set_last_soledad_uid(self.mbox, last)
 
+    def prime_known_uids_to_memstore(self):
+        """
+        Prime memstore with the set of all known uids.
+
+        We do this to be able to filter the requests efficiently.
+        """
+        known_uids = self.messages.all_soledad_uid_iter()
+        self._memstore.set_known_uids(self.mbox, known_uids)
+
     def getUIDValidity(self):
         """
         Return the unique validity identifier for this mailbox.
@@ -525,6 +535,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         return seq_messg
 
     @deferred
+    #@profile
     def fetch(self, messages_asked, uid):
         """
         Retrieve one or more messages in this mailbox.
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index fac66ad..217ad8e 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -149,6 +149,14 @@ class MemoryStore(object):
         """
         self._last_uid = {}
 
+        """
+        known-uids keeps a count of the uids that soledad knows for a given
+        mailbox
+
+        {'mbox-a': set([1,2,3])}
+        """
+        self._known_uids = defaultdict(set)
+
         # New and dirty flags, to set MessageWrapper State.
         self._new = set([])
         self._new_deferreds = {}
@@ -447,10 +455,20 @@ class MemoryStore(object):
 
         :param mbox: the mailbox
         :type mbox: str or unicode
+        :rtype: list
         """
         all_keys = self._msg_store.keys()
         return [uid for m, uid in all_keys if m == mbox]
 
+    def get_soledad_known_uids(self, mbox):
+        """
+        Get all uids that soledad knows about, from the memory cache.
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :rtype: list
+        """
+        return self._known_uids.get(mbox, [])
+
     # last_uid
 
     def get_last_uid(self, mbox):
@@ -496,6 +514,18 @@ class MemoryStore(object):
             if not self._last_uid.get(mbox, None):
                 self._last_uid[mbox] = value
 
+    def set_known_uids(self, mbox, value):
+        """
+        Set the value fo the known-uids set for this mbox.
+
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param value: a sequence of integers to be added to the set.
+        :type value: tuple
+        """
+        current = self._known_uids[mbox]
+        self._known_uids[mbox] = current.union(set(value))
+
     def increment_last_soledad_uid(self, mbox):
         """
         Increment by one the soledad integer cache for the last_uid for
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 356145f..0e5c74a 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -219,6 +219,7 @@ class LeapMessage(fields, MailParser, MBoxParser):
 
     # setFlags not in the interface spec but we use it with store command.
 
+    #@profile
     def setFlags(self, flags, mode):
         """
         Sets the flags for this message
@@ -934,6 +935,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # recent flags
 
+    #@profile
     def _get_recent_flags(self):
         """
         An accessor for the recent-flags set for this mailbox.
@@ -957,13 +959,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                         {'doc_id': rdoc.doc_id, 'set': rflags})
             return rflags
 
-        else:
+        #else:
             # fallback for cases without memory store
-            with self._rdoc_lock:
-                rdoc = self._get_recent_doc()
-                self.__rflags = set(rdoc.content.get(
-                    fields.RECENTFLAGS_KEY, []))
-            return self.__rflags
+            #with self._rdoc_lock:
+                #rdoc = self._get_recent_doc()
+                #self.__rflags = set(rdoc.content.get(
+                    #fields.RECENTFLAGS_KEY, []))
+            #return self.__rflags
 
     def _set_recent_flags(self, value):
         """
@@ -972,21 +974,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         if self.memstore is not None:
             self.memstore.set_recent_flags(self.mbox, value)
 
-        else:
+        #else:
             # fallback for cases without memory store
-            with self._rdoc_lock:
-                rdoc = self._get_recent_doc()
-                newv = set(value)
-                self.__rflags = newv
-                rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
+            #with self._rdoc_lock:
+                #rdoc = self._get_recent_doc()
+                #newv = set(value)
+                #self.__rflags = newv
+                #rdoc.content[fields.RECENTFLAGS_KEY] = list(newv)
                 # XXX should deferLater 0 it?
-                self._soledad.put_doc(rdoc)
+                #self._soledad.put_doc(rdoc)
 
     recent_flags = property(
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
 
     # XXX change naming, indicate soledad query.
+    #@profile
     def _get_recent_doc(self):
         """
         Get recent-flags document from Soledad for this mailbox.
@@ -1027,6 +1030,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             self.recent_flags.difference_update(
                 set([uid]))
 
+    @deferred
     def set_recent_flag(self, uid):
         """
         Set Recent flag for a given uid.
@@ -1095,6 +1099,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # XXX is this working?
         return self._get_uid_from_msgidCb(msgid)
 
+    #@profile
     def set_flags(self, mbox, messages, flags, mode):
         """
         Set flags for a sequence of messages.
@@ -1183,25 +1188,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # FIXME ----------------------------------------------
         return sorted(all_docs, key=lambda item: item.content['uid'])
 
-    def all_uid_iter(self):
+    #@profile
+    def all_soledad_uid_iter(self):
         """
         Return an iterator through the UIDs of all messages, sorted in
         ascending order.
         """
-        # XXX we should get this from the uid table, local-only
-        # XXX FIXME -------------
-        # This should be cached in the memstoretoo
         db_uids = set([doc.content[self.UID_KEY] for doc in
                        self._soledad.get_from_index(
                            fields.TYPE_MBOX_IDX,
                            fields.TYPE_FLAGS_VAL, self.mbox)])
+        return db_uids
+
+    #@profile
+    def all_uid_iter(self):
+        """
+        Return an iterator through the UIDs of all messages, from memory.
+        """
         if self.memstore is not None:
             mem_uids = self.memstore.get_uids(self.mbox)
-            uids = db_uids.union(set(mem_uids))
-        else:
-            uids = db_uids
-
-        return (u for u in sorted(uids))
+            soledad_known_uids = self.memstore.get_soledad_known_uids(
+                self.mbox)
+            combined = tuple(set(mem_uids).union(soledad_known_uids))
+            return combined
 
     # XXX MOVE to memstore
     def all_flags(self):
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index ae5c583..ff5e03b 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -192,7 +192,8 @@ class SoledadStore(ContentDedup):
 
     # IMessageConsumer
 
-    @deferred
+    # It's not thread-safe to defer this to a different thread
+
     def consume(self, queue):
         """
         Creates a new document in soledad db.