diff options
| -rw-r--r-- | mail/src/leap/mail/imap/mailbox.py | 13 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/memorystore.py | 30 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/messages.py | 53 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/soledadstore.py | 3 |
4 files changed, 75 insertions, 24 deletions
diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py index 6c8d78db..802ebf33 100644 --- a/mail/src/leap/mail/imap/mailbox.py +++ b/mail/src/leap/mail/imap/mailbox.py @@ -126,6 +126,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self.setFlags(self.INIT_FLAGS) if self._memstore: + self.prime_known_uids_to_memstore() self.prime_last_uid_to_memstore() @property @@ -263,10 +264,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Prime memstore with last_uid value """ set_exist = set(self.messages.all_uid_iter()) - last = max(set_exist) + 1 if set_exist else 1 + last = max(set_exist) if set_exist else 0 logger.info("Priming Soledad last_uid to %s" % (last,)) self._memstore.set_last_soledad_uid(self.mbox, last) + def prime_known_uids_to_memstore(self): + """ + Prime memstore with the set of all known uids. + + We do this to be able to filter the requests efficiently. + """ + known_uids = self.messages.all_soledad_uid_iter() + self._memstore.set_known_uids(self.mbox, known_uids) + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -525,6 +535,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): return seq_messg @deferred + #@profile def fetch(self, messages_asked, uid): """ Retrieve one or more messages in this mailbox. diff --git a/mail/src/leap/mail/imap/memorystore.py b/mail/src/leap/mail/imap/memorystore.py index fac66adb..217ad8e6 100644 --- a/mail/src/leap/mail/imap/memorystore.py +++ b/mail/src/leap/mail/imap/memorystore.py @@ -149,6 +149,14 @@ class MemoryStore(object): """ self._last_uid = {} + """ + known-uids keeps a count of the uids that soledad knows for a given + mailbox + + {'mbox-a': set([1,2,3])} + """ + self._known_uids = defaultdict(set) + # New and dirty flags, to set MessageWrapper State. self._new = set([]) self._new_deferreds = {} @@ -447,10 +455,20 @@ class MemoryStore(object): :param mbox: the mailbox :type mbox: str or unicode + :rtype: list """ all_keys = self._msg_store.keys() return [uid for m, uid in all_keys if m == mbox] + def get_soledad_known_uids(self, mbox): + """ + Get all uids that soledad knows about, from the memory cache. + :param mbox: the mailbox + :type mbox: str or unicode + :rtype: list + """ + return self._known_uids.get(mbox, []) + # last_uid def get_last_uid(self, mbox): @@ -496,6 +514,18 @@ class MemoryStore(object): if not self._last_uid.get(mbox, None): self._last_uid[mbox] = value + def set_known_uids(self, mbox, value): + """ + Set the value fo the known-uids set for this mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :param value: a sequence of integers to be added to the set. + :type value: tuple + """ + current = self._known_uids[mbox] + self._known_uids[mbox] = current.union(set(value)) + def increment_last_soledad_uid(self, mbox): """ Increment by one the soledad integer cache for the last_uid for diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py index 356145f0..0e5c74a6 100644 --- a/mail/src/leap/mail/imap/messages.py +++ b/mail/src/leap/mail/imap/messages.py @@ -219,6 +219,7 @@ class LeapMessage(fields, MailParser, MBoxParser): # setFlags not in the interface spec but we use it with store command. + #@profile def setFlags(self, flags, mode): """ Sets the flags for this message @@ -934,6 +935,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): # recent flags + #@profile def _get_recent_flags(self): """ An accessor for the recent-flags set for this mailbox. @@ -957,13 +959,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): {'doc_id': rdoc.doc_id, 'set': rflags}) return rflags - else: + #else: # fallback for cases without memory store - with self._rdoc_lock: - rdoc = self._get_recent_doc() - self.__rflags = set(rdoc.content.get( - fields.RECENTFLAGS_KEY, [])) - return self.__rflags + #with self._rdoc_lock: + #rdoc = self._get_recent_doc() + #self.__rflags = set(rdoc.content.get( + #fields.RECENTFLAGS_KEY, [])) + #return self.__rflags def _set_recent_flags(self, value): """ @@ -972,21 +974,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): if self.memstore is not None: self.memstore.set_recent_flags(self.mbox, value) - else: + #else: # fallback for cases without memory store - with self._rdoc_lock: - rdoc = self._get_recent_doc() - newv = set(value) - self.__rflags = newv - rdoc.content[fields.RECENTFLAGS_KEY] = list(newv) + #with self._rdoc_lock: + #rdoc = self._get_recent_doc() + #newv = set(value) + #self.__rflags = newv + #rdoc.content[fields.RECENTFLAGS_KEY] = list(newv) # XXX should deferLater 0 it? - self._soledad.put_doc(rdoc) + #self._soledad.put_doc(rdoc) recent_flags = property( _get_recent_flags, _set_recent_flags, doc="Set of UIDs with the recent flag for this mailbox.") # XXX change naming, indicate soledad query. + #@profile def _get_recent_doc(self): """ Get recent-flags document from Soledad for this mailbox. @@ -1027,6 +1030,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): self.recent_flags.difference_update( set([uid])) + @deferred def set_recent_flag(self, uid): """ Set Recent flag for a given uid. @@ -1095,6 +1099,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): # XXX is this working? return self._get_uid_from_msgidCb(msgid) + #@profile def set_flags(self, mbox, messages, flags, mode): """ Set flags for a sequence of messages. @@ -1183,25 +1188,29 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): # FIXME ---------------------------------------------- return sorted(all_docs, key=lambda item: item.content['uid']) - def all_uid_iter(self): + #@profile + def all_soledad_uid_iter(self): """ Return an iterator through the UIDs of all messages, sorted in ascending order. """ - # XXX we should get this from the uid table, local-only - # XXX FIXME ------------- - # This should be cached in the memstoretoo db_uids = set([doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox)]) + return db_uids + + #@profile + def all_uid_iter(self): + """ + Return an iterator through the UIDs of all messages, from memory. + """ if self.memstore is not None: mem_uids = self.memstore.get_uids(self.mbox) - uids = db_uids.union(set(mem_uids)) - else: - uids = db_uids - - return (u for u in sorted(uids)) + soledad_known_uids = self.memstore.get_soledad_known_uids( + self.mbox) + combined = tuple(set(mem_uids).union(soledad_known_uids)) + return combined # XXX MOVE to memstore def all_flags(self): diff --git a/mail/src/leap/mail/imap/soledadstore.py b/mail/src/leap/mail/imap/soledadstore.py index ae5c583e..ff5e03b6 100644 --- a/mail/src/leap/mail/imap/soledadstore.py +++ b/mail/src/leap/mail/imap/soledadstore.py @@ -192,7 +192,8 @@ class SoledadStore(ContentDedup): # IMessageConsumer - @deferred + # It's not thread-safe to defer this to a different thread + def consume(self, queue): """ Creates a new document in soledad db. |