diff options
| -rw-r--r-- | mail/src/leap/mail/imap/memorystore.py | 187 | ||||
| -rw-r--r-- | mail/src/leap/mail/imap/messages.py | 10 |
2 files changed, 96 insertions, 101 deletions
diff --git a/mail/src/leap/mail/imap/memorystore.py b/mail/src/leap/mail/imap/memorystore.py index 3f3cf83e..b198e128 100644 --- a/mail/src/leap/mail/imap/memorystore.py +++ b/mail/src/leap/mail/imap/memorystore.py @@ -109,13 +109,14 @@ class MemoryStore(object): self._write_period = write_period # Internal Storage: messages - # TODO this probably will have better access times if we - # use msg_store[mbox][uid] insted of the current key scheme. """ - key is str(mbox,uid) + Flags document store. + _fdoc_store[mbox][uid] = { 'content': 'aaa' } """ - self._msg_store = {} + self._fdoc_store = defaultdict(lambda: defaultdict( + lambda: ReferenciableDict({}))) +<<<<<<< HEAD # Sizes """ {'mbox, uid': <int>} @@ -123,10 +124,24 @@ class MemoryStore(object): self._sizes = {} # Internal Storage: payload-hash +======= + # Internal Storage: content-hash:hdoc +>>>>>>> change internal storage and keying scheme in memstore """ - {'phash': weakreaf.proxy(dict)} + hdoc-store keeps references to + the header-documents indexed by content-hash. + + {'chash': { dict-stuff } + } + """ + self._hdoc_store = defaultdict(lambda: ReferenciableDict({})) + + # Internal Storage: payload-hash:cdoc + """ + content-docs stored by payload-hash + {'phash': { dict-stuff } } """ - self._phash_store = {} + self._cdoc_store = defaultdict(lambda: ReferenciableDict({})) # Internal Storage: content-hash:fdoc """ @@ -309,26 +324,12 @@ class MemoryStore(object): Helper method, called by both create_message and put_message. See those for parameter documentation. """ - # XXX have to differentiate between notify_new and notify_dirty - # TODO defaultdict the hell outa here... - - key = mbox, uid msg_dict = message.as_dict() - try: - store = self._msg_store[key] - except KeyError: - self._msg_store[key] = {FDOC: {}, - HDOC: {}, - CDOCS: {}, - DOCS_ID: {}} - store = self._msg_store[key] - fdoc = msg_dict.get(FDOC, None) - if fdoc: - if not store.get(FDOC, None): - store[FDOC] = ReferenciableDict({}) - store[FDOC].update(fdoc) + if fdoc is not None: + fdoc_store = self._fdoc_store[mbox][uid] + fdoc_store.update(fdoc) # content-hash indexing chash = fdoc.get(fields.CONTENT_HASH_KEY) @@ -337,33 +338,21 @@ class MemoryStore(object): chash_fdoc_store[chash] = {} chash_fdoc_store[chash][mbox] = weakref.proxy( - store[FDOC]) + fdoc_store) hdoc = msg_dict.get(HDOC, None) if hdoc is not None: - if not store.get(HDOC, None): - store[HDOC] = ReferenciableDict({}) - store[HDOC].update(hdoc) - - docs_id = msg_dict.get(DOCS_ID, None) - if docs_id: - if not store.get(DOCS_ID, None): - store[DOCS_ID] = {} - store[DOCS_ID].update(docs_id) + chash = hdoc.get(fields.CONTENT_HASH_KEY) + hdoc_store = self._hdoc_store[chash] + hdoc_store.update(hdoc) cdocs = message.cdocs - for cdoc_key in cdocs.keys(): - if not store.get(CDOCS, None): - store[CDOCS] = {} - - cdoc = cdocs[cdoc_key] - # first we make it weak-referenciable - referenciable_cdoc = ReferenciableDict(cdoc) - store[CDOCS][cdoc_key] = referenciable_cdoc + for cdoc in cdocs.values(): phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None) if not phash: continue - self._phash_store[phash] = weakref.proxy(referenciable_cdoc) + cdoc_store = self._cdoc_store[phash] + cdoc_store.update(cdoc) # Update memory store size # XXX this should use [mbox][uid] @@ -371,15 +360,13 @@ class MemoryStore(object): self._sizes[key] = size.get_size(self._fdoc_store[key]) # TODO add hdoc and cdocs sizes too - def prune(seq, store): - for key in seq: - if key in store and empty(store.get(key)): - store.pop(key) - - prune((FDOC, HDOC, CDOCS, DOCS_ID), store) + # XXX what to do with this? + #docs_id = msg_dict.get(DOCS_ID, None) + #if docs_id is not None: + #if not store.get(DOCS_ID, None): + #store[DOCS_ID] = {} + #store[DOCS_ID].update(docs_id) - # Update memory store size - self._sizes[key] = size(self._msg_store[key]) def get_docid_for_fdoc(self, mbox, uid): """ @@ -413,18 +400,20 @@ class MemoryStore(object): :return: MessageWrapper or None """ key = mbox, uid - FDOC = MessagePartType.fdoc.key - msg_dict = self._msg_store.get(key, None) - if empty(msg_dict): + fdoc = self._fdoc_store[mbox][uid] + if empty(fdoc): return None + new, dirty = self._get_new_dirty_state(key) if flags_only: - return MessageWrapper(fdoc=msg_dict[FDOC], + return MessageWrapper(fdoc=fdoc, new=new, dirty=dirty, memstore=weakref.proxy(self)) else: - return MessageWrapper(from_dict=msg_dict, + chash = fdoc.get(fields.CONTENT_HASH_KEY) + hdoc = self._hdoc_store[chash] + return MessageWrapper(fdoc=fdoc, hdoc=hdoc, new=new, dirty=dirty, memstore=weakref.proxy(self)) @@ -448,10 +437,14 @@ class MemoryStore(object): key = mbox, uid self._new.discard(key) self._dirty.discard(key) +<<<<<<< HEAD self._msg_store.pop(key, None) if key in self._sizes: del self._sizes[key] +======= + self._fdoc_store[mbox].pop(uid, None) +>>>>>>> change internal storage and keying scheme in memstore except Exception as exc: logger.exception(exc) @@ -494,8 +487,7 @@ class MemoryStore(object): :type mbox: str or unicode :rtype: list """ - all_keys = self._msg_store.keys() - return [uid for m, uid in all_keys if m == mbox] + return self._fdoc_store[mbox].keys() def get_soledad_known_uids(self, mbox): """ @@ -605,11 +597,9 @@ class MemoryStore(object): """ # We can do direct assignments cause we know this will only # be called during initialization of the mailbox. - msg_store = self._msg_store + fdoc_store = self._fdoc_store[mbox] for uid in flag_docs: - key = mbox, uid - msg_store[key] = {} - msg_store[key][FDOC] = ReferenciableDict(flag_docs[uid]) + fdoc_store[uid] = ReferenciableDict(flag_docs[uid]) def all_flags(self, mbox): """ @@ -621,11 +611,10 @@ class MemoryStore(object): """ flags_dict = {} uids = self.get_uids(mbox) - store = self._msg_store + fdoc_store = self._fdoc_store for uid in uids: - key = mbox, uid try: - flags = store[key][FDOC][fields.FLAGS_KEY] + flags = fdoc_store[uid][fields.FLAGS_KEY] flags_dict[uid] = flags except KeyError: continue @@ -635,7 +624,7 @@ class MemoryStore(object): def count_new_mbox(self, mbox): """ - Count the new messages by inbox. + Count the new messages by mailbox. :param mbox: the mailbox :type mbox: str or unicode @@ -653,6 +642,32 @@ class MemoryStore(object): """ return len(self._new) + def count(self, mbox): + """ + Return the count of messages for a given mbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: number of messages + :rtype: int + """ + return len(self._fdoc_store[mbox]) + + def unseen_iter(self, mbox): + """ + Get an iterator for the message UIDs with no `seen` flag + for a given mailbox. + + :param mbox: the mailbox + :type mbox: str or unicode + :return: iterator through unseen message doc UIDs + :rtype: iterable + """ + fdocs = self._fdoc_store[mbox] + return [uid for uid, value + in fdocs.items() + if fields.SEEN_FLAG not in value["flags"]] + def get_cdoc_from_phash(self, phash): """ Return a content-document by its payload-hash. @@ -661,7 +676,7 @@ class MemoryStore(object): :type phash: str or unicode :rtype: MessagePartDoc """ - doc = self._phash_store.get(phash, None) + doc = self._cdoc_store.get(phash, None) # XXX return None for consistency? @@ -716,15 +731,15 @@ class MemoryStore(object): content=fdoc, doc_id=None) - def all_msg_iter(self): + def iter_fdoc_keys(self): """ - Return generator that iterates through all messages in the store. - - :return: generator of MessageWrappers - :rtype: generator + Return a generator through all the mbox, uid keys in the flags-doc + store. """ - return (self.get_message(*key) - for key in sorted(self._msg_store.keys())) + fdoc_store = self._fdoc_store + for mbox in fdoc_store: + for uid in fdoc_store[mbox]: + yield mbox, uid def all_new_dirty_msg_iter(self): """ @@ -734,23 +749,9 @@ class MemoryStore(object): :rtype: generator """ return (self.get_message(*key) - for key in sorted(self._msg_store.keys()) + for key in sorted(self.iter_fdoc_keys()) if key in self._new or key in self._dirty) - def all_msg_dict_for_mbox(self, mbox): - """ - Return all the message dicts for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: list of dictionaries - :rtype: list - """ - # This *needs* to return a fixed sequence. Otherwise the dictionary len - # will change during iteration, when we modify it - return [self._msg_store[(mb, uid)] - for mb, uid in self._msg_store if mb == mbox] - def all_deleted_uid_iter(self, mbox): """ Return a list with the UIDs for all messags @@ -763,11 +764,10 @@ class MemoryStore(object): """ # This *needs* to return a fixed sequence. Otherwise the dictionary len # will change during iteration, when we modify it - all_deleted = [ - msg['fdoc']['uid'] for msg in self.all_msg_dict_for_mbox(mbox) - if msg.get('fdoc', None) - and fields.DELETED_FLAG in msg['fdoc']['flags']] - return all_deleted + fdocs = self._fdoc_store[mbox] + return [uid for uid, value + in fdocs.items() + if fields.DELETED_FLAG in value["flags"]] # new, dirty flags @@ -780,6 +780,7 @@ class MemoryStore(object): :return: tuple of bools :rtype: tuple """ + # TODO change indexing of sets to [mbox][key] too. # XXX should return *first* the news, and *then* the dirty... return map(lambda _set: key in _set, (self._new, self._dirty)) diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py index 3fbe2adc..3d25598b 100644 --- a/mail/src/leap/mail/imap/messages.py +++ b/mail/src/leap/mail/imap/messages.py @@ -1290,10 +1290,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :rtype: int """ - # XXX get this from a public method in memstore - store = self.memstore._msg_store - return len([uid for (mbox, uid) in store.keys() - if mbox == self.mbox]) + return self.memstore.count(self.mbox) # unseen messages @@ -1305,10 +1302,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): :return: iterator through unseen message doc UIDs :rtype: iterable """ - # XXX get this from a public method in memstore - store = self.memstore._msg_store - return (uid for (mbox, uid), d in store.items() - if mbox == self.mbox and "\\Seen" not in d["fdoc"]["flags"]) + return self.memstore.unseen_iter(self.mbox) def count_unseen(self): """ |