remove all refs during removal, and protect from empty docs

4 files changed, 29 insertions, 11 deletions

diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py
index d18bc9ae..045de820 100644
--- a/mail/src/leap/mail/imap/mailbox.py
+++ b/mail/src/leap/mail/imap/mailbox.py
@@ -609,7 +609,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             logger.debug("Getting msg by index: INEFFICIENT call!")
             raise NotImplementedError
         else:
-            got_msg = [(msgid, getmsg(msgid)) for msgid in seq_messg]
+            got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg)
             result = ((msgid, msg) for msgid, msg in got_msg
                       if msg is not None)
             self.reactor.callLater(0, self.unset_recent_flags, seq_messg)
diff --git a/mail/src/leap/mail/imap/memorystore.py b/mail/src/leap/mail/imap/memorystore.py
index ba444b0a..1e4262ab 100644
--- a/mail/src/leap/mail/imap/memorystore.py
+++ b/mail/src/leap/mail/imap/memorystore.py
@@ -486,15 +486,25 @@ class MemoryStore(object):
         # token to ensure consistency in the removal.
 
         try:
+            del self._fdoc_store[mbox][uid]
+        except KeyError:
+            pass
+
+        try:
             key = mbox, uid
             self._new.discard(key)
             self._dirty.discard(key)
             if key in self._sizes:
                 del self._sizes[key]
-            self._fdoc_store[mbox].pop(uid, None)
+            self._known_uids[mbox].discard(uid)
+        except Exception as exc:
+            logger.error("error while removing message!")
+            logger.exception(exc)
+        try:
             with self._fdoc_docid_lock:
-                self._fdoc_id_store[mbox].pop(uid, None)
+                del self._fdoc_id_store[mbox][uid]
         except Exception as exc:
+            logger.error("error while removing message!")
             logger.exception(exc)
 
     # IMessageStoreWriter
@@ -1124,6 +1134,8 @@ class MemoryStore(object):
             # Stop and trigger last write
             self.stop_and_flush()
             # Wait on the writebacks to finish
+
+            # XXX what if pending deferreds is empty?
             pending_deferreds = (self._new_deferreds.get(mbox, []) +
                                  self._dirty_deferreds.get(mbox, []))
             d1 = defer.gatherResults(pending_deferreds, consumeErrors=True)
@@ -1169,6 +1181,7 @@ class MemoryStore(object):
                 logger.exception(exc)
 
             # 2. Delete all messages marked as deleted in memory.
+            logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,))
             mem_deleted = self.remove_all_deleted(mbox)
 
             all_deleted = set(mem_deleted).union(set(sol_deleted))
diff --git a/mail/src/leap/mail/imap/messageparts.py b/mail/src/leap/mail/imap/messageparts.py
index 6f1376ac..257721cd 100644
--- a/mail/src/leap/mail/imap/messageparts.py
+++ b/mail/src/leap/mail/imap/messageparts.py
@@ -287,7 +287,7 @@ class MessageWrapper(object):
                 logger.debug("Error while walking message...")
                 logger.exception(exc)
 
-        if not empty(self.fdoc.content):
+        if not empty(self.fdoc.content) and 'uid' in self.fdoc.content:
             yield self.fdoc
         if not empty(self.hdoc.content):
             yield self.hdoc
@@ -418,10 +418,8 @@ class MessagePart(object):
         if payload:
             content_type = self._get_ctype_from_document(phash)
             charset = find_charset(content_type)
-            logger.debug("Got charset from header: %s" % (charset,))
             if charset is None:
                 charset = self._get_charset(payload)
-                logger.debug("Got charset: %s" % (charset,))
             try:
                 if isinstance(payload, unicode):
                     payload = payload.encode(charset)
diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py
index c133a6df..0aa40f19 100644
--- a/mail/src/leap/mail/imap/messages.py
+++ b/mail/src/leap/mail/imap/messages.py
@@ -850,7 +850,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
         if not exist:
             exist = self._get_fdoc_from_chash(chash)
-        if exist:
+        if exist and exist.content is not None:
             return exist.content.get(fields.UID_KEY, "unknown-uid")
         else:
             return False
@@ -926,8 +926,13 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             return
 
         uid = self.memstore.increment_last_soledad_uid(self.mbox)
-        # We can say the observer that we're done
+        # We can say the observer that we're done at this point.
+        # Make sure it has no serious consequences if we're issued
+        # a fetch command right after...
         self.reactor.callFromThread(observer.callback, uid)
+        # if we did the notify, we need to invalidate the deferred
+        # so not to try to fire it twice.
+        observer = None
 
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
@@ -952,7 +957,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         msg_container = MessageWrapper(fd, hd, cdocs)
         self.memstore.create_message(
             self.mbox, uid, msg_container,
-            observer=None, notify_on_disk=notify_on_disk)
+            observer=observer, notify_on_disk=notify_on_disk)
 
     #
     # getters: specific queries
@@ -1130,8 +1135,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             if msg is not None:
                 return uid, msg.setFlags(flags, mode)
 
-        result = dict(
-            set_flags(uid, tuple(flags), mode) for uid in messages)
+        setted_flags = [set_flags(uid, flags, mode) for uid in messages]
+        result = dict(filter(None, setted_flags))
 
         reactor.callFromThread(observer.callback, result)
 
@@ -1158,6 +1163,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         """
         msg_container = self.memstore.get_message(
             self.mbox, uid, flags_only=flags_only)
+
         if msg_container is not None:
             if mem_only:
                 msg = LeapMessage(None, uid, self.mbox, collection=self,
@@ -1170,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
                                   collection=self, container=msg_container)
         else:
             msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
+
         if not msg.does_exist():
             return None
         return msg