fix rdoc duplication

4 files changed, 39 insertions, 27 deletions

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index d7be662..59b2b40 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -135,6 +135,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
 
+        from twisted.internet import reactor
+        self.reactor = reactor
+
         self.mbox = self._parse_mailbox_name(mbox)
         self.rw = rw
 
@@ -158,9 +161,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
             self.prime_last_uid_to_memstore()
             self.prime_flag_docs_to_memstore()
 
-        from twisted.internet import reactor
-        self.reactor = reactor
-
         # purge memstore from empty fdocs.
         self._memstore.purge_fdoc_store(mbox)
 
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 875b1b8..aa7da3d 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -506,6 +506,8 @@ class MemoryStore(object):
             if key in self._sizes:
                 del self._sizes[key]
             self._known_uids[mbox].discard(uid)
+        except KeyError:
+            pass
         except Exception as exc:
             logger.error("error while removing message!")
             logger.exception(exc)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 9bd64fc..8c777f5 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -77,7 +77,7 @@ def try_unique_query(curried):
                 # TODO we could take action, like trigger a background
                 # process to kill dupes.
                 name = getattr(curried, 'expected', 'doc')
-                logger.debug(
+                logger.warning(
                     "More than one %s found for this mbox, "
                     "we got a duplicate!!" % (name,))
             return query.pop()
@@ -683,8 +683,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # TODO we would abstract this to a SoledadProperty class
 
-    _rdoc_lock = threading.Lock()
-    _rdoc_property_lock = threading.Lock()
+    _rdoc_lock = defaultdict(lambda: threading.Lock())
+    _rdoc_write_lock = defaultdict(lambda: threading.Lock())
+    _rdoc_read_lock = defaultdict(lambda: threading.Lock())
+    _rdoc_property_lock = defaultdict(lambda: threading.Lock())
 
     _initialized = {}
 
@@ -729,10 +731,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         self.__rflags = None
 
         if not self._initialized.get(mbox, False):
-            self.initialize_db()
-            # ensure that we have a recent-flags and a hdocs-sec doc
-            self._get_or_create_rdoc()
-            self._initialized[mbox] = True
+            try:
+                self.initialize_db()
+                # ensure that we have a recent-flags doc
+                self._get_or_create_rdoc()
+            except Exception:
+                logger.debug("Error initializing %r" % (mbox,))
+            else:
+                self._initialized[mbox] = True
 
         from twisted.internet import reactor
         self.reactor = reactor
@@ -753,12 +759,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         Try to retrieve the recent-flags doc for this MessageCollection,
         and create one if not found.
         """
-        rdoc = self._get_recent_doc()
-        if not rdoc:
-            rdoc = self._get_empty_doc(self.RECENT_DOC)
-            if self.mbox != fields.INBOX_VAL:
-                rdoc[fields.MBOX_KEY] = self.mbox
-            self._soledad.create_doc(rdoc)
+        # XXX should move this to memstore too
+        with self._rdoc_write_lock[self.mbox]:
+            rdoc = self._get_recent_doc_from_soledad()
+            if rdoc is None:
+                rdoc = self._get_empty_doc(self.RECENT_DOC)
+                if self.mbox != fields.INBOX_VAL:
+                    rdoc[fields.MBOX_KEY] = self.mbox
+                self._soledad.create_doc(rdoc)
 
     @deferred_to_thread
     def _do_parse(self, raw):
@@ -976,12 +984,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             return self.__rflags
 
         if self.memstore is not None:
-            with self._rdoc_lock:
+            with self._rdoc_lock[self.mbox]:
                 rflags = self.memstore.get_recent_flags(self.mbox)
                 if not rflags:
                     # not loaded in the memory store yet.
                     # let's fetch them from soledad...
-                    rdoc = self._get_recent_doc()
+                    rdoc = self._get_recent_doc_from_soledad()
                     rflags = set(rdoc.content.get(
                         fields.RECENTFLAGS_KEY, []))
                     # ...and cache them now.
@@ -1001,8 +1009,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         _get_recent_flags, _set_recent_flags,
         doc="Set of UIDs with the recent flag for this mailbox.")
 
-    # XXX change naming, indicate soledad query.
-    def _get_recent_doc(self):
+    def _get_recent_doc_from_soledad(self):
         """
         Get recent-flags document from Soledad for this mailbox.
         :rtype: SoledadDocument or None
@@ -1012,8 +1019,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             fields.TYPE_MBOX_IDX,
             fields.TYPE_RECENT_VAL, self.mbox)
         curried.expected = "rdoc"
-        rdoc = try_unique_query(curried)
-        return rdoc
+        with self._rdoc_read_lock[self.mbox]:
+            return try_unique_query(curried)
 
     # Property-set modification (protected by a different
     # lock to give atomicity to the read/write operation)
@@ -1025,7 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param uids: the uids to unset
         :type uid: sequence
         """
-        with self._rdoc_property_lock:
+        with self._rdoc_property_lock[self.mbox]:
             self.recent_flags.difference_update(
                 set(uids))
 
@@ -1038,7 +1045,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param uid: the uid to unset
         :type uid: int
         """
-        with self._rdoc_property_lock:
+        with self._rdoc_property_lock[self.mbox]:
             self.recent_flags.difference_update(
                 set([uid]))
 
@@ -1050,7 +1057,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         :param uid: the uid to set
         :type uid: int
         """
-        with self._rdoc_property_lock:
+        with self._rdoc_property_lock[self.mbox]:
             self.recent_flags = self.recent_flags.union(
                 set([uid]))
 
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index ed5259a..e047e2e 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -350,6 +350,9 @@ class SoledadStore(ContentDedup):
 
         if call == self._PUT_DOC_FUN:
             doc_id = item.doc_id
+            if doc_id is None:
+                logger.warning("BUG! Dirty doc but has no doc_id!")
+                return
             with put_locks[doc_id]:
                 doc = self._GET_DOC_FUN(doc_id)
 
@@ -438,12 +441,12 @@ class SoledadStore(ContentDedup):
         :return: a tuple with recent-flags doc payload and callable
         :rtype: tuple
         """
-        call = self._CREATE_DOC_FUN
+        call = self._PUT_DOC_FUN
 
         payload = rflags_wrapper.content
         if payload:
             logger.debug("Saving RFLAGS to Soledad...")
-            yield payload, call
+            yield rflags_wrapper, call
 
     # Mbox documents and attributes