Merge remote-tracking branch 'refs/remotes/kali/feature/speedup-select' into develop

1 files changed, 28 insertions, 16 deletions

diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
index 732fe03..25f00bb 100644
--- a/src/leap/mail/imap/soledadstore.py
+++ b/src/leap/mail/imap/soledadstore.py
@@ -133,15 +133,14 @@ A lock per document.
 # Setting this to twice the number of threads in the threadpool
 # should be safe.
 put_locks = defaultdict(lambda: threading.Lock())
+mbox_doc_locks = defaultdict(lambda: threading.Lock())
 
 
 class SoledadStore(ContentDedup):
     """
     This will create docs in the local Soledad database.
     """
-    _soledad_rw_lock = threading.Lock()
     _remove_lock = threading.Lock()
-    _mbox_doc_locks = defaultdict(lambda: threading.Lock())
 
     implements(IMessageConsumer, IMessageStore)
 
@@ -282,9 +281,13 @@ class SoledadStore(ContentDedup):
         def doSoledadCalls(items):
             # we prime the generator, that should return the
             # message or flags wrapper item in the first place.
-            doc_wrapper = items.next()
-            failed = self._soledad_write_document_parts(items)
-            queueNotifyBack(failed, doc_wrapper)
+            try:
+                doc_wrapper = items.next()
+            except StopIteration:
+                pass
+            else:
+                failed = self._soledad_write_document_parts(items)
+                queueNotifyBack(failed, doc_wrapper)
 
         doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper))
 
@@ -309,8 +312,10 @@ class SoledadStore(ContentDedup):
             try:
                 self._try_call(call, item)
             except Exception as exc:
-                logger.debug("ITEM WAS: %s" % str(item))
-                logger.debug("ITEM CONTENT WAS: %s" % str(item.content))
+                logger.debug("ITEM WAS: %s" % repr(item))
+                if hasattr(item, 'content'):
+                        logger.debug("ITEM CONTENT WAS: %s" %
+                                     repr(item.content))
                 logger.exception(exc)
                 failed = True
                 continue
@@ -349,6 +354,9 @@ class SoledadStore(ContentDedup):
 
         if call == self._PUT_DOC_FUN:
             doc_id = item.doc_id
+            if doc_id is None:
+                logger.warning("BUG! Dirty doc but has no doc_id!")
+                return
             with put_locks[doc_id]:
                 doc = self._GET_DOC_FUN(doc_id)
 
@@ -437,12 +445,12 @@ class SoledadStore(ContentDedup):
         :return: a tuple with recent-flags doc payload and callable
         :rtype: tuple
         """
-        call = self._CREATE_DOC_FUN
+        call = self._PUT_DOC_FUN
 
         payload = rflags_wrapper.content
         if payload:
             logger.debug("Saving RFLAGS to Soledad...")
-            yield payload, call
+            yield rflags_wrapper, call
 
     # Mbox documents and attributes
 
@@ -456,7 +464,7 @@ class SoledadStore(ContentDedup):
                  the query failed.
         :rtype: SoledadDocument or None.
         """
-        with self._mbox_doc_locks[mbox]:
+        with mbox_doc_locks[mbox]:
             return self._get_mbox_document(mbox)
 
     def _get_mbox_document(self, mbox):
@@ -471,7 +479,7 @@ class SoledadStore(ContentDedup):
                 return query.pop()
             else:
                 logger.error("Could not find mbox document for %r" %
-                            (self.mbox,))
+                            (mbox,))
         except Exception as exc:
             logger.exception("Unhandled error %r" % exc)
 
@@ -496,7 +504,7 @@ class SoledadStore(ContentDedup):
         :type closed: bool
         """
         leap_assert(isinstance(closed, bool), "closed needs to be boolean")
-        with self._mbox_doc_locks[mbox]:
+        with mbox_doc_locks[mbox]:
             mbox_doc = self._get_mbox_document(mbox)
             if mbox_doc is None:
                 logger.error(
@@ -521,14 +529,18 @@ class SoledadStore(ContentDedup):
         leap_assert_type(value, int)
         key = fields.LAST_UID_KEY
 
-        # XXX change for a lock related to the mbox document
-        # itself.
-        with self._mbox_doc_locks[mbox]:
+        # XXX use accumulator to reduce number of hits
+        with mbox_doc_locks[mbox]:
             mbox_doc = self._get_mbox_document(mbox)
             old_val = mbox_doc.content[key]
             if value > old_val:
                 mbox_doc.content[key] = value
-                self._soledad.put_doc(mbox_doc)
+                try:
+                    self._soledad.put_doc(mbox_doc)
+                except Exception as exc:
+                    logger.error("Error while setting last_uid for %r"
+                                 % (mbox,))
+                    logger.exception(exc)
 
     def get_flags_doc(self, mbox, uid):
         """