From d6ee618534596e76733d3c6b6cf0a75bdb4aa905 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 18 Feb 2014 09:58:54 -0400 Subject: catch soledad error while updating mbox doc --- src/leap/mail/imap/soledadstore.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'src/leap/mail/imap/soledadstore.py') diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py index 732fe03..919f834 100644 --- a/src/leap/mail/imap/soledadstore.py +++ b/src/leap/mail/imap/soledadstore.py @@ -133,15 +133,14 @@ A lock per document. # Setting this to twice the number of threads in the threadpool # should be safe. put_locks = defaultdict(lambda: threading.Lock()) +mbox_doc_locks = defaultdict(lambda: threading.Lock()) class SoledadStore(ContentDedup): """ This will create docs in the local Soledad database. """ - _soledad_rw_lock = threading.Lock() _remove_lock = threading.Lock() - _mbox_doc_locks = defaultdict(lambda: threading.Lock()) implements(IMessageConsumer, IMessageStore) @@ -456,7 +455,7 @@ class SoledadStore(ContentDedup): the query failed. :rtype: SoledadDocument or None. """ - with self._mbox_doc_locks[mbox]: + with mbox_doc_locks[mbox]: return self._get_mbox_document(mbox) def _get_mbox_document(self, mbox): @@ -471,7 +470,7 @@ class SoledadStore(ContentDedup): return query.pop() else: logger.error("Could not find mbox document for %r" % - (self.mbox,)) + (mbox,)) except Exception as exc: logger.exception("Unhandled error %r" % exc) @@ -496,7 +495,7 @@ class SoledadStore(ContentDedup): :type closed: bool """ leap_assert(isinstance(closed, bool), "closed needs to be boolean") - with self._mbox_doc_locks[mbox]: + with mbox_doc_locks[mbox]: mbox_doc = self._get_mbox_document(mbox) if mbox_doc is None: logger.error( @@ -521,14 +520,18 @@ class SoledadStore(ContentDedup): leap_assert_type(value, int) key = fields.LAST_UID_KEY - # XXX change for a lock related to the mbox document - # itself. - with self._mbox_doc_locks[mbox]: + # XXX use accumulator to reduce number of hits + with mbox_doc_locks[mbox]: mbox_doc = self._get_mbox_document(mbox) old_val = mbox_doc.content[key] if value > old_val: mbox_doc.content[key] = value - self._soledad.put_doc(mbox_doc) + try: + self._soledad.put_doc(mbox_doc) + except Exception as exc: + logger.error("Error while setting last_uid for %r" + % (mbox,)) + logger.exception(exc) def get_flags_doc(self, mbox, uid): """ -- cgit v1.2.3 From 95e00da239bd119ae161f249a74af229cb6c7759 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 00:00:24 -0400 Subject: fix attribute error on debug line --- src/leap/mail/imap/soledadstore.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/soledadstore.py') diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py index 919f834..ed5259a 100644 --- a/src/leap/mail/imap/soledadstore.py +++ b/src/leap/mail/imap/soledadstore.py @@ -308,8 +308,10 @@ class SoledadStore(ContentDedup): try: self._try_call(call, item) except Exception as exc: - logger.debug("ITEM WAS: %s" % str(item)) - logger.debug("ITEM CONTENT WAS: %s" % str(item.content)) + logger.debug("ITEM WAS: %s" % repr(item)) + if hasattr(item, 'content'): + logger.debug("ITEM CONTENT WAS: %s" % + repr(item.content)) logger.exception(exc) failed = True continue -- cgit v1.2.3 From 4bcb32639bff9a5aab076dba2bdc7667cea60c7f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 01:11:26 -0400 Subject: fix rdoc duplication --- src/leap/mail/imap/soledadstore.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/soledadstore.py') diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py index ed5259a..e047e2e 100644 --- a/src/leap/mail/imap/soledadstore.py +++ b/src/leap/mail/imap/soledadstore.py @@ -350,6 +350,9 @@ class SoledadStore(ContentDedup): if call == self._PUT_DOC_FUN: doc_id = item.doc_id + if doc_id is None: + logger.warning("BUG! Dirty doc but has no doc_id!") + return with put_locks[doc_id]: doc = self._GET_DOC_FUN(doc_id) @@ -438,12 +441,12 @@ class SoledadStore(ContentDedup): :return: a tuple with recent-flags doc payload and callable :rtype: tuple """ - call = self._CREATE_DOC_FUN + call = self._PUT_DOC_FUN payload = rflags_wrapper.content if payload: logger.debug("Saving RFLAGS to Soledad...") - yield payload, call + yield rflags_wrapper, call # Mbox documents and attributes -- cgit v1.2.3 From 52868a4c67170abbfa19deda9bd20931c21554b7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 01:21:46 -0400 Subject: catch stopiteration --- src/leap/mail/imap/soledadstore.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/soledadstore.py') diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py index e047e2e..25f00bb 100644 --- a/src/leap/mail/imap/soledadstore.py +++ b/src/leap/mail/imap/soledadstore.py @@ -281,9 +281,13 @@ class SoledadStore(ContentDedup): def doSoledadCalls(items): # we prime the generator, that should return the # message or flags wrapper item in the first place. - doc_wrapper = items.next() - failed = self._soledad_write_document_parts(items) - queueNotifyBack(failed, doc_wrapper) + try: + doc_wrapper = items.next() + except StopIteration: + pass + else: + failed = self._soledad_write_document_parts(items) + queueNotifyBack(failed, doc_wrapper) doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper)) -- cgit v1.2.3