Fix: Ignore missing mail data error when creating search index

- Should fix accounts that can currently not log in because of mails
  where part of the docs are missing
- Does only fix the symptoms

3 files changed, 24 insertions, 6 deletions

diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py
index 4a0de6a1..9b62b3ba 100644
--- a/service/pixelated/adapter/mailstore/leap_mailstore.py
+++ b/service/pixelated/adapter/mailstore/leap_mailstore.py
@@ -21,6 +21,7 @@ from uuid import uuid4
 from leap.mail.adaptors.soledad import SoledadMailAdaptor
 from leap.mail.mail import Message
 from twisted.internet import defer
+from twisted.internet.defer import FirstError, DeferredList
 
 from pixelated.adapter.mailstore.body_parser import BodyParser
 from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid
@@ -203,12 +204,18 @@ class LeapMailStore(MailStore):
 
         defer.returnValue(leap_mail)
 
-    def get_mails(self, mail_ids):
+    @defer.inlineCallbacks
+    def get_mails(self, mail_ids, gracefully_ignore_errors=False):
         deferreds = []
         for mail_id in mail_ids:
             deferreds.append(self.get_mail(mail_id, include_body=True))
 
-        return defer.gatherResults(deferreds, consumeErrors=True)
+        if gracefully_ignore_errors:
+            results = yield DeferredList(deferreds, consumeErrors=True)
+            defer.returnValue([mail for ok, mail in results if ok and mail is not None])
+        else:
+            result = yield defer.gatherResults(deferreds, consumeErrors=True)
+            defer.returnValue(result)
 
     @defer.inlineCallbacks
     def update_mail(self, mail):
@@ -218,12 +225,12 @@ class LeapMailStore(MailStore):
         yield self._update_mail(message)  # TODO assert this is yielded (otherwise asynchronous)
 
     @defer.inlineCallbacks
-    def all_mails(self):
+    def all_mails(self, gracefully_ignore_errors=False):
         mdocs = yield self.soledad.get_from_index('by-type', 'meta')
 
         mail_ids = map(lambda doc: doc.doc_id, mdocs)
 
-        mails = yield self.get_mails(mail_ids)
+        mails = yield self.get_mails(mail_ids, gracefully_ignore_errors=gracefully_ignore_errors)
         defer.returnValue(mails)
 
     @defer.inlineCallbacks
diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py
index cf47bfa6..bfe45cad 100644
--- a/service/pixelated/adapter/services/mail_service.py
+++ b/service/pixelated/adapter/services/mail_service.py
@@ -37,7 +37,7 @@ class MailService(object):
 
     @defer.inlineCallbacks
     def all_mails(self):
-        mails = yield self.mail_store.all_mails()
+        mails = yield self.mail_store.all_mails(gracefully_ignore_errors=True)
         defer.returnValue(mails)
 
     def save_attachment(self, content, content_type):
diff --git a/service/test/unit/adapter/mailstore/test_leap_mailstore.py b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
index 0945e0c4..bde4f59b 100644
--- a/service/test/unit/adapter/mailstore/test_leap_mailstore.py
+++ b/service/test/unit/adapter/mailstore/test_leap_mailstore.py
@@ -120,7 +120,7 @@ class TestLeapMailStore(TestCase):
         try:
             yield store.get_mails(['invalid'])
             self.fail('Exception expected')
-        except FirstError:
+        except Exception, e:
             pass
 
     @defer.inlineCallbacks
@@ -384,6 +384,17 @@ class TestLeapMailStore(TestCase):
         self._assert_message_docs_created(expected_message, mail, only_mdoc_and_fdoc=True)
         self._assert_mail_got_deleted(fdoc_id, mail_id)
 
+    @defer.inlineCallbacks
+    def test_all_mail_graceful_error_handling(self):
+        mail_id, fdoc_id = self._add_mail_fixture_to_soledad_from_file('mbox00000000')
+        when(self.soledad).get_from_index('by-type', 'meta').thenReturn(defer.succeed([self.doc_by_id[mail_id]]))
+        when(self.soledad).get_doc(self.doc_by_id[mail_id].content['cdocs'][0]).thenAnswer(lambda: defer.fail(Exception('fail loading attachment')))
+        store = LeapMailStore(self.soledad)
+
+        mails = yield store.all_mails(gracefully_ignore_errors=True)
+
+        self.assertEqual(0, len(mails))
+
     def _assert_mail_got_deleted(self, fdoc_id, mail_id):
         verify(self.soledad).delete_doc(self.doc_by_id[mail_id])
         verify(self.soledad).delete_doc(self.doc_by_id[fdoc_id])