#117 - waiting for soledad sync to be done before indexing mails. Also, removing inbox duplicates

4 files changed, 75 insertions, 7 deletions

diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py
index 42af6d2b..3aab46df 100644
--- a/service/pixelated/adapter/soledad_querier.py
+++ b/service/pixelated/adapter/soledad_querier.py
@@ -21,6 +21,12 @@ class SoledadQuerier:
     def __init__(self, soledad):
         self.soledad = soledad
 
+    def remove_inbox_duplicates(self):
+        inboxes = [d for d in self.soledad.get_from_index('by-type', 'mbox') if d.content['mbox'] == 'INBOX']
+        sorted(inboxes, key=lambda x: x.content['lastuid'], reverse=True)
+        inboxes_to_remove = inboxes[1:len(inboxes)]
+        [self.soledad.delete_doc(inbox) for inbox in inboxes_to_remove]
+
     def all_mails(self):
         fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.soledad.get_from_index('by-type', 'flags')]
         if len(fdocs_chash) == 0:
@@ -60,6 +66,7 @@ class SoledadQuerier:
         return PixelatedMail.from_soledad(fdoc, hdoc, bdoc, soledad_querier=self)
 
     def mails(self, idents):
+        self.remove_inbox_duplicates()
         fdocs_chash = [(self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident), ident) for ident in idents]
         fdocs_chash = [(result[0], ident) for result, ident in fdocs_chash if result]
         return self._build_mails_from_fdocs(fdocs_chash)
diff --git a/service/pixelated/config/app_factory.py b/service/pixelated/config/app_factory.py
index ed752118..1d006506 100644
--- a/service/pixelated/config/app_factory.py
+++ b/service/pixelated/config/app_factory.py
@@ -26,6 +26,18 @@ import pixelated.bitmask_libraries.session as LeapSession
 from pixelated.controllers import *
 from pixelated.adapter.tag_service import TagService
 import os
+from leap.common.events import (
+    register,
+    events_pb2 as proto
+)
+
+
+def init_index_and_delete_duplicate_inboxes(querier, search_engine, mail_service):
+    def wrapper(*args, **kwargs):
+        querier.remove_inbox_duplicates()
+        search_engine.index_mails(mail_service.all_mails())
+
+    return wrapper
 
 
 def _setup_routes(app, home_controller, mails_controller, tags_controller, features_controller):
@@ -50,18 +62,19 @@ def _setup_routes(app, home_controller, mails_controller, tags_controller, featu
 
 
 def create_app(debug_enabled, app):
-
     with app.app_context():
-        leap_session = LeapSession.open(app.config['LEAP_USERNAME'], app.config['LEAP_PASSWORD'],
+        leap_session = LeapSession.open(app.config['LEAP_USERNAME'],
+                                        app.config['LEAP_PASSWORD'],
                                         app.config['LEAP_SERVER_NAME'])
         tag_service = TagService()
+        search_engine = SearchEngine()
+        pixelated_mail_sender = MailSender(leap_session.account_email())
+
         soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad)
         pixelated_mailboxes = Mailboxes(leap_session.account, soledad_querier)
-        pixelated_mail_sender = MailSender(leap_session.account_email())
-        mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, tag_service, soledad_querier)
-        search_engine = SearchEngine()
-        search_engine.index_mails(mail_service.all_mails())
+
         draft_service = DraftService(pixelated_mailboxes)
+        mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, tag_service, soledad_querier)
 
         MailboxIndexerListener.SEARCH_ENGINE = search_engine
         InputMail.FROM_EMAIL_ADDRESS = leap_session.account_email()
@@ -73,6 +86,11 @@ def create_app(debug_enabled, app):
                                            search_engine=search_engine)
         tags_controller = TagsController(search_engine=search_engine)
 
+        register(signal=proto.SOLEDAD_DONE_DATA_SYNC,
+                 callback=init_index_and_delete_duplicate_inboxes(querier=soledad_querier,
+                                                                  search_engine=search_engine,
+                                                                  mail_service=mail_service))
+
         _setup_routes(app, home_controller, mails_controller, tags_controller, features_controller)
 
         app.run(host=app.config['HOST'], debug=debug_enabled,
@@ -83,7 +101,8 @@ def get_static_folder():
     static_folder = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "web-ui", "app"))
     # this is a workaround for packaging
     if not os.path.exists(static_folder):
-        static_folder = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "web-ui", "app"))
+        static_folder = os.path.abspath(
+            os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "web-ui", "app"))
     if not os.path.exists(static_folder):
         static_folder = os.path.join('/', 'usr', 'share', 'pixelated-user-agent')
     return static_folder
diff --git a/service/pixelated/runserver.py b/service/pixelated/runserver.py
index 840d6621..87a22fe0 100644
--- a/service/pixelated/runserver.py
+++ b/service/pixelated/runserver.py
@@ -18,10 +18,12 @@ import os
 import os.path
 import crochet
 from flask import Flask
+from leap.common.events import server as events_server
 from pixelated.config import app_factory
 import pixelated.config.args as input_args
 import pixelated.bitmask_libraries.register as leap_register
 import pixelated.config.reactor_manager as reactor_manager
+import pixelated.support.ext_protobuf  # monkey patch for protobuf in OSX
 
 
 app = Flask(__name__, static_url_path='', static_folder=app_factory.get_static_folder())
@@ -34,6 +36,8 @@ def setup():
         reactor_manager.start_reactor(logging=debug_enabled)
         crochet.setup()
 
+        events_server.ensure_server(port=8090)
+
         app.config.from_pyfile(args.config)
 
         if args.register:
diff --git a/service/pixelated/support/ext_protobuf.py b/service/pixelated/support/ext_protobuf.py
new file mode 100644
index 00000000..8a8e05e6
--- /dev/null
+++ b/service/pixelated/support/ext_protobuf.py
@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+import protobuf.socketrpc.server
+from sys import platform as _platform
+
+
+# protobuf throws a lot of 'Socket is not connected' exceptions on OSX but they are not an issue.
+# refer too https://code.google.com/p/protobuf-socket-rpc/issues/detail?id=10 and
+# or https://leap.se/code/issues/2187
+if _platform == 'darwin':
+    def try_except_decorator(func):
+        def wrapper(*args, **kwargs):
+            try:
+                func(*args, **kwargs)
+                pass
+            except Exception as e:
+                if e.strerror == 'Socket is not connected':
+                    pass
+
+        return wrapper
+
+    protobuf.socketrpc.server.SocketHandler.handle = try_except_decorator(
+        protobuf.socketrpc.server.SocketHandler.handle)