Merge remote-tracking branch 'refs/remotes/ivan/feature/add-backend-alive-check' into develop

7 files changed, 116 insertions, 21 deletions

diff --git a/changes/add-backend-alive-check b/changes/add-backend-alive-check
new file mode 100644
index 00000000..40e12978
--- /dev/null
+++ b/changes/add-backend-alive-check
@@ -0,0 +1 @@
+- Add checks to ensure that the backend is alive or notify the user. Related to #5873.
diff --git a/src/leap/bitmask/app.py b/src/leap/bitmask/app.py
index 88f6bc15..ab49ee37 100644
--- a/src/leap/bitmask/app.py
+++ b/src/leap/bitmask/app.py
@@ -185,7 +185,7 @@ def start_app():
     backend_process.daemon = True
     backend_process.start()
 
-    run_frontend(options, flags_dict)
+    run_frontend(options, flags_dict, backend_pid=backend_process.pid)
 
 
 if __name__ == "__main__":
diff --git a/src/leap/bitmask/backend/api.py b/src/leap/bitmask/backend/api.py
index b8533f36..4f52e470 100644
--- a/src/leap/bitmask/backend/api.py
+++ b/src/leap/bitmask/backend/api.py
@@ -18,10 +18,12 @@
 Backend available API and SIGNALS definition.
 """
 STOP_REQUEST = "stop"
+PING_REQUEST = "PING"
 
 API = (
     STOP_REQUEST,  # this method needs to be defined in order to support the
                    # backend stop action
+    PING_REQUEST,
 
     "eip_can_start",
     "eip_cancel_setup",
diff --git a/src/leap/bitmask/backend/backend.py b/src/leap/bitmask/backend/backend.py
index 833f4368..c895f8f5 100644
--- a/src/leap/bitmask/backend/backend.py
+++ b/src/leap/bitmask/backend/backend.py
@@ -23,7 +23,7 @@ from twisted.internet import defer, reactor, threads
 import zmq
 from zmq.auth.thread import ThreadAuthenticator
 
-from leap.bitmask.backend.api import API
+from leap.bitmask.backend.api import API, PING_REQUEST
 from leap.bitmask.backend.utils import get_backend_certificates
 from leap.bitmask.backend.signaler import Signaler
 
@@ -146,6 +146,10 @@ class Backend(object):
         :param request_json: a json specification of a request.
         :type request_json: str
         """
+        if request_json == PING_REQUEST:
+            # do not process request if it's just a ping
+            return
+
         try:
             # request = zmq.utils.jsonapi.loads(request_json)
             # We use stdlib's json to ensure that we get unicode strings
diff --git a/src/leap/bitmask/backend/backend_proxy.py b/src/leap/bitmask/backend/backend_proxy.py
index f683e465..dc30d2cb 100644
--- a/src/leap/bitmask/backend/backend_proxy.py
+++ b/src/leap/bitmask/backend/backend_proxy.py
@@ -25,7 +25,7 @@ import time
 
 import zmq
 
-from leap.bitmask.backend.api import API, STOP_REQUEST
+from leap.bitmask.backend.api import API, STOP_REQUEST, PING_REQUEST
 from leap.bitmask.backend.utils import get_backend_certificates
 
 import logging
@@ -40,6 +40,11 @@ class BackendProxy(object):
     PORT = '5556'
     SERVER = "tcp://localhost:%s" % PORT
 
+    POLL_TIMEOUT = 4000  # ms
+    POLL_TRIES = 3
+
+    PING_INTERVAL = 2  # secs
+
     def __init__(self):
         self._socket = None
 
@@ -62,6 +67,9 @@ class BackendProxy(object):
         socket.connect(self.SERVER)
         self._socket = socket
 
+        self._ping_at = 0
+        self.online = False
+
         self._call_queue = Queue.Queue()
         self._worker_caller = threading.Thread(target=self._worker)
         self._worker_caller.start()
@@ -82,9 +90,26 @@ class BackendProxy(object):
             except Queue.Empty:
                 pass
             time.sleep(0.01)
+            self._ping()
 
         logger.debug("BackendProxy worker stopped.")
 
+    def _reset_ping(self):
+        """
+        Reset the ping timeout counter.
+        This is called for every ping and request.
+        """
+        self._ping_at = time.time() + self.PING_INTERVAL
+
+    def _ping(self):
+        """
+        Heartbeat helper.
+        Sends a PING request just to know that the server is alive.
+        """
+        if time.time() > self._ping_at:
+            self._send_request(PING_REQUEST)
+            self._reset_ping()
+
     def _api_call(self, *args, **kwargs):
         """
         Call the `api_method` method in backend (through zmq).
@@ -134,16 +159,35 @@ class BackendProxy(object):
         # logger.debug("Sending request to backend: {0}".format(request))
         self._socket.send(request)
 
-        try:
-            # Get the reply.
-            self._socket.recv()
-            # response = self._socket.recv()
-            # msg = "Received reply for '{0}' -> '{1}'"
-            # msg = msg.format(request, response)
-            # logger.debug(msg)
-        except zmq.error.Again as e:
-            msg = "Timeout error contacting backend. {0!r}".format(e)
+        poll = zmq.Poller()
+        poll.register(self._socket, zmq.POLLIN)
+
+        reply = None
+        tries = 0
+
+        while True:
+            socks = dict(poll.poll(self.POLL_TIMEOUT))
+            if socks.get(self._socket) == zmq.POLLIN:
+                reply = self._socket.recv()
+                break
+
+            tries += 1
+            if tries < self.POLL_TRIES:
+                logger.warning('Retrying receive... {0}/{1}'.format(
+                    tries, self.POLL_TRIES))
+            else:
+                break
+
+        if reply is None:
+            msg = "Timeout error contacting backend."
             logger.critical(msg)
+            self.online = False
+        else:
+            # msg = "Received reply for '{0}' -> '{1}'".format(request, reply)
+            # logger.debug(msg)
+            self.online = True
+            # request received, no ping needed for other interval.
+            self._reset_ping()
 
     def __getattribute__(self, name):
         """
diff --git a/src/leap/bitmask/frontend_app.py b/src/leap/bitmask/frontend_app.py
index 51607d0b..5ea89fc9 100644
--- a/src/leap/bitmask/frontend_app.py
+++ b/src/leap/bitmask/frontend_app.py
@@ -54,7 +54,7 @@ def signal_handler(window, pid, signum, frame):
         window.quit()
 
 
-def run_frontend(options, flags_dict):
+def run_frontend(options, flags_dict, backend_pid):
     """
     Run the GUI for the application.
 
@@ -102,7 +102,7 @@ def run_frontend(options, flags_dict):
     timer.start(500)  # You may change this if you wish.
     timer.timeout.connect(lambda: None)  # Let the interpreter run each 500 ms.
 
-    window = MainWindow(start_hidden=start_hidden)
+    window = MainWindow(start_hidden=start_hidden, backend_pid=backend_pid)
 
     my_pid = os.getpid()
     sig_handler = partial(signal_handler, window, my_pid)
diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py
index 6959650b..ff19e17f 100644
--- a/src/leap/bitmask/gui/mainwindow.py
+++ b/src/leap/bitmask/gui/mainwindow.py
@@ -22,6 +22,8 @@ import time
 
 from datetime import datetime
 
+import psutil
+
 from PySide import QtCore, QtGui
 
 from leap.bitmask import __version__ as VERSION
@@ -97,7 +99,7 @@ class MainWindow(QtGui.QMainWindow):
     # We give the services some time to a halt before forcing quit.
     SERVICES_STOP_TIMEOUT = 20000  # in milliseconds
 
-    def __init__(self, start_hidden=False):
+    def __init__(self, start_hidden=False, backend_pid=None):
         """
         Constructor for the client main window
 
@@ -125,6 +127,11 @@ class MainWindow(QtGui.QMainWindow):
 
         self._backend = BackendProxy()
 
+        # periodically check if the backend is alive
+        self._backend_checker = QtCore.QTimer(self)
+        self._backend_checker.timeout.connect(self._check_backend_status)
+        self._backend_checker.start(2000)
+
         self._leap_signaler = LeapSignaler()
         self._leap_signaler.start()
 
@@ -267,6 +274,7 @@ class MainWindow(QtGui.QMainWindow):
         self._logger_window = None
 
         self._start_hidden = start_hidden
+        self._backend_pid = backend_pid
 
         self._mail_conductor = mail_conductor.MailConductor(self._backend)
         self._mail_conductor.connect_mail_signals(self._mail_status)
@@ -334,6 +342,23 @@ class MainWindow(QtGui.QMainWindow):
         logger.error("Bad call to the backend:")
         logger.error(data)
 
+    @QtCore.Slot()
+    def _check_backend_status(self):
+        """
+        TRIGGERS:
+            self._backend_checker.timeout
+
+        Check that the backend is running. Otherwise show an error to the user.
+        """
+        online = self._backend.online
+        if not online:
+            logger.critical("Backend is not online.")
+            QtGui.QMessageBox.critical(
+                self, self.tr("Application error"),
+                self.tr("There is a problem contacting the backend, please "
+                        "restart Bitmask."))
+            self._backend_checker.stop()
+
     def _backend_connect(self, only_tracked=False):
         """
         Connect to backend signals.
@@ -1788,15 +1813,28 @@ class MainWindow(QtGui.QMainWindow):
         # Set this in case that the app is hidden
         QtGui.QApplication.setQuitOnLastWindowClosed(True)
 
-        self._stop_services()
-
         self._really_quit = True
 
+        if not self._backend.online:
+            self.final_quit()
+            return
+
+        self._stop_services()
+
         # call final quit when all the services are stopped
         self.all_services_stopped.connect(self.final_quit)
         # or if we reach the timeout
         QtDelayedCall(self.SERVICES_STOP_TIMEOUT, self.final_quit)
 
+    def _backend_kill(self):
+        """
+        Send a kill signal to the backend process.
+        This is called if the backend does not respond to requests.
+        """
+        if self._backend_pid is not None:
+            logger.debug("Killing backend")
+            psutil.Process(self._backend_pid).kill()
+
     @QtCore.Slot()
     def _remove_service(self, service):
         """
@@ -1825,14 +1863,20 @@ class MainWindow(QtGui.QMainWindow):
         if self._finally_quitting:
             return
 
+        logger.debug('Final quit...')
         self._finally_quitting = True
 
-        logger.debug('Closing soledad...')
-        self._backend.soledad_close()
-        logger.debug('Final quit...')
+        if self._backend.online:
+            logger.debug('Closing soledad...')
+            self._backend.soledad_close()
 
         self._leap_signaler.stop()
-        self._backend.stop()
+
+        if self._backend.online:
+            self._backend.stop()
+        else:
+            self._backend_kill()
+
         time.sleep(0.05)  # give the thread a little time to finish.
 
         # Remove lockfiles on a clean shutdown.