From 68b1be8ef443b088cf5c1f7f964e1bd7ad42408e Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 21 Jul 2014 17:03:47 -0300 Subject: Add heartbeat to check if backend is alive. Send a 'ping' request every 2 secs to ensure that the backend is running. Use polling instead of recv on the backend_proxy. This was already implemented for the signaler. --- changes/add-backend-alive-check | 1 + src/leap/bitmask/backend/api.py | 2 + src/leap/bitmask/backend/backend.py | 6 ++- src/leap/bitmask/backend/backend_proxy.py | 64 ++++++++++++++++++++++++++----- src/leap/bitmask/gui/mainwindow.py | 22 +++++++++++ 5 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 changes/add-backend-alive-check diff --git a/changes/add-backend-alive-check b/changes/add-backend-alive-check new file mode 100644 index 00000000..40e12978 --- /dev/null +++ b/changes/add-backend-alive-check @@ -0,0 +1 @@ +- Add checks to ensure that the backend is alive or notify the user. Related to #5873. diff --git a/src/leap/bitmask/backend/api.py b/src/leap/bitmask/backend/api.py index b8533f36..4f52e470 100644 --- a/src/leap/bitmask/backend/api.py +++ b/src/leap/bitmask/backend/api.py @@ -18,10 +18,12 @@ Backend available API and SIGNALS definition. """ STOP_REQUEST = "stop" +PING_REQUEST = "PING" API = ( STOP_REQUEST, # this method needs to be defined in order to support the # backend stop action + PING_REQUEST, "eip_can_start", "eip_cancel_setup", diff --git a/src/leap/bitmask/backend/backend.py b/src/leap/bitmask/backend/backend.py index 833f4368..c895f8f5 100644 --- a/src/leap/bitmask/backend/backend.py +++ b/src/leap/bitmask/backend/backend.py @@ -23,7 +23,7 @@ from twisted.internet import defer, reactor, threads import zmq from zmq.auth.thread import ThreadAuthenticator -from leap.bitmask.backend.api import API +from leap.bitmask.backend.api import API, PING_REQUEST from leap.bitmask.backend.utils import get_backend_certificates from leap.bitmask.backend.signaler import Signaler @@ -146,6 +146,10 @@ class Backend(object): :param request_json: a json specification of a request. :type request_json: str """ + if request_json == PING_REQUEST: + # do not process request if it's just a ping + return + try: # request = zmq.utils.jsonapi.loads(request_json) # We use stdlib's json to ensure that we get unicode strings diff --git a/src/leap/bitmask/backend/backend_proxy.py b/src/leap/bitmask/backend/backend_proxy.py index f683e465..dc30d2cb 100644 --- a/src/leap/bitmask/backend/backend_proxy.py +++ b/src/leap/bitmask/backend/backend_proxy.py @@ -25,7 +25,7 @@ import time import zmq -from leap.bitmask.backend.api import API, STOP_REQUEST +from leap.bitmask.backend.api import API, STOP_REQUEST, PING_REQUEST from leap.bitmask.backend.utils import get_backend_certificates import logging @@ -40,6 +40,11 @@ class BackendProxy(object): PORT = '5556' SERVER = "tcp://localhost:%s" % PORT + POLL_TIMEOUT = 4000 # ms + POLL_TRIES = 3 + + PING_INTERVAL = 2 # secs + def __init__(self): self._socket = None @@ -62,6 +67,9 @@ class BackendProxy(object): socket.connect(self.SERVER) self._socket = socket + self._ping_at = 0 + self.online = False + self._call_queue = Queue.Queue() self._worker_caller = threading.Thread(target=self._worker) self._worker_caller.start() @@ -82,9 +90,26 @@ class BackendProxy(object): except Queue.Empty: pass time.sleep(0.01) + self._ping() logger.debug("BackendProxy worker stopped.") + def _reset_ping(self): + """ + Reset the ping timeout counter. + This is called for every ping and request. + """ + self._ping_at = time.time() + self.PING_INTERVAL + + def _ping(self): + """ + Heartbeat helper. + Sends a PING request just to know that the server is alive. + """ + if time.time() > self._ping_at: + self._send_request(PING_REQUEST) + self._reset_ping() + def _api_call(self, *args, **kwargs): """ Call the `api_method` method in backend (through zmq). @@ -134,16 +159,35 @@ class BackendProxy(object): # logger.debug("Sending request to backend: {0}".format(request)) self._socket.send(request) - try: - # Get the reply. - self._socket.recv() - # response = self._socket.recv() - # msg = "Received reply for '{0}' -> '{1}'" - # msg = msg.format(request, response) - # logger.debug(msg) - except zmq.error.Again as e: - msg = "Timeout error contacting backend. {0!r}".format(e) + poll = zmq.Poller() + poll.register(self._socket, zmq.POLLIN) + + reply = None + tries = 0 + + while True: + socks = dict(poll.poll(self.POLL_TIMEOUT)) + if socks.get(self._socket) == zmq.POLLIN: + reply = self._socket.recv() + break + + tries += 1 + if tries < self.POLL_TRIES: + logger.warning('Retrying receive... {0}/{1}'.format( + tries, self.POLL_TRIES)) + else: + break + + if reply is None: + msg = "Timeout error contacting backend." logger.critical(msg) + self.online = False + else: + # msg = "Received reply for '{0}' -> '{1}'".format(request, reply) + # logger.debug(msg) + self.online = True + # request received, no ping needed for other interval. + self._reset_ping() def __getattribute__(self, name): """ diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py index 6959650b..a15c4dd8 100644 --- a/src/leap/bitmask/gui/mainwindow.py +++ b/src/leap/bitmask/gui/mainwindow.py @@ -125,6 +125,11 @@ class MainWindow(QtGui.QMainWindow): self._backend = BackendProxy() + # periodically check if the backend is alive + self._backend_checker = QtCore.QTimer(self) + self._backend_checker.timeout.connect(self._check_backend_status) + self._backend_checker.start(2000) + self._leap_signaler = LeapSignaler() self._leap_signaler.start() @@ -334,6 +339,23 @@ class MainWindow(QtGui.QMainWindow): logger.error("Bad call to the backend:") logger.error(data) + @QtCore.Slot() + def _check_backend_status(self): + """ + TRIGGERS: + self._backend_checker.timeout + + Check that the backend is running. Otherwise show an error to the user. + """ + online = self._backend.online + if not online: + logger.critical("Backend is not online.") + QtGui.QMessageBox.critical( + self, self.tr("Application error"), + self.tr("There is a problem contacting the backend, please " + "restart Bitmask.")) + self._backend_checker.stop() + def _backend_connect(self, only_tracked=False): """ Connect to backend signals. -- cgit v1.2.3 From aeacb80a34dc7b8996278e35e9cc888d93e2f853 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Tue, 22 Jul 2014 16:10:10 -0300 Subject: Kill backend on quit if it does not respond. --- src/leap/bitmask/app.py | 2 +- src/leap/bitmask/frontend_app.py | 4 ++-- src/leap/bitmask/gui/mainwindow.py | 21 +++++++++++++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/leap/bitmask/app.py b/src/leap/bitmask/app.py index 88f6bc15..ab49ee37 100644 --- a/src/leap/bitmask/app.py +++ b/src/leap/bitmask/app.py @@ -185,7 +185,7 @@ def start_app(): backend_process.daemon = True backend_process.start() - run_frontend(options, flags_dict) + run_frontend(options, flags_dict, backend_pid=backend_process.pid) if __name__ == "__main__": diff --git a/src/leap/bitmask/frontend_app.py b/src/leap/bitmask/frontend_app.py index 51607d0b..5ea89fc9 100644 --- a/src/leap/bitmask/frontend_app.py +++ b/src/leap/bitmask/frontend_app.py @@ -54,7 +54,7 @@ def signal_handler(window, pid, signum, frame): window.quit() -def run_frontend(options, flags_dict): +def run_frontend(options, flags_dict, backend_pid): """ Run the GUI for the application. @@ -102,7 +102,7 @@ def run_frontend(options, flags_dict): timer.start(500) # You may change this if you wish. timer.timeout.connect(lambda: None) # Let the interpreter run each 500 ms. - window = MainWindow(start_hidden=start_hidden) + window = MainWindow(start_hidden=start_hidden, backend_pid=backend_pid) my_pid = os.getpid() sig_handler = partial(signal_handler, window, my_pid) diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py index a15c4dd8..59a65b1e 100644 --- a/src/leap/bitmask/gui/mainwindow.py +++ b/src/leap/bitmask/gui/mainwindow.py @@ -22,6 +22,8 @@ import time from datetime import datetime +import psutil + from PySide import QtCore, QtGui from leap.bitmask import __version__ as VERSION @@ -97,7 +99,7 @@ class MainWindow(QtGui.QMainWindow): # We give the services some time to a halt before forcing quit. SERVICES_STOP_TIMEOUT = 20000 # in milliseconds - def __init__(self, start_hidden=False): + def __init__(self, start_hidden=False, backend_pid=None): """ Constructor for the client main window @@ -272,6 +274,7 @@ class MainWindow(QtGui.QMainWindow): self._logger_window = None self._start_hidden = start_hidden + self._backend_pid = backend_pid self._mail_conductor = mail_conductor.MailConductor(self._backend) self._mail_conductor.connect_mail_signals(self._mail_status) @@ -1819,6 +1822,15 @@ class MainWindow(QtGui.QMainWindow): # or if we reach the timeout QtDelayedCall(self.SERVICES_STOP_TIMEOUT, self.final_quit) + def _backend_kill(self): + """ + Send a kill signal to the backend process. + This is called if the backend does not respond to requests. + """ + if self._backend_pid is not None: + logger.debug("Killing backend") + psutil.Process(self._backend_pid).kill() + @QtCore.Slot() def _remove_service(self, service): """ @@ -1854,7 +1866,12 @@ class MainWindow(QtGui.QMainWindow): logger.debug('Final quit...') self._leap_signaler.stop() - self._backend.stop() + + if self._backend.online: + self._backend.stop() + else: + self._backend_kill() + time.sleep(0.05) # give the thread a little time to finish. # Remove lockfiles on a clean shutdown. -- cgit v1.2.3 From d97a9804c1a6fb06d7ce39066f92a0259a6ab8c3 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Thu, 24 Jul 2014 12:54:06 -0300 Subject: Don't call the backend if it's not online. --- src/leap/bitmask/gui/mainwindow.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py index 59a65b1e..ff19e17f 100644 --- a/src/leap/bitmask/gui/mainwindow.py +++ b/src/leap/bitmask/gui/mainwindow.py @@ -1813,10 +1813,14 @@ class MainWindow(QtGui.QMainWindow): # Set this in case that the app is hidden QtGui.QApplication.setQuitOnLastWindowClosed(True) - self._stop_services() - self._really_quit = True + if not self._backend.online: + self.final_quit() + return + + self._stop_services() + # call final quit when all the services are stopped self.all_services_stopped.connect(self.final_quit) # or if we reach the timeout @@ -1859,11 +1863,12 @@ class MainWindow(QtGui.QMainWindow): if self._finally_quitting: return + logger.debug('Final quit...') self._finally_quitting = True - logger.debug('Closing soledad...') - self._backend.soledad_close() - logger.debug('Final quit...') + if self._backend.online: + logger.debug('Closing soledad...') + self._backend.soledad_close() self._leap_signaler.stop() -- cgit v1.2.3