retry soledad connections

4 files changed, 84 insertions, 12 deletions

diff --git a/changes/bug_3413-soledad-init-retries b/changes/bug_3413-soledad-init-retries
new file mode 100644
index 00000000..160121dd
--- /dev/null
+++ b/changes/bug_3413-soledad-init-retries
@@ -0,0 +1 @@
+  o Allow soledad initialization to retry if it times out. Closes: #3413
diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py
index b624988f..0197f2f6 100644
--- a/src/leap/bitmask/gui/mainwindow.py
+++ b/src/leap/bitmask/gui/mainwindow.py
@@ -206,6 +206,8 @@ class MainWindow(QtGui.QMainWindow):
         # This thread is similar to the provider bootstrapper
         self._eip_bootstrapper = EIPBootstrapper()
 
+        # TODO change the name of "download_config" signal to
+        # something less confusing (config_ready maybe)
         self._eip_bootstrapper.download_config.connect(
             self._eip_intermediate_stage)
         self._eip_bootstrapper.download_client_certificate.connect(
@@ -216,6 +218,8 @@ class MainWindow(QtGui.QMainWindow):
             self._soledad_intermediate_stage)
         self._soledad_bootstrapper.gen_key.connect(
             self._soledad_bootstrapped_stage)
+        self._soledad_bootstrapper.soledad_timeout.connect(
+            self._retry_soledad_connection)
 
         self._smtp_bootstrapper = SMTPBootstrapper()
         self._smtp_bootstrapper.download_config.connect(
@@ -953,8 +957,22 @@ class MainWindow(QtGui.QMainWindow):
             # TODO: display in the GUI:
             # should pass signal to a slot in status_panel
             # that sets the global status
-            logger.warning("Soledad failed to start: %s" %
-                           (data[self._soledad_bootstrapper.ERROR_KEY],))
+            logger.error("Soledad failed to start: %s" %
+                         (data[self._soledad_bootstrapper.ERROR_KEY],))
+            self._retry_soledad_connection()
+
+    def _retry_soledad_connection(self):
+        """
+        Retries soledad connection.
+        """
+        logger.debug("Retrying soledad connection.")
+        if self._soledad_bootstrapper.should_retry_initialization():
+            self._soledad_bootstrapper.increment_retries_count()
+            threads.deferToThread(
+                self._soledad_bootstrapper.load_and_sync_soledad)
+        else:
+            logger.warning("Max number of soledad initialization "
+                           "retries reached.")
 
     def _soledad_bootstrapped_stage(self, data):
         """
@@ -971,13 +989,14 @@ class MainWindow(QtGui.QMainWindow):
         """
         passed = data[self._soledad_bootstrapper.PASSED_KEY]
         if not passed:
+            logger.debug("ERROR on soledad bootstrapping:")
             logger.error(data[self._soledad_bootstrapper.ERROR_KEY])
             return
+        else:
+            logger.debug("Done bootstrapping Soledad")
 
-        logger.debug("Done bootstrapping Soledad")
-
-        self._soledad = self._soledad_bootstrapper.soledad
-        self._keymanager = self._soledad_bootstrapper.keymanager
+            self._soledad = self._soledad_bootstrapper.soledad
+            self._keymanager = self._soledad_bootstrapper.keymanager
 
         # Ok, now soledad is ready, so we can allow other things that
         # depend on soledad to start.
diff --git a/src/leap/bitmask/services/abstractbootstrapper.py b/src/leap/bitmask/services/abstractbootstrapper.py
index 6f246f47..6d4d319b 100644
--- a/src/leap/bitmask/services/abstractbootstrapper.py
+++ b/src/leap/bitmask/services/abstractbootstrapper.py
@@ -128,7 +128,6 @@ class AbstractBootstrapper(QtCore.QObject):
         :type signal: QtCore.SignalInstance
         """
         if signal:
-            logger.debug("Emitting %s" % (signal,))
             signal.emit({self.PASSED_KEY: True, self.ERROR_KEY: ""})
 
     def _callback_threader(self, cb, res, *args, **kwargs):
diff --git a/src/leap/bitmask/services/soledad/soledadbootstrapper.py b/src/leap/bitmask/services/soledad/soledadbootstrapper.py
index fba74d60..9e150226 100644
--- a/src/leap/bitmask/services/soledad/soledadbootstrapper.py
+++ b/src/leap/bitmask/services/soledad/soledadbootstrapper.py
@@ -21,6 +21,7 @@ Soledad bootstrapping
 
 import logging
 import os
+import socket
 
 from PySide import QtCore
 from u1db import errors as u1db_errors
@@ -49,10 +50,14 @@ class SoledadBootstrapper(AbstractBootstrapper):
 
     PUBKEY_KEY = "user[public_key]"
 
+    MAX_INIT_RETRIES = 10
+
     # All dicts returned are of the form
     # {"passed": bool, "error": str}
     download_config = QtCore.Signal(dict)
     gen_key = QtCore.Signal(dict)
+    soledad_timeout = QtCore.Signal()
+    soledad_failed = QtCore.Signal()
 
     def __init__(self):
         AbstractBootstrapper.__init__(self)
@@ -63,8 +68,11 @@ class SoledadBootstrapper(AbstractBootstrapper):
         self._download_if_needed = False
         self._user = ""
         self._password = ""
+        self._srpauth = None
         self._soledad = None
 
+        self._soledad_retries = 0
+
     @property
     def keymanager(self):
         return self._keymanager
@@ -73,7 +81,32 @@ class SoledadBootstrapper(AbstractBootstrapper):
     def soledad(self):
         return self._soledad
 
-    def _load_and_sync_soledad(self, srp_auth):
+    @property
+    def srpauth(self):
+        leap_assert(self._provider_config is not None,
+                    "We need a provider config")
+        return SRPAuth(self._provider_config)
+
+    # retries
+
+    def should_retry_initialization(self):
+        """
+        Returns True if we should retry the initialization.
+        """
+        logger.debug("current retries: %s, max retries: %s" % (
+            self._soledad_retries,
+            self.MAX_INIT_RETRIES))
+        return self._soledad_retries < self.MAX_INIT_RETRIES
+
+    def increment_retries_count(self):
+        """
+        Increments the count of initialization retries.
+        """
+        self._soledad_retries += 1
+
+    # initialization
+
+    def load_and_sync_soledad(self):
         """
         Once everthing is in the right place, we instantiate and sync
         Soledad
@@ -81,6 +114,7 @@ class SoledadBootstrapper(AbstractBootstrapper):
         :param srp_auth: SRPAuth object used
         :type srp_auth: SRPAuth
         """
+        srp_auth = self.srpauth
         uuid = srp_auth.get_uid()
 
         prefix = os.path.join(self._soledad_config.get_path_prefix(),
@@ -114,8 +148,24 @@ class SoledadBootstrapper(AbstractBootstrapper):
                     cert_file=cert_file,
                     auth_token=srp_auth.get_token())
                 self._soledad.sync()
+
+            # XXX All these errors should be handled by soledad itself,
+            # and return a subclass of SoledadInitializationFailed
+            except socket.timeout:
+                logger.debug("SOLEDAD TIMED OUT...")
+                self.soledad_timeout.emit()
+            except socket.error as exc:
+                logger.error("Socket error while initializing soledad")
+                if exc.errno in (111, ):
+                    self.soledad_failed.emit()
             except u1db_errors.Unauthorized:
-                logger.error("Error while initializing soledad.")
+                logger.error("Error while initializing soledad "
+                             "(unauthorized).")
+                self.soledad_failed.emit()
+            except Exception as exc:
+                logger.error("Unhandled error while initializating "
+                             "soledad: %r" % (exc,))
+                raise
         else:
             raise Exception("No soledad server found")
 
@@ -151,7 +201,8 @@ class SoledadBootstrapper(AbstractBootstrapper):
             api_version)
         logger.debug('Downloading soledad config from: %s' % config_uri)
 
-        srp_auth = SRPAuth(self._provider_config)
+        # TODO factor out this srpauth protected get (make decorator)
+        srp_auth = self.srpauth
         session_id = srp_auth.get_session_id()
         cookies = None
         if session_id:
@@ -183,7 +234,7 @@ class SoledadBootstrapper(AbstractBootstrapper):
                                        self._provider_config.get_domain(),
                                        "soledad-service.json"])
 
-        self._load_and_sync_soledad(srp_auth)
+        self.load_and_sync_soledad()
 
     def _gen_key(self, _):
         """
@@ -197,8 +248,9 @@ class SoledadBootstrapper(AbstractBootstrapper):
 
         logger.debug("Retrieving key for %s" % (address,))
 
-        srp_auth = SRPAuth(self._provider_config)
+        srp_auth = self.srpauth
 
+        # TODO: use which implementation with known paths
         # TODO: Fix for Windows
         gpgbin = "/usr/bin/gpg"
 
@@ -251,6 +303,7 @@ class SoledadBootstrapper(AbstractBootstrapper):
         """
         leap_assert_type(provider_config, ProviderConfig)
 
+        # XXX we should provider a method for setting provider_config
         self._provider_config = provider_config
         self._download_if_needed = download_if_needed
         self._user = user