Merge remote-tracking branch 'kali/bug/start_smtp_without_eip' into develop

9 files changed, 319 insertions, 171 deletions

diff --git a/changes/bug-3847-start-smtp-without-eip b/changes/bug-3847-start-smtp-without-eip
new file mode 100644
index 00000000..5ed959a4
--- /dev/null
+++ b/changes/bug-3847-start-smtp-without-eip
@@ -0,0 +1 @@
+  o Allow SMTP to start even when provider does not offer EIP. Closes: #3847
diff --git a/src/leap/bitmask/crypto/certs.py b/src/leap/bitmask/crypto/certs.py
new file mode 100644
index 00000000..244decfd
--- /dev/null
+++ b/src/leap/bitmask/crypto/certs.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# certs.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for dealing with client certs
+"""
+import logging
+import os
+
+from leap.bitmask.crypto.srpauth import SRPAuth
+from leap.bitmask.util.constants import REQUEST_TIMEOUT
+from leap.common.files import check_and_fix_urw_only
+from leap.common.files import mkdir_p
+
+from leap.common import certs as leap_certs
+
+logger = logging.getLogger(__name__)
+
+
+def download_client_cert(provider_config, path, session):
+    """
+    Downloads the client certificate for each service.
+
+    :param provider_config: instance of a ProviderConfig
+    :type provider_config: ProviderConfig
+    :param path: the path to download the cert to.
+    :type path: str
+    :param session: a fetcher.session instance. For the moment we only
+                   support requests.sessions
+    :type session: requests.sessions.Session
+    """
+    # TODO we should implement the @with_srp_auth decorator
+    # again.
+    srp_auth = SRPAuth(provider_config)
+    session_id = srp_auth.get_session_id()
+    cookies = None
+    if session_id:
+        cookies = {"_session_id": session_id}
+    cert_uri = "%s/%s/cert" % (
+        provider_config.get_api_uri(),
+        provider_config.get_api_version())
+    logger.debug('getting cert from uri: %s' % cert_uri)
+
+    res = session.get(cert_uri,
+                      verify=provider_config
+                      .get_ca_cert_path(),
+                      cookies=cookies,
+                      timeout=REQUEST_TIMEOUT)
+    res.raise_for_status()
+    client_cert = res.content
+
+    if not leap_certs.is_valid_pemfile(client_cert):
+        # XXX raise more specific exception.
+        raise Exception("The downloaded certificate is not a "
+                        "valid PEM file")
+
+    mkdir_p(os.path.dirname(path))
+
+    try:
+        with open(path, "w") as f:
+            f.write(client_cert)
+    except IOError as exc:
+        logger.error(
+            "Error saving client cert: %r" % (exc,))
+        raise
+
+    check_and_fix_urw_only(path)
diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py
index 89069151..1dbf39ef 100644
--- a/src/leap/bitmask/gui/mainwindow.py
+++ b/src/leap/bitmask/gui/mainwindow.py
@@ -1057,17 +1057,6 @@ class MainWindow(QtGui.QMainWindow):
                 self._provider_config,
                 self._smtp_config,
                 True)
-        else:
-            if self._enabled_services.count(self.MX_SERVICE) > 0:
-                pass  # TODO show MX status
-                #self._status_panel.set_eip_status(
-                #    self.tr("%s does not support MX") %
-                #    (self._provider_config.get_domain(),),
-                #                     error=True)
-            else:
-                pass  # TODO show MX status
-                #self._status_panel.set_eip_status(
-                #    self.tr("MX is disabled"))
 
     ###################################################################
     # Service control methods: smtp
@@ -1090,7 +1079,12 @@ class MainWindow(QtGui.QMainWindow):
             logger.error(data[self._smtp_bootstrapper.ERROR_KEY])
             return
         logger.debug("Done bootstrapping SMTP")
+        self._check_smtp_config()
 
+    def _check_smtp_config(self):
+        """
+        Checks smtp config and tries to download smtp client cert if needed.
+        """
         hosts = self._smtp_config.get_hosts()
         # TODO handle more than one host and define how to choose
         if len(hosts) > 0:
@@ -1098,24 +1092,40 @@ class MainWindow(QtGui.QMainWindow):
             logger.debug("Using hostname %s for SMTP" % (hostname,))
             host = hosts[hostname][self.IP_KEY].encode("utf-8")
             port = hosts[hostname][self.PORT_KEY]
-            # TODO move the start to _start_smtp_service
-
-            # TODO Make the encrypted_only configurable
-            # TODO pick local smtp port in a better way
-            # TODO remove hard-coded port and let leap.mail set
-            # the specific default.
-
-            from leap.mail.smtp import setup_smtp_relay
-            client_cert = self._eip_config.get_client_cert_path(
-                self._provider_config)
-            self._smtp_service = setup_smtp_relay(
-                port=2013,
-                keymanager=self._keymanager,
-                smtp_host=host,
-                smtp_port=port,
-                smtp_cert=client_cert,
-                smtp_key=client_cert,
-                encrypted_only=False)
+
+            client_cert = self._smtp_config.get_client_cert_path(
+                self._provider_config,
+                about_to_download=True)
+
+            if not os.path.isfile(client_cert):
+                self._smtp_bootstrapper._download_client_certificates()
+            if os.path.isfile(client_cert):
+                self._start_smtp_service(host, port, client_cert)
+            else:
+                logger.warning("Tried to download email client "
+                               "certificate, but could not find any")
+
+        else:
+            logger.warning("No smtp hosts configured")
+
+    def _start_smtp_service(self, host, port, cert):
+        """
+        Starts the smtp service.
+        """
+        # TODO Make the encrypted_only configurable
+        # TODO pick local smtp port in a better way
+        # TODO remove hard-coded port and let leap.mail set
+        # the specific default.
+
+        from leap.mail.smtp import setup_smtp_relay
+        self._smtp_service = setup_smtp_relay(
+            port=2013,
+            keymanager=self._keymanager,
+            smtp_host=host,
+            smtp_port=port,
+            smtp_cert=cert,
+            smtp_key=cert,
+            encrypted_only=False)
 
     def _stop_smtp_service(self):
         """
diff --git a/src/leap/bitmask/services/__init__.py b/src/leap/bitmask/services/__init__.py
index 339f9cc6..2646235d 100644
--- a/src/leap/bitmask/services/__init__.py
+++ b/src/leap/bitmask/services/__init__.py
@@ -17,8 +17,22 @@
 """
 Services module.
 """
+import logging
+import os
+
 from PySide import QtCore
+
+from leap.bitmask.crypto.srpauth import SRPAuth
+from leap.bitmask.util.constants import REQUEST_TIMEOUT
 from leap.bitmask.util.privilege_policies import is_missing_policy_permissions
+from leap.bitmask.util.request_helpers import get_content
+
+from leap.common.check import leap_assert
+from leap.common.config.baseconfig import BaseConfig
+from leap.common.files import get_mtime
+
+logger = logging.getLogger(__name__)
+
 
 DEPLOYED = ["openvpn", "mx"]
 
@@ -70,3 +84,88 @@ def get_supported(services):
     :rtype: list of str
     """
     return filter(lambda s: s in DEPLOYED, services)
+
+
+def download_service_config(provider_config, service_config,
+                            session,
+                            download_if_needed=True):
+    """
+    Downloads config for a given service.
+
+    :param provider_config: an instance of ProviderConfig
+    :type provider_config: ProviderConfig
+
+    :param service_config: an instance of a particular Service config.
+    :type service_config: BaseConfig
+
+    :param session: an instance of a fetcher.session
+                    (currently we're using requests only, but it can be
+                    anything that implements that interface)
+    :type session: requests.sessions.Session
+    """
+    service_name = service_config.name
+    service_json = "{0}-service.json".format(service_name)
+    headers = {}
+    mtime = get_mtime(os.path.join(service_config.get_path_prefix(),
+                                   "leap",
+                                   "providers",
+                                   provider_config.get_domain(),
+                                   service_json))
+    if download_if_needed and mtime:
+        headers['if-modified-since'] = mtime
+
+    api_version = provider_config.get_api_version()
+
+    config_uri = "%s/%s/config/%s-service.json" % (
+        provider_config.get_api_uri(),
+        api_version,
+        service_name)
+    logger.debug('Downloading %s config from: %s' % (
+        service_name.upper(),
+        config_uri))
+
+    # XXX make and use @with_srp_auth decorator
+    srp_auth = SRPAuth(provider_config)
+    session_id = srp_auth.get_session_id()
+    cookies = None
+    if session_id:
+        cookies = {"_session_id": session_id}
+
+    res = session.get(config_uri,
+                      verify=provider_config.get_ca_cert_path(),
+                      headers=headers,
+                      timeout=REQUEST_TIMEOUT,
+                      cookies=cookies)
+    res.raise_for_status()
+
+    service_config.set_api_version(api_version)
+
+    # Not modified
+    service_path = ("leap", "providers", provider_config.get_domain(),
+                    service_json)
+    if res.status_code == 304:
+        logger.debug(
+            "{0} definition has not been modified".format(
+                service_name.upper()))
+        service_config.load(os.path.join(*service_path))
+    else:
+        service_definition, mtime = get_content(res)
+        service_config.load(data=service_definition, mtime=mtime)
+        service_config.save(service_path)
+
+
+class ServiceConfig(BaseConfig):
+    """
+    Base class used by the different service configs
+    """
+
+    _service_name = None
+
+    @property
+    def name(self):
+        """
+        Getter for the service name.
+        Derived classes should assign it.
+        """
+        leap_assert(self._service_name is not None)
+        return self._service_name
diff --git a/src/leap/bitmask/services/eip/eipbootstrapper.py b/src/leap/bitmask/services/eip/eipbootstrapper.py
index 6393e53a..5a238a1c 100644
--- a/src/leap/bitmask/services/eip/eipbootstrapper.py
+++ b/src/leap/bitmask/services/eip/eipbootstrapper.py
@@ -14,25 +14,22 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
 EIP bootstrapping
 """
-
 import logging
 import os
 
 from PySide import QtCore
 
 from leap.bitmask.config.providerconfig import ProviderConfig
-from leap.bitmask.crypto.srpauth import SRPAuth
-from leap.bitmask.services.eip.eipconfig import EIPConfig
-from leap.bitmask.util.request_helpers import get_content
-from leap.bitmask.util.constants import REQUEST_TIMEOUT
+from leap.bitmask.crypto.certs import download_client_cert
+from leap.bitmask.services import download_service_config
 from leap.bitmask.services.abstractbootstrapper import AbstractBootstrapper
-from leap.common import certs
+from leap.bitmask.services.eip.eipconfig import EIPConfig
+from leap.common import certs as leap_certs
 from leap.common.check import leap_assert, leap_assert_type
-from leap.common.files import check_and_fix_urw_only, get_mtime, mkdir_p
+from leap.common.files import check_and_fix_urw_only
 
 logger = logging.getLogger(__name__)
 
@@ -63,50 +60,15 @@ class EIPBootstrapper(AbstractBootstrapper):
 
         leap_assert(self._provider_config,
                     "We need a provider configuration!")
-
         logger.debug("Downloading EIP config for %s" %
                      (self._provider_config.get_domain(),))
 
-        api_version = self._provider_config.get_api_version()
         self._eip_config = EIPConfig()
-        self._eip_config.set_api_version(api_version)
-
-        headers = {}
-        mtime = get_mtime(os.path.join(self._eip_config
-                                       .get_path_prefix(),
-                                       "leap",
-                                       "providers",
-                                       self._provider_config.get_domain(),
-                                       "eip-service.json"))
-
-        if self._download_if_needed and mtime:
-            headers['if-modified-since'] = mtime
-
-        # there is some confusion with this uri,
-        # it's in 1/config/eip, config/eip and config/1/eip...
-        config_uri = "%s/%s/config/eip-service.json" % (
-            self._provider_config.get_api_uri(),
-            api_version)
-        logger.debug('Downloading eip config from: %s' % config_uri)
-
-        res = self._session.get(config_uri,
-                                verify=self._provider_config
-                                .get_ca_cert_path(),
-                                headers=headers,
-                                timeout=REQUEST_TIMEOUT)
-        res.raise_for_status()
-
-        # Not modified
-        if res.status_code == 304:
-            logger.debug("EIP definition has not been modified")
-        else:
-            eip_definition, mtime = get_content(res)
-
-            self._eip_config.load(data=eip_definition, mtime=mtime)
-            self._eip_config.save(["leap",
-                                   "providers",
-                                   self._provider_config.get_domain(),
-                                   "eip-service.json"])
+        download_service_config(
+            self._provider_config,
+            self._eip_config,
+            self._session,
+            self._download_if_needed)
 
     def _download_client_certificates(self, *args):
         """
@@ -124,40 +86,17 @@ class EIPBootstrapper(AbstractBootstrapper):
 
         # For re-download if something is wrong with the cert
         self._download_if_needed = self._download_if_needed and \
-            not certs.should_redownload(client_cert_path)
+            not leap_certs.should_redownload(client_cert_path)
 
         if self._download_if_needed and \
-                os.path.exists(client_cert_path):
+                os.path.isfile(client_cert_path):
             check_and_fix_urw_only(client_cert_path)
             return
 
-        srp_auth = SRPAuth(self._provider_config)
-        session_id = srp_auth.get_session_id()
-        cookies = None
-        if session_id:
-            cookies = {"_session_id": session_id}
-        cert_uri = "%s/%s/cert" % (
-            self._provider_config.get_api_uri(),
-            self._provider_config.get_api_version())
-        logger.debug('getting cert from uri: %s' % cert_uri)
-        res = self._session.get(cert_uri,
-                                verify=self._provider_config
-                                .get_ca_cert_path(),
-                                cookies=cookies,
-                                timeout=REQUEST_TIMEOUT)
-        res.raise_for_status()
-        client_cert = res.content
-
-        if not certs.is_valid_pemfile(client_cert):
-            raise Exception(self.tr("The downloaded certificate is not a "
-                                    "valid PEM file"))
-
-        mkdir_p(os.path.dirname(client_cert_path))
-
-        with open(client_cert_path, "w") as f:
-            f.write(client_cert)
-
-        check_and_fix_urw_only(client_cert_path)
+        download_client_cert(
+            self._provider_config,
+            client_cert_path,
+            self._session)
 
     def run_eip_setup_checks(self,
                              provider_config,
diff --git a/src/leap/bitmask/services/eip/eipconfig.py b/src/leap/bitmask/services/eip/eipconfig.py
index 1cb7419e..2241290b 100644
--- a/src/leap/bitmask/services/eip/eipconfig.py
+++ b/src/leap/bitmask/services/eip/eipconfig.py
@@ -26,9 +26,9 @@ import time
 import ipaddr
 
 from leap.bitmask.config.providerconfig import ProviderConfig
+from leap.bitmask.services import ServiceConfig
 from leap.bitmask.services.eip.eipspec import get_schema
 from leap.common.check import leap_assert, leap_assert_type
-from leap.common.config.baseconfig import BaseConfig
 
 logger = logging.getLogger(__name__)
 
@@ -144,15 +144,17 @@ class VPNGatewaySelector(object):
         return -local_offset / 3600
 
 
-class EIPConfig(BaseConfig):
+class EIPConfig(ServiceConfig):
     """
     Provider configuration abstraction class
     """
+    _service_name = "eip"
+
     OPENVPN_ALLOWED_KEYS = ("auth", "cipher", "tls-cipher")
     OPENVPN_CIPHERS_REGEX = re.compile("[A-Z0-9\-]+")
 
     def __init__(self):
-        BaseConfig.__init__(self)
+        ServiceConfig.__init__(self)
         self._api_version = None
 
     def _get_schema(self):
diff --git a/src/leap/bitmask/services/mail/smtpbootstrapper.py b/src/leap/bitmask/services/mail/smtpbootstrapper.py
index 0e83424c..032d6357 100644
--- a/src/leap/bitmask/services/mail/smtpbootstrapper.py
+++ b/src/leap/bitmask/services/mail/smtpbootstrapper.py
@@ -14,22 +14,21 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
 SMTP bootstrapping
 """
-
 import logging
 import os
 
 from PySide import QtCore
 
 from leap.bitmask.config.providerconfig import ProviderConfig
-from leap.bitmask.crypto.srpauth import SRPAuth
-from leap.bitmask.util.request_helpers import get_content
+from leap.bitmask.crypto.certs import download_client_cert
+from leap.bitmask.services import download_service_config
 from leap.bitmask.services.abstractbootstrapper import AbstractBootstrapper
+from leap.common import certs as leap_certs
 from leap.common.check import leap_assert, leap_assert_type
-from leap.common.files import get_mtime
+from leap.common.files import check_and_fix_urw_only
 
 logger = logging.getLogger(__name__)
 
@@ -61,55 +60,45 @@ class SMTPBootstrapper(AbstractBootstrapper):
         logger.debug("Downloading SMTP config for %s" %
                      (self._provider_config.get_domain(),))
 
-        headers = {}
-        mtime = get_mtime(os.path.join(self._smtp_config
-                                       .get_path_prefix(),
-                                       "leap",
-                                       "providers",
-                                       self._provider_config.get_domain(),
-                                       "smtp-service.json"))
-
-        if self._download_if_needed and mtime:
-            headers['if-modified-since'] = mtime
-
-        api_version = self._provider_config.get_api_version()
-
-        # there is some confusion with this uri,
-        config_uri = "%s/%s/config/smtp-service.json" % (
-            self._provider_config.get_api_uri(), api_version)
-
-        logger.debug('Downloading SMTP config from: %s' % config_uri)
-
-        srp_auth = SRPAuth(self._provider_config)
-        session_id = srp_auth.get_session_id()
-        cookies = None
-        if session_id:
-            cookies = {"_session_id": session_id}
-
-        res = self._session.get(config_uri,
-                                verify=self._provider_config
-                                .get_ca_cert_path(),
-                                headers=headers,
-                                cookies=cookies)
-        res.raise_for_status()
-
-        self._smtp_config.set_api_version(api_version)
-
-        # Not modified
-        if res.status_code == 304:
-            logger.debug("SMTP definition has not been modified")
-            self._smtp_config.load(os.path.join(
-                "leap", "providers",
-                self._provider_config.get_domain(),
-                "smtp-service.json"))
-        else:
-            smtp_definition, mtime = get_content(res)
-
-            self._smtp_config.load(data=smtp_definition, mtime=mtime)
-            self._smtp_config.save(["leap",
-                                    "providers",
-                                    self._provider_config.get_domain(),
-                                    "smtp-service.json"])
+        download_service_config(
+            self._provider_config,
+            self._smtp_config,
+            self._session,
+            self._download_if_needed)
+
+    def _download_client_certificates(self, *args):
+        """
+        Downloads the SMTP client certificate for the given provider
+
+        We actually are downloading the certificate for the same uri as
+        for the EIP config, but we duplicate these bits to allow mail
+        service to be working in a provider that does not offer EIP.
+        """
+        # TODO factor out with eipboostrapper.download_client_certificates
+        # TODO this shouldn't be a private method, it's called from
+        # mainwindow.
+        leap_assert(self._provider_config, "We need a provider configuration!")
+        leap_assert(self._smtp_config, "We need an smtp configuration!")
+
+        logger.debug("Downloading SMTP client certificate for %s" %
+                     (self._provider_config.get_domain(),))
+
+        client_cert_path = self._smtp_config.\
+            get_client_cert_path(self._provider_config,
+                                 about_to_download=True)
+
+        # For re-download if something is wrong with the cert
+        self._download_if_needed = self._download_if_needed and \
+            not leap_certs.should_redownload(client_cert_path)
+
+        if self._download_if_needed and \
+                os.path.isfile(client_cert_path):
+            check_and_fix_urw_only(client_cert_path)
+            return
+
+        download_client_cert(self._provider_config,
+                             client_cert_path,
+                             self._session)
 
     def run_smtp_setup_checks(self,
                               provider_config,
diff --git a/src/leap/bitmask/services/mail/smtpconfig.py b/src/leap/bitmask/services/mail/smtpconfig.py
index 20041c30..74c9bc94 100644
--- a/src/leap/bitmask/services/mail/smtpconfig.py
+++ b/src/leap/bitmask/services/mail/smtpconfig.py
@@ -14,25 +14,28 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
 SMTP configuration
 """
 import logging
+import os
 
+from leap.bitmask.config.providerconfig import ProviderConfig
+from leap.bitmask.services import ServiceConfig
 from leap.bitmask.services.mail.smtpspec import get_schema
-from leap.common.config.baseconfig import BaseConfig
+from leap.common.check import leap_assert, leap_assert_type
 
 logger = logging.getLogger(__name__)
 
 
-class SMTPConfig(BaseConfig):
+class SMTPConfig(ServiceConfig):
     """
     SMTP configuration abstraction class
     """
+    _service_name = "smtp"
 
     def __init__(self):
-        BaseConfig.__init__(self)
+        ServiceConfig.__init__(self)
 
     def _get_schema(self):
         """
@@ -47,3 +50,28 @@ class SMTPConfig(BaseConfig):
 
     def get_locations(self):
         return self._safe_get_value("locations")
+
+    def get_client_cert_path(self,
+                             providerconfig=None,
+                             about_to_download=False):
+        """
+        Returns the path to the certificate used by smtp
+        """
+
+        leap_assert(providerconfig, "We need a provider")
+        leap_assert_type(providerconfig, ProviderConfig)
+
+        cert_path = os.path.join(self.get_path_prefix(),
+                                 "leap",
+                                 "providers",
+                                 providerconfig.get_domain(),
+                                 "keys",
+                                 "client",
+                                 "smtp.pem")
+
+        if not about_to_download:
+            leap_assert(os.path.exists(cert_path),
+                        "You need to download the certificate first")
+            logger.debug("Using SMTP cert %s" % (cert_path,))
+
+        return cert_path
diff --git a/src/leap/bitmask/services/soledad/soledadconfig.py b/src/leap/bitmask/services/soledad/soledadconfig.py
index 7ed21f77..d3cc7da4 100644
--- a/src/leap/bitmask/services/soledad/soledadconfig.py
+++ b/src/leap/bitmask/services/soledad/soledadconfig.py
@@ -14,25 +14,25 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
 Soledad configuration
 """
 import logging
 
+from leap.bitmask.services import ServiceConfig
 from leap.bitmask.services.soledad.soledadspec import get_schema
-from leap.common.config.baseconfig import BaseConfig
 
 logger = logging.getLogger(__name__)
 
 
-class SoledadConfig(BaseConfig):
+class SoledadConfig(ServiceConfig):
     """
     Soledad configuration abstraction class
     """
+    _service_name = "soledad"
 
     def __init__(self):
-        BaseConfig.__init__(self)
+        ServiceConfig.__init__(self)
 
     def _get_schema(self):
         """