provider service bootstrap and autodiscovery

4 files changed, 328 insertions, 44 deletions

diff --git a/bonafide/src/leap/bonafide/_protocol.py b/bonafide/src/leap/bonafide/_protocol.py
index f1c3834..de959c4 100644
--- a/bonafide/src/leap/bonafide/_protocol.py
+++ b/bonafide/src/leap/bonafide/_protocol.py
@@ -45,6 +45,7 @@ class BonafideProtocol(object):
     _sessions = defaultdict(None)
 
     def _get_api(self, provider_id):
+        # TODO should get deferred
         if provider_id in self._apis:
             return self._apis[provider_id]
 
@@ -73,26 +74,45 @@ class BonafideProtocol(object):
     # Service public methods
 
     def do_signup(self, full_id, password):
+        log.msg('SIGNUP for %s' % full_id)
+        _, provider_id = config.get_username_and_provider(full_id)
+
+        provider = config.Provider(provider_id)
+        d = provider.callWhenReady(self._do_signup, full_id, password)
+        return d
+
+    def _do_signup(self, full_id, password):
+
         # XXX check it's unauthenticated
         def return_user(result, _session):
             return_code, user = result
             if return_code == OK:
                 return user
 
-        log.msg('SIGNUP for %s' % full_id)
+        username, _ = config.get_username_and_provider(full_id)
+        # XXX get deferred?
         session = self._get_session(full_id, password)
-        username, provider_id = config.get_username_and_provider(full_id)
-
         d = session.signup(username, password)
         d.addCallback(return_user, session)
         return d
 
     def do_authenticate(self, full_id, password):
+        log.msg('SIGNUP for %s' % full_id)
+        _, provider_id = config.get_username_and_provider(full_id)
+
+        provider = config.Provider(provider_id)
+        d = provider.callWhenReady(self._do_authenticate, full_id, password)
+        return d
+
+    def _do_authenticate(self, full_id, password):
+
         def return_token_and_uuid(result, _session):
             if result == OK:
                 return str(_session.token), str(_session.uuid)
 
         log.msg('AUTH for %s' % full_id)
+
+        # XXX get deferred?
         session = self._get_session(full_id, password)
         d = session.authenticate()
         d.addCallback(return_token_and_uuid, session)
@@ -130,4 +150,3 @@ class BonafideProtocol(object):
         mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
         return '[+] Bonafide service: [%s sessions] [Mem usage: %s KB]' % (
             len(self._sessions), mem / 1024)
-
diff --git a/bonafide/src/leap/bonafide/config.py b/bonafide/src/leap/bonafide/config.py
index 9490f55..afc59fc 100644
--- a/bonafide/src/leap/bonafide/config.py
+++ b/bonafide/src/leap/bonafide/config.py
@@ -18,14 +18,22 @@
 Configuration for a LEAP provider.
 """
 import datetime
+import json
 import os
 import sys
 
+from twisted.internet import defer, reactor
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python import log
+from twisted.web.client import Agent, downloadPage
+
 from leap.bonafide._http import httpRequest
+from leap.bonafide.provider import Discovery
 
 from leap.common.check import leap_assert
 from leap.common.config import get_path_prefix as common_get_path_prefix
-from leap.common.files import check_and_fix_urw_only, get_mtime, mkdir_p
+from leap.common.files import mkdir_p
+# check_and_fix_urw_only, get_mtime
 
 
 APPNAME = "bonafide"
@@ -36,17 +44,24 @@ def get_path_prefix(standalone=False):
     return common_get_path_prefix(standalone)
 
 
-def get_provider_path(domain):
+def get_provider_path(domain, config='provider.json'):
     """
-    Returns relative path for provider config.
+    Returns relative path for provider configs.
 
     :param domain: the domain to which this providerconfig belongs to.
     :type domain: str
     :returns: the path
     :rtype: str
     """
-    leap_assert(domain is not None, "get_provider_path: We need a domain")
-    return os.path.join("leap", "providers", domain, "provider.json")
+    # TODO sanitize domain
+    leap_assert(domain is not None, 'get_provider_path: We need a domain')
+    return os.path.join('providers', domain, config)
+
+
+def get_ca_cert_path(domain):
+    # TODO sanitize domain
+    leap_assert(domain is not None, 'get_provider_path: We need a domain')
+    return os.path.join('providers', domain, 'keys', 'ca', 'cacert.pem')
 
 
 def get_modification_ts(path):
@@ -99,28 +114,88 @@ def make_address(user, provider):
     :param provider: the provider domain
     :type provider: basestring
     """
-    return "%s@%s" % (user, provider)
+    return '%s@%s' % (user, provider)
 
 
 def get_username_and_provider(full_id):
     return full_id.split('@')
 
 
-class ProviderConfig(object):
-    # TODO add file config for enabled services
+class WebClientContextFactory(ClientContextFactory):
+    def getContext(self, hostname, port):
+        return ClientContextFactory.getContext(self)
+
+
+class Provider(object):
+    # TODO split Provider, ProviderConfig
+    # TODO add validation
 
-    def __init__(self, domain):
-        self._api_base = None
+    SERVICES_MAP = {
+        'openvpn': ['eip'],
+        'mx': ['soledad', 'smtp']}
+
+    def __init__(self, domain, autoconf=True, basedir='~/.config/leap',
+                 check_certificate=True):
         self._domain = domain
+        self._basedir = os.path.expanduser(basedir)
+        self._disco = Discovery('https://%s' % domain)
+        self._provider_config = {}
+
+        is_configured = self.is_configured()
+        if not is_configured:
+            check_certificate = False
+
+        if check_certificate:
+            self.contextFactory = None
+        else:
+            # XXX we should do this only for the FIRST provider download.
+            # For the rest, we should pass the ca cert to the agent.
+            # That means that RIGHT AFTER DOWNLOADING provider_info,
+            # we should instantiate a new Agent...
+            self.contextFactory = WebClientContextFactory()
+        self._agent = Agent(reactor, self.contextFactory)
+
+        self._load_provider_config()
+        # TODO if loaded, setup _get_api_uri on the DISCOVERY
+
+        self._init_deferred = None
+
+        if not is_configured and autoconf:
+            print 'provider %s not configured: downloading files...' % domain
+            self.bootstrap()
+        else:
+            print 'already initialized'
+            self._init_deferred = defer.succeed('already_initialized')
+
+    def callWhenReady(self, cb, *args, **kw):
+        print 'calling when ready', cb
+        d = self._init_deferred
+        d.addCallback(lambda _: cb(*args, **kw))
+        d.addErrback(log.err)
+        return d
 
     def is_configured(self):
         provider_json = self._get_provider_json_path()
         # XXX check if all the services are there
-        if is_file(provider_json):
-            return True
-        return False
+        if not is_file(provider_json):
+            return False
+        if not is_file(self._get_ca_cert_path()):
+            return False
+        return True
+
+    def bootstrap(self):
+        print "Bootstrapping provider %s" % self._domain
+        d = self.maybe_download_provider_info()
+        d.addCallback(self.maybe_download_ca_cert)
+        d.addCallback(self.validate_ca_cert)
+        d.addCallback(self.maybe_download_services_config)
+        d.addCallback(self.load_services_config)
+        self._init_deferred = d
+
+    def has_valid_certificate(self):
+        pass
 
-    def download_provider_info(self):
+    def maybe_download_provider_info(self, replace=False):
         """
         Download the provider.json info from the main domain.
         This SHOULD only be used once with the DOMAIN url.
@@ -128,8 +203,19 @@ class ProviderConfig(object):
         # TODO handle pre-seeded providers?
         # or let client handle that? We could move them to bonafide.
         provider_json = self._get_provider_json_path()
-        if is_file(provider_json):
-            raise RuntimeError('File already exists')
+        if is_file(provider_json) and not replace:
+            return defer.succeed('provider_info_already_exists')
+
+        folders, f = os.path.split(provider_json)
+        mkdir_p(folders)
+
+        uri = self._disco.get_provider_info_uri()
+        met = self._disco.get_provider_info_method()
+
+        d = downloadPage(uri, provider_json, method=met)
+        d.addCallback(lambda _: self._load_provider_config())
+        d.addErrback(log.err)
+        return d
 
     def update_provider_info(self):
         """
@@ -137,16 +223,142 @@ class ProviderConfig(object):
         """
         pass
 
-    def _http_request(self, *args, **kw):
-        # XXX pass if-modified-since header
-        return httpRequest(*args, **kw)
+    def maybe_download_ca_cert(self, ignored):
+        """
+        :rtype: deferred
+        """
+        path = self._get_ca_cert_path()
+        if is_file(path):
+            return defer.succeed('ca_cert_path_already_exists')
+
+        uri = self._get_ca_cert_uri()
+        mkdir_p(os.path.split(path)[0])
+        d = downloadPage(uri, path)
+        d.addErrback(log.err)
+        return d
+
+    def validate_ca_cert(self, ignored):
+        # XXX Need to verify fingerprint against the one in provider.json
+        expected = self._get_expected_ca_cert_fingerprint()
+        print "EXPECTED FINGERPRINT:", expected
+
+    def _get_expected_ca_cert_fingerprint(self):
+        try:
+            fgp = self._provider_config.ca_cert_fingerprint
+        except AttributeError:
+            fgp = None
+        return fgp
+
+
+    def maybe_download_services_config(self, ignored):
+        pass
+
+    def load_services_config(self, ignored):
+        print 'loading services config...'
+        configs_path = self._get_configs_path()
+
+        uri = self._disco.get_configs_uri()
+        met = self._disco.get_configs_method()
+
+        # TODO --- currently, provider on mail.bitmask.net raises 401
+        # UNAUTHENTICATED if we try to # get the services on first boostrap.
+        # See: # https://leap.se/code/issues/7906
+
+        # As a Workaround, these urls work though:
+        # curl -k https://api.mail.bitmask.net:4430/1/config/smtp-service.json 
+        # curl -k https://api.mail.bitmask.net:4430/1/config/soledad-service.json 
+
+        print "GETTING SERVICES FROM...", uri
+
+        d = downloadPage(uri, configs_path, method=met)
+        d.addCallback(lambda _: self._load_provider_config())
+        d.addCallback(lambda _: self._get_config_for_all_services())
+        d.addErrback(log.err)
+        return d
+
+    def offers_service(self, service):
+        if service not in self.SERVICES_MAP.keys():
+            raise RuntimeError('Unknown service: %s' % service)
+        return service in self._provider_config.services
+
+    # TODO is_service_enabled ---> this belongs to core?
 
     def _get_provider_json_path(self):
         domain = self._domain.encode(sys.getfilesystemencoding())
-        provider_json = os.path.join(get_path_prefix(), get_provider_path(domain))
-        return provider_json
+        provider_json_path = os.path.join(
+            self._basedir, get_provider_path(domain, config='provider.json'))
+        return provider_json_path
+
+    def _get_configs_path(self):
+        domain = self._domain.encode(sys.getfilesystemencoding())
+        configs_path = os.path.join(
+            self._basedir, get_provider_path(domain, config='configs.json'))
+        return configs_path
+
+    def _get_service_config_path(self, service):
+        domain = self._domain.encode(sys.getfilesystemencoding())
+        configs_path = os.path.join(
+            self._basedir, get_provider_path(
+                domain, config='%s-service.json' % service))
+        return configs_path
+
+    def _get_ca_cert_path(self):
+        domain = self._domain.encode(sys.getfilesystemencoding())
+        cert_path = os.path.join(self._basedir, get_ca_cert_path(domain))
+        return cert_path
+
+    def _get_ca_cert_uri(self):
+        try:
+            uri = self._provider_config.ca_cert_uri
+            uri = str(uri)
+        except Exception:
+            uri = None
+        return uri
+
+    def _load_provider_config(self):
+        path = self._get_provider_json_path()
+        if not is_file(path):
+            return
+        with open(path, 'r') as config:
+            self._provider_config = Record(**json.load(config))
+
+    def _get_config_for_all_services(self):
+        configs_path = self._get_configs_path()
+        with open(configs_path) as jsonf:
+            services_dict = Record(**json.load(jsonf)).services
+        pending = []
+        base = self._disco.get_base_uri()
+        for service in self._provider_config.services:
+            for subservice in self.SERVICES_MAP[service]:
+                uri = base + str(services_dict[subservice])
+                path = self._get_service_config_path(subservice)
+                d = self._fetch_config_for_service(uri, path)
+                pending.append(d)
+        return defer.gatherResults(pending)
+
+    def _fetch_config_for_service(self, uri, path):
+        log.msg('Downloading config for %s...' % uri)
+        d = downloadPage(uri, path, method='GET')
+        return d
+
+    def _http_request(self, *args, **kw):
+        # XXX pass if-modified-since header
+        return httpRequest(self._agent, *args, **kw)
+
+    def _get_api_uri(self):
+        pass
+
+
+class Record(object):
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
 
 if __name__ == '__main__':
-    config = ProviderConfig('cdev.bitmask.net')
-    config.is_configured()
-    config.download_provider_info()
+
+    def print_done():
+        print '>>> bootstrapping done!!!'
+
+    provider = Provider('cdev.bitmask.net')
+    provider.callWhenReady(print_done)
+    reactor.run()
diff --git a/bonafide/src/leap/bonafide/provider.py b/bonafide/src/leap/bonafide/provider.py
index 1d0b5b7..7e78196 100644
--- a/bonafide/src/leap/bonafide/provider.py
+++ b/bonafide/src/leap/bonafide/provider.py
@@ -38,11 +38,13 @@ class _MetaActionDispatcher(type):
     where `uri_template` is a string that will be formatted with an arbitrary
     number of keyword arguments.
 
-    Any class that uses this one as its metaclass needs to implement two private
-    methods::
+    Any class that uses this one as its metaclass needs to implement two
+    private methods::
 
         _get_uri(self, action_name, **extra_params)
         _get_method(self, action_name)
+
+    Beware that currently they cannot be inherited from bases.
     """
 
     def __new__(meta, name, bases, dct):
@@ -74,17 +76,34 @@ class _MetaActionDispatcher(type):
             meta, name, bases, newdct)
 
 
-class Api(object):
+class BaseProvider(object):
+
+    def __init__(self, netloc, version=1):
+        parsed = urlparse(netloc)
+        if parsed.scheme != 'https':
+            raise ValueError(
+                'ProviderApi needs to be passed a url with https scheme')
+        self.netloc = parsed.netloc
+        self.version = version
+
+    def get_hostname(self):
+        return urlparse(self._get_base_url()).hostname
+
+    def _get_base_url(self):
+        return "https://{0}/{1}".format(self.netloc, self.version)
+
+
+class Api(BaseProvider):
     """
     An object that has all the information that a client needs to communicate
     with the remote methods exposed by the web API of a LEAP provider.
 
     The actions are described in https://leap.se/bonafide
 
-    By using the _MetaActionDispatcher as a metaclass, the _actions dict will be
-    translated dynamically into a set of instance methods that will allow
+    By using the _MetaActionDispatcher as a metaclass, the _actions dict will
+    be translated dynamically into a set of instance methods that will allow
     getting the uri and method for each action.
-    
+
     The keyword arguments specified in the format string will automatically
     raise a KeyError if the needed keyword arguments are not passed to the
     dynamically created methods.
@@ -106,19 +125,48 @@ class Api(object):
         'smtp_cert': ('smtp_cert', 'POST'),
     }
 
-    def __init__(self, netloc, version=1):
-        parsed = urlparse(netloc)
-        if parsed.scheme != 'https':
-            raise ValueError(
-                'ProviderApi needs to be passed a url with https scheme')
-        self.netloc = parsed.netloc
-        self.version = version
+    # Methods expected by the dispatcher metaclass
 
-    def get_hostname(self):
-        return urlparse(self._get_base_url()).hostname
+    def _get_uri(self, action_name, **extra_params):
+        resource, _ = self._actions.get(action_name)
+        uri = '{0}/{1}'.format(
+            bytes(self._get_base_url()),
+            bytes(resource)).format(**extra_params)
+        return uri
+
+    def _get_method(self, action_name):
+        _, method = self._actions.get(action_name)
+        return method
+
+
+class Discovery(BaseProvider):
+    """
+    Discover basic information about a provider, including the provided
+    services.
+    """
+
+    __metaclass__ = _MetaActionDispatcher
+    _actions = {
+        'provider_info': ('provider.json', 'GET'),
+        'configs': ('1/configs.json', 'GET'),
+    }
+
+    api_uri = None
+    api_port = None
 
     def _get_base_url(self):
-        return "https://{0}/{1}".format(self.netloc, self.version)
+        if self.api_uri:
+            base = self.api_uri
+        else:
+            base = self.netloc
+
+        uri = "https://{0}".format(base)
+        if self.api_port:
+            uri = uri + ':%s' % self.api_port
+        return uri
+
+    def get_base_uri(self):
+        return self._get_base_url()
 
     # Methods expected by the dispatcher metaclass
 
diff --git a/bonafide/src/leap/bonafide/service.py b/bonafide/src/leap/bonafide/service.py
index 468721b..bdbf729 100644
--- a/bonafide/src/leap/bonafide/service.py
+++ b/bonafide/src/leap/bonafide/service.py
@@ -25,6 +25,7 @@ from leap.bonafide._protocol import BonafideProtocol
 
 from twisted.application import service
 from twisted.internet import defer
+from twisted.python import log
 
 
 class BonafideService(service.Service):
@@ -63,6 +64,10 @@ class BonafideService(service.Service):
                     this_hook, username=username, password=password)
 
         def notify_bonafide_auth_hook(result):
+            if not result:
+                log.msg("Authentication hook did not return anything")
+                return
+
             this_hook = 'on_bonafide_auth'
             token, uuid = result
             hooked_service = self.get_hooked_service(this_hook)