[feature] pluggable backends and api registry

the idea behind this mechanism (partially implemented for that) is to be
able to check the backend output against some type annotations.

We want to be able to detect if a given backend (real services or
authoritative mocks) have diverged from what's specified in the API
annotations.

7 files changed, 420 insertions, 138 deletions

diff --git a/src/leap/bitmask/cli/bitmask_cli.py b/src/leap/bitmask/cli/bitmask_cli.py
index c5bb1b15..c2b1ba71 100755
--- a/src/leap/bitmask/cli/bitmask_cli.py
+++ b/src/leap/bitmask/cli/bitmask_cli.py
@@ -325,7 +325,7 @@ def send_command(cli):
 
     s = get_zmq_connection()
 
-    d = s.sendMsg(*data, timeout=20)
+    d = s.sendMsg(*data, timeout=60)
     d.addCallback(cb)
     d.addCallback(lambda x: reactor.stop())
     d.addErrback(timeout_handler)
diff --git a/src/leap/bitmask/core/api.py b/src/leap/bitmask/core/api.py
new file mode 100644
index 00000000..9f3725dc
--- /dev/null
+++ b/src/leap/bitmask/core/api.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# api.py
+# Copyright (C) 2016 LEAP Encryption Acess Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Registry for the public API for the Bitmask Backend.
+"""
+from collections import OrderedDict
+
+registry = OrderedDict()
+
+
+class APICommand(type):
+    """
+    A metaclass to keep a global registry of all the methods that compose the
+    public API for the Bitmask Backend.
+    """
+    def __init__(cls, name, bases, attrs):
+        for key, val in attrs.iteritems():
+            properties = getattr(val, 'register', None)
+            label = getattr(cls, 'label', None)
+            if label:
+                name = label
+            if properties is not None:
+                registry['%s.%s' % (name, key)] = properties
+
+
+def register_method(*args):
+    """
+    This method gathers info about all the methods that are supposed to
+    compose the public API to communicate with the backend.
+
+    It sets up a register property for any method that uses it.
+    A type annotation is supposed to be in this property.
+    The APICommand metaclass collects these properties of the methods and
+    stores them in the global api_registry object, where they can be
+    introspected at runtime.
+    """
+    def decorator(f):
+        f.register = tuple(args)
+        return f
+    return decorator
diff --git a/src/leap/bitmask/core/api_contract.py b/src/leap/bitmask/core/api_contract.py
new file mode 100644
index 00000000..86b600c1
--- /dev/null
+++ b/src/leap/bitmask/core/api_contract.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# api_contract.py
+# Copyright (C) 2016 LEAP Encryption Acess Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Display a human-readable representation of the methods that compound the public
+api for Bitmask Core.
+
+The values are meant to be type annotations.
+"""
+
+if __name__ == "__main__":
+    from leap.bitmask.core.service import BitmaskBackend
+    from leap.bitmask.core import api
+    backend = BitmaskBackend()
+
+    print '========= Bitmask Core API =================='
+    print
+
+    for key in api.registry:
+        human_key = key.replace('do_', '').lower()
+        value = api.registry[key]
+
+        print("{}:\t\t{}".format(
+            human_key,
+            ' '.join([x for x in value])))
+    print
+    print '============================================='
diff --git a/src/leap/bitmask/core/dispatcher.py b/src/leap/bitmask/core/dispatcher.py
index 648cbe9b..e7c961fd 100644
--- a/src/leap/bitmask/core/dispatcher.py
+++ b/src/leap/bitmask/core/dispatcher.py
@@ -22,154 +22,214 @@ import json
 from twisted.internet import defer
 from twisted.python import failure, log
 
+from .api import APICommand, register_method
 
-# TODO implement sub-classes to dispatch subcommands (user, mail).
 
+class SubCommand(object):
 
-class CommandDispatcher(object):
+    __metaclass__ = APICommand
 
-    def __init__(self, core):
+    def dispatch(self, service, *parts, **kw):
+        subcmd = parts[1]
 
-        self.core = core
+        _method = getattr(self, 'do_' + subcmd.upper(), None)
+        if not _method:
+            raise RuntimeError('No such subcommand')
+        return _method(service, *parts, **kw)
 
-    def _get_service(self, name):
 
-        try:
-            return self.core.getServiceNamed(name)
-        except KeyError:
-            return None
+class UserCmd(SubCommand):
 
-    def dispatch(self, msg):
-        cmd = msg[0]
+    label = 'user'
 
-        _method = getattr(self, 'do_' + cmd.upper(), None)
+    @register_method("{'srp_token': unicode, 'uuid': unicode}")
+    def do_AUTHENTICATE(self, bonafide, *parts):
+        user, password = parts[2], parts[3]
+        d = defer.maybeDeferred(bonafide.do_authenticate, user, password)
+        return d
 
-        if not _method:
-            return defer.fail(failure.Failure(RuntimeError('No such command')))
+    @register_method("{'signup': 'ok', 'user': str}")
+    def do_SIGNUP(self, bonafide, *parts):
+        user, password = parts[2], parts[3]
+        d = defer.maybeDeferred(bonafide.do_signup, user, password)
+        return d
 
-        return defer.maybeDeferred(_method, *msg)
+    @register_method("{'logout': 'ok'}")
+    def do_LOGOUT(self, bonafide, *parts):
+        user, password = parts[2], parts[3]
+        d = defer.maybeDeferred(bonafide.do_logout, user, password)
+        return d
 
-    def do_STATS(self, *parts):
-        return _format_result(self.core.do_stats())
+    @register_method('str')
+    def do_ACTIVE(self, bonafide, *parts):
+        d = defer.maybeDeferred(bonafide.do_get_active_user)
+        return d
 
-    def do_VERSION(self, *parts):
-        return _format_result(self.core.do_version())
 
-    def do_STATUS(self, *parts):
-        return _format_result(self.core.do_status())
+class EIPCmd(SubCommand):
 
-    def do_SHUTDOWN(self, *parts):
-        return _format_result(self.core.do_shutdown())
+    label = 'eip'
 
-    def do_USER(self, *parts):
+    @register_method('dict')
+    def do_ENABLE(self, service, *parts):
+        d = service.do_enable_service(self.label)
+        return d
 
-        subcmd = parts[1]
-        user, password = parts[2], parts[3]
+    @register_method('dict')
+    def do_DISABLE(self, service, *parts):
+        d = service.do_disable_service(self.label)
+        return d
 
-        bf = self._get_service('bonafide')
+    @register_method('dict')
+    def do_STATUS(self, eip, *parts):
+        d = eip.do_status()
+        return d
 
-        if subcmd == 'authenticate':
-            d = bf.do_authenticate(user, password)
+    @register_method('dict')
+    def do_START(self, eip, *parts):
+        # TODO --- attempt to get active provider
+        # TODO or catch the exception and send error
+        provider = parts[2]
+        d = eip.do_start(provider)
+        return d
 
-        elif subcmd == 'signup':
-            d = bf.do_signup(user, password)
+    @register_method('dict')
+    def do_STOP(self, eip, *parts):
+        d = eip.do_stop()
+        return d
 
-        elif subcmd == 'logout':
-            d = bf.do_logout(user, password)
 
-        elif subcmd == 'active':
-            d = bf.do_get_active_user()
+class MailCmd(SubCommand):
 
-        d.addCallbacks(_format_result, _format_error)
+    label = 'mail'
+
+    @register_method('dict')
+    def do_ENABLE(self, service, *parts):
+        d = service.do_enable_service(self.label)
         return d
 
-    def do_EIP(self, *parts):
+    @register_method('dict')
+    def do_DISABLE(self, service, *parts):
+        d = service.do_disable_service(self.label)
+        return d
 
-        subcmd = parts[1]
-        eip_label = 'eip'
+    @register_method('dict')
+    def do_STATUS(self, mail, *parts):
+        d = mail.do_status()
+        return d
 
-        if subcmd == 'enable':
-            return _format_result(
-                self.core.do_enable_service(eip_label))
+    @register_method('dict')
+    def do_GET_IMAP_TOKEN(self, mail, *parts):
+        d = mail.get_imap_token()
+        return d
 
-        eip = self._get_service(eip_label)
-        if not eip:
-            return _format_result('eip: disabled')
+    @register_method('dict')
+    def do_GET_SMTP_TOKEN(self, mail, *parts):
+        d = mail.get_smtp_token()
+        return d
 
-        if subcmd == 'status':
-            return _format_result(eip.do_status())
+    @register_method('dict')
+    def do_GET_SMTP_CERTIFICATE(self, mail, *parts, **kw):
+        # TODO move to mail service
+        # TODO should ask for confirmation? like --force or something,
+        # if we already have a valid one. or better just refuse if cert
+        # exists.
+        # TODO how should we pass the userid??
+        # - Keep an 'active' user in bonafide (last authenticated)
+        # (doing it now)
+        # - Get active user from Mail Service (maybe preferred?)
+        # - Have a command/method to set 'active' user.
+
+        @defer.inlineCallbacks
+        def save_cert(cert_data):
+            userid, cert_str = cert_data
+            cert_path = yield mail.do_get_smtp_cert_path(userid)
+            with open(cert_path, 'w') as outf:
+                outf.write(cert_str)
+            defer.returnValue('certificate saved to %s' % cert_path)
+
+        bonafide = kw['bonafide']
+        d = bonafide.do_get_smtp_cert()
+        d.addCallback(save_cert)
+        return d
 
-        elif subcmd == 'disable':
-            return _format_result(
-                self.core.do_disable_service(eip_label))
 
-        elif subcmd == 'start':
-            # TODO --- attempt to get active provider
-            # TODO or catch the exception and send error
-            provider = parts[2]
-            d = eip.do_start(provider)
-            d.addCallbacks(_format_result, _format_error)
-            return d
+class CommandDispatcher(object):
 
-        elif subcmd == 'stop':
-            d = eip.do_stop()
-            d.addCallbacks(_format_result, _format_error)
-            return d
+    __metaclass__ = APICommand
 
-    def do_MAIL(self, *parts):
+    label = 'core'
 
+    def __init__(self, core):
+
+        self.core = core
+        self.subcommand_user = UserCmd()
+        self.subcommand_eip = EIPCmd()
+        self.subcommand_mail = MailCmd()
+
+    # XXX --------------------------------------------
+    # TODO move general services to another subclass
+
+    @register_method("{'mem_usage': str}")
+    def do_STATS(self, *parts):
+        return _format_result(self.core.do_stats())
+
+    @register_method("{version_core': '0.0.0'}")
+    def do_VERSION(self, *parts):
+        return _format_result(self.core.do_version())
+
+    @register_method("{'mail': 'running'}")
+    def do_STATUS(self, *parts):
+        return _format_result(self.core.do_status())
+
+    @register_method("{'shutdown': 'ok'}")
+    def do_SHUTDOWN(self, *parts):
+        return _format_result(self.core.do_shutdown())
+
+    # -----------------------------------------------
+
+    def do_USER(self, *parts):
+        bonafide = self._get_service('bonafide')
+        d = self.subcommand_user.dispatch(bonafide, *parts)
+        d.addCallbacks(_format_result, _format_error)
+        return d
+
+    def do_EIP(self, *parts):
+        eip = self._get_service(self.subcommand_eip.label)
+        if not eip:
+            return _format_result('eip: disabled')
         subcmd = parts[1]
-        mail_label = 'mail'
 
-        if subcmd == 'enable':
-            return _format_result(
-                self.core.do_enable_service(mail_label))
+        dispatch = self._subcommand_eip.dispatch
+        if subcmd in ('enable', 'disable'):
+            d = dispatch(self.core, *parts)
+        else:
+            d = dispatch(eip, *parts)
 
-        m = self._get_service(mail_label)
-        bf = self._get_service('bonafide')
+        d.addCallbacks(_format_result, _format_error)
+        return d
 
-        if not m:
-            return _format_result('mail: disabled')
+    def do_MAIL(self, *parts):
+        subcmd = parts[1]
+        dispatch = self.subcommand_mail.dispatch
 
-        if subcmd == 'status':
-            return _format_result(m.do_status())
+        if subcmd == 'enable':
+            d = dispatch(self.core, *parts)
 
-        elif subcmd == 'disable':
-            return _format_result(self.core.do_disable_service(mail_label))
+        mail = self._get_service(self.subcommand_mail.label)
+        bonafide = self._get_service('bonafide')
+        kw = {'bonafide': bonafide}
 
-        elif subcmd == 'get_imap_token':
-            d = m.get_imap_token()
-            d.addCallbacks(_format_result, _format_error)
-            return d
+        if not mail:
+            return _format_result('mail: disabled')
 
-        elif subcmd == 'get_smtp_token':
-            d = m.get_smtp_token()
-            d.addCallbacks(_format_result, _format_error)
-            return d
+        if subcmd == 'disable':
+            d = dispatch(self.core)
+        else:
+            d = dispatch(mail, *parts, **kw)
 
-        elif subcmd == 'get_smtp_certificate':
-            # TODO move to mail service
-            # TODO should ask for confirmation? like --force or something,
-            # if we already have a valid one. or better just refuse if cert
-            # exists.
-            # TODO how should we pass the userid??
-            # - Keep an 'active' user in bonafide (last authenticated)
-            # (doing it now)
-            # - Get active user from Mail Service (maybe preferred?)
-            # - Have a command/method to set 'active' user.
-
-            @defer.inlineCallbacks
-            def save_cert(cert_data):
-                userid, cert_str = cert_data
-                cert_path = yield m.do_get_smtp_cert_path(userid)
-                with open(cert_path, 'w') as outf:
-                    outf.write(cert_str)
-                defer.returnValue('certificate saved to %s' % cert_path)
-
-            d = bf.do_get_smtp_cert()
-            d.addCallback(save_cert)
-            d.addCallbacks(_format_result, _format_error)
-            return d
+        d.addCallbacks(_format_result, _format_error)
+        return d
 
     def do_KEYS(self, *parts):
         subcmd = parts[1]
@@ -187,6 +247,22 @@ class CommandDispatcher(object):
             d.addCallbacks(_format_result, _format_error)
             return d
 
+    def dispatch(self, msg):
+        cmd = msg[0]
+
+        _method = getattr(self, 'do_' + cmd.upper(), None)
+
+        if not _method:
+            return defer.fail(failure.Failure(RuntimeError('No such command')))
+
+        return defer.maybeDeferred(_method, *msg)
+
+    def _get_service(self, name):
+        try:
+            return self.core.getServiceNamed(name)
+        except KeyError:
+            return None
+
 
 def _format_result(result):
     return json.dumps({'error': None, 'result': result})
diff --git a/src/leap/bitmask/core/dummy.py b/src/leap/bitmask/core/dummy.py
new file mode 100644
index 00000000..99dfafa5
--- /dev/null
+++ b/src/leap/bitmask/core/dummy.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# dummy.py
+# Copyright (C) 2016 LEAP Encryption Acess Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+An authoritative dummy backend for tests.
+"""
+import json
+
+from leap.common.service_hooks import HookableService
+
+
+class BackendCommands(object):
+
+    """
+    General commands for the BitmaskBackend Core Service.
+    """
+
+    def __init__(self, core):
+        self.core = core
+
+    def do_status(self):
+        return json.dumps(
+            {'soledad': 'running',
+             'keymanager': 'running',
+             'mail': 'running',
+             'eip': 'stopped',
+             'backend': 'dummy'})
+
+    def do_version(self):
+        return {'version_core': '0.0.1'}
+
+    def do_stats(self):
+        return {'mem_usage': '01 KB'}
+
+    def do_shutdown(self):
+        return {'shutdown': 'ok'}
+
+
+class mail_services(object):
+
+    class SoledadService(HookableService):
+        pass
+
+    class KeymanagerService(HookableService):
+        pass
+
+    class StandardMailService(HookableService):
+        pass
+
+
+class BonafideService(HookableService):
+
+    def __init__(self, basedir):
+        pass
+
+    def do_authenticate(self, user, password):
+        return {u'srp_token': u'deadbeef123456789012345678901234567890123',
+                u'uuid': u'01234567890abcde01234567890abcde'}
+
+    def do_signup(self, user, password):
+        return {'signup': 'ok', 'user': 'dummyuser@provider.example.org'}
+
+    def do_logout(self, user, password):
+        return {'logout': 'ok'}
+
+    def do_get_active_user(self):
+        return 'dummyuser@provider.example.org'
diff --git a/src/leap/bitmask/core/mail_services.py b/src/leap/bitmask/core/mail_services.py
index 3495aa39..fb9ee698 100644
--- a/src/leap/bitmask/core/mail_services.py
+++ b/src/leap/bitmask/core/mail_services.py
@@ -64,10 +64,10 @@ class ImproperlyConfigured(Exception):
 
 class SoledadContainer(Container):
 
-    def __init__(self, basedir=DEFAULT_BASEDIR):
+    def __init__(self, service=None, basedir=DEFAULT_BASEDIR):
         self._basedir = os.path.expanduser(basedir)
         self._usermap = UserMap()
-        super(SoledadContainer, self).__init__()
+        super(SoledadContainer, self).__init__(service=service)
 
     def add_instance(self, userid, passphrase, uuid=None, token=None):
 
@@ -89,7 +89,7 @@ class SoledadContainer(Container):
             uuid, passphrase, soledad_path, soledad_url,
             cert_path, token)
 
-        self.add_instances(userid, soledad)
+        super(SoledadContainer, self).add_instance(userid, soledad)
 
         data = {'user': userid, 'uuid': uuid, 'token': token,
                 'soledad': soledad}
@@ -202,9 +202,9 @@ class SoledadService(HookableService):
 
 class KeymanagerContainer(Container):
 
-    def __init__(self, basedir):
+    def __init__(self, service=None, basedir=DEFAULT_BASEDIR):
         self._basedir = os.path.expanduser(basedir)
-        super(KeymanagerContainer, self).__init__()
+        super(KeymanagerContainer, self).__init__(service=service)
 
     def add_instance(self, userid, token, uuid, soledad):
 
diff --git a/src/leap/bitmask/core/service.py b/src/leap/bitmask/core/service.py
index fca51048..99132c2d 100644
--- a/src/leap/bitmask/core/service.py
+++ b/src/leap/bitmask/core/service.py
@@ -25,19 +25,30 @@ from twisted.python import log
 
 from leap.bitmask import __version__
 from leap.bitmask.core import configurable
-from leap.bitmask.core import mail_services
 from leap.bitmask.core import _zmq
 from leap.bitmask.core import flags
-from leap.bonafide.service import BonafideService
 from leap.common.events import server as event_server
 # from leap.vpn import EIPService
 
 
+backend = flags.BACKEND
+
+if backend == 'default':
+    from leap.bitmask.core import mail_services
+    from leap.bonafide.service import BonafideService
+elif backend == 'dummy':
+    from leap.bitmask.core.dummy import mail_services
+    from leap.bitmask.core.dummy import BonafideService
+else:
+    raise RuntimeError('Backend not supported')
+
+
 class BitmaskBackend(configurable.ConfigurableService):
 
     def __init__(self, basedir=configurable.DEFAULT_BASEDIR):
 
         configurable.ConfigurableService.__init__(self, basedir)
+        self.core_commands = BackendCommands(self)
 
         def enabled(service):
             return self.get_config('services', service, False, boolean=True)
@@ -117,38 +128,19 @@ class BitmaskBackend(configurable.ConfigurableService):
             service.setServiceParent(self)
             return service
 
-    # General commands for the BitmaskBackend Core Service
-
     def do_stats(self):
-        log.msg('BitmaskCore Service STATS')
-        mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-        return 'BitmaskCore: [Mem usage: %s KB]' % (mem / 1024)
+        return self.core_commands.do_stats()
 
     def do_status(self):
-        # we may want to make this tuple a class member
-        services = ('soledad', 'keymanager', 'mail', 'eip')
-
-        status = {}
-        for name in services:
-            _status = 'stopped'
-            try:
-                if self.getServiceNamed(name).running:
-                    _status = 'running'
-            except KeyError:
-                pass
-            status[name] = _status
-        status['backend'] = flags.BACKEND
-
-        return json.dumps(status)
+        return self.core_commands.do_status()
 
     def do_version(self):
-        version = __version__
-        return 'BitmaskCore: %s' % version
+        return self.core_commands.do_version()
 
     def do_shutdown(self):
-        self.stopService()
-        reactor.callLater(1, reactor.stop)
-        return 'shutting down...'
+        return self.core_commands.do_shutdown()
+
+    # Service Toggling
 
     def do_enable_service(self, service):
         assert service in self.service_names
@@ -175,3 +167,43 @@ class BitmaskBackend(configurable.ConfigurableService):
         # TODO -- should stop also?
         self.set_config('services', service, 'False')
         return 'ok'
+
+
+class BackendCommands(object):
+
+    """
+    General commands for the BitmaskBackend Core Service.
+    """
+
+    def __init__(self, core):
+        self.core = core
+
+    def do_status(self):
+        # we may want to make this tuple a class member
+        services = ('soledad', 'keymanager', 'mail', 'eip')
+
+        status = {}
+        for name in services:
+            _status = 'stopped'
+            try:
+                if self.core.getServiceNamed(name).running:
+                    _status = 'running'
+            except KeyError:
+                pass
+            status[name] = _status
+        status['backend'] = flags.BACKEND
+
+        return json.dumps(status)
+
+    def do_version(self):
+        return {'version_core': __version__}
+
+    def do_stats(self):
+        log.msg('BitmaskCore Service STATS')
+        mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        return {'mem_usage': '%s KB' % (mem / 1024)}
+
+    def do_shutdown(self):
+        self.core.stopService()
+        reactor.callLater(1, reactor.stop)
+        return {'shutdown': 'ok'}