[feat] add web/js core API

Implements http REST API for the core and bitmask.js generic library to
use this API. For events it uses long polling.

- Resolves: #8265

8 files changed, 482 insertions, 50 deletions

diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst
new file mode 100644
index 0000000..26e9cdc
--- /dev/null
+++ b/changes/next-changelog.rst
@@ -0,0 +1,31 @@
+0.10.0 - xxx
++++++++++++++++++++++++++++++++
+
+Please add lines to this file, they will be moved to the CHANGELOG.rst during
+the next release.
+
+There are two template lines for each category, use them as reference.
+
+I've added a new category `Misc` so we can track doc/style/packaging stuff.
+
+Features
+~~~~~~~~
+- `#7965 <https://leap.se/code/issues/7965>`_: Add basic keymanagement to the cli.
+- `#8265 <https://leap.se/code/issues/8265>`_: Add a REST API and bitmask.js library for it.
+- Use mail_auth token in the core instead of imap/smtp tokens.
+
+- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
+- New feature without related issue number.
+
+Bugfixes
+~~~~~~~~
+- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
+- Bugfix without related issue number.
+
+Misc
+~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
+- Some change without issue number.
+
+Known Issues
+~~~~~~~~~~~~
diff --git a/src/leap/bitmask/core/_web.py b/src/leap/bitmask/core/_web.py
new file mode 100644
index 0000000..5107ad1
--- /dev/null
+++ b/src/leap/bitmask/core/_web.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# _web.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+HTTP REST Dispatcher Service.
+"""
+
+import json
+import os
+import pkg_resources
+
+from twisted.internet import reactor
+from twisted.application import service
+
+from twisted.web.resource import Resource
+from twisted.web.server import Site, NOT_DONE_YET
+from twisted.web.static import File
+
+from leap.bitmask.core.dispatcher import CommandDispatcher
+
+
+class HTTPDispatcherService(service.Service):
+
+    """
+    A Dispatcher for BitmaskCore exposing a REST API.
+    """
+
+    def __init__(self, core, port=7070, debug=False):
+        self._core = core
+        self.port = port
+        self.debug = debug
+
+    def startService(self):
+        webdir = os.path.abspath(
+            pkg_resources.resource_filename("leap.bitmask.core", "web"))
+        root = File(webdir)
+
+        api = Api(CommandDispatcher(self._core))
+        root.putChild(u"API", api)
+
+        site = Site(root)
+        self.site = site
+        self.listener = reactor.listenTCP(self.port, site,
+                                          interface='127.0.0.1')
+
+    def stopService(self):
+        self.site.stopFactory()
+        self.listener.stopListening()
+
+
+class Api(Resource):
+    isLeaf = True
+
+    def __init__(self, dispatcher):
+        Resource.__init__(self)
+        self.dispatcher = dispatcher
+
+    def render_POST(self, request):
+        command = request.uri.split('/')[2:]
+        params = request.content.getvalue()
+        if params:
+            # json.loads returns unicode strings and the rest of the code
+            # expects strings. This 'str(param)' conversion can be removed
+            # if we move to python3
+            for param in json.loads(params):
+                command.append(str(param))
+
+        d = self.dispatcher.dispatch(command)
+        d.addCallback(self._write_response, request)
+        return NOT_DONE_YET
+
+    def _write_response(self, response, request):
+        request.setHeader('Content-Type', 'application/json')
+        request.write(response)
+        request.finish()
diff --git a/src/leap/bitmask/core/configurable.py b/src/leap/bitmask/core/configurable.py
index 8e33de9..8bd2ecf 100644
--- a/src/leap/bitmask/core/configurable.py
+++ b/src/leap/bitmask/core/configurable.py
@@ -38,7 +38,7 @@ class MissingConfigEntry(Exception):
 class ConfigurableService(service.MultiService):
 
     config_file = u"bitmaskd.cfg"
-    service_names = ('mail', 'eip', 'zmq', 'web')
+    service_names = ('mail', 'eip', 'zmq', 'web', 'websockets')
 
     def __init__(self, basedir=DEFAULT_BASEDIR):
         service.MultiService.__init__(self)
@@ -102,5 +102,6 @@ DEFAULT_CONFIG = """
 mail = True
 eip = True
 zmq = True
-web = False
+web = True
+websockets = False
 """
diff --git a/src/leap/bitmask/core/dispatcher.py b/src/leap/bitmask/core/dispatcher.py
index e81cad6..0ea4e47 100644
--- a/src/leap/bitmask/core/dispatcher.py
+++ b/src/leap/bitmask/core/dispatcher.py
@@ -19,9 +19,18 @@ Command dispatcher.
 """
 import json
 
+try:
+    from Queue import Queue
+except ImportError:
+    from queue import Queue
+
 from twisted.internet import defer
 from twisted.python import failure, log
 
+from leap.common.events import register_async as register
+from leap.common.events import unregister_async as unregister
+from leap.common.events import catalog
+
 from .api import APICommand, register_method
 
 
@@ -45,25 +54,21 @@ class UserCmd(SubCommand):
     @register_method("{'srp_token': unicode, 'uuid': unicode}")
     def do_AUTHENTICATE(self, bonafide, *parts):
         user, password = parts[2], parts[3]
-        d = defer.maybeDeferred(bonafide.do_authenticate, user, password)
-        return d
+        return bonafide.do_authenticate(user, password)
 
     @register_method("{'signup': 'ok', 'user': str}")
     def do_SIGNUP(self, bonafide, *parts):
         user, password = parts[2], parts[3]
-        d = defer.maybeDeferred(bonafide.do_signup, user, password)
-        return d
+        return bonafide.do_signup(user, password)
 
     @register_method("{'logout': 'ok'}")
     def do_LOGOUT(self, bonafide, *parts):
         user = parts[2]
-        d = defer.maybeDeferred(bonafide.do_logout, user)
-        return d
+        bonafide.do_logout(user)
 
     @register_method('str')
     def do_ACTIVE(self, bonafide, *parts):
-        d = defer.maybeDeferred(bonafide.do_get_active_user)
-        return d
+        return bonafide.do_get_active_user()
 
 
 class EIPCmd(SubCommand):
@@ -209,6 +214,45 @@ class KeysCmd(SubCommand):
         return d
 
 
+class EventsCmd(SubCommand):
+
+    label = 'events'
+
+    def __init__(self):
+        self.queue = Queue()
+        self.waiting = []
+
+    @register_method("")
+    def do_REGISTER(self, _, *parts, **kw):
+        event = getattr(catalog, parts[2])
+        register(event, self._callback)
+
+    @register_method("")
+    def do_UNREGISTER(self, _, *parts, **kw):
+        event = getattr(catalog, parts[2])
+        unregister(event)
+
+    @register_method("(str, [])")
+    def do_POLL(self, _, *parts, **kw):
+        if not self.queue.empty():
+            return self.queue.get()
+
+        d = defer.Deferred()
+        self.waiting.append(d)
+        return d
+
+    @register_method("")
+    def _callback(self, event, *content):
+        payload = (str(event), content)
+        if not self.waiting:
+            self.queue.put(payload)
+            return
+
+        while self.waiting:
+            d = self.waiting.pop()
+            d.callback(payload)
+
+
 class CommandDispatcher(object):
 
     __metaclass__ = APICommand
@@ -222,6 +266,7 @@ class CommandDispatcher(object):
         self.subcommand_eip = EIPCmd()
         self.subcommand_mail = MailCmd()
         self.subcommand_keys = KeysCmd()
+        self.subcommand_events = EventsCmd()
 
     # XXX --------------------------------------------
     # TODO move general services to another subclass
@@ -302,6 +347,12 @@ class CommandDispatcher(object):
         d.addCallbacks(_format_result, _format_error)
         return d
 
+    def do_EVENTS(self, *parts):
+        dispatch = self.subcommand_events.dispatch
+        d = dispatch(None, *parts)
+        d.addCallbacks(_format_result, _format_error)
+        return d
+
     def dispatch(self, msg):
         cmd = msg[0]
 
diff --git a/src/leap/bitmask/core/service.py b/src/leap/bitmask/core/service.py
index 79465a8..ee918d4 100644
--- a/src/leap/bitmask/core/service.py
+++ b/src/leap/bitmask/core/service.py
@@ -26,6 +26,7 @@ from twisted.python import log
 from leap.bitmask import __version__
 from leap.bitmask.core import configurable
 from leap.bitmask.core import _zmq
+from leap.bitmask.core import _web
 from leap.bitmask.core import flags
 from leap.common.events import server as event_server
 # from leap.vpn import EIPService
@@ -72,6 +73,9 @@ class BitmaskBackend(configurable.ConfigurableService):
         if enabled('web'):
             on_start(self.init_web)
 
+        if enabled('websockets'):
+            on_start(self.init_websockets)
+
     def init_events(self):
         event_server.ensure_server()
 
@@ -115,6 +119,10 @@ class BitmaskBackend(configurable.ConfigurableService):
         zs.setServiceParent(self)
 
     def init_web(self):
+        http = _web.HTTPDispatcherService(self)
+        http.setServiceParent(self)
+
+    def init_websockets(self):
         from leap.bitmask.core import websocket
         ws = websocket.WebSocketsDispatcherService(self)
         ws.setServiceParent(self)
@@ -160,6 +168,8 @@ class BitmaskBackend(configurable.ConfigurableService):
         elif service == 'web':
             self.init_web()
 
+        self.init_http()
+
         return 'ok'
 
     def do_disable_service(self, service):
diff --git a/src/leap/bitmask/core/web/bitmask.js b/src/leap/bitmask/core/web/bitmask.js
new file mode 100644
index 0000000..39e677f
--- /dev/null
+++ b/src/leap/bitmask/core/web/bitmask.js
@@ -0,0 +1,257 @@
+// bitmask.js
+// Copyright (C) 2016 LEAP
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * bitmask object
+ *
+ * Contains all the bitmask API mapped by sections
+ * - user. User management like login, creation, ...
+ * - mail. Email service control.
+ * - keys. Keyring operations.
+ * - events. For registering to events.
+ *
+ * Every function returns a Promise that will be triggered once the request is
+ * finished or will fail if there was any error. Errors are always user readable
+ * strings.
+ */
+var bitmask = function(){
+    var event_handlers = {};
+    
+    function call(command) {
+        var url = '/API/' + command.slice(0, 2).join('/');
+        var data = JSON.stringify(command.slice(2));
+
+        return new Promise(function(resolve, reject) {
+            var req = new XMLHttpRequest();
+            req.open('POST', url);
+            
+            req.onload = function() {
+                if (req.status == 200) {
+                    parseResponse(req.response, resolve, reject);
+                }
+                else {
+                    reject(Error(req.statusText));
+                }
+            };
+            
+            req.onerror = function() {
+                reject(Error("Network Error"));
+            };
+            
+            req.send(data);
+        });
+    };
+
+    function parseResponse(raw_response, resolve, reject) {
+        var response = JSON.parse(raw_response);
+        if (response.error === null) {
+            resolve(response.result);
+        } else {
+            reject(response.error);
+        }
+    };
+
+    function event_polling() {
+        call(['events', 'poll']).then(function(response) {
+            if (response !== null) {
+                evnt = response[0];
+                content = response[1];
+                if (evnt in event_handlers) {
+                    event_handlers[evnt](evnt, content);
+                }
+            }
+            event_polling();
+        }, function(error) {
+            setTimeout(event_polling, 5000);
+        });
+    };
+    event_polling();
+
+    function private_str(priv) {
+        if (priv) {
+            return 'private'
+        }
+        return 'public'
+    };
+
+    return {
+        /**
+         * uids are of the form user@provider.net
+         */
+        user: {
+            /**
+             * Check wich user is active
+             *
+             * @return {Promise<string>} The uid of the active user
+             */
+            active: function() {
+                return call(['user', 'active']);
+            },
+
+            /**
+             * Register a new user
+             *
+             * @param {string} uid The uid to be created
+             * @param {string} password The user password
+             */
+            create: function(uid, password) {
+                return call(['user', 'create', uid, password]);
+            },
+
+            /**
+             * Login
+             *
+             * @param {string} uid The uid to log in
+             * @param {string} password The user password
+             */
+            auth: function(uid, password) {
+                return call(['user', 'authenticate', uid, password]);
+            },
+
+            /**
+             * Logout
+             *
+             * @param {string} uid The uid to log out.
+             *                     If no uid is provided the active user will be used
+             */
+            logout: function(uid) {
+                if (typeof uid !== 'string') {
+                    uid = "";
+                }
+                return call(['user', 'logout', uid]);
+            }
+        },
+
+        mail: {
+            /**
+             * Check the status of the email service
+             *
+             * @return {Promise<string>} User readable status
+             */
+            status: function() {
+                return call(['mail', 'status']);
+            },
+
+            /**
+             * Get the token of the active user.
+             *
+             * This token is used as password to authenticate in the IMAP and SMTP services.
+             *
+             * @return {Promise<string>} The token
+             */
+            get_token: function() {
+                return call(['mail', 'get-token']);
+            }
+        },
+
+        /**
+         * A KeyObject have the following attributes:
+         *   - address {string} the email address for wich this key is active
+         *   - fingerprint {string} the fingerprint of the key
+         *   - length {number} the size of the key bits
+         *   - private {bool} if the key is private
+         *   - uids {[string]} the uids in the key
+         *   - key_data {string} the key content
+         *   - validation {string} the validation level which this key was found
+         *   - expiry_date {string} date when the key expires
+         *   - refreshed_at {string} date of the last refresh of the key
+         *   - audited_at {string} date of the last audit (unused for now)
+         *   - sign_used {bool} if has being used to checking signatures
+         *   - enc_used {bool} if has being used to encrypt
+         */
+        keys: {
+            /**
+             * List all the keys in the keyring
+             *
+             * @param {boolean} priv Should list private keys?
+             *                       If it's not provided the public ones will be listed.
+             *
+             * @return {Promise<[KeyObject]>} List of keys in the keyring
+             */
+            list: function(priv) {
+                return call(['keys', 'list', private_str(priv)]);
+            },
+
+            /**
+             * Export key
+             *
+             * @param {string} address The email address of the key
+             * @param {boolean} priv Should get the private key?
+             *                       If it's not provided the public one will be fetched.
+             *
+             * @return {Promise<KeyObject>} The key
+             */
+            exprt: function(address, priv) {
+                return call(['keys', 'export', address, private_str(priv)]);
+            },
+
+            /**
+             * Insert key
+             *
+             * @param {string} address The email address of the key
+             * @param {string} rawkey The key material
+             * @param {string} validation The validation level of the key
+             *                            If it's not provided 'Fingerprint' level will be used.
+             *
+             * @return {Promise<KeyObject>} The key
+             */
+            insert: function(address, rawkey, validation) {
+                if (typeof validation !== 'string') {
+                    validation = 'Fingerprint';
+                }
+                return call(['keys', 'insert', address, validation, rawkey]);
+            },
+
+            /**
+             * Delete a key
+             *
+             * @param {string} address The email address of the key
+             * @param {boolean} priv Should get the private key?
+             *                       If it's not provided the public one will be deleted.
+             *
+             * @return {Promise<KeyObject>} The key
+             */
+            del: function(address, priv) {
+                return call(['keys', 'delete', address, private_str(priv)]);
+            }
+        },
+
+        events: {
+            /**
+             * Register func for an event
+             *
+             * @param {string} evnt The event to register
+             * @param {function} func The function that will be called on each event.
+             *                        It has to be like: function(event, content) {}
+             *                        Where content will be a list of strings.
+             */
+            register: function(evnt, func) {
+                event_handlers[evnt] = func;
+                return call(['events', 'register', evnt])
+            },
+
+            /**
+             * Unregister from an event
+             *
+             * @param {string} evnt The event to unregister
+             */
+            unregister: function(evnt) {
+                delete event_handlers[evnt];
+                return call(['events', 'unregister', evnt])
+            }
+        }
+    };
+}();
diff --git a/src/leap/bitmask/core/web/index.html b/src/leap/bitmask/core/web/index.html
index 9490eca..6ac9b29 100644
--- a/src/leap/bitmask/core/web/index.html
+++ b/src/leap/bitmask/core/web/index.html
@@ -2,53 +2,44 @@
 <html>
    <head>
       <title>Bitmask WebSockets Endpoint</title>
+      <script src="bitmask.js"></script>
       <script type="text/javascript">
-         var sock = null;
          var ellog = null;
 
          window.onload = function() {
-
             ellog = document.getElementById('log');
 
-            var wsuri;
-            if (window.location.protocol === "file:") {
-               wsuri = "ws://127.0.0.1:8080/ws";
-            } else {
-               wsuri = "ws://" + window.location.hostname + ":8080/bitmask";
-            }
-
-            if ("WebSocket" in window) {
-               sock = new WebSocket(wsuri);
-            } else if ("MozWebSocket" in window) {
-               sock = new MozWebSocket(wsuri);
-            } else {
-               log("Browser does not support WebSocket!");
-               window.location = "http://autobahn.ws/unsupportedbrowser";
-            }
-
-            if (sock) {
-               sock.onopen = function() {
-                  log("Connected to " + wsuri);
-               }
-
-               sock.onclose = function(e) {
-                  log("Connection closed (wasClean = " + e.wasClean + ", code = " + e.code + ", reason = '" + e.reason + "')");
-                  sock = null;
-               }
+            bitmask.events.register("KEYMANAGER_KEY_FOUND", event_handler);
+         };
 
-               sock.onmessage = function(e) {
-                  log("[res] " + e.data + '\n');
-               }
-            }
+         function login() {
+            var email = document.getElementById('email').value;
+            var password = document.getElementById('password').value;
+            bitmask.user.auth(email, password).then(function(response) {
+                log("We are logged in: ");
+                for (k in response) {
+                    log("    " + k + ": " + response[k]);
+                }
+            }, function(error) {
+                log("Some error ocurred: " + error);
+            });
          };
 
-         function send() {
-            var msg = document.getElementById('message').value;
-            if (sock) {
-               sock.send(msg);
-               log("[cmd] " + msg);
-            } else {
-               log("Not connected.");
+         function logout() {
+            bitmask.user.logout().then(function(response) {
+                log("We are logged out: ");
+                for (k in response) {
+                    log("    " + k + ": " + response[k]);
+                }
+            }, function(error) {
+                log("Some error ocurred: " + error);
+            });
+	 };
+
+         function event_handler(evnt, content) {
+            log("Event: " + evnt);
+            for (i in content) {
+               log("    " + content[i]);
             }
          };
 
@@ -62,9 +53,11 @@
       <h1>Bitmask Control Panel</h1>
       <noscript>You must enable JavaScript</noscript>
       <form>
-         <p>Command: <input id="message" type="text" size="50" maxlength="50" value="status"></p>
+         <p>Email address: <input id="email" type="text" size="50" maxlength="50" value="user@mail.bitmask.net"></p>
+         <p>Password: <input id="password" type="password" size="50" maxlength="50" ></p>
       </form>
-      <button onclick='send();'>Send Command</button>
+      <button onclick='login();'>Log In</button>
+      <button onclick='logout();'>Log Out</button>
       <pre id="log" style="height: 20em; overflow-y: scroll; background-color: #faa;"></pre>
    </body>
 </html>
diff --git a/src/leap/bitmask/core/web/root.py b/src/leap/bitmask/core/web/root.py
deleted file mode 100644
index e69de29..0000000
--- a/src/leap/bitmask/core/web/root.py
+++ /dev/null