[test] add e2e test for incoming mail pipeline

I had to include part of the bonafide source code because it was the
easiest way to interact with the webapp.

Closes: #8941

8 files changed, 1087 insertions, 0 deletions

diff --git a/tests/e2e/bonafide/__init__.py b/tests/e2e/bonafide/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/e2e/bonafide/__init__.py
diff --git a/tests/e2e/bonafide/_http.py b/tests/e2e/bonafide/_http.py
new file mode 100644
index 00000000..095aca37
--- /dev/null
+++ b/tests/e2e/bonafide/_http.py
@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# _http.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+twisted.web utils for bonafide.
+"""
+import cookielib
+import urllib
+
+from twisted.internet import defer, protocol, reactor
+from twisted.internet.ssl import Certificate
+from twisted.python.filepath import FilePath
+from twisted.web.client import Agent, CookieAgent
+from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import IBodyProducer
+from zope.interface import implements
+
+
+def cookieAgentFactory(verify_path, connectTimeout=30):
+    customPolicy = BrowserLikePolicyForHTTPS(
+        Certificate.loadPEM(FilePath(verify_path).getContent()))
+    agent = Agent(reactor, customPolicy, connectTimeout=connectTimeout)
+    cookiejar = cookielib.CookieJar()
+    return CookieAgent(agent, cookiejar)
+
+
+def httpRequest(agent, url, values={}, headers={}, method='POST', token=None):
+    data = ''
+    if values:
+        data = urllib.urlencode(values)
+        headers['Content-Type'] = ['application/x-www-form-urlencoded']
+
+    if token:
+        headers['Authorization'] = ['Token token="%s"' % (bytes(token))]
+
+    def handle_response(response):
+        # print "RESPONSE CODE", response.code
+        if response.code == 204:
+            d = defer.succeed('')
+        else:
+            class SimpleReceiver(protocol.Protocol):
+                def __init__(s, d):
+                    s.buf = ''
+                    s.d = d
+
+                def dataReceived(s, data):
+                    s.buf += data
+
+                def connectionLost(s, reason):
+                    # TODO: test if reason is twisted.web.client.ResponseDone,
+                    # if not, do an errback
+                    s.d.callback(s.buf)
+            d = defer.Deferred()
+            response.deliverBody(SimpleReceiver(d))
+        return d
+
+    d = agent.request(method, url, Headers(headers),
+                      StringProducer(data) if data else None)
+    d.addCallback(handle_response)
+    return d
+
+
+class StringProducer(object):
+
+    implements(IBodyProducer)
+
+    def __init__(self, body):
+        self.body = body
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return defer.succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
diff --git a/tests/e2e/bonafide/_srp.py b/tests/e2e/bonafide/_srp.py
new file mode 100644
index 00000000..1ec40d82
--- /dev/null
+++ b/tests/e2e/bonafide/_srp.py
@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+# _srp.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+SRP Authentication.
+"""
+
+from twisted.logger import Logger
+
+import binascii
+import json
+
+import srp
+
+
+log = Logger()
+
+
+class SRPAuthMechanism(object):
+
+    """
+    Implement a protocol-agnostic SRP Authentication mechanism.
+    """
+
+    def __init__(self, username, password):
+        self.username = username
+        self.srp_user = srp.User(username, password,
+                                 srp.SHA256, srp.NG_1024)
+        _, A = self.srp_user.start_authentication()
+        self.A = A
+        self.M = None
+        self.M2 = None
+
+    def get_handshake_params(self):
+        return {'login': bytes(self.username),
+                'A': binascii.hexlify(self.A)}
+
+    def process_handshake(self, handshake_response):
+        challenge = json.loads(handshake_response)
+        self._check_for_errors(challenge)
+        salt = challenge.get('salt', None)
+        B = challenge.get('B', None)
+        unhex_salt, unhex_B = self._unhex_salt_B(salt, B)
+        self.M = self.srp_user.process_challenge(unhex_salt, unhex_B)
+
+    def get_authentication_params(self):
+        # It looks A is not used server side
+        return {'client_auth': binascii.hexlify(self.M),
+                'A': binascii.hexlify(self.A)}
+
+    def process_authentication(self, authentication_response):
+        auth = json.loads(authentication_response)
+        self._check_for_errors(auth)
+        uuid = auth.get('id', None)
+        token = auth.get('token', None)
+        self.M2 = auth.get('M2', None)
+        self._check_auth_params(uuid, token, self.M2)
+        return uuid, token
+
+    def verify_authentication(self):
+        unhex_M2 = _safe_unhexlify(self.M2)
+        self.srp_user.verify_session(unhex_M2)
+        assert self.srp_user.authenticated()
+
+    def _check_for_errors(self, response):
+        if 'errors' in response:
+            msg = response['errors']['base']
+            raise SRPAuthError(unicode(msg).encode('utf-8'))
+
+    def _unhex_salt_B(self, salt, B):
+        if salt is None:
+            raise SRPAuthNoSalt()
+        if B is None:
+            raise SRPAuthNoB()
+        try:
+            unhex_salt = _safe_unhexlify(salt)
+            unhex_B = _safe_unhexlify(B)
+        except (TypeError, ValueError) as e:
+            raise SRPAuthBadDataFromServer(str(e))
+        return unhex_salt, unhex_B
+
+    def _check_auth_params(self, uuid, token, M2):
+        if not all((uuid, token, M2)):
+            msg = '%s' % str((M2, uuid, token))
+            raise SRPAuthBadDataFromServer(msg)
+
+
+class SRPSignupMechanism(object):
+
+    """
+    Implement a protocol-agnostic SRP Registration mechanism.
+    """
+
+    def get_signup_params(self, username, password, invite=None):
+        salt, verifier = _get_salt_verifier(username, password)
+        user_data = {
+            'user[login]': username,
+            'user[password_salt]': binascii.hexlify(salt),
+            'user[password_verifier]': binascii.hexlify(verifier)}
+        if invite is not None:
+            user_data.update({'user[invite_code]': invite})
+        return user_data
+
+    def process_signup(self, signup_response):
+        signup = json.loads(signup_response)
+        errors = signup.get('errors')
+        if errors:
+            errmsg = json.dumps(errors)
+            log.error('Oops! Errors during signup: {data!r}', data=errmsg)
+            msg = errors.get('invite_code')
+            if msg:
+                msg = msg[0]
+            else:
+                msg = errors.get('login')
+                if msg:
+                    # there is a bug  https://leap.se/code/issues/8504
+                    # the server tells us 'has already been taken' several
+                    # times
+                    msg = 'username ' + msg[0]
+                else:
+                    msg = 'unknown signup error'
+            error = SRPRegistrationError(msg)
+            error.expected = True
+            raise error
+        else:
+            username = signup.get('login')
+            return username
+
+
+class SRPPasswordChangeMechanism(object):
+
+    """
+    Implement a protocol-agnostic SRP password change mechanism.
+    """
+
+    def get_password_params(self, username, password):
+        salt, verifier = _get_salt_verifier(username, password)
+        user_data = {
+            'user[password_salt]': binascii.hexlify(salt),
+            'user[password_verifier]': binascii.hexlify(verifier)}
+        return user_data
+
+
+class SRPRecoveryCodeUpdateMechanism(object):
+
+    """
+    Implement a protocol-agnostic SRP recovery code update mechanism.
+    """
+
+    def get_recovery_code_params(self, username, recovery_code):
+        salt, verifier = _get_salt_verifier(username, recovery_code)
+        user_data = {
+            'user[recovery_code_salt]': binascii.hexlify(salt),
+            'user[recovery_code_verifier]': binascii.hexlify(verifier)}
+        return user_data
+
+
+def _get_salt_verifier(username, password):
+    return srp.create_salted_verification_key(bytes(username), bytes(password),
+                                              srp.SHA256, srp.NG_1024)
+
+
+def _safe_unhexlify(val):
+    return binascii.unhexlify(val) \
+        if (len(val) % 2 == 0) else binascii.unhexlify('0' + val)
+
+
+class SRPAuthError(Exception):
+    """
+    Base exception for srp authentication errors
+    """
+
+
+class SRPAuthNoSalt(SRPAuthError):
+    message = 'The server didn\'t send the salt parameter'
+
+
+class SRPAuthNoB(SRPAuthError):
+    message = 'The server didn\'t send the B parameter'
+
+
+class SRPAuthBadDataFromServer(SRPAuthError):
+    pass
+
+
+class SRPRegistrationError(Exception):
+    pass
diff --git a/tests/e2e/bonafide/cred_srp.py b/tests/e2e/bonafide/cred_srp.py
new file mode 100644
index 00000000..9fcb8d9e
--- /dev/null
+++ b/tests/e2e/bonafide/cred_srp.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+# srp_cred.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Credential module for authenticating SRP requests against the LEAP platform.
+"""
+
+# ----------------- DOC ------------------------------------------------------
+# See examples of cred modules:
+# https://github.com/oubiwann-unsupported/txBrowserID/blob/master/browserid/checker.py
+# http://stackoverflow.com/questions/19171686/book-twisted-network-programming-essentials-example-9-1-does-not-work
+# ----------------- DOC ------------------------------------------------------
+
+from zope.interface import implements, implementer, Interface
+
+from twisted.cred import portal, credentials, error as credError
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.internet import defer
+
+from session import Session
+
+
+@implementer(ICredentialsChecker)
+class SRPCredentialsChecker(object):
+
+    # TODO need to decide if the credentials that we pass here are per provider
+    # or not.
+    # I think it's better to have the credentials passed with the full user_id,
+    # and here split user/provider.
+    # XXX then we need to check if the provider is properly configured, to get
+    # the right api info AND the needed certificates.
+    # XXX might need to initialize credential checker with a ProviderAPI
+
+    credentialInterfaces = (credentials.IUsernamePassword,)
+
+    def requestAvatarId(self, credentials):
+        # TODO If we are already authenticated, we should just
+        # return the session object, somehow.
+        # XXX If not authenticated (ie, no cached credentials?)
+        # we pass credentials to srpauth.authenticate method
+        # another srpauth class should interface with the blocking srpauth
+        # library, and chain all the calls needed to do the handshake.
+        # Therefore:
+        # should keep reference to the srpauth instances somewhere
+        # TODO If we want to return an anonymous user (useful for getting the
+        # anon-vpn cert), we should return an empty tuple from here.
+
+        return defer.maybeDeferred(_get_leap_session(credentials)).addCallback(
+            self._check_srp_auth)
+
+    def _check_srp_auth(session, username):
+        if session.is_authenticated:
+            # is ok! --- should add it to some global cache?
+            return defer.succeed(username)
+        else:
+            return defer.fail(credError.UnauthorizedLogin(
+                "Bad username/password combination"))
+
+
+def _get_leap_session(credentials):
+    session = Session(credentials)
+    d = session.authenticate()
+    d.addCallback(lambda _: session)
+    return d
+
+
+class ILeapUserAvatar(Interface):
+
+    # TODO add attributes for username, uuid, token, session_id
+
+    def logout():
+        """
+        Clean up per-login resource allocated to this avatar.
+        """
+
+
+@implementer(ILeapUserAvatar)
+class LeapUserAvatar(object):
+
+    # TODO initialize with: username, uuid, token, session_id
+    # TODO initialize provider data (for api)
+    # TODO how does this relate to LeapSession? maybe we should get one passed?
+
+    def logout(self):
+
+        # TODO reset the (global?) srpauth object.
+        # https://leap.se/en/docs/design/bonafide#logout
+        # DELETE API_BASE/logout(.json)
+        pass
+
+
+class LeapAuthRealm(object):
+    """
+    The realm corresponds to an application domain and is in charge of avatars,
+    which are network-accessible business logic objects.
+    """
+
+    # TODO should be initialized with provider API objects.
+
+    implements(portal.IRealm)
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+
+        if ILeapUserAvatar in interfaces:
+            # XXX how should we get the details for the requested avatar?
+            avatar = LeapUserAvatar()
+            return ILeapUserAvatar, avatar, avatar.logout
+
+        raise NotImplementedError(
+            "This realm only supports the ILeapUserAvatar interface.")
diff --git a/tests/e2e/bonafide/provider.py b/tests/e2e/bonafide/provider.py
new file mode 100644
index 00000000..1c6fbbcb
--- /dev/null
+++ b/tests/e2e/bonafide/provider.py
@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+# provier.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+LEAP Provider API.
+"""
+
+from copy import deepcopy
+import re
+from urlparse import urlparse
+
+
+"""
+Maximum API version number supported by bonafide
+"""
+MAX_API_VERSION = 1
+
+
+class _MetaActionDispatcher(type):
+
+    """
+    A metaclass that will create dispatcher methods dynamically for each
+    action made available by the LEAP provider API.
+
+    The new methods will be created according to the values contained in an
+    `_actions` dictionary, with the following format::
+
+        {'action_name': (uri_template, method)}
+
+    where `uri_template` is a string that will be formatted with an arbitrary
+    number of keyword arguments.
+
+    Any class that uses this one as its metaclass needs to implement two
+    private methods::
+
+        _get_uri(self, action_name, **extra_params)
+        _get_method(self, action_name)
+
+    Beware that currently they cannot be inherited from bases.
+    """
+
+    def __new__(meta, name, bases, dct):
+
+        def _generate_action_funs(dct):
+            _get_uri = dct['_get_uri']
+            _get_method = dct['_get_method']
+            newdct = deepcopy(dct)
+            actions = dct['_actions']
+
+            def create_uri_fun(action_name):
+                return lambda self, **kw: _get_uri(
+                    self, action_name=action_name, **kw)
+
+            def create_met_fun(action_name):
+                return lambda self: _get_method(
+                    self, action_name=action_name)
+
+            for action in actions:
+                uri, method = actions[action]
+                _action_uri = 'get_%s_uri' % action
+                _action_met = 'get_%s_method' % action
+                newdct[_action_uri] = create_uri_fun(action)
+                newdct[_action_met] = create_met_fun(action)
+            return newdct
+
+        newdct = _generate_action_funs(dct)
+        return super(_MetaActionDispatcher, meta).__new__(
+            meta, name, bases, newdct)
+
+
+class BaseProvider(object):
+
+    def __init__(self, netloc, version=1):
+        parsed = urlparse(netloc)
+        if parsed.scheme != 'https':
+            raise ValueError(
+                'BaseProvider needs to be passed a url with https scheme')
+        self.netloc = parsed.netloc
+
+        self.version = version
+        if version > MAX_API_VERSION:
+            self.version = MAX_API_VERSION
+
+    def get_hostname(self):
+        return urlparse(self._get_base_url()).hostname
+
+    def _get_base_url(self):
+        return "https://{0}/{1}".format(self.netloc, self.version)
+
+
+class Api(BaseProvider):
+    """
+    An object that has all the information that a client needs to communicate
+    with the remote methods exposed by the web API of a LEAP provider.
+
+    The actions are described in https://leap.se/bonafide
+
+    By using the _MetaActionDispatcher as a metaclass, the _actions dict will
+    be translated dynamically into a set of instance methods that will allow
+    getting the uri and method for each action.
+
+    The keyword arguments specified in the format string will automatically
+    raise a KeyError if the needed keyword arguments are not passed to the
+    dynamically created methods.
+    """
+
+    # TODO when should the provider-api object be created?
+    # TODO pass a Provider object to constructor, with autoconf flag.
+    # TODO make the actions attribute depend on the api version
+    # TODO missing UPDATE USER RECORD
+
+    __metaclass__ = _MetaActionDispatcher
+    _actions = {
+        'signup': ('users', 'POST'),
+        'update_user': ('users/{uid}', 'PUT'),
+        'handshake': ('sessions', 'POST'),
+        'authenticate': ('sessions/{login}', 'PUT'),
+        'logout': ('logout', 'DELETE'),
+        'vpn_cert': ('cert', 'POST'),
+        'smtp_cert': ('smtp_cert', 'POST'),
+    }
+
+    # Methods expected by the dispatcher metaclass
+
+    def _get_uri(self, action_name, **extra_params):
+        resource, _ = self._actions.get(action_name)
+        uri = '{0}/{1}'.format(
+            bytes(self._get_base_url()),
+            bytes(resource)).format(**extra_params)
+        return uri
+
+    def _get_method(self, action_name):
+        _, method = self._actions.get(action_name)
+        return method
+
+
+class Discovery(BaseProvider):
+    """
+    Discover basic information about a provider, including the provided
+    services.
+    """
+
+    __metaclass__ = _MetaActionDispatcher
+    _actions = {
+        'provider_info': ('provider.json', 'GET'),
+        'configs': ('1/configs.json', 'GET'),
+    }
+
+    def _get_base_url(self):
+        return "https://{0}".format(self.netloc)
+
+    def get_base_uri(self):
+        return self._get_base_url()
+
+    # Methods expected by the dispatcher metaclass
+
+    def _get_uri(self, action_name, **extra_params):
+        resource, _ = self._actions.get(action_name)
+        uri = '{0}/{1}'.format(
+            bytes(self._get_base_url()),
+            bytes(resource)).format(**extra_params)
+        return uri
+
+    def _get_method(self, action_name):
+        _, method = self._actions.get(action_name)
+        return method
+
+
+def validate_username(username):
+    accepted_characters = '^[a-z0-9\-\_\.]*$'
+    if not re.match(accepted_characters, username):
+        raise ValueError('Only lowercase letters, digits, . - and _ allowed.')
diff --git a/tests/e2e/bonafide/session.py b/tests/e2e/bonafide/session.py
new file mode 100644
index 00000000..e4129609
--- /dev/null
+++ b/tests/e2e/bonafide/session.py
@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+# session.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+LEAP Session management.
+"""
+from twisted.internet import defer, reactor
+from twisted.logger import Logger
+
+from . import _srp
+from . import provider
+from ._http import httpRequest, cookieAgentFactory
+
+
+OK = 'ok'
+
+
+def _auth_required(func):
+    """
+    Decorate a method so that it will not be called if the instance
+    attribute `is_authenticated` does not evaluate to True.
+    """
+    def decorated(*args, **kwargs):
+        instance = args[0]
+        allowed = getattr(instance, 'is_authenticated')
+        if not allowed:
+            raise RuntimeError('This method requires authentication')
+        return func(*args, **kwargs)
+    return decorated
+
+
+class Session(object):
+
+    log = Logger()
+
+    def __init__(self, credentials, api, provider_cert):
+        # TODO check if an anonymous credentials is passed.
+        # TODO move provider_cert to api object.
+        # On creation, it should be able to retrieve all the info it needs
+        # (calling bootstrap).
+        # TODO could get a "provider" object instead.
+        # this provider can have an api attribute,
+        # and a "autoconfig" attribute passed on initialization.
+        # TODO get a file-descriptor for password if not in credentials
+        # TODO merge self._request with config.Provider._http_request ?
+
+        self.username = credentials.username
+        self.password = credentials.password
+        self._provider_cert = provider_cert
+        self._api = api
+        self._initialize_session()
+
+    def _initialize_session(self):
+        self._agent = cookieAgentFactory(self._provider_cert)
+        username = self.username or ''
+        password = self.password or ''
+        self._srp_auth = _srp.SRPAuthMechanism(username, password)
+        self._srp_signup = _srp.SRPSignupMechanism()
+        self._srp_password = _srp.SRPPasswordChangeMechanism()
+        self._srp_recovery_code = _srp.SRPRecoveryCodeUpdateMechanism()
+        self._token = None
+        self._uuid = None
+
+    # Session
+
+    @property
+    def token(self):
+        return self._token
+
+    @property
+    def uuid(self):
+        return self._uuid
+
+    @property
+    def is_authenticated(self):
+        return self._srp_auth.srp_user.authenticated()
+
+    @defer.inlineCallbacks
+    def authenticate(self):
+        uri = self._api.get_handshake_uri()
+        met = self._api.get_handshake_method()
+        self.log.debug('%s to %s' % (met, uri))
+        params = self._srp_auth.get_handshake_params()
+
+        handshake = yield self._request(self._agent, uri, values=params,
+                                        method=met)
+
+        self._srp_auth.process_handshake(handshake)
+        uri = self._api.get_authenticate_uri(login=self.username)
+        met = self._api.get_authenticate_method()
+
+        self.log.debug('%s to %s' % (met, uri))
+        params = self._srp_auth.get_authentication_params()
+
+        auth = yield self._request(self._agent, uri, values=params,
+                                   method=met)
+
+        uuid, token = self._srp_auth.process_authentication(auth)
+        self._srp_auth.verify_authentication()
+
+        self._uuid = uuid
+        self._token = token
+        defer.returnValue(OK)
+
+    @_auth_required
+    @defer.inlineCallbacks
+    def logout(self):
+        uri = self._api.get_logout_uri()
+        met = self._api.get_logout_method()
+        yield self._request(self._agent, uri, method=met)
+        self.username = None
+        self.password = None
+        self._initialize_session()
+        defer.returnValue(OK)
+
+    @_auth_required
+    @defer.inlineCallbacks
+    def change_password(self, password):
+        uri = self._api.get_update_user_uri(uid=self._uuid)
+        met = self._api.get_update_user_method()
+        params = self._srp_password.get_password_params(
+            self.username, password)
+        yield self._request(self._agent, uri, values=params, method=met)
+        self.password = password
+        self._srp_auth = _srp.SRPAuthMechanism(self.username, password)
+        defer.returnValue(OK)
+
+    @_auth_required
+    @defer.inlineCallbacks
+    def update_recovery_code(self, recovery_code):
+        uri = self._api.get_update_user_uri(uid=self._uuid)
+        met = self._api.get_update_user_method()
+        params = self._srp_recovery_code.get_recovery_code_params(
+            self.username, recovery_code)
+        update = yield self._request(self._agent, uri, values=params,
+                                     method=met)
+        defer.returnValue(update)
+
+    # User certificates
+
+    def get_vpn_cert(self):
+        # TODO pass it to the provider object so that it can save it in the
+        # right path.
+        uri = self._api.get_vpn_cert_uri()
+        met = self._api.get_vpn_cert_method()
+        return self._request(self._agent, uri, method=met)
+
+    @_auth_required
+    def get_smtp_cert(self):
+        # TODO pass it to the provider object so that it can save it in the
+        # right path.
+        uri = self._api.get_smtp_cert_uri()
+        met = self._api.get_smtp_cert_method()
+        print met, "to", uri
+        return self._request(self._agent, uri, method=met)
+
+    def _request(self, *args, **kw):
+        # TODO: we are not pinning the TLS cert of the API
+        # maybe we can use leap.common.http
+        kw['token'] = self._token
+        return httpRequest(*args, **kw)
+
+    # User management
+
+    @defer.inlineCallbacks
+    def signup(self, username, password, invite=None):
+        # XXX should check that it_IS_NOT_authenticated
+        provider.validate_username(username)
+        uri = self._api.get_signup_uri()
+        met = self._api.get_signup_method()
+        params = self._srp_signup.get_signup_params(
+            username, password, invite)
+
+        signup = yield self._request(self._agent, uri, values=params,
+                                     method=met)
+        registered_user = self._srp_signup.process_signup(signup)
+        self.username = username
+        self.password = password
+        defer.returnValue((OK, registered_user))
+
+    @_auth_required
+    def update_user_record(self):
+        # FIXME to be implemented
+        pass
+
+    # Authentication-protected configuration
+
+    @defer.inlineCallbacks
+    def fetch_provider_configs(self, uri, path):
+        config = yield self._request(self._agent, uri)
+        with open(path, 'w') as cf:
+            cf.write(config)
+        defer.returnValue('ok')
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+    from twisted.cred.credentials import UsernamePassword
+
+    log = Logger()
+
+    if len(sys.argv) != 4:
+        print "Usage:", sys.argv[0], "provider", "username", "password"
+        sys.exit()
+    _provider, username, password = sys.argv[1], sys.argv[2], sys.argv[3]
+    api = provider.Api('https://api.%s:4430' % _provider)
+    credentials = UsernamePassword(username, password)
+    cdev_pem = os.path.expanduser(
+        '~/.config/leap/providers/%s/keys/ca/cacert.pem' % _provider)
+    session = Session(credentials, api, cdev_pem)
+
+    def print_result(result):
+        print result
+        return result
+
+    def cbShutDown(ignored):
+        reactor.stop()
+
+    def auth_eb(failure):
+        log.error(failure)
+
+    d = session.authenticate()
+    d.addCallback(print_result)
+    d.addErrback(auth_eb)
+
+    d.addCallback(lambda _: session.get_smtp_cert())
+    # d.addCallback(lambda _: session.get_vpn_cert())
+    d.addCallback(print_result)
+    d.addErrback(auth_eb)
+
+    d.addCallback(lambda _: session.logout())
+    d.addErrback(auth_eb)
+    d.addBoth(cbShutDown)
+    reactor.run()
diff --git a/tests/e2e/test_incoming_mail_pipeline.py b/tests/e2e/test_incoming_mail_pipeline.py
new file mode 100644
index 00000000..a5201a46
--- /dev/null
+++ b/tests/e2e/test_incoming_mail_pipeline.py
@@ -0,0 +1,56 @@
+# This script does the following:
+#
+# - create a user using bonafide and and invite code given as an environment
+#   variable.
+#
+# - create and upload an OpenPGP key manually, as that would be
+#   a responsibility of bitmask-dev.
+#
+# - send an email to the user using sendmail, with a secret in the body.
+#
+# - start a soledad client using the created user.
+#
+# - download pending blobs. There should be only one.
+#
+# - look inside the blob, parse the email message.
+#
+# - compare the token in the incoming message with the token in the sent
+#   message and succeed if the tokens are the same.
+#
+# - delete the user (even if the test failed). (TODO)
+
+
+import pytest
+
+from utils import get_session
+from utils import gen_key
+from utils import put_key
+from utils import send_email
+from utils import get_incoming_fd
+from utils import get_received_secret
+
+
+@pytest.inlineCallbacks
+def test_incoming_mail_pipeline(soledad_client, tmpdir):
+
+    # create a user and login
+    session = yield get_session(tmpdir)
+
+    # create a OpenPGP key and upload it
+    key = gen_key(session.username)
+    yield put_key(session.uuid, session.token, str(key.pubkey))
+
+    # get a soledad client for that user
+    client = soledad_client(
+        uuid=session.uuid,
+        passphrase='123',
+        token=session.token)
+
+    # send the email
+    sent_secret = send_email(session.username)
+
+    # check the incoming blob and compare sent and received secrets
+    fd = yield get_incoming_fd(client)
+    received_secret = get_received_secret(key, fd)
+    assert sent_secret == received_secret
+    # TODO: delete user in the end
diff --git a/tests/e2e/utils.py b/tests/e2e/utils.py
new file mode 100644
index 00000000..3fca28b8
--- /dev/null
+++ b/tests/e2e/utils.py
@@ -0,0 +1,178 @@
+import email
+import json
+import os
+import pytest
+import random
+import time
+import treq
+import urllib
+
+from string import ascii_lowercase
+from email.mime.text import MIMEText
+from subprocess import Popen, PIPE
+
+from twisted.internet import reactor
+from twisted.internet.defer import returnValue
+from twisted.web.client import Agent
+from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.internet.ssl import Certificate
+from twisted.cred.credentials import UsernamePassword
+
+import pgpy
+from pgpy.constants import (
+    PubKeyAlgorithm,
+    KeyFlags,
+    HashAlgorithm,
+    SymmetricKeyAlgorithm,
+    CompressionAlgorithm
+)
+
+from bonafide import provider
+from bonafide.session import Session
+
+from leap.soledad.common.blobs import Flags
+
+
+_provider = 'cdev.bitmask.net'
+
+uri = "https://api.%s:4430/1/" % _provider
+ca = "https://%s/ca.crt" % _provider
+
+
+random.seed()
+
+
+#
+# session management: user creation and authentication
+#
+
+def _get_invite_code():
+    invite = os.environ.get('INVITE_CODE')
+    if not invite:
+        raise Exception('The INVITE_CODE environment variable is empty, but '
+                        'we need it set to interact with the provider.')
+    return invite
+
+
+@pytest.inlineCallbacks
+def _get_ca_file(tmpdir):
+    response = yield treq.get(ca)
+    pemdata = yield response.text()
+    fname = os.path.join(tmpdir.strpath, 'cacert.pem')
+    with open(fname, 'w') as f:
+        f.write(pemdata)
+    returnValue(fname)
+
+
+@pytest.inlineCallbacks
+def get_session(tmpdir):
+    # setup user params
+    invite = _get_invite_code()
+    username = ''.join(random.choice(ascii_lowercase) for i in range(20))
+    # users starting with "test_user" get removed by cron on a regular basis
+    username = 'tmp_user_e2e_' + username
+    passphrase = ''.join(random.choice(ascii_lowercase) for i in range(20))
+
+    # create user and login
+    credentials = UsernamePassword(username, passphrase)
+    api = provider.Api('https://api.%s:4430' % _provider)
+    cdev_pem = yield _get_ca_file(tmpdir)
+    session = Session(credentials, api, cdev_pem)
+    print("creating user")
+    yield session.signup(username, passphrase, invite=invite)
+    print("logging in")
+    yield session.authenticate()
+    returnValue(session)
+
+
+#
+# OpenPGP key creation and upload
+#
+
+def gen_key(username):
+    print("generating OpenPGP key pair")
+    key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 4096)
+    uid = pgpy.PGPUID.new(username, email='%s@%s' % (username, _provider))
+    key.add_uid(
+        uid,
+        usage={KeyFlags.EncryptCommunications},
+        hashes=[HashAlgorithm.SHA512],
+        ciphers=[SymmetricKeyAlgorithm.AES256],
+        compression=[CompressionAlgorithm.Uncompressed]
+    )
+    return key
+
+
+@pytest.inlineCallbacks
+def _get_http_client():
+    response = yield treq.get(ca)
+    pemdata = yield response.text()
+    cert = Certificate.loadPEM(pemdata)
+    policy = BrowserLikePolicyForHTTPS(trustRoot=cert)
+    agent = Agent(reactor, contextFactory=policy)
+    client = treq.client.HTTPClient(agent)
+    returnValue(client)
+
+
+@pytest.inlineCallbacks
+def put_key(uuid, token, data):
+    print("uploading public key to server")
+    client = yield _get_http_client()
+    headers = {
+        'Authorization': [str('Token token=%s' % token)],
+        'Content-Type': ['application/x-www-form-urlencoded'],
+    }
+    data = str(urllib.urlencode({'user[public_key]': data}))
+    response = yield client.put(
+        '%s/users/%s.json' % (uri, uuid),
+        headers=headers,
+        data=data)
+    assert response.code == 204
+
+
+#
+# mail sending
+#
+
+def send_email(username):
+    address = "%s@%s" % (username, _provider)
+    print("sending email to %s" % address)
+    secret = ''.join(random.choice(ascii_lowercase) for i in range(20))
+    msg = MIMEText(secret)
+    msg["To"] = address
+    msg["Subject"] = "e2e test token"
+    p = Popen(["/usr/sbin/sendmail", "-t"], stdin=PIPE)
+    p.communicate(msg.as_string())
+    return secret
+
+
+#
+# incoming message retrieval
+#
+
+@pytest.inlineCallbacks
+def get_incoming_fd(client):
+    pending = []
+    attempts = 1
+    while not pending:
+        print("attempting to fetch incoming blob (%d/10)" % attempts)
+        pending = yield client.blobmanager.remote_list(
+            namespace='MX', filter_flags=Flags.PENDING)
+        if not pending and attempts == 10:
+            raise Exception("Timed out waiting for message to get delivered.")
+        attempts += 1
+        time.sleep(1)
+    assert len(pending) == 1
+    fd = yield client.blobmanager.get(pending.pop(), namespace='MX')
+    returnValue(fd)
+
+
+def get_received_secret(key, fd):
+    print("decoding incoming blob to get the secret")
+    encrypted = pgpy.PGPMessage.from_blob(fd.read())
+    decrypted = key.decrypt(encrypted)
+    doc_content = json.loads(decrypted.message)
+    content = doc_content['content']
+    email_message = email.message_from_string(content)
+    received_secret = email_message.get_payload().strip()
+    return received_secret