diff options
Diffstat (limited to 'src/leap/bitmask/core/mail_services.py')
| -rw-r--r-- | src/leap/bitmask/core/mail_services.py | 672 |
1 files changed, 672 insertions, 0 deletions
diff --git a/src/leap/bitmask/core/mail_services.py b/src/leap/bitmask/core/mail_services.py new file mode 100644 index 00000000..9858d498 --- /dev/null +++ b/src/leap/bitmask/core/mail_services.py @@ -0,0 +1,672 @@ +""" +Mail services. + +This is quite moving work still. +This should be moved to the different packages when it stabilizes. +""" +import json +import os +from glob import glob +from collections import defaultdict +from collections import namedtuple + +from twisted.application import service +from twisted.internet import defer +from twisted.python import log + +from leap.bonafide import config +from leap.common.service_hooks import HookableService +from leap.keymanager import KeyManager, openpgp +from leap.keymanager.errors import KeyNotFound +from leap.soledad.client.api import Soledad +from leap.mail.constants import INBOX_NAME +from leap.mail.mail import Account +from leap.mail.imap.service import imap +from leap.mail.incoming.service import IncomingMail, INCOMING_CHECK_PERIOD +from leap.mail import smtp + +from leap.bitmask.core.uuid_map import UserMap + + +class Container(object): + + def __init__(self): + self._instances = defaultdict(None) + + def get_instance(self, key): + return self._instances.get(key, None) + + +class ImproperlyConfigured(Exception): + pass + + +def get_all_soledad_uuids(): + return [os.path.split(p)[-1].split('.db')[0] for p in + glob(os.path.expanduser('~/.config/leap/soledad/*.db'))] + # FIXME do not hardcode basedir + + +class SoledadContainer(Container): + + def __init__(self, basedir='~/.config/leap'): + # FIXME do not hardcode basedir + self._basedir = os.path.expanduser(basedir) + self._usermap = UserMap() + super(SoledadContainer, self).__init__() + + def add_instance(self, userid, passphrase, uuid=None, token=None): + + if not uuid: + bootstrapped_uuid = self._usermap.lookup_uuid(userid, passphrase) + uuid = bootstrapped_uuid + if not uuid: + return + else: + self._usermap.add(userid, uuid, passphrase) + + user, provider = userid.split('@') + + soledad_path = os.path.join(self._basedir, 'soledad') + soledad_url = _get_soledad_uri(self._basedir, provider) + cert_path = _get_ca_cert_path(self._basedir, provider) + + soledad = self._create_soledad_instance( + uuid, passphrase, soledad_path, soledad_url, + cert_path, token) + + self._instances[userid] = soledad + + data = {'user': userid, 'uuid': uuid, 'token': token, + 'soledad': soledad} + self.service.trigger_hook('on_new_soledad_instance', **data) + + def _create_soledad_instance(self, uuid, passphrase, basedir, server_url, + cert_file, token): + # setup soledad info + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + + if token is None: + syncable = False + token = '' + else: + syncable = True + + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True, + syncable=syncable) + + def set_remote_auth_token(self, userid, token): + self.get_instance(userid).token = token + + def set_syncable(self, userid, state): + # TODO should check that there's a token! + self.get_instance(userid).set_syncable(bool(state)) + + def sync(self, userid): + self.get_instance(userid).sync() + + +def _get_provider_from_full_userid(userid): + _, provider_id = config.get_username_and_provider(userid) + return config.Provider(provider_id) + + +def is_service_ready(service, provider): + """ + Returns True when the following conditions are met: + - Provider offers that service. + - We have the config files for the service. + - The service is enabled. + """ + has_service = provider.offers_service(service) + has_config = provider.has_config_for_service(service) + is_enabled = provider.is_service_enabled(service) + return has_service and has_config and is_enabled + + +class SoledadService(service.Service, HookableService): + + subscribed_to_hooks = ('on_bonafide_auth', 'on_passphrase_entry') + + def __init__(self, basedir): + service.Service.__init__(self) + self._basedir = basedir + + def startService(self): + log.msg('Starting Soledad Service') + self._container = SoledadContainer() + self._container.service = self + super(SoledadService, self).startService() + + # hooks + + def hook_on_passphrase_entry(self, **kw): + userid = kw.get('username') + provider = _get_provider_from_full_userid(userid) + provider.callWhenReady(self._hook_on_passphrase_entry, provider, **kw) + + def _hook_on_passphrase_entry(self, provider, **kw): + if is_service_ready('mx', provider): + userid = kw.get('username') + password = kw.get('password') + uuid = kw.get('uuid') + container = self._container + log.msg("on_passphrase_entry: New Soledad Instance: %s" % userid) + if not container.get_instance(userid): + container.add_instance(userid, password, uuid=uuid, token=None) + else: + log.msg('Service MX is not ready...') + + def hook_on_bonafide_auth(self, **kw): + userid = kw['username'] + provider = _get_provider_from_full_userid(userid) + provider.callWhenReady(self._hook_on_bonafide_auth, provider, **kw) + + def _hook_on_bonafide_auth(self, provider, **kw): + if provider.offers_service('mx'): + userid = kw['username'] + password = kw['password'] + token = kw['token'] + uuid = kw['uuid'] + + container = self._container + if container.get_instance(userid): + log.msg("Passing a new SRP Token to Soledad: %s" % userid) + container.set_remote_auth_token(userid, token) + container.set_syncable(userid, True) + else: + log.msg("Adding a new Soledad Instance: %s" % userid) + container.add_instance( + userid, password, uuid=uuid, token=token) + + +class KeymanagerContainer(Container): + + def __init__(self, basedir): + self._basedir = os.path.expanduser(basedir) + super(KeymanagerContainer, self).__init__() + + def add_instance(self, userid, token, uuid, soledad): + + keymanager = self._create_keymanager_instance( + userid, token, uuid, soledad) + + d = self._get_or_generate_keys(keymanager, userid) + d.addCallback(self._on_keymanager_ready_cb, userid, soledad) + return d + + def set_remote_auth_token(self, userid, token): + self.get_instance(userid)._token = token + + def _on_keymanager_ready_cb(self, keymanager, userid, soledad): + # TODO use onready-deferreds instead + self._instances[userid] = keymanager + + log.msg("Adding Keymanager instance for: %s" % userid) + data = {'userid': userid, 'soledad': soledad, 'keymanager': keymanager} + self.service.trigger_hook('on_new_keymanager_instance', **data) + + def _get_or_generate_keys(self, keymanager, userid): + + def if_not_found_generate(failure): + # TODO -------------- should ONLY generate if INITIAL_SYNC_DONE. + # ie: put callback on_soledad_first_sync_ready ----------------- + # -------------------------------------------------------------- + failure.trap(KeyNotFound) + log.msg("Core: Key not found. Generating key for %s" % (userid,)) + d = keymanager.gen_key(openpgp.OpenPGPKey) + d.addCallbacks(send_key, log_key_error("generating")) + return d + + def send_key(ignored): + # ---------------------------------------------------------------- + # It might be the case that we have generated a key-pair + # but this hasn't been successfully uploaded. How do we know that? + # XXX Should this be a method of bonafide instead? + # ----------------------------------------------------------------- + d = keymanager.send_key(openpgp.OpenPGPKey) + d.addCallbacks( + lambda _: log.msg( + "Key generated successfully for %s" % userid), + log_key_error("sending")) + return d + + def log_key_error(step): + def log_error(failure): + log.err("Error while %s key!" % step) + log.err(failure) + return failure + return log_error + + d = keymanager.get_key( + userid, openpgp.OpenPGPKey, private=True, fetch_remote=False) + d.addErrback(if_not_found_generate) + d.addCallback(lambda _: keymanager) + return d + + def _create_keymanager_instance(self, userid, token, uuid, soledad): + user, provider = userid.split('@') + nickserver_uri = self._get_nicknym_uri(provider) + + cert_path = _get_ca_cert_path(self._basedir, provider) + api_uri = self._get_api_uri(provider) + + if not token: + token = self.service.tokens.get(userid) + + km_args = (userid, nickserver_uri, soledad) + km_kwargs = { + "token": token, "uid": uuid, + "api_uri": api_uri, "api_version": "1", + "ca_cert_path": cert_path, + "gpgbinary": "/usr/bin/gpg" + } + keymanager = KeyManager(*km_args, **km_kwargs) + return keymanager + + def _get_api_uri(self, provider): + # TODO get this from service.json (use bonafide service) + api_uri = "https://api.{provider}:4430".format( + provider=provider) + return api_uri + + def _get_nicknym_uri(self, provider): + return 'https://nicknym.{provider}:6425'.format( + provider=provider) + + +class KeymanagerService(service.Service, HookableService): + + subscribed_to_hooks = ('on_new_soledad_instance', 'on_bonafide_auth') + + def __init__(self, basedir='~/.config/leap'): + service.Service.__init__(self) + self._basedir = basedir + + def startService(self): + log.msg('Starting Keymanager Service') + self._container = KeymanagerContainer(self._basedir) + self._container.service = self + self.tokens = {} + super(KeymanagerService, self).startService() + + # hooks + + def hook_on_new_soledad_instance(self, **kw): + container = self._container + user = kw['user'] + token = kw['token'] + uuid = kw['uuid'] + soledad = kw['soledad'] + if not container.get_instance(user): + log.msg('Adding a new Keymanager instance for %s' % user) + if not token: + token = self.tokens.get(user) + container.add_instance(user, token, uuid, soledad) + + def hook_on_bonafide_auth(self, **kw): + userid = kw['username'] + provider = _get_provider_from_full_userid(userid) + provider.callWhenReady(self._hook_on_bonafide_auth, provider, **kw) + + def _hook_on_bonafide_auth(self, provider, **kw): + if provider.offers_service('mx'): + userid = kw['username'] + token = kw['token'] + + container = self._container + if container.get_instance(userid): + log.msg('Passing a new SRP Token to Keymanager: %s' % userid) + container.set_remote_auth_token(userid, token) + else: + log.msg('storing the keymanager token...') + self.tokens[userid] = token + + # commands + + def do_list_keys(self, userid): + km = self._container.get_instance(userid) + d = km.get_all_keys() + d.addCallback( + lambda keys: [ + (key.uids, key.fingerprint) for key in keys]) + return d + + +class StandardMailService(service.MultiService, HookableService): + """ + A collection of Services. + + This is the parent service, that launches 3 different services that expose + Encrypted Mail Capabilities on specific ports: + + - SMTP service, on port 2013 + - IMAP service, on port 1984 + - The IncomingMail Service, which doesn't listen on any port, but + watches and processes the Incoming Queue and saves the processed mail + into the matching INBOX. + """ + + name = 'mail' + + # TODO factor out Mail Service to inside mail package. + + subscribed_to_hooks = ('on_new_keymanager_instance',) + + def __init__(self, basedir): + self._basedir = basedir + self._soledad_sessions = {} + self._keymanager_sessions = {} + self._sendmail_opts = {} + self._imap_tokens = {} + self._smtp_tokens = {} + self._active_user = None + super(StandardMailService, self).__init__() + self.initializeChildrenServices() + + def initializeChildrenServices(self): + self.addService(IMAPService(self._soledad_sessions)) + self.addService(IncomingMailService(self)) + self.addService(SMTPService( + self._soledad_sessions, self._keymanager_sessions, + self._sendmail_opts)) + + def startService(self): + log.msg('Starting Mail Service...') + super(StandardMailService, self).startService() + + def stopService(self): + super(StandardMailService, self).stopService() + + def startInstance(self, userid, soledad, keymanager): + username, provider = userid.split('@') + + self._soledad_sessions[userid] = soledad + self._keymanager_sessions[userid] = keymanager + + sendmail_opts = _get_sendmail_opts(self._basedir, provider, username) + self._sendmail_opts[userid] = sendmail_opts + + incoming = self.getServiceNamed('incoming_mail') + incoming.startInstance(userid) + + def registerIMAPToken(token): + self._imap_tokens[userid] = token + self._active_user = userid + return token + + def registerSMTPToken(token): + self._smtp_tokens[userid] = token + return token + + d = soledad.get_or_create_service_token('imap') + d.addCallback(registerIMAPToken) + d.addCallback( + lambda _: soledad.get_or_create_service_token('smtp')) + d.addCallback(registerSMTPToken) + return d + + def stopInstance(self): + pass + + # hooks + + def hook_on_new_keymanager_instance(self, **kw): + # XXX we can specify this as a waterfall, or just AND the two + # conditions. + userid = kw['userid'] + soledad = kw['soledad'] + keymanager = kw['keymanager'] + + # TODO --- only start instance if "autostart" is True. + self.startInstance(userid, soledad, keymanager) + + # commands + + def do_status(self): + return 'mail: %s' % 'running' if self.running else 'disabled' + + def get_imap_token(self): + active_user = self._active_user + if not active_user: + return defer.succeed('NO ACTIVE USER') + token = self._imap_tokens.get(active_user) + return defer.succeed("IMAP TOKEN (%s): %s" % (active_user, token)) + + def get_smtp_token(self): + active_user = self._active_user + if not active_user: + return defer.succeed('NO ACTIVE USER') + token = self._smtp_tokens.get(active_user) + return defer.succeed("SMTP TOKEN (%s): %s" % (active_user, token)) + + def do_get_smtp_cert_path(self, userid): + username, provider = userid.split('@') + return _get_smtp_client_cert_path(self._basedir, provider, username) + + # access to containers + + def get_soledad_session(self, userid): + return self._soledad_sessions.get(userid) + + def get_keymanager_session(self, userid): + return self._keymanager_sessions.get(userid) + + +class IMAPService(service.Service): + + name = 'imap' + + def __init__(self, soledad_sessions): + port, factory = imap.run_service(soledad_sessions) + + self._port = port + self._factory = factory + self._soledad_sessions = soledad_sessions + super(IMAPService, self).__init__() + + def startService(self): + log.msg('Starting IMAP Service') + super(IMAPService, self).startService() + + def stopService(self): + self._port.stopListening() + self._factory.doStop() + super(IMAPService, self).stopService() + + +class SMTPService(service.Service): + + name = 'smtp' + + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + basedir='~/.config/leap'): + + self._basedir = os.path.expanduser(basedir) + port, factory = smtp.run_service( + soledad_sessions, keymanager_sessions, sendmail_opts) + self._port = port + self._factory = factory + self._soledad_sessions = soledad_sessions + self._keymanager_sessions = keymanager_sessions + self._sendmail_opts = sendmail_opts + super(SMTPService, self).__init__() + + def startService(self): + log.msg('Starting SMTP Service') + super(SMTPService, self).startService() + + def stopService(self): + # TODO cleanup all instances + super(SMTPService, self).stopService() + + +class IncomingMailService(service.Service): + + name = 'incoming_mail' + + def __init__(self, mail_service): + super(IncomingMailService, self).__init__() + self._mail = mail_service + self._instances = {} + + def startService(self): + log.msg('Starting IncomingMail Service') + super(IncomingMailService, self).startService() + + def stopService(self): + super(IncomingMailService, self).stopService() + + # Individual accounts + + # TODO IncomingMail *IS* already a service. + # I think we should better model the current Service + # as a startInstance inside a container, and get this + # multi-tenant service inside the leap.mail.incoming.service. + # ... or just simply make it a multiService and set per-user + # instances as Child of this parent. + + def startInstance(self, userid): + soledad = self._mail.get_soledad_session(userid) + keymanager = self._mail.get_keymanager_session(userid) + + log.msg('Starting Incoming Mail instance for %s' % userid) + self._start_incoming_mail_instance( + keymanager, soledad, userid) + + def stopInstance(self, userid): + # TODO toggle offline! + pass + + def _start_incoming_mail_instance(self, keymanager, soledad, + userid, start_sync=True): + + def setUpIncomingMail(inbox): + incoming_mail = IncomingMail( + keymanager, soledad, + inbox, userid, + check_period=INCOMING_CHECK_PERIOD) + return incoming_mail + + def registerInstance(incoming_instance): + self._instances[userid] = incoming_instance + if start_sync: + incoming_instance.startService() + + acc = Account(soledad) + d = acc.callWhenReady( + lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) + d.addCallback(setUpIncomingMail) + d.addCallback(registerInstance) + d.addErrback(log.err) + return d + +# -------------------------------------------------------------------- +# +# config utilities. should be moved to bonafide +# + +SERVICES = ('soledad', 'smtp', 'eip') + + +Provider = namedtuple( + 'Provider', ['hostname', 'ip_address', 'location', 'port']) + +SendmailOpts = namedtuple( + 'SendmailOpts', ['cert', 'key', 'hostname', 'port']) + + +def _get_ca_cert_path(basedir, provider): + path = os.path.join( + basedir, 'providers', provider, 'keys', 'ca', 'cacert.pem') + return path + + +def _get_sendmail_opts(basedir, provider, username): + cert = _get_smtp_client_cert_path(basedir, provider, username) + key = cert + prov = _get_provider_for_service('smtp', basedir, provider) + hostname = prov.hostname + port = prov.port + opts = SendmailOpts(cert, key, hostname, port) + return opts + + +def _get_smtp_client_cert_path(basedir, provider, username): + path = os.path.join( + basedir, 'providers', provider, 'keys', 'client', 'stmp_%s.pem' % + username) + return path + + +def _get_config_for_service(service, basedir, provider): + if service not in SERVICES: + raise ImproperlyConfigured('Tried to use an unknown service') + + config_path = os.path.join( + basedir, 'providers', provider, '%s-service.json' % service) + try: + with open(config_path) as config: + config = json.loads(config.read()) + except IOError: + # FIXME might be that the provider DOES NOT offer this service! + raise ImproperlyConfigured( + 'could not open config file %s' % config_path) + else: + return config + + +def first(xs): + return xs[0] + + +def _pick_server(config, strategy=first): + """ + Picks a server from a list of possible choices. + The service files have a <describe>. + This implementation just picks the FIRST available server. + """ + servers = config['hosts'].keys() + choice = config['hosts'][strategy(servers)] + return choice + + +def _get_subdict(d, keys): + return {key: d.get(key) for key in keys} + + +def _get_provider_for_service(service, basedir, provider): + + if service not in SERVICES: + raise ImproperlyConfigured('Tried to use an unknown service') + + config = _get_config_for_service(service, basedir, provider) + p = _pick_server(config) + attrs = _get_subdict(p, ('hostname', 'ip_address', 'location', 'port')) + provider = Provider(**attrs) + return provider + + +def _get_smtp_uri(basedir, provider): + prov = _get_provider_for_service('smtp', basedir, provider) + url = 'https://{hostname}:{port}'.format( + hostname=prov.hostname, port=prov.port) + return url + + +def _get_soledad_uri(basedir, provider): + prov = _get_provider_for_service('soledad', basedir, provider) + url = 'https://{hostname}:{port}'.format( + hostname=prov.hostname, port=prov.port) + return url |