From 9cdd52be577fff75830c854bd7738ee1649e7083 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Fri, 19 Aug 2016 21:37:34 -0300 Subject: Started deferring leap session creation #759 Started adapting get_leap_session to deferreds Soledad and keymanager setup calls will now happen in deferreds and leap session creation itself is a deferred with callbacks This is a start in breaking the big blocking calls we were doing on the main thread, this was done without changing code inside the leap libraries yet so things can be further optimized This breaks the ~4 seconds get_leap_session piece into smaller 1 seconds one, that can be further optimized and deferred to even smaller calls There are requests calls happening on the main thread that should get this number even further down Also moved some pieces from bitmask libraries to our bootstrap, because they are not bitmask libraries anymore and that was causing confusion --- service/pixelated/config/leap.py | 17 +- service/pixelated/config/leap_config.py | 42 +++++ service/pixelated/config/services.py | 8 +- service/pixelated/config/sessions.py | 304 ++++++++++++++++++++++++++++++++ 4 files changed, 358 insertions(+), 13 deletions(-) create mode 100644 service/pixelated/config/leap_config.py create mode 100644 service/pixelated/config/sessions.py (limited to 'service/pixelated/config') diff --git a/service/pixelated/config/leap.py b/service/pixelated/config/leap.py index 9d0a35c4..b14aacc7 100644 --- a/service/pixelated/config/leap.py +++ b/service/pixelated/config/leap.py @@ -6,10 +6,10 @@ from leap.soledad.common.errors import InvalidAuthTokenError from leap.auth import SRPAuth from pixelated.config import credentials -from pixelated.bitmask_libraries.config import LeapConfig +from pixelated.config import leap_config from pixelated.bitmask_libraries.certs import LeapCertificate from pixelated.bitmask_libraries.provider import LeapProvider -from pixelated.bitmask_libraries.session import LeapSessionFactory +from pixelated.config.sessions import LeapSessionFactory log = logging.getLogger(__name__) @@ -17,13 +17,12 @@ log = logging.getLogger(__name__) def initialize_leap_provider(provider_hostname, provider_cert, provider_fingerprint, leap_home): LeapCertificate.set_cert_and_fingerprint(provider_cert, provider_fingerprint) - - config = LeapConfig(leap_home=leap_home, start_background_jobs=True) - provider = LeapProvider(provider_hostname, config) + leap_config.set_leap_home(leap_home) + provider = LeapProvider(provider_hostname) provider.download_certificate() LeapCertificate(provider).setup_ca_bundle() - return config, provider + return provider @defer.inlineCallbacks @@ -40,16 +39,16 @@ def initialize_leap_multi_user(provider_hostname, @defer.inlineCallbacks def create_leap_session(provider, username, password, auth=None): - leap_session = LeapSessionFactory(provider).create(username, password, auth) + leap_session = yield LeapSessionFactory(provider).create(username, password, auth) try: - yield leap_session.initial_sync() + yield leap_session.first_required_sync() except InvalidAuthTokenError: try: leap_session.close() except Exception, e: log.error(e) leap_session = LeapSessionFactory(provider).create(username, password, auth) - yield leap_session.initial_sync() + yield leap_session.first_required_sync() defer.returnValue(leap_session) diff --git a/service/pixelated/config/leap_config.py b/service/pixelated/config/leap_config.py new file mode 100644 index 00000000..324edc5e --- /dev/null +++ b/service/pixelated/config/leap_config.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + +import os +from distutils.spawn import find_executable + + +def discover_gpg_binary(): + path = find_executable('gpg') + if path is None: + raise Exception('Did not find a gpg executable!') + + if os.path.islink(path): + path = os.path.realpath(path) + + return path + + +SYSTEM_CA_BUNDLE = True +leap_home = '~/.leap/' +gpg_binary = discover_gpg_binary() + + +def set_leap_home(new_home): + leap_home = new_home + + +def set_gpg_binary(new_binary): + gpg_binary = binary diff --git a/service/pixelated/config/services.py b/service/pixelated/config/services.py index a49e1df9..494be703 100644 --- a/service/pixelated/config/services.py +++ b/service/pixelated/config/services.py @@ -12,14 +12,14 @@ from pixelated.adapter.listeners.mailbox_indexer_listener import listen_all_mail from twisted.internet import defer, reactor from pixelated.adapter.search.index_storage_key import SearchIndexStorageKey from pixelated.adapter.services.feedback_service import FeedbackService - +from pixelated.config import leap_config logger = logging.getLogger(__name__) class Services(object): def __init__(self, leap_session): - self._leap_home = leap_session.config.leap_home + self._leap_home = leap_config.leap_home self._leap_session = leap_session @defer.inlineCallbacks @@ -33,7 +33,7 @@ class Services(object): self.mail_service = self._setup_mail_service(self.search_engine) - self.keymanager = self._leap_session.nicknym + self.keymanager = self._leap_session.keymanager self.draft_service = self._setup_draft_service(self._leap_session.mail_store) self.feedback_service = self._setup_feedback_service() @@ -61,7 +61,7 @@ class Services(object): self.search_engine = search_engine def _setup_mail_service(self, search_engine): - pixelated_mail_sender = MailSender(self._leap_session.smtp_config, self._leap_session.nicknym.keymanager) + pixelated_mail_sender = MailSender(self._leap_session.smtp_config, self._leap_session.keymanager.keymanager) return MailService( pixelated_mail_sender, diff --git a/service/pixelated/config/sessions.py b/service/pixelated/config/sessions.py new file mode 100644 index 00000000..ed492ea9 --- /dev/null +++ b/service/pixelated/config/sessions.py @@ -0,0 +1,304 @@ +from __future__ import absolute_import + +import os +import errno +import requests +import logging +from twisted.internet import defer, threads, reactor +from leap.soledad.common.crypto import WrongMacError, UnknownMacMethodError +from leap.soledad.client import Soledad +from pixelated.bitmask_libraries.keymanager import Keymanager +from leap.mail.incoming.service import IncomingMail +from leap.mail.mail import Account +import leap.common.certs as leap_certs +from leap.common.events import ( + register, unregister, + catalog as events +) + +from pixelated.adapter.mailstore import LeapMailStore +from pixelated.config import leap_config +from pixelated.bitmask_libraries.certs import LeapCertificate +from pixelated.bitmask_libraries.smtp import LeapSMTPConfig + +logger = logging.getLogger(__name__) + + +class LeapSessionFactory(object): + def __init__(self, provider): + self._provider = provider + + @defer.inlineCallbacks + def create(self, username, password, auth): + key = SessionCache.session_key(self._provider, username) + session = SessionCache.lookup_session(key) + if not session: + session = yield self._create_new_session(username, password, auth) + SessionCache.remember_session(key, session) + defer.returnValue(session) + + @defer.inlineCallbacks + def _create_new_session(self, username, password, auth): + account_email = self._provider.address_for(username) + + self._create_database_dir(auth.uuid) + + api_cert = LeapCertificate(self._provider).provider_api_cert + + soledad = yield self.setup_soledad(auth.token, auth.uuid, password, api_cert) + + mail_store = LeapMailStore(soledad) + + keymanager = yield self.setup_keymanager(self._provider, soledad, account_email, auth.token, auth.uuid) + + smtp_client_cert = self._download_smtp_cert(auth) + smtp_host, smtp_port = self._provider.smtp_info() + smtp_config = LeapSMTPConfig(account_email, smtp_client_cert, smtp_host, smtp_port) + + leap_session = LeapSession(self._provider, auth, mail_store, soledad, keymanager, smtp_config) + + defer.returnValue(leap_session) + + @defer.inlineCallbacks + def setup_soledad(self, + user_token, + user_uuid, + password, + api_cert): + secrets = self._secrets_path(user_uuid) + local_db = self._local_db_path(user_uuid) + server_url = self._provider.discover_soledad_server(user_uuid) + try: + soledad = yield threads.deferToThread(Soledad, + user_uuid, + passphrase=unicode(password), + secrets_path=secrets, + local_db_path=local_db, + server_url=server_url, + cert_file=api_cert, + shared_db=None, + auth_token=user_token, + defer_encryption=False) + defer.returnValue(soledad) + except (WrongMacError, UnknownMacMethodError), e: + raise SoledadWrongPassphraseException(e) + + @defer.inlineCallbacks + def setup_keymanager(self, provider, soledad, account_email, token, uuid): + keymanager = yield threads.deferToThread(Keymanager, + provider, + soledad, + account_email, + token, + uuid) + defer.returnValue(keymanager) + + def _download_smtp_cert(self, auth): + cert = SmtpClientCertificate(self._provider, auth, self._user_path(auth.uuid)) + return cert.cert_path() + + def _create_dir(self, path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + def _user_path(self, user_uuid): + return os.path.join(leap_config.leap_home, user_uuid) + + def _soledad_path(self, user_uuid): + return os.path.join(leap_config.leap_home, user_uuid, 'soledad') + + def _secrets_path(self, user_uuid): + return os.path.join(self._soledad_path(user_uuid), 'secrets') + + def _local_db_path(self, user_uuid): + return os.path.join(self._soledad_path(user_uuid), 'soledad.db') + + def _create_database_dir(self, user_uuid): + try: + os.makedirs(self._soledad_path(user_uuid)) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(self._soledad_path(user_uuid)): + pass + else: + raise + + +class LeapSession(object): + + def __init__(self, provider, user_auth, mail_store, soledad, keymanager, smtp_config): + self.smtp_config = smtp_config + self.provider = provider + self.user_auth = user_auth + self.mail_store = mail_store + self.soledad = soledad + self.keymanager = keymanager + self.fresh_account = False + self.incoming_mail_fetcher = None + self.account = None + self._has_been_initially_synced = False + self._is_closed = False + register(events.KEYMANAGER_FINISHED_KEY_GENERATION, self._set_fresh_account, uid=self.account_email()) + + @defer.inlineCallbacks + def first_required_sync(self): + yield self.sync() + yield self.finish_bootstrap() + + @defer.inlineCallbacks + def finish_bootstrap(self): + yield self.keymanager.generate_openpgp_key() + yield self._create_account(self.soledad, self.user_auth.uuid) + self.incoming_mail_fetcher = yield self._create_incoming_mail_fetcher( + self.keymanager, + self.soledad, + self.account, + self.account_email()) + reactor.callFromThread(self.incoming_mail_fetcher.startService) + + def _create_account(self, soledad, user_id): + self.account = Account(soledad, user_id) + return self.account.deferred_initialization + + def _set_fresh_account(self, event, email_address): + logger.debug('Key for email %s has been generated' % email_address) + if email_address == self.account_email(): + self.fresh_account = True + + def account_email(self): + name = self.user_auth.username + return self.provider.address_for(name) + + def close(self): + self._is_closed = True + self.stop_background_jobs() + unregister(events.KEYMANAGER_FINISHED_KEY_GENERATION, uid=self.account_email()) + self.soledad.close() + self.remove_from_cache() + self._close_account() + + @property + def is_closed(self): + return self._is_closed + + def _close_account(self): + if self.account: + self.account.end_session() + + def remove_from_cache(self): + key = SessionCache.session_key(self.provider, self.user_auth.username) + SessionCache.remove_session(key) + + @defer.inlineCallbacks + def _create_incoming_mail_fetcher(self, keymanager, soledad, account, user_mail): + inbox = yield account.callWhenReady(lambda _: account.get_collection_by_mailbox('INBOX')) + defer.returnValue(IncomingMail(keymanager.keymanager, + soledad, + inbox, + user_mail)) + + def stop_background_jobs(self): + if self.incoming_mail_fetcher: + reactor.callFromThread(self.incoming_mail_fetcher.stopService) + self.incoming_mail_fetcher = None + + def sync(self): + try: + return self.soledad.sync() + except: + traceback.print_exc(file=sys.stderr) + raise + + +class SessionCache(object): + + sessions = {} + + @staticmethod + def lookup_session(key): + session = SessionCache.sessions.get(key, None) + if session is not None and session.is_closed: + SessionCache.remove_session(key) + return None + else: + return session + + @staticmethod + def remember_session(key, session): + SessionCache.sessions[key] = session + + @staticmethod + def remove_session(key): + if key in SessionCache.sessions: + del SessionCache.sessions[key] + + @staticmethod + def session_key(provider, username): + return hash((provider, username)) + + +class SmtpClientCertificate(object): + def __init__(self, provider, auth, user_path): + self._provider = provider + self._auth = auth + self._user_path = user_path + + def cert_path(self): + if not self._is_cert_already_downloaded() or self._should_redownload(): + self._download_smtp_cert() + + return self._smtp_client_cert_path() + + def _is_cert_already_downloaded(self): + return os.path.exists(self._smtp_client_cert_path()) + + def _should_redownload(self): + return leap_certs.should_redownload(self._smtp_client_cert_path()) + + def _download_smtp_cert(self): + cert_path = self._smtp_client_cert_path() + + if not os.path.exists(os.path.dirname(cert_path)): + os.makedirs(os.path.dirname(cert_path)) + + self.download_to(cert_path) + + def _smtp_client_cert_path(self): + return os.path.join( + self._user_path, + "providers", + self._provider.domain, + "keys", "client", "smtp.pem") + + def download(self): + cert_url = '%s/%s/smtp_cert' % (self._provider.api_uri, self._provider.api_version) + headers = {} + headers["Authorization"] = 'Token token="{0}"'.format(self._auth.token) + params = {'address': self._auth.username} + response = requests.post( + cert_url, + params=params, + data=params, + verify=LeapCertificate(self._provider).provider_api_cert, + timeout=15, + headers=headers) + response.raise_for_status() + + client_cert = response.content + + return client_cert + + def download_to(self, target_file): + client_cert = self.download() + + with open(target_file, 'w') as f: + f.write(client_cert) + + +class SoledadWrongPassphraseException(Exception): + def __init__(self, *args, **kwargs): + super(SoledadWrongPassphraseException, self).__init__(*args, **kwargs) -- cgit v1.2.3