diff options
author | Bruno Wagner <bwagner@riseup.net> | 2016-08-19 21:37:34 -0300 |
---|---|---|
committer | Bruno Wagner <bwagner@riseup.net> | 2016-08-19 21:37:34 -0300 |
commit | 9cdd52be577fff75830c854bd7738ee1649e7083 (patch) | |
tree | ef560dd628bda40832503250e1325283c49ede83 /service/pixelated/bitmask_libraries | |
parent | 9c5811c6b760415372c6cc67a9d34680c990cdd8 (diff) |
Started deferring leap session creation #759
Started adapting get_leap_session to deferreds
Soledad and keymanager setup calls will now
happen in deferreds and leap session creation
itself is a deferred with callbacks
This is a start in breaking the big blocking
calls we were doing on the main thread, this
was done without changing code inside the
leap libraries yet so things can be further
optimized
This breaks the ~4 seconds get_leap_session
piece into smaller 1 seconds one, that can be
further optimized and deferred to even smaller
calls
There are requests calls happening on the main
thread that should get this number even further
down
Also moved some pieces from bitmask libraries
to our bootstrap, because they are not bitmask
libraries anymore and that was causing confusion
Diffstat (limited to 'service/pixelated/bitmask_libraries')
-rw-r--r-- | service/pixelated/bitmask_libraries/certs.py | 6 | ||||
-rw-r--r-- | service/pixelated/bitmask_libraries/config.py | 46 | ||||
-rw-r--r-- | service/pixelated/bitmask_libraries/keymanager.py (renamed from service/pixelated/bitmask_libraries/nicknym.py) | 13 | ||||
-rw-r--r-- | service/pixelated/bitmask_libraries/provider.py | 35 | ||||
-rw-r--r-- | service/pixelated/bitmask_libraries/session.py | 301 | ||||
-rw-r--r-- | service/pixelated/bitmask_libraries/soledad.py | 46 |
6 files changed, 25 insertions, 422 deletions
diff --git a/service/pixelated/bitmask_libraries/certs.py b/service/pixelated/bitmask_libraries/certs.py index 9d543672..e3466d05 100644 --- a/service/pixelated/bitmask_libraries/certs.py +++ b/service/pixelated/bitmask_libraries/certs.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. import os +from pixelated.config import leap_config class LeapCertificate(object): @@ -22,7 +23,6 @@ class LeapCertificate(object): LEAP_FINGERPRINT = None def __init__(self, provider): - self._config = provider.config self._server_name = provider.server_name self._provider = provider @@ -41,10 +41,10 @@ class LeapCertificate(object): @property def provider_api_cert(self): - return str(os.path.join(self._provider.config.leap_home, 'providers', self._server_name, 'keys', 'client', 'api.pem')) + return str(os.path.join(leap_config.leap_home, 'providers', self._server_name, 'keys', 'client', 'api.pem')) def setup_ca_bundle(self): - path = os.path.join(self._provider.config.leap_home, 'providers', self._server_name, 'keys', 'client') + path = os.path.join(leap_config.leap_home, 'providers', self._server_name, 'keys', 'client') if not os.path.isdir(path): os.makedirs(path, 0700) self._download_cert(self.provider_api_cert) diff --git a/service/pixelated/bitmask_libraries/config.py b/service/pixelated/bitmask_libraries/config.py deleted file mode 100644 index c521a093..00000000 --- a/service/pixelated/bitmask_libraries/config.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. - -import os -from distutils.spawn import find_executable - - -def discover_gpg_binary(): - path = find_executable('gpg') - if path is None: - raise Exception('Did not find a gpg executable!') - - if os.path.islink(path): - path = os.path.realpath(path) - - return path - - -SYSTEM_CA_BUNDLE = True - - -class LeapConfig(object): - - def __init__(self, - leap_home=None, - timeout_in_s=15, - start_background_jobs=False, - gpg_binary=discover_gpg_binary()): - - self.leap_home = leap_home - self.timeout_in_s = timeout_in_s - self.start_background_jobs = start_background_jobs - self.gpg_binary = gpg_binary diff --git a/service/pixelated/bitmask_libraries/nicknym.py b/service/pixelated/bitmask_libraries/keymanager.py index aeedab2c..78d6e935 100644 --- a/service/pixelated/bitmask_libraries/nicknym.py +++ b/service/pixelated/bitmask_libraries/keymanager.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. from leap.keymanager import KeyManager, KeyNotFound +from pixelated.config import leap_config from .certs import LeapCertificate from twisted.internet import defer import logging @@ -21,15 +22,15 @@ import logging logger = logging.getLogger(__name__) -class NickNym(object): - def __init__(self, provider, config, soledad, email_address, token, uuid): - nicknym_url = _discover_nicknym_server(provider) +class Keymanager(object): + def __init__(self, provider, soledad, email_address, token, uuid): + nicknym_url = provider._discover_nicknym_server() self._email = email_address self.keymanager = KeyManager(self._email, nicknym_url, soledad, token=token, ca_cert_path=LeapCertificate(provider).provider_api_cert, api_uri=provider.api_uri, api_version=provider.api_version, - uid=uuid, gpgbinary=config.gpg_binary) + uid=uuid, gpgbinary=leap_config.gpg_binary) @defer.inlineCallbacks def generate_openpgp_key(self): @@ -55,7 +56,3 @@ class NickNym(object): def _send_key_to_leap(self): return self.keymanager.send_key() - - -def _discover_nicknym_server(provider): - return 'https://nicknym.%s:6425/' % provider.domain diff --git a/service/pixelated/bitmask_libraries/provider.py b/service/pixelated/bitmask_libraries/provider.py index 9c889287..07791624 100644 --- a/service/pixelated/bitmask_libraries/provider.py +++ b/service/pixelated/bitmask_libraries/provider.py @@ -19,16 +19,18 @@ import os from leap.common.certs import get_digest import requests from .certs import LeapCertificate +from pixelated.config import leap_config from pixelated.support.tls_adapter import EnforceTLSv1Adapter -from pixelated.bitmask_libraries.soledad import SoledadDiscoverException + +REQUESTS_TIMEOUT = 15 class LeapProvider(object): - def __init__(self, server_name, config): + def __init__(self, server_name): self.server_name = server_name - self.config = config - self.local_ca_crt = '%s/ca.crt' % self.config.leap_home + self.local_ca_crt = '%s/ca.crt' % leap_config.leap_home self.provider_json = self.fetch_provider_json() + self.soledad_json = self.fetch_soledad_json() @property def api_uri(self): @@ -119,7 +121,7 @@ class LeapProvider(object): session = requests.session() try: session.mount('https://', EnforceTLSv1Adapter(assert_fingerprint=LeapCertificate.LEAP_FINGERPRINT)) - response = session.get(url, verify=LeapCertificate(self).provider_web_cert, timeout=self.config.timeout_in_s) + response = session.get(url, verify=LeapCertificate(self).provider_web_cert, timeout=REQUESTS_TIMEOUT) response.raise_for_status() return response finally: @@ -134,14 +136,14 @@ class LeapProvider(object): def fetch_soledad_json(self): service_url = "%s/%s/config/soledad-service.json" % ( self.api_uri, self.api_version) - response = requests.get(service_url, verify=LeapCertificate(self).provider_api_cert, timeout=self.config.timeout_in_s) + response = requests.get(service_url, verify=LeapCertificate(self).provider_api_cert, timeout=REQUESTS_TIMEOUT) response.raise_for_status() return json.loads(response.content) def fetch_smtp_json(self): service_url = '%s/%s/config/smtp-service.json' % ( self.api_uri, self.api_version) - response = requests.get(service_url, verify=LeapCertificate(self).provider_api_cert, timeout=self.config.timeout_in_s) + response = requests.get(service_url, verify=LeapCertificate(self).provider_api_cert, timeout=REQUESTS_TIMEOUT) response.raise_for_status() return json.loads(response.content) @@ -152,14 +154,11 @@ class LeapProvider(object): return '%s@%s' % (username, self.domain) def discover_soledad_server(self, user_uuid): - try: - json_data = self.fetch_soledad_json() - - hosts = json_data['hosts'] - host = hosts.keys()[0] - server_url = 'https://%s:%d/user-%s' % \ - (hosts[host]['hostname'], hosts[host]['port'], - user_uuid) - return server_url - except Exception, e: - raise SoledadDiscoverException(e) + hosts = self.soledad_json['hosts'] + host = hosts.keys()[0] + server_url = 'https://%s:%d/user-%s' % \ + (hosts[host]['hostname'], hosts[host]['port'], user_uuid) + return server_url + + def _discover_nicknym_server(self): + return 'https://nicknym.%s:6425/' % self.domain diff --git a/service/pixelated/bitmask_libraries/session.py b/service/pixelated/bitmask_libraries/session.py deleted file mode 100644 index 72ad8520..00000000 --- a/service/pixelated/bitmask_libraries/session.py +++ /dev/null @@ -1,301 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -import errno -import traceback -import sys -import os -import requests -import logging - -from twisted.internet import reactor, defer -from pixelated.bitmask_libraries.certs import LeapCertificate -from pixelated.adapter.mailstore import LeapMailStore -from leap.mail.incoming.service import IncomingMail -from leap.mail.mail import Account -from leap.auth import SRPAuth -from .nicknym import NickNym -from .smtp import LeapSMTPConfig -from .soledad import SoledadFactory -import leap.common.certs as leap_certs - -from leap.common.events import ( - register, unregister, - catalog as events -) - - -log = logging.getLogger(__name__) - - -class LeapSession(object): - - def __init__(self, provider, user_auth, mail_store, soledad, nicknym, smtp_config): - self.smtp_config = smtp_config - self.config = provider.config - self.provider = provider - self.user_auth = user_auth - self.mail_store = mail_store - self.soledad = soledad - self.nicknym = nicknym - self.fresh_account = False - self.incoming_mail_fetcher = None - self.account = None - self._has_been_initially_synced = False - self._sem_intial_sync = defer.DeferredLock() - self._is_closed = False - register(events.KEYMANAGER_FINISHED_KEY_GENERATION, self._set_fresh_account, uid=self.account_email()) - - @defer.inlineCallbacks - def initial_sync(self): - yield self._sem_intial_sync.acquire() - try: - yield self.sync() - if not self._has_been_initially_synced: - yield self.after_first_sync() - self._has_been_initially_synced = True - finally: - yield self._sem_intial_sync.release() - defer.returnValue(self) - - @defer.inlineCallbacks - def after_first_sync(self): - yield self.nicknym.generate_openpgp_key() - yield self._create_account(self.soledad, self.user_auth.uuid) - self.incoming_mail_fetcher = yield self._create_incoming_mail_fetcher( - self.nicknym, - self.soledad, - self.account, - self.account_email()) - reactor.callFromThread(self.incoming_mail_fetcher.startService) - - def _create_account(self, soledad, user_id): - self.account = Account(soledad, user_id) - return self.account.deferred_initialization - - def _set_fresh_account(self, event, email_address): - log.debug('Key for email %s has been generated' % email_address) - if email_address == self.account_email(): - self.fresh_account = True - - def account_email(self): - name = self.user_auth.username - return self.provider.address_for(name) - - def close(self): - self._is_closed = True - self.stop_background_jobs() - unregister(events.KEYMANAGER_FINISHED_KEY_GENERATION, uid=self.account_email()) - self.soledad.close() - self.remove_from_cache() - self._close_account() - - @property - def is_closed(self): - return self._is_closed - - def _close_account(self): - if self.account: - self.account.end_session() - - def remove_from_cache(self): - key = SessionCache.session_key(self.provider, self.user_auth.username) - SessionCache.remove_session(key) - - @defer.inlineCallbacks - def _create_incoming_mail_fetcher(self, nicknym, soledad, account, user_mail): - inbox = yield account.callWhenReady(lambda _: account.get_collection_by_mailbox('INBOX')) - defer.returnValue(IncomingMail(nicknym.keymanager, - soledad, - inbox, - user_mail)) - - def stop_background_jobs(self): - if self.incoming_mail_fetcher: - reactor.callFromThread(self.incoming_mail_fetcher.stopService) - self.incoming_mail_fetcher = None - - def sync(self): - try: - return self.soledad.sync() - except: - traceback.print_exc(file=sys.stderr) - raise - - -class SmtpClientCertificate(object): - def __init__(self, provider, auth, user_path): - self._provider = provider - self._auth = auth - self._user_path = user_path - - def cert_path(self): - if not self._is_cert_already_downloaded() or self._should_redownload(): - self._download_smtp_cert() - - return self._smtp_client_cert_path() - - def _is_cert_already_downloaded(self): - return os.path.exists(self._smtp_client_cert_path()) - - def _should_redownload(self): - return leap_certs.should_redownload(self._smtp_client_cert_path()) - - def _download_smtp_cert(self): - cert_path = self._smtp_client_cert_path() - - if not os.path.exists(os.path.dirname(cert_path)): - os.makedirs(os.path.dirname(cert_path)) - - SmtpCertDownloader(self._provider, self._auth).download_to(cert_path) - - def _smtp_client_cert_path(self): - return os.path.join( - self._user_path, - "providers", - self._provider.domain, - "keys", "client", "smtp.pem") - - -class SmtpCertDownloader(object): - - def __init__(self, provider, auth): - self._provider = provider - self._auth = auth - - def download(self): - cert_url = '%s/%s/smtp_cert' % (self._provider.api_uri, self._provider.api_version) - headers = {} - headers["Authorization"] = 'Token token="{0}"'.format(self._auth.token) - params = {'address': self._auth.username} - response = requests.post( - cert_url, - params=params, - data=params, - verify=LeapCertificate(self._provider).provider_api_cert, - timeout=self._provider.config.timeout_in_s, - headers=headers) - response.raise_for_status() - - client_cert = response.content - - return client_cert - - def download_to(self, target_file): - client_cert = self.download() - - with open(target_file, 'w') as f: - f.write(client_cert) - - -class LeapSessionFactory(object): - def __init__(self, provider): - self._provider = provider - self._config = provider.config - - def create(self, username, password, auth): - key = SessionCache.session_key(self._provider, username) - session = SessionCache.lookup_session(key) - if not session: - session = self._create_new_session(username, password, auth) - SessionCache.remember_session(key, session) - - return session - - def _create_new_session(self, username, password, auth): - account_email = self._provider.address_for(username) - - self._create_database_dir(auth.uuid) - - soledad = SoledadFactory.create(auth.token, - auth.uuid, - password, - self._secrets_path(auth.uuid), - self._local_db_path(auth.uuid), - self._provider.discover_soledad_server(auth.uuid), - LeapCertificate(self._provider).provider_api_cert) - - mail_store = LeapMailStore(soledad) - nicknym = self._create_nicknym(account_email, auth.token, auth.uuid, soledad) - - smtp_client_cert = self._download_smtp_cert(auth) - smtp_host, smtp_port = self._provider.smtp_info() - smtp_config = LeapSMTPConfig(account_email, smtp_client_cert, smtp_host, smtp_port) - - return LeapSession(self._provider, auth, mail_store, soledad, nicknym, smtp_config) - - def _download_smtp_cert(self, auth): - cert = SmtpClientCertificate(self._provider, auth, self._user_path(auth.uuid)) - return cert.cert_path() - - def _create_dir(self, path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def _create_nicknym(self, email_address, token, uuid, soledad): - return NickNym(self._provider, self._config, soledad, email_address, token, uuid) - - def _user_path(self, user_uuid): - return os.path.join(self._config.leap_home, user_uuid) - - def _soledad_path(self, user_uuid): - return os.path.join(self._config.leap_home, user_uuid, 'soledad') - - def _secrets_path(self, user_uuid): - return os.path.join(self._soledad_path(user_uuid), 'secrets') - - def _local_db_path(self, user_uuid): - return os.path.join(self._soledad_path(user_uuid), 'soledad.db') - - def _create_database_dir(self, user_uuid): - try: - os.makedirs(self._soledad_path(user_uuid)) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(self._soledad_path(user_uuid)): - pass - else: - raise - - -class SessionCache(object): - - sessions = {} - - @staticmethod - def lookup_session(key): - session = SessionCache.sessions.get(key, None) - if session is not None and session.is_closed: - SessionCache.remove_session(key) - return None - else: - return session - - @staticmethod - def remember_session(key, session): - SessionCache.sessions[key] = session - - @staticmethod - def remove_session(key): - if key in SessionCache.sessions: - del SessionCache.sessions[key] - - @staticmethod - def session_key(provider, username): - return hash((provider, username)) diff --git a/service/pixelated/bitmask_libraries/soledad.py b/service/pixelated/bitmask_libraries/soledad.py deleted file mode 100644 index 406e9fc1..00000000 --- a/service/pixelated/bitmask_libraries/soledad.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from leap.soledad.client import Soledad -from leap.soledad.common.crypto import WrongMacError, UnknownMacMethodError - - -class SoledadDiscoverException(Exception): - def __init__(self, *args, **kwargs): - super(SoledadDiscoverException, self).__init__(*args, **kwargs) - - -class SoledadWrongPassphraseException(Exception): - def __init__(self, *args, **kwargs): - super(SoledadWrongPassphraseException, self).__init__(*args, **kwargs) - - -class SoledadFactory(object): - - @classmethod - def create(cls, user_token, user_uuid, encryption_passphrase, secrets, local_db, server_url, api_cert): - try: - return Soledad(user_uuid, - passphrase=unicode(encryption_passphrase), - secrets_path=secrets, - local_db_path=local_db, - server_url=server_url, - cert_file=api_cert, - shared_db=None, - auth_token=user_token, - defer_encryption=False) - - except (WrongMacError, UnknownMacMethodError), e: - raise SoledadWrongPassphraseException(e) |