diff options
Diffstat (limited to 'service')
24 files changed, 1221 insertions, 9 deletions
diff --git a/service/.gitignore b/service/.gitignore new file mode 100644 index 00000000..a21a0f95 --- /dev/null +++ b/service/.gitignore @@ -0,0 +1 @@ +.virtualenv diff --git a/service/app/bitmask_libraries/__init__.py b/service/app/bitmask_libraries/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/service/app/bitmask_libraries/__init__.py @@ -0,0 +1 @@ + diff --git a/service/app/bitmask_libraries/auth.py b/service/app/bitmask_libraries/auth.py new file mode 100644 index 00000000..4958c586 --- /dev/null +++ b/service/app/bitmask_libraries/auth.py @@ -0,0 +1,22 @@ +from .leap_srp import LeapSecureRemotePassword +from .certs import which_bundle + +USE_PASSWORD = None + + +class LeapCredentials(object): + def __init__(self, user_name, password, db_passphrase=USE_PASSWORD): + self.user_name = user_name + self.password = password + self.db_passphrase = db_passphrase if db_passphrase is not None else password + + +class LeapAuthenticator(object): + def __init__(self, provider): + self._provider = provider + + def authenticate(self, credentials): + config = self._provider.config + srp = LeapSecureRemotePassword(ca_bundle=which_bundle(self._provider), timeout_in_s=config.timeout_in_s) + srp_session = srp.authenticate(self._provider.api_uri, credentials.user_name, credentials.password) + return srp_session diff --git a/service/app/bitmask_libraries/certs.py b/service/app/bitmask_libraries/certs.py new file mode 100644 index 00000000..a814ec82 --- /dev/null +++ b/service/app/bitmask_libraries/certs.py @@ -0,0 +1,33 @@ +import os + +from leap.common import ca_bundle + +from .config import AUTO_DETECT_CA_BUNDLE + + +def which_bundle(provider): + return LeapCertificate(provider).auto_detect_ca_bundle() + + +class LeapCertificate(object): + def __init__(self, provider): + self._config = provider.config + self._server_name = provider.server_name + self._leap_home = self._config.leap_home + + def auto_detect_ca_bundle(self): + if self._config.ca_cert_bundle == AUTO_DETECT_CA_BUNDLE: + local_cert = self._local_server_cert() + if local_cert: + return local_cert + else: + return ca_bundle.where() + else: + return self._config.ca_cert_bundle + + def _local_server_cert(self): + cert_file = os.path.join(self._leap_home, '%s.ca.crt' % self._server_name) + if os.path.isfile(cert_file): + return cert_file + else: + return None diff --git a/service/app/bitmask_libraries/config.py b/service/app/bitmask_libraries/config.py new file mode 100644 index 00000000..5baa7808 --- /dev/null +++ b/service/app/bitmask_libraries/config.py @@ -0,0 +1,66 @@ +import os +from os.path import expanduser +from distutils.spawn import find_executable + + +def discover_gpg_binary(): + path = find_executable('gpg') + if path is None: + raise Exception('Did not find a gpg executable!') + + if os.path.islink(path): + path = os.path.realpath(path) + + return path + + +DEFAULT_LEAP_HOME = os.path.join(expanduser("~"), '.leap') + +SYSTEM_CA_BUNDLE = True +AUTO_DETECT_CA_BUNDLE = None + + +class LeapConfig(object): + """ + LEAP client configuration + + """ + + def __init__(self, leap_home=DEFAULT_LEAP_HOME, ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, verify_ssl=True, + fetch_interval_in_s=30, + timeout_in_s=15, start_background_jobs=True, gpg_binary=discover_gpg_binary()): + """ + Constructor. + + :param server_name: The LEAP server name, e.g. demo.leap.se + :type server_name: str + + :param user_name: The LEAP account user name, normally the first part of your email, e.g. foobar for foobar@demo.leap.se + :type user_name: str + + :param user_password: The LEAP account password + :type user_password: str + + :param db_passphrase: The passphrase used to encrypt the local soledad database + :type db_passphrase: str + + :param verify_ssl: Set to false to disable strict SSL certificate validation + :type verify_ssl: bool + + :param fetch_interval_in_s: Polling interval for fetching incoming mail from LEAP server + :type fetch_interval_in_s: int + + :param timeout_in_s: Timeout for network operations, e.g. HTTP calls + :type timeout_in_s: int + + :param gpg_binary: Path to the GPG binary (must not be a symlink) + :type gpg_binary: str + + """ + self.leap_home = leap_home + self.ca_cert_bundle = ca_cert_bundle + self.verify_ssl = verify_ssl + self.timeout_in_s = timeout_in_s + self.start_background_jobs = start_background_jobs + self.gpg_binary = gpg_binary + self.fetch_interval_in_s = fetch_interval_in_s diff --git a/service/app/bitmask_libraries/leap_srp.py b/service/app/bitmask_libraries/leap_srp.py new file mode 100644 index 00000000..a1de7de3 --- /dev/null +++ b/service/app/bitmask_libraries/leap_srp.py @@ -0,0 +1,105 @@ +import binascii +import json + +from requests import Session +from srp import User, srp +from requests.exceptions import HTTPError, SSLError, Timeout +from config import SYSTEM_CA_BUNDLE + + +class LeapAuthException(Exception): + def __init__(self, *args, **kwargs): + super(LeapAuthException, self).__init__(*args, **kwargs) + + +class LeapSRPSession(object): + def __init__(self, user_name, api_server_name, uuid, token, session_id, api_version='1'): + self.user_name = user_name + self.api_server_name = api_server_name + self.uuid = uuid + self.token = token + self.session_id = session_id + self.api_version = api_version + + def __str__(self): + return 'LeapSRPSession(%s, %s, %s, %s, %s, %s)' % (self.user_name, self.api_server_name, self.uuid, self.token, self.session_id, self.api_version) + + +class LeapSecureRemotePassword(object): + def __init__(self, hash_alg=srp.SHA256, ng_type=srp.NG_1024, ca_bundle=SYSTEM_CA_BUNDLE, timeout_in_s=15, + leap_api_version='1'): + + self.hash_alg = hash_alg + self.ng_type = ng_type + self.timeout_in_s = timeout_in_s + self.ca_bundle = ca_bundle + self.leap_api_version = leap_api_version + + def authenticate(self, api_uri, username, password): + session = Session() + try: + return self._authenticate_with_session(session, api_uri, username, password) + except Timeout, e: + raise LeapAuthException(e) + finally: + session.close() + + def _authenticate_with_session(self, http_session, api_uri, username, password): + try: + srp_user = User(username.encode('utf-8'), password.encode('utf-8'), self.hash_alg, self.ng_type) + + salt, B_challenge = self._begin_authentication(srp_user, http_session, api_uri) + M2_verfication_code, leap_session = self._process_challenge(srp_user, http_session, api_uri, salt, + B_challenge) + self._verify_session(srp_user, M2_verfication_code) + + return leap_session + except (HTTPError, SSLError), e: + raise LeapAuthException(e) + + def _begin_authentication(self, user, session, api_uri): + _, A = user.start_authentication() + + auth_data = { + "login": user.get_username(), + "A": binascii.hexlify(A) + } + session_url = '%s/%s/sessions' % (api_uri, self.leap_api_version) + response = session.post(session_url, data=auth_data, verify=self.ca_bundle, timeout=self.timeout_in_s) + response.raise_for_status() + json_content = json.loads(response.content) + + salt = _safe_unhexlify(json_content.get('salt')) + B = _safe_unhexlify(json_content.get('B')) + + return salt, B + + def _process_challenge(self, user, session, api_uri, salt, B): + M = user.process_challenge(salt, B) + + auth_data = { + "client_auth": binascii.hexlify(M) + } + + auth_url = '%s/%s/sessions/%s' % (api_uri, self.leap_api_version, user.get_username()) + response = session.put(auth_url, data=auth_data, verify=self.ca_bundle, timeout=self.timeout_in_s) + response.raise_for_status() + auth_json = json.loads(response.content) + + M2 = _safe_unhexlify(auth_json.get('M2')) + uuid = auth_json.get('id') + token = auth_json.get('token') + session_id = response.cookies.get('_session_id') + + return M2, LeapSRPSession(user.get_username(), api_uri, uuid, token, session_id) + + def _verify_session(self, user, M2): + user.verify_session(M2) + if not user.authenticated(): + raise LeapAuthException() + + +def _safe_unhexlify(hex_str): + return binascii.unhexlify(hex_str) \ + if (len(hex_str) % 2 == 0) else binascii.unhexlify('0' + hex_str) + diff --git a/service/app/bitmask_libraries/nicknym.py b/service/app/bitmask_libraries/nicknym.py new file mode 100644 index 00000000..5d9c5308 --- /dev/null +++ b/service/app/bitmask_libraries/nicknym.py @@ -0,0 +1,18 @@ +from leap.keymanager import KeyManager +from .certs import which_bundle + +SOLEDAD_CERT = '/tmp/ca.crt' + + +class NickNym(object): + + def __init__(self, provider, config, soledad_session, srp_session): + nicknym_url = _discover_nicknym_server(provider) + self.keymanager = KeyManager('%s@%s' % (srp_session.user_name, provider.domain), nicknym_url, soledad_session.soledad, + srp_session.session_id, which_bundle(provider), provider.api_uri, + provider.api_version, + srp_session.uuid, config.gpg_binary) + + +def _discover_nicknym_server(provider): + return 'https://nicknym.%s:6425/' % provider.domain diff --git a/service/app/bitmask_libraries/provider.py b/service/app/bitmask_libraries/provider.py new file mode 100644 index 00000000..1bfd69fb --- /dev/null +++ b/service/app/bitmask_libraries/provider.py @@ -0,0 +1,112 @@ +import json + +from leap.common.certs import get_digest +import requests + +from .certs import which_bundle + + +class LeapProvider(object): + def __init__(self, server_name, config): + self.server_name = server_name + self.config = config + + self.provider_json = self.fetch_provider_json() + + @property + def api_uri(self): + return self.provider_json.get('api_uri') + + @property + def ca_cert_fingerprint(self): + return self.provider_json.get('ca_cert_fingerprint') + + @property + def ca_cert_uri(self): + return self.provider_json.get('ca_cert_uri') + + @property + def api_version(self): + return self.provider_json.get('api_version') + + @property + def domain(self): + return self.provider_json.get('domain') + + @property + def services(self): + return self.provider_json.get('services') + + def __hash__(self): + return hash(self.server_name) + + def __eq__(self, other): + return self.server_name == other.server_name + + def ensure_supports_mx(self): + if not 'mx' in self.services: + raise Exception + + def download_certificate_to(self, filename): + """ + Downloads the server certificate, validates it against the provided fingerprint and stores it to file + """ + cert = self.fetch_valid_certificate() + with open(filename, 'w') as out: + out.write(cert) + + def fetch_valid_certificate(self): + cert = self._fetch_certificate() + self.validate_certificate(cert) + return cert + + def _fetch_certificate(self): + session = requests.session() + try: + cert_url = '%s/ca.crt' % self._provider_base_url() + response = session.get(cert_url, verify=which_bundle(self), timeout=self.config.timeout_in_s) + response.raise_for_status() + + cert_data = response.content + return cert_data + finally: + session.close() + + def validate_certificate(self, cert_data=None): + if cert_data is None: + cert_data = self._fetch_certificate() + + parts = str(self.ca_cert_fingerprint).split(':') + method = parts[0].strip() + fingerprint = parts[1].strip() + + digest = get_digest(cert_data, method) + + if fingerprint.strip() != digest: + raise Exception('Certificate fingerprints don\'t match') + + def fetch_provider_json(self): + + url = '%s/provider.json' % self._provider_base_url() + response = requests.get(url, verify=which_bundle(self), timeout=self.config.timeout_in_s) + response.raise_for_status() + + json_data = json.loads(response.content) + return json_data + + def fetch_soledad_json(self): + service_url = "%s/%s/config/soledad-service.json" % ( + self.api_uri, self.api_version) + response = requests.get(service_url, verify=which_bundle(self), timeout=self.config.timeout_in_s) + response.raise_for_status() + return json.loads(response.content) + + def fetch_smtp_json(self): + service_url = '%s/%s/config/smtp-service.json' % ( + self.api_uri, self.api_version) + response = requests.get(service_url, verify=which_bundle(self), timeout=self.config.timeout_in_s) + response.raise_for_status() + return json.loads(response.content) + + def _provider_base_url(self): + return 'https://%s' % self.server_name diff --git a/service/app/bitmask_libraries/session.py b/service/app/bitmask_libraries/session.py new file mode 100644 index 00000000..66e4fcab --- /dev/null +++ b/service/app/bitmask_libraries/session.py @@ -0,0 +1,153 @@ +import os +import errno +import traceback +from leap.mail.imap.fetch import LeapIncomingMail +from leap.mail.imap.server import SoledadBackedAccount +import sys +from twisted.internet import reactor +from .nicknym import NickNym + +from .auth import LeapAuthenticator +from .soledad import SoledadSessionFactory, SoledadSession +from .smtp import LeapSmtp + +SESSIONS = {} + + +class LeapSession(object): + """ + A LEAP session. + + + Properties: + + - ``leap_config`` the configuration for this session (LeapClientConfig). + + - ``srp_session`` the secure remote password session to authenticate with LEAP. See http://en.wikipedia.org/wiki/Secure_Remote_Password_protocol (LeapSecureRemotePassword) + + - ``soledad_session`` the soledad session. See https://leap.se/soledad (LeapSecureRemotePassword) + + - ``nicknym`` the nicknym instance. See https://leap.se/nicknym (NickNym) + + - ``account`` the actual leap mail account. Implements Twisted imap4.IAccount and imap4.INamespacePresenter (SoledadBackedAccount) + + - ``incoming_mail_fetcher`` Background job for fetching incoming mails from LEAP server (LeapIncomingMail) + """ + + def __init__(self, provider, srp_session, soledad_session, nicknym, soledad_account, incoming_mail_fetcher): + """ + Constructor. + + :param leap_config: The config for this LEAP session + :type leap_config: LeapConfig + + """ + self.config = provider.config + self.provider = provider + self.srp_session = srp_session + self.soledad_session = soledad_session + self.nicknym = nicknym + self.account = soledad_account + self.incoming_mail_fetcher = incoming_mail_fetcher + + if self.config.start_background_jobs: + self.start_background_jobs() + + def account_email(self): + domain = self.provider.domain + name = self.srp_session.user_name + return '%s@%s' % (name, domain) + + def close(self): + self.stop_background_jobs() + + def start_background_jobs(self): + reactor.callFromThread(self.incoming_mail_fetcher.start_loop) + + def stop_background_jobs(self): + reactor.callFromThread(self.incoming_mail_fetcher.stop) + + def sync(self): + try: + self.soledad_session.sync() + except: + traceback.print_exc(file=sys.stderr) + raise + + +class LeapSessionFactory(object): + def __init__(self, provider): + self._provider = provider + self._config = provider.config + + def create(self, credentials): + key = self._session_key(credentials) + session = self._lookup_session(key) + if not session: + session = self._create_new_session(credentials) + self._remember_session(key, session) + + return session + + def _create_new_session(self, credentials): + self._create_dir(self._provider.config.leap_home) + self._provider.download_certificate_to('%s/ca.crt' % self._provider.config.leap_home) + + auth = LeapAuthenticator(self._provider).authenticate(credentials) + soledad = SoledadSessionFactory.create(self._provider, auth, credentials.db_passphrase) + + nicknym = self._create_nicknym(auth, soledad) + account = self._create_account(auth, soledad) + incoming_mail_fetcher = self._create_incoming_mail_fetcher(nicknym, soledad, + account, auth) + + smtp = self._create_smtp_service(nicknym, auth) + smtp.start() + + session = LeapSession(self._provider, auth, soledad, nicknym, account, incoming_mail_fetcher) + + return session + + def _lookup_session(self, key): + global SESSIONS + if key in SESSIONS: + return SESSIONS[key] + else: + return None + + def _remember_session(self, key, session): + global SESSIONS + SESSIONS[key] = session + + def _session_key(self, credentials): + return hash((self._provider, credentials.user_name)) + + def _create_dir(self, path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + def _create_soledad_session(self, srp_session, db_passphrase): + return SoledadSession(self._provider, db_passphrase, srp_session) + + def _create_nicknym(self, srp_session, soledad_session): + return NickNym(self._provider, self._config, soledad_session, srp_session) + + def _create_account(self, srp_session, soledad_session): + return SoledadBackedAccount(srp_session.uuid, soledad_session.soledad) + + def _create_incoming_mail_fetcher(self, nicknym, soledad_session, account, auth): + return LeapIncomingMail(nicknym.keymanager, soledad_session.soledad, account, + self._config.fetch_interval_in_s, self._account_email(auth)) + + def _create_smtp_service(self, nicknym, auth): + return LeapSmtp(self._provider, nicknym.keymanager, auth) + + def _account_email(self, auth): + domain = self._provider.domain + name = auth.user_name + return '%s@%s' % (name, domain) diff --git a/service/app/bitmask_libraries/smtp.py b/service/app/bitmask_libraries/smtp.py new file mode 100644 index 00000000..0315b40a --- /dev/null +++ b/service/app/bitmask_libraries/smtp.py @@ -0,0 +1,77 @@ +import os +import requests +from .certs import which_bundle +from leap.mail.smtp import setup_smtp_gateway + +class LeapSmtp(object): + + SMTP_PORT = 2014 + + def __init__(self, provider, keymanager=None, leap_srp_session=None): + self._provider = provider + self._keymanager = keymanager + self._srp_session = leap_srp_session + self._hostname, self._port = self._discover_smtp_server() + self._smtp_port = None + self._smtp_service = None + + def smtp_info(self): + return ('localhost', LeapSmtp.SMTP_PORT) + + def _discover_smtp_server(self): + json_data = self._provider.fetch_smtp_json() + hosts = json_data['hosts'] + hostname = hosts.keys()[0] + host = hosts[hostname] + + hostname = host['hostname'] + port = host['port'] + + return hostname, port + + def _download_client_certificates(self): + cert_path = self._client_cert_path() + + if not os.path.exists(os.path.dirname(cert_path)): + os.makedirs(os.path.dirname(cert_path)) + + session = requests.session() + cert_url = '%s/%s/cert' % (self._provider.api_uri, self._provider.api_version) + cookies = { "_session_id": self._srp_session.session_id } + + response = requests.get(cert_url, verify=which_bundle(self._provider), cookies=cookies, timeout=self._provider.config.timeout_in_s) + response.raise_for_status() + + client_cert = response.content + + with open(cert_path, 'w') as f: + f.write(client_cert) + + def _client_cert_path(self): + return os.path.join(self._provider.config.leap_home, + "providers", + self._provider.domain, + "keys", "client", "smtp.pem") + + def start(self): + self._download_client_certificates() + cert_path = self._client_cert_path() + email = '%s@%s' % (self._srp_session.user_name, self._provider.domain) + + self._smtp_service, self._smtp_port = setup_smtp_gateway( + port=LeapSmtp.SMTP_PORT, + userid=email, + keymanager=self._keymanager, + smtp_host=self._hostname.encode('UTF-8'), + smtp_port=self._port, + smtp_cert=cert_path, + smtp_key=cert_path, + encrypted_only=False) + + def stop(self): + if self._smtp_service is not None: + self._smtp_port.stopListening() + self._smtp_service.doStop() + self._smtp_port = None + self._smtp_service = None + diff --git a/service/app/bitmask_libraries/soledad.py b/service/app/bitmask_libraries/soledad.py new file mode 100644 index 00000000..132e671f --- /dev/null +++ b/service/app/bitmask_libraries/soledad.py @@ -0,0 +1,95 @@ +import json +import os +import errno +from leap.keymanager import KeyManager +from leap.soledad.client import Soledad +from leap.soledad.common.crypto import WrongMac, UnknownMacMethod, MacMethods +import requests +import sys +import time +from .certs import which_bundle + +SOLEDAD_TIMEOUT = 120 +SOLEDAD_CERT = '/tmp/ca.crt' + + +class SoledadDiscoverException(Exception): + def __init__(self, *args, **kwargs): + super(SoledadDiscoverException, self).__init__(*args, **kwargs) + + +class SoledadWrongPassphraseException(Exception): + def __init__(self, *args, **kwargs): + super(SoledadWrongPassphraseException, self).__init__(*args, **kwargs) + + +class LeapKeyManager(object): + def __init__(self, soledad, leap_session, nicknym_url): + provider = leap_session.provider + self.keymanager = KeyManager(leap_session.account_email(), nicknym_url, soledad, + leap_session.session_id, leap_session.leap_home + '/ca.crt', provider.api_uri, leap_session.api_version, + leap_session.uuid, leap_session.leap_config.gpg_binary) + + +class SoledadSessionFactory(object): + @classmethod + def create(cls, provider, srp_session, encryption_passphrase): + return SoledadSession(provider, encryption_passphrase, srp_session) + + +class SoledadSession(object): + def __init__(self, provider, encryption_passphrase, leap_srp_session): + self.provider = provider + self.config = provider.config + self.leap_srp_session = leap_srp_session + + self.soledad = self._init_soledad(encryption_passphrase) + + def _init_soledad(self, encryption_passphrase): + try: + server_url = self._discover_soledad_server() + + self._create_database_dir() + secrets = self._secrets_path() + local_db = self._local_db_path() + + return Soledad(self.leap_srp_session.uuid, unicode(encryption_passphrase), secrets, + local_db, server_url, which_bundle(self.provider), self.leap_srp_session.token) + + except (WrongMac, UnknownMacMethod, MacMethods), e: + raise SoledadWrongPassphraseException(e) + + def _leap_path(self): + return "%s/soledad" % self.config.leap_home + + def _secrets_path(self): + return "%s/%s.secret" % (self._leap_path(), self.leap_srp_session.uuid) + + def _local_db_path(self): + return "%s/%s.db" % (self._leap_path(), self.leap_srp_session.uuid) + + def _create_database_dir(self): + try: + os.makedirs(self._leap_path()) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(self._leap_path()): + pass + else: + raise + + def sync(self): + if self.soledad.need_sync(self.soledad.server_url): + self.soledad.sync() + + def _discover_soledad_server(self): + try: + json_data = self.provider.fetch_soledad_json() + + hosts = json_data['hosts'] + host = hosts.keys()[0] + server_url = 'https://%s:%d/user-%s' % \ + (hosts[host]['hostname'], hosts[host]['port'], + self.leap_srp_session.uuid) + return server_url + except Exception, e: + raise SoledadDiscoverException(e) diff --git a/service/app/leap/client.py b/service/app/leap/client.py index 5f9020fd..031f7526 100644 --- a/service/app/leap/client.py +++ b/service/app/leap/client.py @@ -1,8 +1,24 @@ + class Client: + def __init__(self, config, username, password, server_name, mailbox_name): + try: + self.username = username + self.password = password + self.server_name = server_name + self.mailbox_name = mailbox_name + self.leapdir = '%s/leap' % config.workdir + + self._open_leap_session() + except: + traceback.print_exc(file=sys.stdout) + raise + + def _open_leap_session(self): + self.leap_config = LeapConfig(leap_home=self.leapdir) + self.provider = LeapProvider(self.server_name, self.leap_config) + self.leap_session = LeapSessionFactory(self.provider).create(LeapCredentials(self.username, self.password)) + self.mbx = self.leap_session.account.getMailbox(self.mailbox_name) - def __init__(self, account): - pass - def mails(self, query): raise NotImplementedError() @@ -58,6 +74,3 @@ class Client: def all_contacts(self, query): raise NotImplementedError() - - - @@ -1,4 +1,4 @@ #!/bin/bash export PIXELATED_UA_CFG=../config/pixelated_ua.cfg -python app/pixelated_usar_agent.py +python app/pixelated_user_agent.py diff --git a/service/requirements.txt b/service/requirements.txt index e1ec3242..468be9d4 100644 --- a/service/requirements.txt +++ b/service/requirements.txt @@ -2,3 +2,17 @@ Twisted==12.2.0 flask==0.10.1 scanner==0.0.5 requests==2.3.0 +pytest==2.6.0 +#requirements for bitmask libraries +mock==1.0.1 +httmock==1.2.2 +srp==1.0.4 +--allow-external dirspec +--allow-unverified dirspec +dirspec==13.10 +--allow-external u1db +--allow-unverified u1db +u1db==13.10 +leap.keymanager==0.3.8 +leap.soledad.common==0.5.0 +leap.mail==0.3.8 diff --git a/service/runtests b/service/runtests index 64316a29..6b10459a 100755 --- a/service/runtests +++ b/service/runtests @@ -1 +1,2 @@ -APP_ROOT=`pwd`/app py.test test/ +#!/bin/bash +APP_ROOT=`pwd` py.test test/ diff --git a/service/test/bitmask_libraries/__init__.py b/service/test/bitmask_libraries/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/test/bitmask_libraries/__init__.py diff --git a/service/test/bitmask_libraries/abstract_leap_test.py b/service/test/bitmask_libraries/abstract_leap_test.py new file mode 100644 index 00000000..007fe06a --- /dev/null +++ b/service/test/bitmask_libraries/abstract_leap_test.py @@ -0,0 +1,26 @@ +import os +import tempfile +import unittest +from uuid import uuid4 +from mock import Mock, MagicMock + + +class AbstractLeapTest(unittest.TestCase): + uuid = str(uuid4()) + session_id = str(uuid4()) + token = str(uuid4()) + + leap_home = os.path.join(tempfile.mkdtemp(), 'leap') + + config = Mock(leap_home=leap_home, ca_cert_bundle='/some/path/to/ca_cert', gpg_binary='/path/to/gpg') + provider = Mock(config=config, server_name='some-server.test', domain='some-server.test', + api_uri='https://api.some-server.test:4430', api_version='1') + soledad = Mock() + soledad_session = Mock(soledad=soledad) + srp_session = Mock(user_name='test_user', api_server_name='some-server.test', uuid=uuid, session_id=session_id, token=token) + + nicknym = MagicMock() + + soledad_account = MagicMock() + + mail_fetcher_mock = MagicMock() diff --git a/service/test/bitmask_libraries/leap_srp_test.py b/service/test/bitmask_libraries/leap_srp_test.py new file mode 100644 index 00000000..a8b5b3fb --- /dev/null +++ b/service/test/bitmask_libraries/leap_srp_test.py @@ -0,0 +1,106 @@ +import sys +import os +sys.path.insert(0, os.environ['APP_ROOT']) + +import json +import unittest +import binascii +from urlparse import parse_qs + +from httmock import urlmatch, all_requests, HTTMock, response +from requests.exceptions import Timeout +import srp + +from app.bitmask_libraries.leap_srp import LeapSecureRemotePassword, LeapAuthException + + + +(salt_bytes, verification_key_bytes) = srp.create_salted_verification_key('username', 'password', hash_alg=srp.SHA256, ng_type=srp.NG_1024) +verifier = None + + +@all_requests +def not_found_mock(url, request): + return {'status_code': 404, + 'content': 'foobar'} + + +@all_requests +def timeout_mock(url, request): + raise Timeout() + +@urlmatch(netloc=r'(.*\.)?leap\.local$') +def srp_login_server_simulator_mock(url, request): + global verifier + + data = parse_qs(request.body) + if 'login' in data: + # SRP Authentication Step 1 + A = binascii.unhexlify(data.get('A')[0]) + + verifier = srp.Verifier('username', salt_bytes, verification_key_bytes, A, hash_alg=srp.SHA256, ng_type=srp.NG_1024) + (salt, B) = verifier.get_challenge() + + content = { + 'salt': binascii.hexlify(salt), + 'B': binascii.hexlify(B) + } + + return {'status_code': 200, + 'content': json.dumps(content)} + + else: + # SRP Authentication Step 2 + data = parse_qs(request.body) + client_auth = binascii.unhexlify(data.get('client_auth')[0]) + + M2 = verifier.verify_session(client_auth) + + if not verifier.authenticated(): + return {'status_code': 404, + 'content': ''} + + content = { + 'M2': binascii.hexlify(M2), + 'id': 'some id', + 'token': 'some token' + } + headers = { + 'Content-Type': 'application/json', + 'Set-Cookie': '_session_id=some_session_id;'} + return response(200, content, headers, None, 5, request) + + +class LeapSRPTest(unittest.TestCase): + + def test_status_code_is_checked(self): + with HTTMock(not_found_mock): + lsrp = LeapSecureRemotePassword() + self.assertRaises(LeapAuthException, lsrp.authenticate, 'https://api.leap.local', 'username', 'password') + + def test_invalid_username(self): + with HTTMock(srp_login_server_simulator_mock): + lsrp = LeapSecureRemotePassword() + self.assertRaises(LeapAuthException, lsrp.authenticate, 'https://api.leap.local', 'invalid_user', 'password') + + def test_invalid_password(self): + with HTTMock(srp_login_server_simulator_mock): + lsrp = LeapSecureRemotePassword() + self.assertRaises(LeapAuthException, lsrp.authenticate, 'https://api.leap.local', 'username', 'invalid') + + def test_login(self): + with HTTMock(srp_login_server_simulator_mock): + lsrp = LeapSecureRemotePassword() + leap_session = lsrp.authenticate('https://api.leap.local', 'username', 'password') + + self.assertIsNotNone(leap_session) + self.assertEqual('username', leap_session.user_name) + self.assertEqual('1', leap_session.api_version) + self.assertEqual('https://api.leap.local', leap_session.api_server_name) + self.assertEqual('some token', leap_session.token) + self.assertEqual('some_session_id', leap_session.session_id) + + def test_timeout(self): + with HTTMock(timeout_mock): + lrsp = LeapSecureRemotePassword() + self.assertRaises(LeapAuthException, lrsp.authenticate, 'https://api.leap.local', 'username', 'password') diff --git a/service/test/bitmask_libraries/nicknym_test.py b/service/test/bitmask_libraries/nicknym_test.py new file mode 100644 index 00000000..a087874a --- /dev/null +++ b/service/test/bitmask_libraries/nicknym_test.py @@ -0,0 +1,19 @@ +from mock import patch + +from app.bitmask_libraries.nicknym import NickNym +from abstract_leap_test import AbstractLeapTest + + +class NickNymTest(AbstractLeapTest): + @patch('app.bitmask_libraries.nicknym.KeyManager.__init__', return_value=None) + def test_that_keymanager_is_created(self, init_mock): + #given + + #when + NickNym(self.provider, self.config, self.soledad_session, self.srp_session) + + #then + init_mock.assert_called_with('test_user@some-server.test', 'https://nicknym.some-server.test:6425/', + self.soledad, self.session_id, '/some/path/to/ca_cert', + 'https://api.some-server.test:4430', '1', self.uuid, + '/path/to/gpg') diff --git a/service/test/bitmask_libraries/provider_test.py b/service/test/bitmask_libraries/provider_test.py new file mode 100644 index 00000000..16abbd0d --- /dev/null +++ b/service/test/bitmask_libraries/provider_test.py @@ -0,0 +1,170 @@ +import json + +from httmock import all_requests, HTTMock, urlmatch +from requests import HTTPError + +from app.bitmask_libraries.config import LeapConfig +from app.bitmask_libraries.provider import LeapProvider +from abstract_leap_test import AbstractLeapTest + + +@all_requests +def not_found_mock(url, request): + return {'status_code': 404, + 'content': 'foobar'} + +@urlmatch(netloc=r'(.*\.)?some-provider\.test$', path='/provider.json') +def provider_json_mock(url, request): + return provider_json_response("SHA256: 06e2300bdbc118c290eda0dc977c24080718f4eeca68c8b0ad431872a2baa22d") + + +@urlmatch(netloc=r'(.*\.)?some-provider\.test$', path='/provider.json') +def provider_json_invalid_fingerprint_mock(url, request): + return provider_json_response("SHA256: 0123456789012345678901234567890123456789012345678901234567890123") + + +def provider_json_response(fingerprint): + content = { + "api_uri": "https://api.some-provider.test:4430", + "api_version": "1", + "ca_cert_fingerprint": fingerprint, + "ca_cert_uri": "https://some-provider.test/ca.crt", + "domain": "some-provider.test", + "services": [ + "mx" + ] + } + return { + "status_code": 200, + "content": json.dumps(content) + } + + +@urlmatch(netloc=r'api\.some-provider\.test:4430$', path='/1/config/soledad-service.json') +def soledad_json_mock(url, request): + content = { + "some key": "some value", + } + return { + "status_code": 200, + "content": json.dumps(content) + } + +@urlmatch(netloc=r'api\.some-provider\.test:4430$', path='/1/config/smtp-service.json') +def smtp_json_mock(url, request): + content = { + "hosts": { + "leap-mx": { + "hostname": "mx.some-provider.test", + "ip_address": "0.0.0.0", + "port": 465 + } + }, + "locations": { + }, + "serial": 1, + "version": 1 + } + return { + "status_code": 200, + "content": json.dumps(content) + } + + +@urlmatch(netloc=r'(.*\.)?some-provider\.test$', path='/ca.crt') +def ca_cert_mock(url, request): + return { + "status_code": 200, + "content": ca_crt + } + + +ca_crt = """ +-----BEGIN CERTIFICATE----- +MIIFbzCCA1egAwIBAgIBATANBgkqhkiG9w0BAQ0FADBKMREwDwYDVQQKDAhXYXpv +a2F6aTEaMBgGA1UECwwRaHR0cHM6Ly9kZmkubG9jYWwxGTAXBgNVBAMMEFdhem9r +YXppIFJvb3QgQ0EwHhcNMTQwMzI1MDAwMDAwWhcNMjQwMzI1MDAwMDAwWjBKMREw +DwYDVQQKDAhXYXpva2F6aTEaMBgGA1UECwwRaHR0cHM6Ly9kZmkubG9jYWwxGTAX +BgNVBAMMEFdhem9rYXppIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDSPyaslC6SNVsKpGoXllInPXbjiq7rJaV08Xg+64FJU/257BZZEJ/j +r33r0xlt2kj85PcbPySLKy0omXAQt9bs273hwAQXExdY41FxMD3wP/dmLqd55KYa +LDV4GUw0QPZ0QUyWVrRHkrdCDyjpRG+6GbowmtygJKLflYmUFC1PYQ3492esr0jC ++Q6L6+/D2+hBiH3NPI22Yk0kQmuPfnu2pvo+EYQ3It81qZE0Jo8u/BqOMgN2f9DS +GvSNfZcKAP18A41/VRrYFa/WUcdDxt/uP5nO1dm2vfLorje3wcMGtGRcDKG/+GAm +S0nYKKQeWYc6z5SDvPM1VlNdn1gOejhAoggT3Hr5Dq8kxW/lQZbOz+HLbz15qGjz +gL4KHKuDE6hOuqxpHdMTY4WZBBQ8/6ICBxaXH9587/nNDdZiom+XukVD4mrSMJS7 +PRr14Hw57433AJDJcZRwZNRRAGgDPNsCoR2caKB6/Uwkp+dWVndj5Ad8MEjyM1yV ++fYU6PSQWNig7qqN5VhNY+zUCcez5gL6volMuW00iOkXISW4lBrcZmEAQTTcWT1D +U7EkLlwITQce63LcuvK7ZWsEm5XCqD+yUz9oQfugmIhxAlTdqt3De9FA0WT9WxGt +zLeswCNKjnMpRgTerq6elwB03EBJVc7k1QRn4+s6C30sXR12dYnEMwIDAQABo2Aw +XjAdBgNVHQ4EFgQU8ItSdI5pSqMDjgRjgYI3Nj0SwxQwDgYDVR0PAQH/BAQDAgIE +MAwGA1UdEwQFMAMBAf8wHwYDVR0jBBgwFoAU8ItSdI5pSqMDjgRjgYI3Nj0SwxQw +DQYJKoZIhvcNAQENBQADggIBALdSPUrIqyIlSMr4R7pWd6Ep0BZH5RztVUcoXtei +x2MFi/rsw7aL9qZqACYIE8Gkkh6Z6GQph0fIqhAlNFvJXKkguL3ri5xh0XmPfbv/ +OLIvaUAixATivdm8ro/IqYQWdL3P6mDZOv4O6POdBEJ9JLc9RXUt1LiQ5Xb9QiLs +l/yOthhp5dJHqC8s6CDEUHRe3s9Q/4cwNB4td47I+mkLsNtVNXqi4lOzuQamqiFt +cFIqOLTFtBJ7G3k9iaDuN6RPS6LMRbqabwg4gafQTmJ+roHpnsaiHkfomI4MZOVi +TLQKOAJ3/pRGm5cGzkzQ+z4sUiCSQxtIWs7EnQCCE8agqpef6zArAvKEO+139+f2 +u1BhWOm/aHT5a3INnJEbuFr8V9MlbZSxSzU3UH7hby+9PxWKYesc6KUAu6Icooci +gEQqrVhVKmfaYMLL7UZHhw56yv/6B10SSmeAMiJhtTExjjrTRLSCaKCPa2ISAUDB +aPR3t8ZoUESWRAFQGj5NvWOomTaXfyE8Or2WfNemvdlWsKvlLeVsjts+iaTgQRU9 +VXcrUhrHhaXhYXeWrWkDDcl8VUlDWXzoUGV9SczOGwr6hONJWMn1HNxNV7ywFWf0 +QXH1g3LBW7qNgRaGhbIX4a1WoNQDmbbKaLgKWs74atZ8o4A2aUEjomclgZWPsc5l +VeJ6 +-----END CERTIFICATE----- +""" + + +class LeapProviderTest(AbstractLeapTest): + def setUp(self): + self.config = LeapConfig(verify_ssl=False, leap_home='/tmp/foobar', ca_cert_bundle='/tmp/ca.crt') + + def test_provider_fetches_provider_json(self): + with HTTMock(provider_json_mock): + provider = LeapProvider('some-provider.test', self.config) + + self.assertEqual("1", provider.api_version) + self.assertEqual("some-provider.test", provider.domain) + self.assertEqual("https://api.some-provider.test:4430", provider.api_uri) + self.assertEqual("https://some-provider.test/ca.crt", provider.ca_cert_uri) + self.assertEqual("SHA256: 06e2300bdbc118c290eda0dc977c24080718f4eeca68c8b0ad431872a2baa22d", + provider.ca_cert_fingerprint) + self.assertEqual(["mx"], provider.services) + + def test_provider_json_throws_exception_on_status_code(self): + with HTTMock(not_found_mock): + self.assertRaises(HTTPError, LeapProvider, 'some-provider.test', self.config) + + def test_fetch_soledad_json(self): + with HTTMock(provider_json_mock, soledad_json_mock, not_found_mock): + provider = LeapProvider('some-provider.test', self.config) + soledad = provider.fetch_soledad_json() + + self.assertEqual("some value", soledad.get('some key')) + + def test_throw_exception_for_fetch_soledad_status_code(self): + with HTTMock(provider_json_mock, not_found_mock): + provider = LeapProvider('some-provider.test', self.config) + + self.assertRaises(HTTPError, provider.fetch_soledad_json) + + def test_fetch_smtp_json(self): + with HTTMock(provider_json_mock, smtp_json_mock, not_found_mock): + provider = LeapProvider('some-provider.test', self.config) + smtp = provider.fetch_smtp_json() + self.assertEqual('mx.some-provider.test', smtp.get('hosts').get('leap-mx').get('hostname')) + + def test_throw_exception_for_fetch_smtp_status_code(self): + with HTTMock(provider_json_mock, not_found_mock): + provider = LeapProvider('some-provider.test', self.config) + self.assertRaises(HTTPError, provider.fetch_smtp_json) + + def test_fetch_valid_certificate(self): + with HTTMock(provider_json_mock, ca_cert_mock, not_found_mock): + provider = LeapProvider('some-provider.test', self.config) + provider.fetch_valid_certificate() + + def test_throw_exception_for_invalid_certificate(self): + with HTTMock(provider_json_invalid_fingerprint_mock, ca_cert_mock, not_found_mock): + provider = LeapProvider('some-provider.test', self.config) + self.assertRaises(Exception, provider.fetch_valid_certificate) diff --git a/service/test/bitmask_libraries/session_test.py b/service/test/bitmask_libraries/session_test.py new file mode 100644 index 00000000..aa5f012d --- /dev/null +++ b/service/test/bitmask_libraries/session_test.py @@ -0,0 +1,49 @@ +from mock import patch + +from app.bitmask_libraries.session import LeapSession +from abstract_leap_test import AbstractLeapTest + + +class SessionTest(AbstractLeapTest): + def test_background_jobs_are_started(self): + self.config.start_background_jobs = True + + with patch('app.bitmask_libraries.session.reactor.callFromThread', new=_execute_func) as _: + self._create_session() + + self.mail_fetcher_mock.start_loop.assert_called_once_with() + + def test_background_jobs_are_not_started(self): + self.config.start_background_jobs = False + + with patch('app.bitmask_libraries.session.reactor.callFromThread', new=_execute_func) as _: + self._create_session() + + self.assertFalse(self.mail_fetcher_mock.start_loop.called) + + def test_that_close_stops_background_jobs(self): + with patch('app.bitmask_libraries.session.reactor.callFromThread', new=_execute_func) as _: + session = self._create_session() + + session.close() + + self.mail_fetcher_mock.stop.assert_called_once_with() + + def test_that_sync_deferes_to_soledad(self): + session = self._create_session() + + session.sync() + + self.soledad_session.sync.assert_called_once_with() + + def test_account_email(self): + session = self._create_session() + self.assertEqual('test_user@some-server.test', session.account_email()) + + def _create_session(self): + return LeapSession(self.provider, self.srp_session, self.soledad_session, self.nicknym, self.soledad_account, + self.mail_fetcher_mock) + + +def _execute_func(func): + func() diff --git a/service/test/bitmask_libraries/smtp_test.py b/service/test/bitmask_libraries/smtp_test.py new file mode 100644 index 00000000..3982a50c --- /dev/null +++ b/service/test/bitmask_libraries/smtp_test.py @@ -0,0 +1,77 @@ +from mock import MagicMock, patch +from abstract_leap_test import AbstractLeapTest +from app.bitmask_libraries.smtp import LeapSmtp +from httmock import all_requests, HTTMock, urlmatch +import os +import sys + +@all_requests +def not_found_mock(url, request): + sys.stderr.write('url=%s\n' % url.netloc) + sys.stderr.write('path=%s\n' % url.path) + return {'status_code': 404, + 'content': 'foobar'} + +@urlmatch(netloc='api.some-server.test:4430', path='/1/cert') +def ca_cert_mock(url, request): + return { + "status_code": 200, + "content": "some content" + } + +class LeapSmtpTest(AbstractLeapTest): + keymanager = MagicMock() + + def setUp(self): + self.provider.fetch_smtp_json.return_value = { + 'hosts': { + 'leap-mx': { + 'hostname': 'smtp.some-sever.test', + 'port': '1234' + } + } + } + self.config.timeout_in_s = 15 + + def test_that_client_cert_gets_downloaded(self): + smtp = LeapSmtp(self.provider, self.keymanager, self.srp_session) + + with HTTMock(ca_cert_mock, not_found_mock): + smtp._download_client_certificates() + + path = self._client_cert_path() + self.assertTrue(os.path.isfile(path)) + + def _client_cert_path(self): + return os.path.join(self.leap_home, 'providers', 'some-server.test', 'keys', 'client', 'smtp.pem') + + @patch('app.bitmask_libraries.smtp.setup_smtp_gateway') + def test_that_start_calls_setup_smtp_gateway(self, gateway_mock): + smtp = LeapSmtp(self.provider, self.keymanager, self.srp_session) + gateway_mock.return_value = (None, None) + with HTTMock(ca_cert_mock, not_found_mock): + smtp.start() + + cert_path = self._client_cert_path() + gateway_mock.assert_called_with(keymanager=self.keymanager, smtp_cert=cert_path, smtp_key=cert_path, userid='test_user@some-server.test', smtp_port='1234', encrypted_only=False, smtp_host='smtp.some-sever.test', port=2014) + + def test_that_client_stop_does_nothing_if_not_started(self): + smtp = LeapSmtp(self.provider, self.keymanager, self.srp_session) + + with HTTMock(not_found_mock): + smtp.stop() + + @patch('app.bitmask_libraries.smtp.setup_smtp_gateway') + def test_that_running_smtp_sevice_is_stopped(self, gateway_mock): + smtp = LeapSmtp(self.provider, self.keymanager, self.srp_session) + + smtp_service = MagicMock() + smtp_port = MagicMock() + gateway_mock.return_value = (smtp_service, smtp_port) + + with HTTMock(ca_cert_mock, not_found_mock): + smtp.start() + smtp.stop() + + smtp_port.stopListening.assert_called_with() + smtp_service.doStop.assert_called_with() diff --git a/service/test/bitmask_libraries/soledad_test.py b/service/test/bitmask_libraries/soledad_test.py new file mode 100644 index 00000000..1c1c105e --- /dev/null +++ b/service/test/bitmask_libraries/soledad_test.py @@ -0,0 +1,54 @@ +from mock import patch +from app.bitmask_libraries.soledad import SoledadSession +from abstract_leap_test import AbstractLeapTest + + +@patch('app.bitmask_libraries.soledad.Soledad') +class SoledadSessionTest(AbstractLeapTest): + + def setUp(self): + #given + self.provider.fetch_soledad_json.return_value = {'hosts': { + 'couch1': { + 'hostname': 'couch1.some-server.test', + 'ip_address': '192.168.1.1', + 'port': 1234 + } + }} + + @patch('app.bitmask_libraries.soledad.Soledad.__init__') + def test_that_soledad_is_created_with_required_params(self, soledad_mock, init_mock): + #when + SoledadSession(self.provider, 'any-passphrase', self.srp_session) + + #then + init_mock.assert_called_with(self.uuid, 'any-passphrase', '%s/soledad/%s.secret' % (self.leap_home, self.uuid), + '%s/soledad/%s.db' % (self.leap_home, self.uuid), + 'https://couch1.some-server.test:1234/user-%s' % self.uuid, + '/some/path/to/ca_cert', self.token) + + def test_that_sync_is_called(self, soledad_mock): + instance = soledad_mock.return_value + instance.server_url = '/foo/bar' + instance.need_sync.return_value = True + soledad_session = SoledadSession(self.provider, 'any-passphrase', self.srp_session) + + #when + soledad_session.sync() + + #then + instance.need_sync.assert_called_with('/foo/bar') + instance.sync.assert_called_with() + + def test_that_sync_not_called_if_not_needed(self, mock): + instance = mock.return_value + instance.server_url = '/foo/bar' + instance.need_sync.return_value = False + soledad_session = SoledadSession(self.provider, 'any-passphrase', self.srp_session) + + #when + soledad_session.sync() + + #then + instance.need_sync.assert_called_with('/foo/bar') + self.assertFalse(instance.sync.called) diff --git a/service/test/search/test_search_query.py b/service/test/search/test_search_query.py index d980c3f0..85d9681d 100644 --- a/service/test/search/test_search_query.py +++ b/service/test/search/test_search_query.py @@ -1,7 +1,7 @@ import sys, os sys.path.insert(0, os.environ['APP_ROOT']) -from search import SearchQuery +from app.search import SearchQuery def test_one_tag(): assert SearchQuery.compile(u"in:inbox")["tags"] == ["inbox"] |