diff options
Diffstat (limited to 'service')
| -rw-r--r-- | service/pixelated/adapter/model/mail.py | 38 | ||||
| -rw-r--r-- | service/pixelated/bitmask_libraries/certs.py | 34 | ||||
| -rw-r--r-- | service/pixelated/bitmask_libraries/config.py | 3 | ||||
| -rw-r--r-- | service/pixelated/bitmask_libraries/register.py | 3 | ||||
| -rw-r--r-- | service/pixelated/bitmask_libraries/session.py | 4 | ||||
| -rw-r--r-- | service/pixelated/config/leap_cert.py | 2 | ||||
| -rw-r--r-- | service/pixelated/support/ext_requests_urllib3.py | 107 | ||||
| -rw-r--r-- | service/pixelated/support/tls_adapter.py | 2 | ||||
| -rw-r--r-- | service/test/unit/adapter/test_mail.py | 34 | ||||
| -rw-r--r-- | service/test/unit/bitmask_libraries/test_certs.py | 6 |
10 files changed, 152 insertions, 81 deletions
diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py index f23c2708..618b980a 100644 --- a/service/pixelated/adapter/model/mail.py +++ b/service/pixelated/adapter/model/mail.py @@ -21,6 +21,7 @@ from email.header import decode_header from leap.mail.imap.fields import fields import leap.mail.walk as walk import dateutil.parser as dateparser +from datetime import datetime from pixelated.adapter.model.status import Status import pixelated.support.date from email.MIMEMultipart import MIMEMultipart @@ -291,7 +292,10 @@ class PixelatedMail(Mail): for header in ['From', 'Subject']: _headers[header] = self._decode_header(hdoc_headers.get(header)) - _headers['Date'] = self._get_date() + try: + _headers['Date'] = self._get_date() + except Exception, e: + _headers['Date'] = pixelated.support.date.iso_now() if self.parts and len(self.parts['alternatives']) > 1: _headers['content_type'] = 'multipart/alternative; boundary="%s"' % self.boundary @@ -303,25 +307,35 @@ class PixelatedMail(Mail): return _headers + def _decode_header_with_fallback(self, entry): + try: + return decode_header(entry)[0][0] + except Exception, e: + return entry.encode('ascii', 'ignore') + def _decode_header(self, header): if not header: return None if isinstance(header, list): - return [decode_header(entry)[0][0] for entry in header] + return [self._decode_header_with_fallback(entry) for entry in header] else: - return decode_header(header)[0][0] + return self._decode_header_with_fallback(header) def _get_date(self): date = self.hdoc.content.get('date', None) - if not date: - received = self.hdoc.content.get('received', None) - if received: - date = received.split(";")[-1].strip() - else: - # we can't get a date for this mail, so lets just use now - logger.warning('Encountered a mail with missing date and received header fields. Subject %s' % self.hdoc.content.get('subject', None)) - date = pixelated.support.date.iso_now() - return dateparser.parse(date).isoformat() + try: + if not date: + received = self.hdoc.content.get('received', None) + if received: + date = received.split(";")[-1].strip() + else: + # we can't get a date for this mail, so lets just use now + logger.warning('Encountered a mail with missing date and received header fields. Subject %s' % self.hdoc.content.get('subject', None)) + date = pixelated.support.date.iso_now() + return dateparser.parse(date).isoformat() + except (ValueError, TypeError) as e: + date = pixelated.support.date.iso_now() + return dateparser.parse(date).isoformat() @property def security_casing(self): diff --git a/service/pixelated/bitmask_libraries/certs.py b/service/pixelated/bitmask_libraries/certs.py index 31e68d1c..a321e00e 100644 --- a/service/pixelated/bitmask_libraries/certs.py +++ b/service/pixelated/bitmask_libraries/certs.py @@ -14,13 +14,15 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. import os - +import requests +import json from leap.common import ca_bundle from .config import AUTO_DETECT_CA_BUNDLE LEAP_CERT = None LEAP_FINGERPRINT = None +PACKAGED_CERTS_HOME = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "certificates")) def which_api_CA_bundle(provider): @@ -45,7 +47,6 @@ class LeapCertificate(object): def __init__(self, provider): self._config = provider.config self._server_name = provider.server_name - self._certs_home = self._config.certs_home self._provider = provider def auto_detect_bootstrap_ca_bundle(self): @@ -84,12 +85,35 @@ class LeapCertificate(object): return path def _local_bootstrap_server_cert(self): - cert_file = os.path.join(self._certs_home, '%s.ca.crt' % self._server_name) - if not os.path.isfile(cert_file): - self._download_server_cert(cert_file) + cert_file = self._bootstrap_certs_cert_file() + if os.path.isfile(cert_file): + return cert_file + + cert_file = os.path.join(PACKAGED_CERTS_HOME, '%s.ca.crt' % self._server_name) + if os.path.exists(cert_file): + return cert_file + + # else download the file + cert_file = self._bootstrap_certs_cert_file() + response = requests.get('https://%s/provider.json' % self._server_name) + provider_data = json.loads(response.content) + ca_cert_uri = str(provider_data['ca_cert_uri']) + + response = requests.get(ca_cert_uri) + with open(cert_file, 'w') as file: + file.write(response.content) return cert_file + def _bootstrap_certs_cert_file(self): + path = os.path.join(self._provider.config.leap_home, 'providers', self._server_name) + if not os.path.isdir(path): + os.makedirs(path, 0700) + + file_path = os.path.join(path, '%s.ca.crt' % self._server_name) + + return file_path + def _download_server_cert(self, cert_file_name): cert = self._provider.fetch_valid_certificate() diff --git a/service/pixelated/bitmask_libraries/config.py b/service/pixelated/bitmask_libraries/config.py index 56f28706..8c862d0a 100644 --- a/service/pixelated/bitmask_libraries/config.py +++ b/service/pixelated/bitmask_libraries/config.py @@ -45,7 +45,7 @@ class LeapConfig(object): def __init__(self, leap_home=DEFAULT_LEAP_HOME, bootstrap_ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, verify_ssl=True, fetch_interval_in_s=30, - timeout_in_s=15, start_background_jobs=False, gpg_binary=discover_gpg_binary(), certs_home=None): + timeout_in_s=15, start_background_jobs=False, gpg_binary=discover_gpg_binary()): """ Constructor. @@ -75,7 +75,6 @@ class LeapConfig(object): """ self.leap_home = leap_home - self.certs_home = certs_home self.bootstrap_ca_cert_bundle = bootstrap_ca_cert_bundle self.ca_cert_bundle = ca_cert_bundle self.verify_ssl = verify_ssl diff --git a/service/pixelated/bitmask_libraries/register.py b/service/pixelated/bitmask_libraries/register.py index 258394da..bb54477f 100644 --- a/service/pixelated/bitmask_libraries/register.py +++ b/service/pixelated/bitmask_libraries/register.py @@ -25,8 +25,7 @@ from pixelated.bitmask_libraries.auth import LeapAuthenticator, LeapCredentials def register_new_user(username, server_name): - certs_home = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "certificates")) - config = LeapConfig(certs_home=certs_home) + config = LeapConfig() provider = LeapProvider(server_name, config) password = getpass.getpass('Please enter password for %s: ' % username) LeapAuthenticator(provider).register(LeapCredentials(username, password)) diff --git a/service/pixelated/bitmask_libraries/session.py b/service/pixelated/bitmask_libraries/session.py index c0c8f712..eb6f39d7 100644 --- a/service/pixelated/bitmask_libraries/session.py +++ b/service/pixelated/bitmask_libraries/session.py @@ -37,9 +37,7 @@ SESSIONS = {} def open(username, password, server_name, leap_home=DEFAULT_LEAP_HOME): - certs_home = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "certificates")) - - config = LeapConfig(leap_home=leap_home, certs_home=certs_home) + config = LeapConfig(leap_home=leap_home) provider = LeapProvider(server_name, config) refresh_ca_bundle(provider) session = LeapSessionFactory(provider).create(LeapCredentials(username, password)) diff --git a/service/pixelated/config/leap_cert.py b/service/pixelated/config/leap_cert.py index 3172c953..22f73720 100644 --- a/service/pixelated/config/leap_cert.py +++ b/service/pixelated/config/leap_cert.py @@ -19,7 +19,7 @@ import pixelated.bitmask_libraries.certs as certs def init_leap_cert(args): if args.leap_cert_fingerprint is None: - certs.LEAP_CERT = args.leap_cert + certs.LEAP_CERT = args.leap_cert or True certs.LEAP_FINGERPRINT = None else: certs.LEAP_FINGERPRINT = args.leap_cert_fingerprint diff --git a/service/pixelated/support/ext_requests_urllib3.py b/service/pixelated/support/ext_requests_urllib3.py index a836d6fd..c4ec2438 100644 --- a/service/pixelated/support/ext_requests_urllib3.py +++ b/service/pixelated/support/ext_requests_urllib3.py @@ -15,65 +15,68 @@ # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. import requests -import requests.packages.urllib3.connectionpool -from socket import error as SocketError, timeout as SocketTimeout -from requests.packages.urllib3.packages.ssl_match_hostname import CertificateError, match_hostname -import socket -import ssl -from requests.packages.urllib3.exceptions import ( - ClosedPoolError, - ConnectTimeoutError, - EmptyPoolError, - HostChangedError, - MaxRetryError, - SSLError, - ReadTimeoutError, - ProxyError, -) -from requests.packages.urllib3.util import ( - assert_fingerprint, - get_host, - is_connection_dropped, - resolve_cert_reqs, - resolve_ssl_version, - ssl_wrap_socket, - Timeout, -) +if requests.__version__ == '2.0.0': + try: + import requests.packages.urllib3.connectionpool + from socket import error as SocketError, timeout as SocketTimeout + from requests.packages.urllib3.packages.ssl_match_hostname import CertificateError, match_hostname + import socket + import ssl + from requests.packages.urllib3.exceptions import ( + ClosedPoolError, + ConnectTimeoutError, + EmptyPoolError, + HostChangedError, + MaxRetryError, + SSLError, + ReadTimeoutError, + ProxyError, + ) -def patched_connect(self): - # Add certificate verification - try: - sock = socket.create_connection(address=(self.host, self.port), timeout=self.timeout) - except SocketTimeout: - raise ConnectTimeoutError(self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) + from requests.packages.urllib3.util import ( + assert_fingerprint, + get_host, + is_connection_dropped, + resolve_cert_reqs, + resolve_ssl_version, + ssl_wrap_socket, + Timeout, + ) - resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) - resolved_ssl_version = resolve_ssl_version(self.ssl_version) + def patched_connect(self): + # Add certificate verification + try: + sock = socket.create_connection(address=(self.host, self.port), timeout=self.timeout) + except SocketTimeout: + raise ConnectTimeoutError(self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) - if self._tunnel_host: - self.sock = sock - # Calls self._set_hostport(), so self.host is - # self._tunnel_host below. - self._tunnel() + resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) + resolved_ssl_version = resolve_ssl_version(self.ssl_version) - # Wrap socket using verification with the root certs in - # trusted_root_certs - self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, - cert_reqs=resolved_cert_reqs, - ca_certs=self.ca_certs, - server_hostname=self.host, - ssl_version=resolved_ssl_version) + if self._tunnel_host: + self.sock = sock + # Calls self._set_hostport(), so self.host is + # self._tunnel_host below. + self._tunnel() - if self.assert_fingerprint: - assert_fingerprint(self.sock.getpeercert(binary_form=True), - self.assert_fingerprint) - elif resolved_cert_reqs != ssl.CERT_NONE and self.assert_hostname is not False: - match_hostname(self.sock.getpeercert(), - self.assert_hostname or self.host) + # Wrap socket using verification with the root certs in + # trusted_root_certs + self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, + cert_reqs=resolved_cert_reqs, + ca_certs=self.ca_certs, + server_hostname=self.host, + ssl_version=resolved_ssl_version) + if self.assert_fingerprint: + assert_fingerprint(self.sock.getpeercert(binary_form=True), + self.assert_fingerprint) + elif resolved_cert_reqs != ssl.CERT_NONE and self.assert_hostname is not False: + match_hostname(self.sock.getpeercert(), + self.assert_hostname or self.host) -if requests.__version__ == '2.0.0': - requests.packages.urllib3.connectionpool.VerifiedHTTPSConnection.connect = patched_connect + requests.packages.urllib3.connectionpool.VerifiedHTTPSConnection.connect = patched_connect + except ImportError: + pass # The patch is specific for the debian package. Ignore it if it can't be found diff --git a/service/pixelated/support/tls_adapter.py b/service/pixelated/support/tls_adapter.py index f543bf4d..301a2123 100644 --- a/service/pixelated/support/tls_adapter.py +++ b/service/pixelated/support/tls_adapter.py @@ -41,7 +41,7 @@ class EnforceTLSv1Adapter(HTTPAdapter): def init_poolmanager(self, connections, maxsize, block=False): self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, ssl_version=latest_available_ssl_version(), + block=block, assert_hostname=self._assert_hostname, assert_fingerprint=self._assert_fingerprint, cert_reqs=ssl.CERT_REQUIRED) diff --git a/service/test/unit/adapter/test_mail.py b/service/test/unit/adapter/test_mail.py index c7910b7f..d77816cd 100644 --- a/service/test/unit/adapter/test_mail.py +++ b/service/test/unit/adapter/test_mail.py @@ -1,3 +1,4 @@ +# -*- coding: UTF-8 -*- # # Copyright (c) 2014 ThoughtWorks, Inc. # @@ -66,6 +67,28 @@ class TestPixelatedMail(unittest.TestCase): self.assertEqual(str(mail.headers['Date']), leap_mail_date_in_iso_format) + def test_use_datetime_now_as_fallback_for_invalid_date(self): + leap_mail_date = u'söme däte' + date_expected = "2014-09-03T13:11:15-03:00" + + when(pixelated.support.date).iso_now().thenReturn(date_expected) + leap_mail = test_helper.leap_mail(headers={'date': leap_mail_date}) + + mail = PixelatedMail.from_soledad(*leap_mail, soledad_querier=self.querier) + + self.assertEqual(str(mail.headers['Date']), date_expected) + + def test_fall_back_to_ascii_if_invalid_received_header(self): + leap_mail_received_header = u"söme invalid received heäder\n" + date_expected = "2014-09-03T13:11:15-03:00" + + when(pixelated.support.date).iso_now().thenReturn(date_expected) + leap_mail = test_helper.leap_mail(headers={'received': leap_mail_received_header}) + + mail = PixelatedMail.from_soledad(*leap_mail, soledad_querier=self.querier) + + self.assertEqual(mail.headers['Date'], date_expected) + def test_update_tags_return_a_set_with_the_current_tags(self): soledad_docs = test_helper.leap_mail(extra_headers={'X-tags': '["custom_1", "custom_2"]'}) pixelated_mail = PixelatedMail.from_soledad(*soledad_docs, soledad_querier=self.querier) @@ -334,6 +357,17 @@ class TestPixelatedMail(unittest.TestCase): mail.as_dict() + def test_parse_UTF8_headers_with_CharsetAscii(self): + leap_mail_from = u'"söme ümläuds" <lisa5@dev.pixelated-project.org>' + leap_mail_to = u'"söme ümläuds" <lisa5@dev.pixelated-project.org>,\n"söme ümläuds" <lisa5@dev.pixelated-project.org>' + + leap_mail = test_helper.leap_mail(extra_headers={'From': leap_mail_from, 'Subject': "some subject", 'To': leap_mail_to}) + + mail = PixelatedMail.from_soledad(*leap_mail, soledad_querier=self.querier) + + mail.headers['From'].encode('ascii') + self.assertEqual(mail.headers['To'], ['"sme mluds" <lisa5@dev.pixelated-project.org>', '"sme mluds" <lisa5@dev.pixelated-project.org>']) + def simple_mail_dict(): return { diff --git a/service/test/unit/bitmask_libraries/test_certs.py b/service/test/unit/bitmask_libraries/test_certs.py index ba56d5c2..4a06649d 100644 --- a/service/test/unit/bitmask_libraries/test_certs.py +++ b/service/test/unit/bitmask_libraries/test_certs.py @@ -12,12 +12,12 @@ class CertsTest(unittest.TestCase): def test_that_which_bootstrap_cert_bundle_returns_byte_string(self, mock_isdir, mock_isfile): mock_isfile.return_value = True mock_isdir.return_value = True - config = MagicMock(bootstrap_ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, certs_home='/some/path') + config = MagicMock(bootstrap_ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, leap_home='/leap/home') provider = MagicMock(server_name=u'test.leap.net', config=config) bundle = which_bootstrap_CA_bundle(provider) - self.assertEqual('/some/path/test.leap.net.ca.crt', bundle) + self.assertEqual('/leap/home/providers/test.leap.net/test.leap.net.ca.crt', bundle) self.assertEqual(str, type(bundle)) @patch('pixelated.bitmask_libraries.certs.os.path.isfile') @@ -26,7 +26,7 @@ class CertsTest(unittest.TestCase): mock_isfile.return_value = True mock_isdir.return_value = True - config = MagicMock(bootstrap_ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, ca_cert_bundle=None, leap_home='/some/leap/home', certs_home='/some/path') + config = MagicMock(bootstrap_ca_cert_bundle=AUTO_DETECT_CA_BUNDLE, ca_cert_bundle=None, leap_home='/some/leap/home') provider = MagicMock(server_name=u'test.leap.net', config=config) bundle = which_api_CA_bundle(provider) |